From 16239bef00ddce62553f57a3adfb65191b2999da Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 12:49:53 +0800 Subject: [PATCH 001/148] Add fidelity-aware verification pilot --- ...idelity-aware-harness-headroom-20260714.md | 95 + ...idelity-aware-harness-protocol-20260714.md | 193 + runs/fidelity-headroom/analyze_existing.py | 508 +++ runs/fidelity-headroom/analyze_pilot.py | 293 ++ runs/fidelity-headroom/analyze_prefixes.py | 629 +++ runs/fidelity-headroom/freeze_models.py | 93 + runs/fidelity-headroom/frozen-models.json | 515 +++ runs/fidelity-headroom/metrics.json | 1142 +++++ runs/fidelity-headroom/pilot_controller.py | 436 ++ runs/fidelity-headroom/prefix-metrics.json | 3696 +++++++++++++++++ runs/fidelity-headroom/prepare_pilot.py | 351 ++ runs/fidelity-headroom/test_analysis.py | 40 + runs/fidelity-headroom/test_pilot_tools.py | 85 + .../fidelity-headroom/test_prefix_analysis.py | 72 + runs/opprof-phase6/opprof_phase6_client.py | 5 + src/aituner/slo.py | 1 + src/aituner/worker.py | 4 + tests/test_core_flow.py | 12 +- 18 files changed, 8165 insertions(+), 5 deletions(-) create mode 100644 docs/fidelity-aware-harness-headroom-20260714.md create mode 100644 docs/fidelity-aware-harness-protocol-20260714.md create mode 100644 runs/fidelity-headroom/analyze_existing.py create mode 100644 runs/fidelity-headroom/analyze_pilot.py create mode 100644 runs/fidelity-headroom/analyze_prefixes.py create mode 100644 runs/fidelity-headroom/freeze_models.py create mode 100644 runs/fidelity-headroom/frozen-models.json create mode 100644 runs/fidelity-headroom/metrics.json create mode 100644 runs/fidelity-headroom/pilot_controller.py create mode 100644 runs/fidelity-headroom/prefix-metrics.json create mode 100644 runs/fidelity-headroom/prepare_pilot.py create mode 100644 runs/fidelity-headroom/test_analysis.py create mode 100644 runs/fidelity-headroom/test_pilot_tools.py create mode 100644 runs/fidelity-headroom/test_prefix_analysis.py diff --git a/docs/fidelity-aware-harness-headroom-20260714.md b/docs/fidelity-aware-harness-headroom-20260714.md new file mode 100644 index 0000000..a3cb832 --- /dev/null +++ b/docs/fidelity-aware-harness-headroom-20260714.md @@ -0,0 +1,95 @@ +# Fidelity-aware harness headroom audit + +Status: **PROMISING PREMISE, NO CONTRIBUTION CLAIM**. + +The audit answers whether engine instrumentation has enough incremental signal +to justify a prospective experiment. It does not establish generalization. + +## Simulator shortlist lower bound + +On the frozen 12-cell SimFid task, the strongest calibrated SLO simulator +reading places TP2/MNS32 and TP2/MNS64 in the same first tie bucket. Real-final +evaluation of that two-cell bucket selects TP2/MNS32 and has zero real regret. +A method requiring a real calibration probe plus final verification cannot beat +two real cell evaluations on this task. Therefore “better initial selection” +is not a viable claim here; the remaining headroom is shorter real verification +inside the same shortlist. + +## Five-second prefix result + +The retrospective Phase-6 dataset contains 37 primary anchors across 12 cells. +Stable labels use the frozen same-placement 2-of-3 adjudication: 28 feasible and +9 infeasible. Three TP4 primary measurements disagree with their repeated +labels, so single-run feasibility is not treated as ground truth. + +Using leave-one-cell-out folds, identical L2 logistic models, and a 5-second +prefix: + +| Metric | Outcome-only | Instrumentation-aware | Delta | +|---|---:|---:|---:| +| Accuracy | 78.38% | 89.19% | +10.81 pp | +| Balanced accuracy | 70.63% | 81.55% | +10.92 pp | +| Brier score | 0.1297 | 0.0901 | -0.0396 | +| Correct only in this model | 0 | 4 | +4 | +| McNemar exact two-sided p | — | 0.125 | not significant | + +At the frozen conservative threshold 0.95, both policies make zero false +accepts and zero false rejects on this retrospective set. Outcome-only safely +cuts 36.35% of measured primary-trial cost; instrumentation-aware safely cuts +61.10%, an additional 24.75 percentage points. Regularization sensitivity for +accuracy delta is `[0.00, +10.81]` percentage points, so the sign is +non-negative but the magnitude is not stable. + +Longer prefixes do not strengthen the case monotonically. At 10 seconds, +headline accuracy is 91.89% outcome-only versus 89.19% instrumentation-aware; +at 15 seconds it is 88.89% versus 91.67%; at 20 seconds it is 86.11% versus +91.67%, but both 0.95 policies make one false reject. Five seconds is therefore +a training-selected operating point, not a test result. + +## Interpretation + +There is enough headroom to run a held-out pilot, but not enough evidence to +claim the harness contribution: + +- the 5-second cost gap is operationally large; +- only four paired classifications differ, so significance is absent; +- all examples share one workload/SLO/engine task; +- completion timestamps are reconstructed from arrival + TTFT + TPOT rather + than recorded directly; +- three adjudication disagreements are concentrated in transient TP4 runs; +- outcome-only already recovers the simulator shortlist oracle with very few + real cells. + +The next experiment must therefore freeze the 5-second model and threshold, +record exact monotonic completions, use a held-out trace, and label each anchor +with three full repetitions. The registered protocol is +`docs/fidelity-aware-harness-protocol-20260714.md`. + +## Artifacts + +- `runs/fidelity-headroom/analyze_existing.py` +- `runs/fidelity-headroom/metrics.json` +- `runs/fidelity-headroom/analyze_prefixes.py` +- `runs/fidelity-headroom/prefix-metrics.json` +- `runs/fidelity-headroom/test_analysis.py` +- `runs/fidelity-headroom/test_prefix_analysis.py` + +## Sanity block + +| Family | n | Min | Max | Distinct | Invariant/result | +|---|---:|---:|---:|---:|---| +| Real SimFid cell scores | 12 | 1.2833 | 3.2833 | 7 | Non-negative; not identical | +| Prefix examples at 5 s | 37 | 5 s | 5 s | 1 expected | All 12 cells represented | +| Adjudicated labels | 37 | 0 | 1 | 2 | 28 positive / 9 negative | +| Primary/adjudicated disagreement | 37 | 0 | 1 | 2 | 3 TP4 disagreements retained | +| Full primary elapsed time | 37 | 14.566 s | 62.064 s | 37 | Every 5 s prefix is in range | +| Outcome probability | 37 | in `[0,1]` | in `[0,1]` | >1 | Checked before metrics | +| Instrumentation probability | 37 | in `[0,1]` | in `[0,1]` | >1 | Checked before metrics | +| Layer-1 streams | 12 | 14,174 records | 58,725 records | 12 | Contiguous, zero drops | + +Checked invariants: same folds/model family and cutoff; no full verdict in a +feature; prefix-only Layer-1 slicing; non-negative costs/counters; bounded +ratios/probabilities; both labels present; per-config results not identical; +tie expansion before top-k; no imputation of non-monotonic frontiers. The main +limitation is reconstructed request completion time, explicitly marked on all +37 five-second examples. diff --git a/docs/fidelity-aware-harness-protocol-20260714.md b/docs/fidelity-aware-harness-protocol-20260714.md new file mode 100644 index 0000000..1bdb1b2 --- /dev/null +++ b/docs/fidelity-aware-harness-protocol-20260714.md @@ -0,0 +1,193 @@ +# Fidelity-aware real-verification harness protocol + +Status: **PRE-REGISTERED STAGED EVALUATION; CONTRIBUTION NOT YET ESTABLISHED**. + +Date frozen: 2026-07-14 (Asia/Singapore). + +## Research question and contribution bar + +The harness has an independent systems contribution only if engine-internal +instrumentation improves a tuning decision beyond what is already achievable +with a simulator shortlist and external benchmark outcomes. The intended +claim is therefore deliberately stronger than “telemetry explains a run”: + +> Given the same simulator ranking, the same candidate order, and the same +> short real-GPU probe, a learned instrumentation-aware verifier reaches a +> configuration with at most 5% real SLO-goodput regret using materially fewer +> H20-hours than both (a) simulator top-k followed by full real evaluation and +> (b) an outcome-only verifier given exactly the same probe. + +The paper-facing gate is: + +- at least 20% lower real-verification H20-hours than outcome-only calibration; +- at least 30% lower real-verification H20-hours than simulator top-k plus full + real final evaluation; +- paired 95% task-bootstrap confidence interval for the outcome-only cost + reduction strictly above zero; +- selected-configuration SLO-goodput regret at most 5% on every headline task; +- no false-safe early accept in the pilot and at most 1% in the expanded suite; +- profiling, warm-up, confirmation, instrumentation, and failed-run costs are + included rather than amortized away. An amortized profile-cost view may be + reported only as a secondary result. + +If these conditions fail, instrumentation remains a debugging facility. It is +not an independent tuning-harness contribution. + +## What is learned, and what is not a rule + +The decision target is a stable, repeated real verdict, not a hand-authored +diagnosis such as “queue length above N means reject.” Each anchor receives +three full real repetitions and a frozen 2-of-3 feasibility label. A nested +pair of regularized models predicts that label from a fixed prefix: + +- **Outcome-only input X:** configuration, offered rate, admitted/completed + progress, observed TTFT/TPOT margins, failures, and known workload lengths. +- **Instrumentation input Z:** the same X plus generic engine state: running and + waiting queues, decode-batch shape, KV usage, graph mode and padding, prefill + share, preemptions, and model-step rate. + +Both models use the same L2 logistic family, train split, standardization, +regularization, cutoff, and probability threshold. The only experimental +difference is Z. The initial family is intentionally simple: a positive result +then demonstrates value in the engine signal rather than capacity in a larger +learner. A sequence model is admissible only as a later, paired ablation. + +The frozen first policy uses a 5-second prefix, L2 regularization 1.0, and a +two-sided abstaining threshold of 0.95: accept at `p(feasible)>=0.95`, reject at +`p(feasible)<=0.05`, otherwise continue the exact same trial to completion. +Threshold and cutoff were selected on the historical training task and are +therefore not evidence; all claims come from subsequent held-out tasks. + +## Fair baselines + +| Method | Simulator | 5-second real prefix | External outcomes | Engine state | Full real continuation | +|---|---:|---:|---:|---:|---:| +| Real-only oracle | no | no | full | optional diagnostic | every candidate/anchor | +| Sim top-k + real final | yes | included in full run | full | no decision use | every shortlisted candidate/anchor | +| Outcome-only calibration | yes | yes | yes | no | only on abstention | +| Instrumentation-aware | yes | yes | yes | yes | only on abstention | + +Tie buckets are expanded before top-k. `k` is selected on training tasks and +is fixed on held-out tasks; an oracle per-task k is forbidden. Outcome-only +receives all information available outside the engine, including config and +workload features. Instrumentation cannot use any record submitted after the +cutoff. The full label, confirmation votes, simulator error, and later +requests are never model features. + +## Staged experiment + +### R0: historical premise and headroom audit + +The frozen SimFid surface has 12 cells. The strongest calibrated SLO simulator +reading has a top tie bucket `{TP2/MNS32, TP2/MNS64}`; full real evaluation of +those two cells already finds the oracle with zero regret. Consequently this +single task cannot demonstrate a selection-count advantage: any method needing +one real calibration probe and one real final verification has a lower bound of +two real cells. + +The viable estimand is instead the duration and number of full real frontier +evaluations inside a fixed shortlist. Historical Phase-6 prefixes are analyzed +only as training/premise data. Their request completion times are reconstructed +from arrival, TTFT, TPOT, and token count, so they cannot support a final claim. + +### P1: exact-timestamp prospective pilot + +- Engine/model/hardware: patched vLLM 0.24.1.dev3, Qwen3-30B-A3B, one solo + server/client on dash0, NVIDIA H20, `TP in {1,2,4}`. +- Held-out workload: `chat_w20260312_1000`, 60-second replay after the frozen + 0.1 time scale, raw input `[0,8192]`, exactly 128 output tokens. +- SLO: stepped TTFT 2/4/6 seconds, TPOT 50 ms, 95% request pass rate. +- Cells: TP1/MNS8, TP1/MNS64, TP2/MNS8, TP2/MNS64, TP4/MNS16, TP4/MNS64. +- Per cell: one attainable low offered rate near 0.85x the historical v0.24 + frontier and one high rate near 1.25x. The exact threshold and selected + request hashes are frozen by a CPU preflight before launch. +- Each cell uses a fresh server, the accepted long-request warm-up, one + unmeasured full-window burn-in, then three repetitions per rate. Rate order + alternates and reverses across cells to prevent a fixed warm-state/order + confound. +- The first repetition supplies the exact prefix. All three repetitions supply + the 2-of-3 label. Every request records a monotonic completion timestamp; + Layer-1 records are cut at the same monotonic boundary. +- Placement is serialized. Co-location is forbidden because Phase 6 observed + up to 92.86 percentage-point pass-rate shifts under co-location. +- Hard cap: 3.5 H20-hours, including startup, warm-up, burn-in, all repetitions, + failures, and cleanup. Projected cap violation stops before the next cell. + +P1 opens P2 only if all data invariants pass and instrumentation-aware has zero +false accept/reject, is no worse than outcome-only, and either makes at least +three additional correct early decisions or improves total valid trial-cost +reduction by at least 15 absolute percentage points. The pilot is a gate, not +paper evidence. + +### P2: held-out task replication + +If P1 passes, freeze the model and run at least six independent task groups: +three trace windows spanning distinct date/slot combinations and two SLO +regimes. No task used for threshold/model selection enters the headline test. +The candidate surface is the full 12-cell `TP={1,2,4} x MNS={8,16,32,64}` +surface. Splits are by complete task, never by anchor or request. A task-level +paired bootstrap (10,000 repetitions, fixed seed) estimates cost and regret +intervals. Non-monotonic or split 2-of-3 anchors remain explicit; no frontier +is imputed. + +### P3: end-to-end shortlist and search replay + +For each P2 task, run the same frozen simulator and tie-expanded top-k policy. +Replay the real binary/frontier search under all three verification policies: +full real, outcome-only, and instrumentation-aware. The policy consumes only +prefixes that would have been available at that decision point. Report: + +- selected cell and real SLO-goodput regret; +- number of real cells, anchors, and confirmations; +- measured H20-hours and wall time; +- false accept, false reject, and abstention counts; +- profile, startup/warm-up, probe, full-continuation, confirmation, logging, and + failure cost breakdowns. + +### P4: simulator-rank-error attribution + +This phase distinguishes an outdated implementation/profile from a structural +simulator limitation. For each held-out task compare: + +1. the original simulator/profile; +2. a version-matched re-profiled simulator; +3. a trajectory-conditioned run supplied with the realized arrival and request + length sequence; +4. outcome-only residual calibration; +5. instrumentation-aware residual calibration. + +The engine trace is extended only as needed with a worker-level step UID and +CUDA-event duration, because current async submit-to-complete spans overlap and +are not GPU step time. Residuals are decomposed into operator-profile error, +scheduler/state error, and run-to-run noise. If re-profiling alone restores the +ranking, the old 30% loss was an implementation/profile defect. If exact +profiles and realized trajectories still mis-rank cells, and the residual is +systematically explained by queue/KV/graph/batch state unavailable to the +simulator, that is evidence of a structural state-abstraction gap. Correlation +alone is not called causal. + +## Failure modes that reject the route + +- Outcome-only matches or beats instrumentation-aware under the same cutoff. +- Instrumentation gains average accuracy but introduces false-safe decisions. +- Gains disappear under task-level rather than request/anchor-level splitting. +- Savings come only from excluding startup, warm-up, profiling, confirmations, + or failed trials. +- A different cutoff/threshold must be selected after seeing each test task. +- The simulator top-k baseline already reaches the target with equal or lower + total H20-hours. +- Exact instrumentation overhead exceeds 1% throughput or materially changes + p95/p99 latency. +- Results depend on TP4 transient/non-monotonic trials and do not replicate on + held-out tasks. + +## Data sanity contract + +Every analysis ends with n, min/max, distinct count, label balance, and these +invariants: non-negative counters/costs; probabilities and ratios in `[0,1]`; +per-config results not all identical; timestamps monotonic; every prefix record +at or before its cutoff; selected request ID/arrival/length hashes stable across +repetitions; exact 128-token completion or counted failure; no dropped Layer-1 +records; 2-of-3 labels reproducible; no co-resident GPU process; total H20-hours +below the hard cap; final GPUs idle. A red flag is reported first and blocks +the contribution claim. diff --git a/runs/fidelity-headroom/analyze_existing.py b/runs/fidelity-headroom/analyze_existing.py new file mode 100644 index 0000000..7fcc74f --- /dev/null +++ b/runs/fidelity-headroom/analyze_existing.py @@ -0,0 +1,508 @@ +#!/usr/bin/env python3 +"""Retrospective headroom audit for a fidelity-aware tuning harness. + +This analysis intentionally separates two questions: + +1. How many real cell evaluations does a simulator top-k shortlist already + need to recover the real optimum on the frozen SimFid surface? +2. On the P6 anchor ladder, do Layer-1 engine features predict the next + anchor's feasibility better than outcome-only features from the same + current anchor? + +The second question is diagnostic rather than decision-bearing: it uses a +small, already-observed single-workload surface and full current-anchor +summaries. It is a premise check for a future prospective early-probe study. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable + +import numpy as np + + +SCHEMA = "fidelity-headroom-v1" +DEFAULT_REGULARIZATION = 1.0 +REGULARIZATION_SENSITIVITY = (0.1, 1.0, 10.0) +BOOTSTRAP_SEED = 20260714 +BOOTSTRAP_REPLICATES = 10_000 + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def numeric(values: Iterable[float | int]) -> dict[str, Any]: + array = [float(value) for value in values] + return { + "n": len(array), + "min": min(array) if array else None, + "max": max(array) if array else None, + "distinct_n": len(set(array)), + } + + +def score_buckets(scores: dict[str, float], tolerance: float) -> dict[str, int]: + if tolerance <= 0: + raise ValueError("score tolerance must be positive") + return {cell: math.floor(float(score) / tolerance) for cell, score in scores.items()} + + +def topk_curve( + real_scores: dict[str, float], + simulated_scores: dict[str, float], + tolerance: float, +) -> dict[str, Any]: + if set(real_scores) != set(simulated_scores): + raise ValueError("real and simulator score cells differ") + buckets = score_buckets(simulated_scores, tolerance) + ordered = sorted( + simulated_scores, + key=lambda cell: (-buckets[cell], -float(simulated_scores[cell]), cell), + ) + real_best = max(float(value) for value in real_scores.values()) + points = [] + for nominal_k in range(1, len(ordered) + 1): + cutoff_bucket = buckets[ordered[nominal_k - 1]] + candidates = [cell for cell in ordered if buckets[cell] >= cutoff_bucket] + selected = max(candidates, key=lambda cell: (float(real_scores[cell]), cell)) + selected_score = float(real_scores[selected]) + points.append( + { + "nominal_k": nominal_k, + "expanded_k": len(candidates), + "candidates": candidates, + "selected_cell_after_real_final": selected, + "selected_real_score": selected_score, + "real_regret": 1.0 - selected_score / real_best, + } + ) + + minimum_k = {} + for name, threshold in (("zero", 1e-15), ("one_percent", 0.01), ("five_percent", 0.05)): + eligible = [point for point in points if point["real_regret"] <= threshold] + minimum_k[name] = ( + { + "nominal_k": eligible[0]["nominal_k"], + "expanded_k": eligible[0]["expanded_k"], + } + if eligible + else None + ) + return { + "real_best": real_best, + "minimum_k": minimum_k, + "points": points, + } + + +@dataclass(frozen=True) +class Transition: + cell: str + current_anchor: float + next_anchor: float + external: tuple[float, ...] + instrumentation: tuple[float, ...] + next_feasible: int + + +EXTERNAL_FEATURES = ( + "log_current_rate_per_gpu", + "log_next_over_current_rate", + "log2_tp", + "log2_mns", + "current_pass_rate", + "ttft_max_over_6s", + "tpot_max_over_50ms", + "exact_output_fraction", + "early_stopped", +) + +INSTRUMENTATION_FEATURES = ( + "waiting_mean", + "waiting_max", + "decode_batch_mean", + "decode_batch_cv", + "kv_usage_mean", + "kv_usage_max", + "graph_none_share", + "graph_full_share", + "padding_fraction", + "prefill_token_fraction", + "model_steps_per_second", +) + + +def _finite(value: float | int | None) -> float: + if value is None: + return 0.0 + result = float(value) + if not math.isfinite(result): + raise ValueError(f"non-finite feature: {value}") + return result + + +def build_transitions(phase6: dict[str, Any]) -> list[Transition]: + transitions = [] + for cell, cell_result in sorted(phase6["cells"].items()): + anchors = sorted(cell_result["anchors"], key=lambda item: float(item["anchor"])) + for current, following in zip(anchors, anchors[1:]): + if following["accepted_feasible"] is None: + continue + primary = current["primary"] + next_primary = following["primary"] + layer = current["layer1"] + rate = float(primary["selection"]["offered_req_s_per_gpu"]) + next_rate = float(next_primary["selection"]["offered_req_s_per_gpu"]) + selected_count = int(primary["selection"]["count"]) + if rate <= 0 or next_rate <= 0 or selected_count <= 0: + raise ValueError("rates and selected counts must be positive") + external = ( + math.log(rate), + math.log(next_rate / rate), + math.log2(float(cell_result["tp"])), + math.log2(float(cell_result["mns"])), + float(primary["pass_rate"]), + _finite(primary["ttft_ms"]["max"]) / 6000.0, + _finite(primary["tpot_ms"]["max"]) / 50.0, + float(primary["exact_output_count"]) / selected_count, + float(bool(primary["early_stopped"])), + ) + graph_shares = layer.get("graph_mode_shares", {}) + prefill_tokens = _finite(layer["prefill_tokens"]) + decode_tokens = _finite(layer["decode_tokens"]) + instrumentation = ( + _finite(layer["waiting_mean"]), + _finite(layer["waiting_max"]), + _finite(layer["decode_B_mean"]), + _finite(layer["decode_B_cv"]), + _finite(layer["kv_usage_mean"]), + _finite(layer["kv_usage_max"]), + float(graph_shares.get("NONE", 0.0)), + float(graph_shares.get("FULL", 0.0)), + _finite(layer["padding_fraction"]), + prefill_tokens / max(1.0, prefill_tokens + decode_tokens), + _finite(layer["model_steps"]) / float(primary["interval"]["elapsed_s"]), + ) + transitions.append( + Transition( + cell=cell, + current_anchor=float(current["anchor"]), + next_anchor=float(following["anchor"]), + external=external, + instrumentation=instrumentation, + next_feasible=int(bool(following["accepted_feasible"])), + ) + ) + return transitions + + +def _sigmoid(values: np.ndarray) -> np.ndarray: + clipped = np.clip(values, -30.0, 30.0) + return 1.0 / (1.0 + np.exp(-clipped)) + + +def _fit_logistic(x: np.ndarray, y: np.ndarray, regularization: float) -> np.ndarray: + weights = np.zeros(x.shape[1], dtype=np.float64) + penalty = np.eye(x.shape[1], dtype=np.float64) + penalty[0, 0] = 0.0 + for _ in range(100): + probability = _sigmoid(x @ weights) + gradient = x.T @ (probability - y) / len(y) + gradient += regularization * penalty @ weights / len(y) + curvature = probability * (1.0 - probability) + hessian = (x.T * curvature) @ x / len(y) + hessian += regularization * penalty / len(y) + step = np.linalg.lstsq(hessian, gradient, rcond=None)[0] + weights -= step + if float(np.max(np.abs(step))) < 1e-9: + break + return weights + + +def _classification_metrics(y: np.ndarray, probability: np.ndarray) -> dict[str, Any]: + if np.any(probability < 0.0) or np.any(probability > 1.0): + raise ValueError("classification probabilities must be in [0, 1]") + prediction = probability >= 0.5 + true_positive = int(np.sum(prediction & (y == 1))) + true_negative = int(np.sum(~prediction & (y == 0))) + false_positive = int(np.sum(prediction & (y == 0))) + false_negative = int(np.sum(~prediction & (y == 1))) + positive_total = true_positive + false_negative + negative_total = true_negative + false_positive + balanced = 0.5 * ( + true_positive / positive_total + true_negative / negative_total + ) + clipped = np.clip(probability, 1e-12, 1.0 - 1e-12) + return { + "accuracy": float(np.mean(prediction == y)), + "balanced_accuracy": float(balanced), + "brier": float(np.mean((probability - y) ** 2)), + "log_loss": float(np.mean(-(y * np.log(clipped) + (1 - y) * np.log(1 - clipped)))), + "confusion": { + "true_positive": true_positive, + "true_negative": true_negative, + "false_positive": false_positive, + "false_negative": false_negative, + }, + } + + +def _mcnemar_exact_p(outcome_only_correct: int, instrumentation_only_correct: int) -> float: + discordant = outcome_only_correct + instrumentation_only_correct + if discordant == 0: + return 1.0 + tail = sum( + math.comb(discordant, value) + for value in range(min(outcome_only_correct, instrumentation_only_correct) + 1) + ) / (2**discordant) + return min(1.0, 2.0 * tail) + + +def grouped_predictions( + transitions: list[Transition], + *, + instrumentation_aware: bool, + regularization: float, +) -> tuple[np.ndarray, np.ndarray, list[str]]: + probabilities = [] + labels = [] + test_cells = [] + for held_out in sorted({transition.cell for transition in transitions}): + train = [transition for transition in transitions if transition.cell != held_out] + test = [transition for transition in transitions if transition.cell == held_out] + + def row(transition: Transition) -> np.ndarray: + values = transition.external + if instrumentation_aware: + values += transition.instrumentation + return np.asarray((1.0, *values), dtype=np.float64) + + x_train = np.stack([row(transition) for transition in train]) + x_test = np.stack([row(transition) for transition in test]) + y_train = np.asarray([transition.next_feasible for transition in train], dtype=np.float64) + mean = x_train[:, 1:].mean(axis=0) + standard_deviation = x_train[:, 1:].std(axis=0) + standard_deviation[standard_deviation < 1e-8] = 1.0 + x_train[:, 1:] = (x_train[:, 1:] - mean) / standard_deviation + x_test[:, 1:] = (x_test[:, 1:] - mean) / standard_deviation + weights = _fit_logistic(x_train, y_train, regularization) + probabilities.extend(_sigmoid(x_test @ weights).tolist()) + labels.extend(transition.next_feasible for transition in test) + test_cells.extend(held_out for _ in test) + return ( + np.asarray(labels, dtype=np.int64), + np.asarray(probabilities, dtype=np.float64), + test_cells, + ) + + +def _group_bootstrap_delta( + y: np.ndarray, + outcome_probability: np.ndarray, + instrumentation_probability: np.ndarray, + cells: list[str], +) -> dict[str, Any]: + groups = sorted(set(cells)) + indices = {group: np.asarray([i for i, cell in enumerate(cells) if cell == group]) for group in groups} + random = np.random.default_rng(BOOTSTRAP_SEED) + accuracy_deltas = [] + brier_deltas = [] + for _ in range(BOOTSTRAP_REPLICATES): + sampled = random.choice(groups, size=len(groups), replace=True) + selected = np.concatenate([indices[group] for group in sampled]) + selected_y = y[selected] + outcome = outcome_probability[selected] + instrumentation = instrumentation_probability[selected] + accuracy_deltas.append( + float(np.mean((instrumentation >= 0.5) == selected_y)) + - float(np.mean((outcome >= 0.5) == selected_y)) + ) + brier_deltas.append( + float(np.mean((instrumentation - selected_y) ** 2)) + - float(np.mean((outcome - selected_y) ** 2)) + ) + return { + "semantics": "group bootstrap over cells; diagnostic confidence interval", + "replicates": BOOTSTRAP_REPLICATES, + "seed": BOOTSTRAP_SEED, + "accuracy_delta_instrumentation_minus_outcome": { + "point": float(np.mean((instrumentation_probability >= 0.5) == y)) + - float(np.mean((outcome_probability >= 0.5) == y)), + "ci95": [float(x) for x in np.percentile(accuracy_deltas, [2.5, 97.5])], + }, + "brier_delta_instrumentation_minus_outcome": { + "point": float(np.mean((instrumentation_probability - y) ** 2)) + - float(np.mean((outcome_probability - y) ** 2)), + "ci95": [float(x) for x in np.percentile(brier_deltas, [2.5, 97.5])], + }, + } + + +def transition_analysis(transitions: list[Transition]) -> dict[str, Any]: + sensitivity = {} + headline_payload = None + for regularization in REGULARIZATION_SENSITIVITY: + y, outcome_probability, cells = grouped_predictions( + transitions, + instrumentation_aware=False, + regularization=regularization, + ) + instrumentation_y, instrumentation_probability, instrumentation_cells = grouped_predictions( + transitions, + instrumentation_aware=True, + regularization=regularization, + ) + if not np.array_equal(y, instrumentation_y) or cells != instrumentation_cells: + raise AssertionError("model folds or labels differ") + outcome_correct = (outcome_probability >= 0.5) == y + instrumentation_correct = (instrumentation_probability >= 0.5) == y + payload = { + "outcome_only": _classification_metrics(y, outcome_probability), + "instrumentation_aware": _classification_metrics(y, instrumentation_probability), + "paired_correctness": { + "both_correct": int(np.sum(outcome_correct & instrumentation_correct)), + "outcome_only_correct": int(np.sum(outcome_correct & ~instrumentation_correct)), + "instrumentation_only_correct": int(np.sum(~outcome_correct & instrumentation_correct)), + "both_wrong": int(np.sum(~outcome_correct & ~instrumentation_correct)), + }, + "bootstrap": _group_bootstrap_delta( + y, + outcome_probability, + instrumentation_probability, + cells, + ), + } + payload["paired_correctness"]["mcnemar_exact_two_sided_p"] = _mcnemar_exact_p( + payload["paired_correctness"]["outcome_only_correct"], + payload["paired_correctness"]["instrumentation_only_correct"], + ) + sensitivity[str(regularization)] = payload + if regularization == DEFAULT_REGULARIZATION: + headline_payload = payload + assert headline_payload is not None + labels = [transition.next_feasible for transition in transitions] + accuracy_deltas = [ + value["instrumentation_aware"]["accuracy"] - value["outcome_only"]["accuracy"] + for value in sensitivity.values() + ] + brier_deltas = [ + value["instrumentation_aware"]["brier"] - value["outcome_only"]["brier"] + for value in sensitivity.values() + ] + return { + "status": "RETROSPECTIVE_DIAGNOSTIC_ONLY", + "estimand": "next-anchor feasibility from the full current-anchor summary", + "split": "leave-one-cell-out", + "model": "L2 logistic regression with train-fold standardization", + "external_features": list(EXTERNAL_FEATURES), + "instrumentation_features": list(INSTRUMENTATION_FEATURES), + "headline_regularization": DEFAULT_REGULARIZATION, + "headline": headline_payload, + "regularization_sensitivity": sensitivity, + "sensitivity_summary": { + "accuracy_delta_min_max": [min(accuracy_deltas), max(accuracy_deltas)], + "brier_delta_min_max": [min(brier_deltas), max(brier_deltas)], + "incremental_signal_verdict": "NEEDS_PROSPECTIVE_EVIDENCE", + }, + "label_sanity": { + **numeric(labels), + "positive": sum(labels), + "negative": len(labels) - sum(labels), + }, + } + + +def analyze(simfid_path: Path, phase6_path: Path) -> dict[str, Any]: + simfid = json.loads(simfid_path.read_text()) + phase6 = json.loads(phase6_path.read_text()) + real_scores = {cell: float(score) for cell, score in simfid["real_scores"].items()} + topk = {} + for reading, payload in sorted(simfid["analyses"].items()): + tie = payload["metrics"]["tie_buckets"]["simulator"] + topk[reading] = topk_curve( + real_scores, + {cell: float(score) for cell, score in payload["simulated_scores"].items()}, + float(tie["tolerance"]), + ) + transitions = build_transitions(phase6) + transition_result = transition_analysis(transitions) + red_flags = [] + if len(real_scores) != 12: + red_flags.append("unexpected_simfid_cell_count") + if len(transitions) == 0 or len(set(x.next_feasible for x in transitions)) != 2: + red_flags.append("transition_labels_missing_or_single_class") + if any(not math.isfinite(value) or value < 0 for value in real_scores.values()): + red_flags.append("invalid_real_score") + return { + "schema": SCHEMA, + "status": "PASS" if not red_flags else "STOP", + "scope": "retrospective single-workload premise audit; not prospective contribution evidence", + "provenance": { + "simfid_metrics": str(simfid_path.resolve()), + "simfid_sha256": sha256_file(simfid_path), + "phase6_metrics": str(phase6_path.resolve()), + "phase6_sha256": sha256_file(phase6_path), + }, + "topk_headroom": topk, + "next_anchor_prediction": transition_result, + "decision": { + "current_surface_can_show_selection_contribution": False, + "reason": ( + "The strongest frozen-calibrated SLO reading reaches zero real regret " + "after real evaluation of its first two-cell tie bucket. A method that " + "requires one calibration probe and one final verification cannot use " + "this single task to demonstrate fewer real cell evaluations." + ), + "prospective_target": ( + "Test whether internal features from a short, shared real probe reduce " + "the number or duration of full frontier evaluations relative to an " + "outcome-only model given the same probe." + ), + }, + "sanity": { + "real_scores": numeric(real_scores.values()), + "simulator_readings": len(topk), + "transitions": len(transitions), + "transition_cells": len({transition.cell for transition in transitions}), + "red_flags": red_flags, + "invariants": { + "same_cells_all_readings": all( + set(payload["simulated_scores"]) == set(real_scores) + for payload in simfid["analyses"].values() + ), + "scores_nonnegative": all(value >= 0 for value in real_scores.values()), + "transition_features_finite": all( + all(math.isfinite(value) for value in (*item.external, *item.instrumentation)) + for item in transitions + ), + "probabilities_bounded": True, + }, + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--simfid-metrics", type=Path, required=True) + parser.add_argument("--phase6-metrics", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + result = analyze(args.simfid_metrics, args.phase6_metrics) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") + print(json.dumps({"status": result["status"], "output": str(args.output)}, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/runs/fidelity-headroom/analyze_pilot.py b/runs/fidelity-headroom/analyze_pilot.py new file mode 100644 index 0000000..3797e18 --- /dev/null +++ b/runs/fidelity-headroom/analyze_pilot.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +"""Evaluate frozen outcome-only and instrumentation-aware policies on P1.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +import numpy as np + +from analyze_existing import _classification_metrics, _mcnemar_exact_p +from analyze_prefixes import ( + PrefixExample, + _load_jsonl, + _prefix_features, + numeric, + policy_metrics, + predict_frozen_model, + sha256_file, +) + + +def result_path(run_root: Path, cell: str, level: str, replicate: int) -> Path: + return run_root / "cells" / cell / f"{level}-rep{replicate}" / "result.json" + + +def requests_path(run_root: Path, cell: str, level: str, replicate: int) -> Path: + return run_root / "cells" / cell / f"{level}-rep{replicate}" / "requests.jsonl" + + +def selection_for( + manifest: dict[str, Any], cell: str, level: str, replicate: int +) -> dict[str, Any]: + role = f"{level}{replicate}" + return manifest["cells"][cell]["targets"][level]["selections"][role] + + +def build_pilot_examples( + manifest: dict[str, Any], run_root: Path, cutoff_s: float +) -> tuple[list[PrefixExample], list[dict[str, Any]], list[str]]: + examples = [] + details = [] + red_flags = [] + for cell, config in sorted(manifest["cells"].items()): + stream_path = next((run_root / "cells" / cell / "opprof").glob("*.jsonl")) + stream = _load_jsonl(stream_path, require_key="submit_mono_ns") + for level in ("low", "high"): + results = [ + json.loads(result_path(run_root, cell, level, replicate).read_text()) + for replicate in (1, 2, 3) + ] + votes = [bool(result["feasible"]) for result in results] + adjudicated = sum(votes) >= 2 + primary = results[0] + requests = _load_jsonl(requests_path(run_root, cell, level, 1)) + exact_timestamps = sum( + request.get("completed_elapsed_s") is not None for request in requests + ) + actual_outcomes = sum( + request.get("completed_mono_ns") is not None for request in requests + ) + if exact_timestamps != actual_outcomes: + red_flags.append(f"timestamp_count_mismatch_{cell}_{level}") + expected = selection_for(manifest, cell, level, 1) + if int(primary["selection"]["count"]) != int(expected["selected_count"]): + red_flags.append(f"selection_count_mismatch_{cell}_{level}") + for result_key, manifest_key in ( + ("request_id_order_sha256", "request_id_order_sha256"), + ("arrival_order_sha256", "arrival_order_sha256"), + ("raw_length_order_sha256", "input_length_order_sha256"), + ): + if primary["selection"][result_key] != expected[manifest_key]: + red_flags.append(f"selection_hash_mismatch_{cell}_{level}_{result_key}") + start_ns = int(primary["interval"]["start_mono_ns"]) + end_ns = start_ns + int(cutoff_s * 1e9) + records = [ + record + for record in stream + if record.get("model_executed") + and start_ns <= int(record["submit_mono_ns"]) <= end_ns + ] + outcome, instrumentation, completion_source = _prefix_features( + primary=primary, + tp=int(config["tp"]), + max_num_seqs=int(config["mns"]), + requests=requests, + records=records, + cutoff_s=cutoff_s, + ) + example = PrefixExample( + cell=cell, + anchor=float(primary["anchor"]), + cutoff_s=cutoff_s, + tp=int(config["tp"]), + full_elapsed_s=float(primary["interval"]["elapsed_s"]), + feasible=int(adjudicated), + primary_feasible=int(bool(primary["feasible"])), + outcome=outcome, + instrumentation=instrumentation, + completion_time_source=completion_source, + ) + examples.append(example) + details.append( + { + "cell": cell, + "level": level, + "anchor_rep1": primary["anchor"], + "selected_count_rep1": primary["selection"]["count"], + "votes": votes, + "pass_rates": [result["pass_rate"] for result in results], + "adjudicated_feasible": adjudicated, + "primary_feasible": bool(primary["feasible"]), + "actual_timestamped_outcomes": actual_outcomes, + "selected_outcomes": len(requests), + "prefix_layer1_records": len(records), + "completion_time_source": completion_source, + } + ) + return examples, details, red_flags + + +def analyze( + manifest_path: Path, + model_path: Path, + run_root: Path, +) -> dict[str, Any]: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + models = json.loads(model_path.read_text(encoding="utf-8")) + state_path = run_root / "controller-state.json" + state = json.loads(state_path.read_text(encoding="utf-8")) + cutoff_s = float(models["cutoff_s"]) + threshold = float(models["accept_probability"]) + examples, details, red_flags = build_pilot_examples(manifest, run_root, cutoff_s) + labels = np.asarray([example.feasible for example in examples], dtype=np.int64) + outcome_probability = predict_frozen_model(models["models"]["outcome_only"], examples) + instrumentation_probability = predict_frozen_model( + models["models"]["instrumentation_aware"], examples + ) + outcome_policy = policy_metrics( + examples, labels, outcome_probability, threshold + ) + instrumentation_policy = policy_metrics( + examples, labels, instrumentation_probability, threshold + ) + outcome_correct = (outcome_probability >= 0.5) == labels + instrumentation_correct = (instrumentation_probability >= 0.5) == labels + paired = { + "both_correct": int(np.sum(outcome_correct & instrumentation_correct)), + "outcome_only_correct": int(np.sum(outcome_correct & ~instrumentation_correct)), + "instrumentation_only_correct": int(np.sum(~outcome_correct & instrumentation_correct)), + "both_wrong": int(np.sum(~outcome_correct & ~instrumentation_correct)), + } + paired["mcnemar_exact_two_sided_p"] = _mcnemar_exact_p( + paired["outcome_only_correct"], paired["instrumentation_only_correct"] + ) + for detail, outcome_p, instrumentation_p in zip( + details, outcome_probability, instrumentation_probability + ): + detail["outcome_probability_feasible"] = float(outcome_p) + detail["instrumentation_probability_feasible"] = float(instrumentation_p) + + positive = int(np.sum(labels)) + negative = len(labels) - positive + if state["status"] != "complete" or int(state["completed_cells"]) != 6: + red_flags.append("campaign_incomplete") + if positive < 3 or negative < 3: + red_flags.append("insufficient_label_balance") + if any( + detail["actual_timestamped_outcomes"] == 0 for detail in details + ): + red_flags.append("no_exact_request_timestamps") + if float(state["gpu_hours_total"]) >= float(state["hard_cap_h20_hours"]): + red_flags.append("hard_cap_exceeded") + + outcome_errors = outcome_policy["false_accept"] + outcome_policy["false_reject"] + instrumentation_errors = ( + instrumentation_policy["false_accept"] + + instrumentation_policy["false_reject"] + ) + outcome_decisions = outcome_policy["early_accept"] + outcome_policy["early_reject"] + instrumentation_decisions = ( + instrumentation_policy["early_accept"] + + instrumentation_policy["early_reject"] + ) + outcome_reduction = outcome_policy["valid_cost_reduction_fraction"] + instrumentation_reduction = instrumentation_policy["valid_cost_reduction_fraction"] + cost_delta = ( + instrumentation_reduction - outcome_reduction + if outcome_reduction is not None and instrumentation_reduction is not None + else None + ) + data_valid = not red_flags + safety_gate = instrumentation_errors == 0 and instrumentation_errors <= outcome_errors + incremental_gate = ( + instrumentation_decisions - outcome_decisions >= 3 + or (cost_delta is not None and cost_delta >= 0.15) + ) + pilot_pass = data_valid and safety_gate and incremental_gate + + return { + "schema": "fidelity-prefix-pilot-result-v1", + "status": "PILOT_PASS" if pilot_pass else "PILOT_FAIL", + "scope": "held-out single-task gate; not paper-facing contribution evidence", + "provenance": { + "manifest": str(manifest_path.resolve()), + "manifest_sha256": sha256_file(manifest_path), + "frozen_models": str(model_path.resolve()), + "frozen_models_sha256": sha256_file(model_path), + "controller_state": str(state_path.resolve()), + "controller_state_sha256": sha256_file(state_path), + }, + "cutoff_s": cutoff_s, + "threshold": threshold, + "examples": details, + "outcome_only": { + "classification": _classification_metrics(labels, outcome_probability), + "policy": outcome_policy, + }, + "instrumentation_aware": { + "classification": _classification_metrics(labels, instrumentation_probability), + "policy": instrumentation_policy, + }, + "paired_correctness": paired, + "gate": { + "data_valid": data_valid, + "safety_gate": safety_gate, + "incremental_gate": incremental_gate, + "additional_early_decisions": instrumentation_decisions - outcome_decisions, + "valid_cost_reduction_fraction_delta": cost_delta, + "opens_expanded_p2": pilot_pass, + }, + "gpu": { + "actual_h20_hours": state["gpu_hours_total"], + "hard_cap_h20_hours": state["hard_cap_h20_hours"], + }, + "sanity": { + "red_flags": red_flags, + "labels": { + **numeric(labels.tolist()), + "positive": positive, + "negative": negative, + }, + "full_elapsed_s": numeric(example.full_elapsed_s for example in examples), + "remaining_h20_hours": numeric( + example.remaining_h20_hours for example in examples + ), + "outcome_probability": numeric(outcome_probability.tolist()), + "instrumentation_probability": numeric( + instrumentation_probability.tolist() + ), + "invariants": { + "examples_12": len(examples) == 12, + "cells_6": len({example.cell for example in examples}) == 6, + "ratios_bounded": bool( + np.all((outcome_probability >= 0) & (outcome_probability <= 1)) + and np.all( + (instrumentation_probability >= 0) + & (instrumentation_probability <= 1) + ) + ), + "costs_nonnegative": all( + example.remaining_h20_hours >= 0 for example in examples + ), + "all_cell_validations": all( + all(cell["validation"]["invariants"].values()) + for cell in state["cells"].values() + ), + }, + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--frozen-models", type=Path, required=True) + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + result = analyze(args.manifest, args.frozen_models, args.run_root) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") + print(json.dumps({ + "status": result["status"], + "gate": result["gate"], + "sanity_red_flags": result["sanity"]["red_flags"], + }, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/runs/fidelity-headroom/analyze_prefixes.py b/runs/fidelity-headroom/analyze_prefixes.py new file mode 100644 index 0000000..8fb480b --- /dev/null +++ b/runs/fidelity-headroom/analyze_prefixes.py @@ -0,0 +1,629 @@ +#!/usr/bin/env python3 +"""Retrospective, leakage-bounded audit of short real-probe prefixes. + +The outcome-only and instrumentation-aware models receive the same trial +prefix. The latter differs only by Layer-1 engine state. Existing Phase-6 +request artifacts predate exact completion timestamps, so their completion +time is reconstructed from arrival + TTFT + token intervals and is explicitly +marked approximate. New artifacts use ``completed_elapsed_s`` directly. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable + +import numpy as np + +from analyze_existing import ( + DEFAULT_REGULARIZATION, + REGULARIZATION_SENSITIVITY, + _classification_metrics, + _fit_logistic, + _group_bootstrap_delta, + _mcnemar_exact_p, + _sigmoid, +) + + +SCHEMA = "fidelity-prefix-v1" +DEFAULT_CUTOFFS = (5.0, 10.0, 15.0, 20.0) +POLICY_THRESHOLDS = (0.8, 0.9, 0.95) + +OUTCOME_FEATURES = ( + "log_offered_rate_per_gpu", + "log2_tp", + "log2_max_num_seqs", + "admitted_fraction", + "completed_over_admitted", + "completed_pass_rate", + "completed_fail_fraction_of_total", + "outstanding_over_admitted", + "ttft_max_over_slo_max", + "ttft_mean_over_slo_max", + "tpot_max_over_slo", + "tpot_mean_over_slo", + "admitted_input_tokens_mean_over_limit", +) + +INSTRUMENTATION_FEATURES = ( + "model_steps_per_second", + "waiting_mean", + "waiting_max", + "waiting_nonzero_share", + "running_mean", + "running_max", + "decode_batch_mean", + "decode_batch_max", + "decode_batch_cv", + "kv_usage_mean", + "kv_usage_max", + "kv_usage_end_minus_start", + "graph_none_share", + "graph_full_share", + "padding_fraction", + "prefill_token_fraction", + "preemptions", +) + + +@dataclass(frozen=True) +class PrefixExample: + cell: str + anchor: float + cutoff_s: float + tp: int + full_elapsed_s: float + feasible: int + primary_feasible: int + outcome: tuple[float, ...] + instrumentation: tuple[float, ...] + completion_time_source: str + + @property + def remaining_h20_hours(self) -> float: + return self.tp * max(0.0, self.full_elapsed_s - self.cutoff_s) / 3600.0 + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def numeric(values: Iterable[float | int]) -> dict[str, Any]: + array = [float(value) for value in values] + return { + "n": len(array), + "min": min(array) if array else None, + "max": max(array) if array else None, + "distinct_n": len(set(array)), + } + + +def _cv(values: list[float]) -> float: + if not values: + return 0.0 + array = np.asarray(values, dtype=np.float64) + mean = float(array.mean()) + return float(array.std(ddof=0) / mean) if mean else 0.0 + + +def completion_elapsed_s(request: dict[str, Any]) -> tuple[float | None, str]: + exact = request.get("completed_elapsed_s") + if exact is not None: + value = float(exact) + if value < 0 or not math.isfinite(value): + raise ValueError(f"invalid completed_elapsed_s={exact}") + return value, "exact_monotonic" + if not request.get("success"): + return None, "unobserved_failure" + required = ( + request.get("arrival_s"), + request.get("ttft_ms"), + request.get("tpot_ms"), + request.get("completion_tokens"), + ) + if any(value is None for value in required): + return None, "unobserved_failure" + arrival_s, ttft_ms, tpot_ms, completion_tokens = required + value = float(arrival_s) + ( + float(ttft_ms) + max(int(completion_tokens) - 1, 0) * float(tpot_ms) + ) / 1000.0 + if value < 0 or not math.isfinite(value): + raise ValueError(f"invalid reconstructed completion time={value}") + return value, "reconstructed_from_latency" + + +def _load_jsonl(path: Path, *, require_key: str | None = None) -> list[dict[str, Any]]: + records = [] + with path.open(encoding="utf-8") as source: + for line in source: + item = json.loads(line) + if require_key is None or require_key in item: + records.append(item) + return records + + +def _anchor_directory(cell_root: Path, anchor: float) -> Path: + matches = [] + for result_path in cell_root.glob("anchor-*/result.json"): + payload = json.loads(result_path.read_text(encoding="utf-8")) + if math.isclose(float(payload["anchor"]), anchor, rel_tol=0.0, abs_tol=1e-15): + matches.append(result_path.parent) + if len(matches) != 1: + raise ValueError(f"expected one primary directory for anchor {anchor}: {matches}") + return matches[0] + + +def _prefix_features( + *, + primary: dict[str, Any], + tp: int, + max_num_seqs: int, + requests: list[dict[str, Any]], + records: list[dict[str, Any]], + cutoff_s: float, +) -> tuple[tuple[float, ...], tuple[float, ...], str]: + admitted = [request for request in requests if float(request["arrival_s"]) <= cutoff_s] + completed = [] + sources = set() + for request in requests: + completed_s, source = completion_elapsed_s(request) + if completed_s is None or completed_s > cutoff_s: + continue + completed.append(request) + sources.add(source) + if not admitted or not records: + raise ValueError("prefix has no admitted requests or Layer-1 records") + if any(request not in admitted for request in completed): + raise ValueError("completed request was not admitted inside prefix") + + total = len(requests) + passed = sum(bool(request["slo_pass"]) for request in completed) + ttft = [float(request["ttft_ms"]) for request in completed if request["ttft_ms"] is not None] + tpot = [float(request["tpot_ms"]) for request in completed if request["tpot_ms"] is not None] + offered_rate = float(primary["selection"]["offered_req_s_per_gpu"]) + if offered_rate <= 0 or total <= 0: + raise ValueError("offered rate and selected request count must be positive") + + outcome = ( + math.log(offered_rate), + math.log2(float(tp)), + math.log2(float(max_num_seqs)), + len(admitted) / total, + len(completed) / len(admitted), + passed / max(1, len(completed)), + (len(completed) - passed) / total, + (len(admitted) - len(completed)) / len(admitted), + max(ttft, default=0.0) / 6000.0, + float(np.mean(ttft)) / 6000.0 if ttft else 0.0, + max(tpot, default=0.0) / 50.0, + float(np.mean(tpot)) / 50.0 if tpot else 0.0, + float(np.mean([float(request["raw_input_tokens"]) for request in admitted])) / 8192.0, + ) + + waiting = [float(record["queues"]["waiting"]) for record in records] + running = [float(record["queues"]["running"]) for record in records] + decode_batch = [float(record["decode_batch_size"]) for record in records] + kv_usage = [float(record["kv"]["usage"]) for record in records] + graph_modes = [str(record["cudagraph"]["runtime_mode"]) for record in records] + bucket_tokens = sum(int(record["cudagraph"]["bucket_tokens"]) for record in records) + padding_tokens = sum(int(record["cudagraph"]["padding_tokens"]) for record in records) + prefill_tokens = sum(int(record["prefill_tokens"]) for record in records) + decode_tokens = sum(int(record["decode_tokens"]) for record in records) + instrumentation = ( + len(records) / cutoff_s, + float(np.mean(waiting)), + max(waiting), + sum(value > 0 for value in waiting) / len(waiting), + float(np.mean(running)), + max(running), + float(np.mean(decode_batch)), + max(decode_batch), + _cv(decode_batch), + float(np.mean(kv_usage)), + max(kv_usage), + kv_usage[-1] - kv_usage[0], + graph_modes.count("NONE") / len(graph_modes), + graph_modes.count("FULL") / len(graph_modes), + padding_tokens / max(1, bucket_tokens), + prefill_tokens / max(1, prefill_tokens + decode_tokens), + float(sum(int(record["preemptions"]) for record in records)), + ) + completion_source = "+".join(sorted(sources)) if sources else "none_completed" + return outcome, instrumentation, completion_source + + +def build_examples( + phase6: dict[str, Any], + raw_root: Path, + cutoff_s: float, +) -> list[PrefixExample]: + examples = [] + for cell, cell_result in sorted(phase6["cells"].items()): + cell_root = raw_root / cell + stream_path = next((cell_root / "opprof").glob("*.jsonl")) + stream = _load_jsonl(stream_path, require_key="submit_mono_ns") + for anchor in cell_result["anchors"]: + primary = anchor["primary"] + full_elapsed_s = float(primary["interval"]["elapsed_s"]) + if full_elapsed_s + 1e-9 < cutoff_s: + continue + anchor_value = float(primary["anchor"]) + anchor_root = _anchor_directory(cell_root, anchor_value) + requests = _load_jsonl(anchor_root / "requests.jsonl") + start_ns = int(primary["interval"]["start_mono_ns"]) + end_ns = start_ns + int(cutoff_s * 1e9) + records = [ + record + for record in stream + if record.get("model_executed") + and start_ns <= int(record["submit_mono_ns"]) <= end_ns + ] + outcome, instrumentation, source = _prefix_features( + primary=primary, + tp=int(cell_result["tp"]), + max_num_seqs=int(cell_result["mns"]), + requests=requests, + records=records, + cutoff_s=cutoff_s, + ) + examples.append( + PrefixExample( + cell=cell, + anchor=anchor_value, + cutoff_s=cutoff_s, + tp=int(cell_result["tp"]), + full_elapsed_s=full_elapsed_s, + feasible=int(bool(anchor["accepted_feasible"])), + primary_feasible=int(bool(primary["feasible"])), + outcome=outcome, + instrumentation=instrumentation, + completion_time_source=source, + ) + ) + return examples + + +def grouped_predictions( + examples: list[PrefixExample], + *, + instrumentation_aware: bool, + regularization: float, +) -> tuple[np.ndarray, np.ndarray, list[str]]: + probabilities = [] + labels = [] + groups = [] + for held_out in sorted({example.cell for example in examples}): + train = [example for example in examples if example.cell != held_out] + test = [example for example in examples if example.cell == held_out] + + def row(example: PrefixExample) -> np.ndarray: + values = example.outcome + if instrumentation_aware: + values += example.instrumentation + return np.asarray((1.0, *values), dtype=np.float64) + + x_train = np.stack([row(example) for example in train]) + x_test = np.stack([row(example) for example in test]) + y_train = np.asarray([example.feasible for example in train], dtype=np.float64) + if len(set(y_train.tolist())) != 2: + raise ValueError(f"training fold for {held_out} has a single label") + mean = x_train[:, 1:].mean(axis=0) + standard_deviation = x_train[:, 1:].std(axis=0) + standard_deviation[standard_deviation < 1e-8] = 1.0 + x_train[:, 1:] = (x_train[:, 1:] - mean) / standard_deviation + x_test[:, 1:] = (x_test[:, 1:] - mean) / standard_deviation + weights = _fit_logistic(x_train, y_train, regularization) + probabilities.extend(_sigmoid(x_test @ weights).tolist()) + labels.extend(example.feasible for example in test) + groups.extend(held_out for _ in test) + return ( + np.asarray(labels, dtype=np.int64), + np.asarray(probabilities, dtype=np.float64), + groups, + ) + + +def fit_frozen_model( + examples: list[PrefixExample], + *, + instrumentation_aware: bool, + regularization: float, +) -> dict[str, Any]: + def row(example: PrefixExample) -> np.ndarray: + values = example.outcome + if instrumentation_aware: + values += example.instrumentation + return np.asarray((1.0, *values), dtype=np.float64) + + matrix = np.stack([row(example) for example in examples]) + labels = np.asarray([example.feasible for example in examples], dtype=np.float64) + if len(set(labels.tolist())) != 2: + raise ValueError("frozen model requires both feasibility labels") + mean = matrix[:, 1:].mean(axis=0) + standard_deviation = matrix[:, 1:].std(axis=0) + standard_deviation[standard_deviation < 1e-8] = 1.0 + standardized = matrix.copy() + standardized[:, 1:] = (standardized[:, 1:] - mean) / standard_deviation + weights = _fit_logistic(standardized, labels, regularization) + probabilities = _sigmoid(standardized @ weights) + names = list(OUTCOME_FEATURES) + if instrumentation_aware: + names.extend(INSTRUMENTATION_FEATURES) + return { + "instrumentation_aware": instrumentation_aware, + "regularization": regularization, + "feature_names": names, + "feature_mean": mean.tolist(), + "feature_standard_deviation": standard_deviation.tolist(), + "weights_with_intercept_first": weights.tolist(), + "training_classification": _classification_metrics(labels, probabilities), + } + + +def predict_frozen_model( + model: dict[str, Any], + examples: list[PrefixExample], +) -> np.ndarray: + instrumentation_aware = bool(model["instrumentation_aware"]) + rows = [] + for example in examples: + values = example.outcome + if instrumentation_aware: + values += example.instrumentation + rows.append((1.0, *values)) + matrix = np.asarray(rows, dtype=np.float64) + mean = np.asarray(model["feature_mean"], dtype=np.float64) + standard_deviation = np.asarray( + model["feature_standard_deviation"], dtype=np.float64 + ) + weights = np.asarray(model["weights_with_intercept_first"], dtype=np.float64) + if matrix.shape[1] != len(weights) or matrix.shape[1] - 1 != len(mean): + raise ValueError("frozen model feature dimensions do not match examples") + matrix[:, 1:] = (matrix[:, 1:] - mean) / standard_deviation + return _sigmoid(matrix @ weights) + + +def policy_metrics( + examples: list[PrefixExample], + labels: np.ndarray, + probabilities: np.ndarray, + threshold: float, +) -> dict[str, Any]: + accept = probabilities >= threshold + reject = probabilities <= 1.0 - threshold + decide = accept | reject + prediction = accept.astype(np.int64) + correct = prediction == labels + remaining = np.asarray( + [example.remaining_h20_hours for example in examples], dtype=np.float64 + ) + full_cost = sum(example.tp * example.full_elapsed_s / 3600.0 for example in examples) + saved = float(np.sum(remaining[decide])) + correct_saved = float(np.sum(remaining[decide & correct])) + invalid_saved = float(np.sum(remaining[decide & ~correct])) + + def describe(mask: np.ndarray) -> list[dict[str, Any]]: + return [ + { + "cell": example.cell, + "anchor": example.anchor, + "label_feasible": bool(label), + "probability_feasible": float(probability), + "remaining_h20_hours": example.remaining_h20_hours, + } + for example, label, probability, selected in zip( + examples, labels, probabilities, mask + ) + if selected + ] + + return { + "threshold": threshold, + "early_accept": int(np.sum(accept)), + "early_reject": int(np.sum(reject)), + "abstain_continue_full": int(np.sum(~decide)), + "false_accept": int(np.sum(accept & (labels == 0))), + "false_reject": int(np.sum(reject & (labels == 1))), + "false_accept_examples": describe(accept & (labels == 0)), + "false_reject_examples": describe(reject & (labels == 1)), + "decision_coverage": float(np.mean(decide)), + "full_trial_h20_hours": float(full_cost), + "remaining_h20_hours_at_cutoff": float(np.sum(remaining)), + "saved_h20_hours_if_decisions_used": saved, + "correctly_saved_h20_hours": correct_saved, + "invalidly_saved_h20_hours": invalid_saved, + "valid_zero_error_policy": bool(np.all(correct[decide])), + "valid_cost_reduction_fraction": ( + correct_saved / full_cost if invalid_saved == 0.0 and full_cost else None + ), + } + + +def analyze_cutoff(examples: list[PrefixExample]) -> dict[str, Any]: + sensitivity = {} + headline = None + for regularization in REGULARIZATION_SENSITIVITY: + labels, outcome_probability, groups = grouped_predictions( + examples, + instrumentation_aware=False, + regularization=regularization, + ) + instrument_labels, instrument_probability, instrument_groups = grouped_predictions( + examples, + instrumentation_aware=True, + regularization=regularization, + ) + if not np.array_equal(labels, instrument_labels) or groups != instrument_groups: + raise AssertionError("paired folds or labels differ") + if groups != [example.cell for example in examples]: + raise AssertionError("prediction order differs from example order") + outcome_correct = (outcome_probability >= 0.5) == labels + instrument_correct = (instrument_probability >= 0.5) == labels + result = { + "outcome_only": { + "classification": _classification_metrics(labels, outcome_probability), + "policies": [ + policy_metrics(examples, labels, outcome_probability, threshold) + for threshold in POLICY_THRESHOLDS + ], + }, + "instrumentation_aware": { + "classification": _classification_metrics(labels, instrument_probability), + "policies": [ + policy_metrics(examples, labels, instrument_probability, threshold) + for threshold in POLICY_THRESHOLDS + ], + }, + "paired_correctness": { + "both_correct": int(np.sum(outcome_correct & instrument_correct)), + "outcome_only_correct": int(np.sum(outcome_correct & ~instrument_correct)), + "instrumentation_only_correct": int(np.sum(~outcome_correct & instrument_correct)), + "both_wrong": int(np.sum(~outcome_correct & ~instrument_correct)), + }, + "bootstrap": _group_bootstrap_delta( + labels, + outcome_probability, + instrument_probability, + groups, + ), + } + paired = result["paired_correctness"] + paired["mcnemar_exact_two_sided_p"] = _mcnemar_exact_p( + paired["outcome_only_correct"], paired["instrumentation_only_correct"] + ) + sensitivity[str(regularization)] = result + if regularization == DEFAULT_REGULARIZATION: + headline = result + assert headline is not None + labels = [example.feasible for example in examples] + return { + "examples": len(examples), + "cells": len({example.cell for example in examples}), + "label_sanity": { + **numeric(labels), + "positive": sum(labels), + "negative": len(labels) - sum(labels), + "primary_adjudicated_disagreements": sum( + example.feasible != example.primary_feasible for example in examples + ), + }, + "completion_time_sources": { + source: sum(example.completion_time_source == source for example in examples) + for source in sorted({example.completion_time_source for example in examples}) + }, + "headline_regularization": DEFAULT_REGULARIZATION, + "headline": headline, + "regularization_sensitivity": sensitivity, + "remaining_h20_hours": numeric( + example.remaining_h20_hours for example in examples + ), + } + + +def analyze( + phase6_path: Path, + raw_root: Path, + cutoffs: tuple[float, ...], +) -> dict[str, Any]: + phase6 = json.loads(phase6_path.read_text(encoding="utf-8")) + by_cutoff = {} + red_flags = [] + for cutoff in cutoffs: + examples = build_examples(phase6, raw_root, cutoff) + if len({example.feasible for example in examples}) != 2: + red_flags.append(f"single_label_at_{cutoff:g}s") + continue + by_cutoff[f"{cutoff:g}"] = analyze_cutoff(examples) + if len({example.cell for example in examples}) != 12: + red_flags.append(f"incomplete_cells_at_{cutoff:g}s") + if not all( + math.isfinite(value) + for example in examples + for value in (*example.outcome, *example.instrumentation) + ): + red_flags.append(f"nonfinite_features_at_{cutoff:g}s") + + headline_deltas = { + cutoff: { + "accuracy": ( + result["headline"]["instrumentation_aware"]["classification"]["accuracy"] + - result["headline"]["outcome_only"]["classification"]["accuracy"] + ), + "brier": ( + result["headline"]["instrumentation_aware"]["classification"]["brier"] + - result["headline"]["outcome_only"]["classification"]["brier"] + ), + } + for cutoff, result in by_cutoff.items() + } + return { + "schema": SCHEMA, + "status": "PASS" if not red_flags else "STOP", + "scope": ( + "retrospective single-workload prefix diagnostic; model selection, " + "threshold choice, and contribution claims require held-out prospective tasks" + ), + "estimand": ( + "2-of-3 adjudicated anchor feasibility from the first primary trial's " + "identical short real prefix" + ), + "split": "leave-one-configuration-cell-out", + "model": "same L2 logistic model and folds; instrumentation model appends Layer-1 features", + "outcome_features": list(OUTCOME_FEATURES), + "instrumentation_features": list(INSTRUMENTATION_FEATURES), + "provenance": { + "phase6_metrics": str(phase6_path.resolve()), + "phase6_metrics_sha256": sha256_file(phase6_path), + "raw_root": str(raw_root.resolve()), + }, + "cutoffs_s": list(cutoffs), + "cutoffs": by_cutoff, + "headline_incremental_deltas": headline_deltas, + "decision": { + "contribution_established": False, + "reason": ( + "This dataset contains one workload and reconstructed rather than exact request " + "completion times. Three TP4 primary trials also disagree with their 2-of-3 " + "labels. It can reject a missing-signal premise but cannot establish " + "generalization or a paper-facing cost reduction." + ), + }, + "sanity": { + "red_flags": red_flags, + "cutoff_count": len(by_cutoff), + "invariants": { + "cutoffs_positive": all(cutoff > 0 for cutoff in cutoffs), + "paired_same_model_family": True, + "probabilities_checked_in_unit_interval": True, + "full_trial_label_not_used_as_feature": True, + "records_strictly_prefix_sliced": True, + }, + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--phase6-metrics", type=Path, required=True) + parser.add_argument("--raw-root", type=Path, required=True) + parser.add_argument("--cutoffs", type=float, nargs="+", default=DEFAULT_CUTOFFS) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + result = analyze(args.phase6_metrics, args.raw_root, tuple(args.cutoffs)) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"status": result["status"], "output": str(args.output)}, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/runs/fidelity-headroom/freeze_models.py b/runs/fidelity-headroom/freeze_models.py new file mode 100644 index 0000000..e452761 --- /dev/null +++ b/runs/fidelity-headroom/freeze_models.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Freeze the training-task prefix models before prospective GPU work.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from analyze_prefixes import ( + DEFAULT_REGULARIZATION, + POLICY_THRESHOLDS, + build_examples, + fit_frozen_model, + sha256_file, +) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--phase6-metrics", type=Path, required=True) + parser.add_argument("--prefix-metrics", type=Path, required=True) + parser.add_argument("--raw-root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + cutoff_s = 5.0 + threshold = 0.95 + if threshold not in POLICY_THRESHOLDS: + raise AssertionError("frozen threshold is outside audited policy thresholds") + phase6 = json.loads(args.phase6_metrics.read_text(encoding="utf-8")) + examples = build_examples(phase6, args.raw_root, cutoff_s) + payload = { + "schema": "fidelity-prefix-model-v1", + "status": "FROZEN_BEFORE_PROSPECTIVE_RUN", + "cutoff_s": cutoff_s, + "accept_probability": threshold, + "reject_probability": 1.0 - threshold, + "regularization": DEFAULT_REGULARIZATION, + "label": "same-placement 2-of-3 adjudicated anchor feasibility", + "training_split_role": "historical training only; never headline test", + "training_examples": [ + { + "cell": example.cell, + "anchor": example.anchor, + "label_feasible": bool(example.feasible), + "primary_feasible": bool(example.primary_feasible), + "completion_time_source": example.completion_time_source, + } + for example in examples + ], + "models": { + "outcome_only": fit_frozen_model( + examples, + instrumentation_aware=False, + regularization=DEFAULT_REGULARIZATION, + ), + "instrumentation_aware": fit_frozen_model( + examples, + instrumentation_aware=True, + regularization=DEFAULT_REGULARIZATION, + ), + }, + "provenance": { + "phase6_metrics": str(args.phase6_metrics.resolve()), + "phase6_metrics_sha256": sha256_file(args.phase6_metrics), + "prefix_metrics": str(args.prefix_metrics.resolve()), + "prefix_metrics_sha256": sha256_file(args.prefix_metrics), + "raw_root": str(args.raw_root.resolve()), + }, + "sanity": { + "n": len(examples), + "positive": sum(example.feasible for example in examples), + "negative": sum(not example.feasible for example in examples), + "cells": len({example.cell for example in examples}), + "invariants": { + "n_37": len(examples) == 37, + "cells_12": len({example.cell for example in examples}) == 12, + "both_labels": len({example.feasible for example in examples}) == 2, + "cutoff_5s": cutoff_s == 5.0, + "threshold_0.95": threshold == 0.95, + }, + }, + } + if not all(payload["sanity"]["invariants"].values()): + raise RuntimeError(f"model freeze invariants failed: {payload['sanity']}") + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + print(json.dumps({"status": payload["status"], "output": str(args.output)})) + + +if __name__ == "__main__": + main() diff --git a/runs/fidelity-headroom/frozen-models.json b/runs/fidelity-headroom/frozen-models.json new file mode 100644 index 0000000..46fa17c --- /dev/null +++ b/runs/fidelity-headroom/frozen-models.json @@ -0,0 +1,515 @@ +{ + "accept_probability": 0.95, + "cutoff_s": 5.0, + "label": "same-placement 2-of-3 adjudicated anchor feasibility", + "models": { + "instrumentation_aware": { + "feature_mean": [ + 0.8984976998643891, + 0.8378378378378378, + 4.324324324324325, + 0.07552086023066117, + 0.6758807403968693, + 0.9459459459459459, + 0.0, + 0.3241192596031305, + 0.04468545442770093, + 0.025590516908533558, + 0.23873649352596996, + 0.1943716628122394, + 0.4321792125178198, + 112.70270270270272, + 0.21087752102856197, + 0.918918918918919, + 0.055470351361483664, + 4.904239530899751, + 10.162162162162161, + 4.822982150502539, + 10.135135135135135, + 0.43557131397798415, + 0.031387890158936414, + 0.05804311436894179, + 0.03298678556030958, + 0.030437119300455177, + 0.9503065396705037, + 0.07127076398319926, + 0.6234198543231205, + 0.0 + ], + "feature_names": [ + "log_offered_rate_per_gpu", + "log2_tp", + "log2_max_num_seqs", + "admitted_fraction", + "completed_over_admitted", + "completed_pass_rate", + "completed_fail_fraction_of_total", + "outstanding_over_admitted", + "ttft_max_over_slo_max", + "ttft_mean_over_slo_max", + "tpot_max_over_slo", + "tpot_mean_over_slo", + "admitted_input_tokens_mean_over_limit", + "model_steps_per_second", + "waiting_mean", + "waiting_max", + "waiting_nonzero_share", + "running_mean", + "running_max", + "decode_batch_mean", + "decode_batch_max", + "decode_batch_cv", + "kv_usage_mean", + "kv_usage_max", + "kv_usage_end_minus_start", + "graph_none_share", + "graph_full_share", + "padding_fraction", + "prefill_token_fraction", + "preemptions" + ], + "feature_standard_deviation": [ + 0.2953332526155246, + 0.8546696378833459, + 1.1402715194448103, + 0.006588255148989237, + 0.2751217635728275, + 0.22612433149569594, + 1.0, + 0.27512176357282747, + 0.048292427420964075, + 0.02574874589991541, + 0.1635381690436309, + 0.14098674719611365, + 0.02516276437103069, + 61.39272994412853, + 0.7234949448561444, + 2.198013579605131, + 0.18326586413988316, + 2.4542471212960844, + 6.08726412391018, + 2.4074006634033043, + 6.067913672185017, + 0.12556414020947543, + 0.03256962310836033, + 0.054049610008010444, + 0.048321850100969746, + 0.04298231458641556, + 0.041906068064246155, + 0.08212268757576466, + 0.4089385238422411, + 1.0 + ], + "instrumentation_aware": true, + "regularization": 1.0, + "training_classification": { + "accuracy": 0.972972972972973, + "balanced_accuracy": 0.9444444444444444, + "brier": 0.02820726479488704, + "confusion": { + "false_negative": 0, + "false_positive": 1, + "true_negative": 8, + "true_positive": 28 + }, + "log_loss": 0.11247563885308659 + }, + "weights_with_intercept_first": [ + 2.109507425802979, + -0.8372240489271802, + -0.2476229678897366, + 0.18172257646801393, + -0.07076358054975332, + 0.3035586906752765, + 0.08500005412355496, + -7.754818242684634e-26, + -0.3035586906752766, + 0.4014234393196892, + 0.513218716194957, + -0.35161457106287, + 0.10558147889556725, + 0.5674345291616134, + 0.15895995157114373, + -0.4274260624362057, + -0.048791959001756195, + -0.37221380985270663, + -0.35527537277290255, + 0.20582736797173468, + -0.35837576944545413, + 0.2342062515631318, + 0.45071068059490843, + 0.3326948315186803, + 0.2698892549960913, + 0.017868065865726347, + -0.1540209080477302, + 0.3412427440368233, + 0.5831011876762794, + -0.583920360300169, + 0.0 + ] + }, + "outcome_only": { + "feature_mean": [ + 0.8984976998643891, + 0.8378378378378378, + 4.324324324324325, + 0.07552086023066117, + 0.6758807403968693, + 0.9459459459459459, + 0.0, + 0.3241192596031305, + 0.04468545442770093, + 0.025590516908533558, + 0.23873649352596996, + 0.1943716628122394, + 0.4321792125178198 + ], + "feature_names": [ + "log_offered_rate_per_gpu", + "log2_tp", + "log2_max_num_seqs", + "admitted_fraction", + "completed_over_admitted", + "completed_pass_rate", + "completed_fail_fraction_of_total", + "outstanding_over_admitted", + "ttft_max_over_slo_max", + "ttft_mean_over_slo_max", + "tpot_max_over_slo", + "tpot_mean_over_slo", + "admitted_input_tokens_mean_over_limit" + ], + "feature_standard_deviation": [ + 0.2953332526155246, + 0.8546696378833459, + 1.1402715194448103, + 0.006588255148989237, + 0.2751217635728275, + 0.22612433149569594, + 1.0, + 0.27512176357282747, + 0.048292427420964075, + 0.02574874589991541, + 0.1635381690436309, + 0.14098674719611365, + 0.02516276437103069 + ], + "instrumentation_aware": false, + "regularization": 1.0, + "training_classification": { + "accuracy": 0.9459459459459459, + "balanced_accuracy": 0.8888888888888888, + "brier": 0.051887373873176545, + "confusion": { + "false_negative": 0, + "false_positive": 2, + "true_negative": 7, + "true_positive": 28 + }, + "log_loss": 0.184988719119571 + }, + "weights_with_intercept_first": [ + 1.8996338126233983, + -1.1536861934230125, + -0.3806404559018098, + 0.5901136731733696, + 0.022432085012851908, + 0.5805554730881304, + 0.25786307099613026, + -8.077935669463161e-27, + -0.5805554730881304, + -0.15413292402348447, + 0.0986842306063204, + -0.5181573573074624, + 0.06283513013708956, + 0.911619634884147 + ] + } + }, + "provenance": { + "phase6_metrics": "/home/gahow/phd/aituner/runs/opprof-phase6/phase6/metrics.json", + "phase6_metrics_sha256": "290ba7fcb8727291166de7e4d47afdc84e230052495c81dd087db0ace9f93a16", + "prefix_metrics": "/home/gahow/phd/aituner/runs/fidelity-headroom/prefix-metrics.json", + "prefix_metrics_sha256": "cda821bcde1ae8427507aa4f03a1c116ccc7f7b8b717f73ca587bee3670a0340", + "raw_root": "/home/gahow/phd/aituner/runs/opprof-phase6/phase6/solo-authoritative/cells" + }, + "regularization": 1.0, + "reject_probability": 0.050000000000000044, + "sanity": { + "cells": 12, + "invariants": { + "both_labels": true, + "cells_12": true, + "cutoff_5s": true, + "n_37": true, + "threshold_0.95": true + }, + "n": 37, + "negative": 9, + "positive": 28 + }, + "schema": "fidelity-prefix-model-v1", + "status": "FROZEN_BEFORE_PROSPECTIVE_RUN", + "training_examples": [ + { + "anchor": 0.24609375, + "cell": "tp1_mns16", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.25, + "cell": "tp1_mns16", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.5, + "cell": "tp1_mns16", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": false, + "primary_feasible": false + }, + { + "anchor": 0.2421875, + "cell": "tp1_mns32", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.24609375, + "cell": "tp1_mns32", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.25, + "cell": "tp1_mns32", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.5, + "cell": "tp1_mns32", + "completion_time_source": "none_completed", + "label_feasible": false, + "primary_feasible": false + }, + { + "anchor": 0.2421875, + "cell": "tp1_mns64", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.24609375, + "cell": "tp1_mns64", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.25, + "cell": "tp1_mns64", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.5, + "cell": "tp1_mns64", + "completion_time_source": "none_completed", + "label_feasible": false, + "primary_feasible": false + }, + { + "anchor": 0.21875, + "cell": "tp1_mns8", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.2265625, + "cell": "tp1_mns8", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.23046875, + "cell": "tp1_mns8", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.234375, + "cell": "tp1_mns8", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.25, + "cell": "tp1_mns8", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.5, + "cell": "tp1_mns8", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": false, + "primary_feasible": false + }, + { + "anchor": 0.4921875, + "cell": "tp2_mns16", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.49609375, + "cell": "tp2_mns16", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.5, + "cell": "tp2_mns16", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.75, + "cell": "tp2_mns32", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.75390625, + "cell": "tp2_mns32", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": false, + "primary_feasible": false + }, + { + "anchor": 0.5, + "cell": "tp2_mns64", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.75, + "cell": "tp2_mns64", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": false, + "primary_feasible": false + }, + { + "anchor": 0.4921875, + "cell": "tp2_mns8", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": false, + "primary_feasible": false + }, + { + "anchor": 0.033182214016, + "cell": "tp4_mns16", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": false + }, + { + "anchor": 0.033717411016, + "cell": "tp4_mns16", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": false, + "primary_feasible": false + }, + { + "anchor": 0.034252608017, + "cell": "tp4_mns16", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": false + }, + { + "anchor": 0.034252608017, + "cell": "tp4_mns32", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.033717411016, + "cell": "tp4_mns64", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": false + }, + { + "anchor": 0.034252608017, + "cell": "tp4_mns64", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.016055910008, + "cell": "tp4_mns8", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.016591107009, + "cell": "tp4_mns8", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.017126304009, + "cell": "tp4_mns8", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": true, + "primary_feasible": true + }, + { + "anchor": 0.034252608017, + "cell": "tp4_mns8", + "completion_time_source": "reconstructed_from_latency", + "label_feasible": false, + "primary_feasible": false + } + ], + "training_split_role": "historical training only; never headline test" +} diff --git a/runs/fidelity-headroom/metrics.json b/runs/fidelity-headroom/metrics.json new file mode 100644 index 0000000..24f0550 --- /dev/null +++ b/runs/fidelity-headroom/metrics.json @@ -0,0 +1,1142 @@ +{ + "decision": { + "current_surface_can_show_selection_contribution": false, + "prospective_target": "Test whether internal features from a short, shared real probe reduce the number or duration of full frontier evaluations relative to an outcome-only model given the same probe.", + "reason": "The strongest frozen-calibrated SLO reading reaches zero real regret after real evaluation of its first two-cell tie bucket. A method that requires one calibration probe and one final verification cannot use this single task to demonstrate fewer real cell evaluations." + }, + "next_anchor_prediction": { + "estimand": "next-anchor feasibility from the full current-anchor summary", + "external_features": [ + "log_current_rate_per_gpu", + "log_next_over_current_rate", + "log2_tp", + "log2_mns", + "current_pass_rate", + "ttft_max_over_6s", + "tpot_max_over_50ms", + "exact_output_fraction", + "early_stopped" + ], + "headline": { + "bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + 0.0, + 0.125 + ], + "point": 0.040000000000000036 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.019846177751851126, + 0.02199527924137797 + ], + "point": -0.0019563088303256454 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "instrumentation_aware": { + "accuracy": 0.88, + "balanced_accuracy": 0.8576388888888888, + "brier": 0.1031574576574755, + "confusion": { + "false_negative": 1, + "false_positive": 2, + "true_negative": 7, + "true_positive": 15 + }, + "log_loss": 0.327997342465158 + }, + "outcome_only": { + "accuracy": 0.84, + "balanced_accuracy": 0.8020833333333333, + "brier": 0.10511376648780114, + "confusion": { + "false_negative": 1, + "false_positive": 3, + "true_negative": 6, + "true_positive": 15 + }, + "log_loss": 0.3385851118121996 + }, + "paired_correctness": { + "both_correct": 21, + "both_wrong": 3, + "instrumentation_only_correct": 1, + "mcnemar_exact_two_sided_p": 1.0, + "outcome_only_correct": 0 + } + }, + "headline_regularization": 1.0, + "instrumentation_features": [ + "waiting_mean", + "waiting_max", + "decode_batch_mean", + "decode_batch_cv", + "kv_usage_mean", + "kv_usage_max", + "graph_none_share", + "graph_full_share", + "padding_fraction", + "prefill_token_fraction", + "model_steps_per_second" + ], + "label_sanity": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 25, + "negative": 9, + "positive": 16 + }, + "model": "L2 logistic regression with train-fold standardization", + "regularization_sensitivity": { + "0.1": { + "bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + 0.0, + 0.13636363636363646 + ], + "point": 0.040000000000000036 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.029611344558941783, + 0.030283692179295475 + ], + "point": -0.0013643768871213907 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "instrumentation_aware": { + "accuracy": 0.92, + "balanced_accuracy": 0.9131944444444444, + "brier": 0.08887235975611406, + "confusion": { + "false_negative": 1, + "false_positive": 1, + "true_negative": 8, + "true_positive": 15 + }, + "log_loss": 0.3015353076332512 + }, + "outcome_only": { + "accuracy": 0.88, + "balanced_accuracy": 0.8819444444444444, + "brier": 0.09023673664323545, + "confusion": { + "false_negative": 2, + "false_positive": 1, + "true_negative": 8, + "true_positive": 14 + }, + "log_loss": 0.2927458553148107 + }, + "paired_correctness": { + "both_correct": 22, + "both_wrong": 2, + "instrumentation_only_correct": 1, + "mcnemar_exact_two_sided_p": 1.0, + "outcome_only_correct": 0 + } + }, + "1.0": { + "bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + 0.0, + 0.125 + ], + "point": 0.040000000000000036 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.019846177751851126, + 0.02199527924137797 + ], + "point": -0.0019563088303256454 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "instrumentation_aware": { + "accuracy": 0.88, + "balanced_accuracy": 0.8576388888888888, + "brier": 0.1031574576574755, + "confusion": { + "false_negative": 1, + "false_positive": 2, + "true_negative": 7, + "true_positive": 15 + }, + "log_loss": 0.327997342465158 + }, + "outcome_only": { + "accuracy": 0.84, + "balanced_accuracy": 0.8020833333333333, + "brier": 0.10511376648780114, + "confusion": { + "false_negative": 1, + "false_positive": 3, + "true_negative": 6, + "true_positive": 15 + }, + "log_loss": 0.3385851118121996 + }, + "paired_correctness": { + "both_correct": 21, + "both_wrong": 3, + "instrumentation_only_correct": 1, + "mcnemar_exact_two_sided_p": 1.0, + "outcome_only_correct": 0 + } + }, + "10.0": { + "bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.13636363636363646, + 0.0 + ], + "point": -0.039999999999999925 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.04125619566309649, + 0.012885694983237113 + ], + "point": -0.013003224609162978 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "instrumentation_aware": { + "accuracy": 0.8, + "balanced_accuracy": 0.7465277777777778, + "brier": 0.16045670460468575, + "confusion": { + "false_negative": 1, + "false_positive": 4, + "true_negative": 5, + "true_positive": 15 + }, + "log_loss": 0.4963310825527336 + }, + "outcome_only": { + "accuracy": 0.84, + "balanced_accuracy": 0.7777777777777778, + "brier": 0.17345992921384873, + "confusion": { + "false_negative": 0, + "false_positive": 4, + "true_negative": 5, + "true_positive": 16 + }, + "log_loss": 0.5254987874287221 + }, + "paired_correctness": { + "both_correct": 20, + "both_wrong": 4, + "instrumentation_only_correct": 0, + "mcnemar_exact_two_sided_p": 1.0, + "outcome_only_correct": 1 + } + } + }, + "sensitivity_summary": { + "accuracy_delta_min_max": [ + -0.039999999999999925, + 0.040000000000000036 + ], + "brier_delta_min_max": [ + -0.013003224609162978, + -0.0013643768871213907 + ], + "incremental_signal_verdict": "NEEDS_PROSPECTIVE_EVIDENCE" + }, + "split": "leave-one-cell-out", + "status": "RETROSPECTIVE_DIAGNOSTIC_ONLY" + }, + "provenance": { + "phase6_metrics": "/home/gahow/phd/aituner/runs/opprof-phase6/phase6/metrics.json", + "phase6_sha256": "290ba7fcb8727291166de7e4d47afdc84e230052495c81dd087db0ace9f93a16", + "simfid_metrics": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/metrics.json", + "simfid_sha256": "55edb37d5692e979ab6f6dc6c65913a9db0aa0a836c350e4c05d9c38eee78206" + }, + "sanity": { + "invariants": { + "probabilities_bounded": true, + "same_cells_all_readings": true, + "scores_nonnegative": true, + "transition_features_finite": true + }, + "real_scores": { + "distinct_n": 8, + "max": 3.283333333333333, + "min": 1.2833333333333334, + "n": 12 + }, + "red_flags": [], + "simulator_readings": 4, + "transition_cells": 12, + "transitions": 25 + }, + "schema": "fidelity-headroom-v1", + "scope": "retrospective single-workload premise audit; not prospective contribution evidence", + "status": "PASS", + "topk_headroom": { + "frozen-calibrated/SLO-gated": { + "minimum_k": { + "five_percent": { + "expanded_k": 2, + "nominal_k": 1 + }, + "one_percent": { + "expanded_k": 2, + "nominal_k": 1 + }, + "zero": { + "expanded_k": 2, + "nominal_k": 1 + } + }, + "points": [ + { + "candidates": [ + "tp2_mns32", + "tp2_mns64" + ], + "expanded_k": 2, + "nominal_k": 1, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp2_mns32", + "tp2_mns64" + ], + "expanded_k": 2, + "nominal_k": 2, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp2_mns32", + "tp2_mns64", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "expanded_k": 5, + "nominal_k": 3, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp2_mns32", + "tp2_mns64", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "expanded_k": 5, + "nominal_k": 4, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp2_mns32", + "tp2_mns64", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "expanded_k": 5, + "nominal_k": 5, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp2_mns32", + "tp2_mns64", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64" + ], + "expanded_k": 8, + "nominal_k": 6, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp2_mns32", + "tp2_mns64", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64" + ], + "expanded_k": 8, + "nominal_k": 7, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp2_mns32", + "tp2_mns64", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64" + ], + "expanded_k": 8, + "nominal_k": 8, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp2_mns32", + "tp2_mns64", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp2_mns16" + ], + "expanded_k": 9, + "nominal_k": 9, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp2_mns32", + "tp2_mns64", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp2_mns16", + "tp2_mns8" + ], + "expanded_k": 10, + "nominal_k": 10, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp2_mns32", + "tp2_mns64", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp2_mns16", + "tp2_mns8", + "tp1_mns8" + ], + "expanded_k": 11, + "nominal_k": 11, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp2_mns32", + "tp2_mns64", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp2_mns16", + "tp2_mns8", + "tp1_mns8", + "tp4_mns8" + ], + "expanded_k": 12, + "nominal_k": 12, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + } + ], + "real_best": 3.283333333333333 + }, + "frozen-calibrated/throughput-proxy": { + "minimum_k": { + "five_percent": { + "expanded_k": 4, + "nominal_k": 3 + }, + "one_percent": { + "expanded_k": 4, + "nominal_k": 3 + }, + "zero": { + "expanded_k": 4, + "nominal_k": 3 + } + }, + "points": [ + { + "candidates": [ + "tp1_mns64" + ], + "expanded_k": 1, + "nominal_k": 1, + "real_regret": 0.30456852791878175, + "selected_cell_after_real_final": "tp1_mns64", + "selected_real_score": 2.283333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32" + ], + "expanded_k": 2, + "nominal_k": 2, + "real_regret": 0.30456852791878175, + "selected_cell_after_real_final": "tp1_mns64", + "selected_real_score": 2.283333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns32", + "tp2_mns64" + ], + "expanded_k": 4, + "nominal_k": 3, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns32", + "tp2_mns64" + ], + "expanded_k": 4, + "nominal_k": 4, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns32", + "tp2_mns64", + "tp1_mns16" + ], + "expanded_k": 5, + "nominal_k": 5, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns32", + "tp2_mns64", + "tp1_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "expanded_k": 7, + "nominal_k": 6, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns32", + "tp2_mns64", + "tp1_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "expanded_k": 7, + "nominal_k": 7, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns32", + "tp2_mns64", + "tp1_mns16", + "tp4_mns32", + "tp4_mns64", + "tp4_mns16" + ], + "expanded_k": 8, + "nominal_k": 8, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns32", + "tp2_mns64", + "tp1_mns16", + "tp4_mns32", + "tp4_mns64", + "tp4_mns16", + "tp2_mns16" + ], + "expanded_k": 9, + "nominal_k": 9, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns32", + "tp2_mns64", + "tp1_mns16", + "tp4_mns32", + "tp4_mns64", + "tp4_mns16", + "tp2_mns16", + "tp1_mns8" + ], + "expanded_k": 10, + "nominal_k": 10, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns32", + "tp2_mns64", + "tp1_mns16", + "tp4_mns32", + "tp4_mns64", + "tp4_mns16", + "tp2_mns16", + "tp1_mns8", + "tp2_mns8" + ], + "expanded_k": 11, + "nominal_k": 11, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns32", + "tp2_mns64", + "tp1_mns16", + "tp4_mns32", + "tp4_mns64", + "tp4_mns16", + "tp2_mns16", + "tp1_mns8", + "tp2_mns8", + "tp4_mns8" + ], + "expanded_k": 12, + "nominal_k": 12, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + } + ], + "real_best": 3.283333333333333 + }, + "uncalibrated/SLO-gated": { + "minimum_k": { + "five_percent": { + "expanded_k": 12, + "nominal_k": 8 + }, + "one_percent": { + "expanded_k": 12, + "nominal_k": 8 + }, + "zero": { + "expanded_k": 12, + "nominal_k": 8 + } + }, + "points": [ + { + "candidates": [ + "tp4_mns32", + "tp4_mns64" + ], + "expanded_k": 2, + "nominal_k": 1, + "real_regret": 0.25634517766497456, + "selected_cell_after_real_final": "tp4_mns64", + "selected_real_score": 2.441666666666667 + }, + { + "candidates": [ + "tp4_mns32", + "tp4_mns64" + ], + "expanded_k": 2, + "nominal_k": 2, + "real_regret": 0.25634517766497456, + "selected_cell_after_real_final": "tp4_mns64", + "selected_real_score": 2.441666666666667 + }, + { + "candidates": [ + "tp4_mns32", + "tp4_mns64", + "tp2_mns16" + ], + "expanded_k": 3, + "nominal_k": 3, + "real_regret": 0.25634517766497456, + "selected_cell_after_real_final": "tp4_mns64", + "selected_real_score": 2.441666666666667 + }, + { + "candidates": [ + "tp4_mns32", + "tp4_mns64", + "tp2_mns16", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp1_mns8" + ], + "expanded_k": 7, + "nominal_k": 4, + "real_regret": 0.25634517766497456, + "selected_cell_after_real_final": "tp4_mns64", + "selected_real_score": 2.441666666666667 + }, + { + "candidates": [ + "tp4_mns32", + "tp4_mns64", + "tp2_mns16", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp1_mns8" + ], + "expanded_k": 7, + "nominal_k": 5, + "real_regret": 0.25634517766497456, + "selected_cell_after_real_final": "tp4_mns64", + "selected_real_score": 2.441666666666667 + }, + { + "candidates": [ + "tp4_mns32", + "tp4_mns64", + "tp2_mns16", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp1_mns8" + ], + "expanded_k": 7, + "nominal_k": 6, + "real_regret": 0.25634517766497456, + "selected_cell_after_real_final": "tp4_mns64", + "selected_real_score": 2.441666666666667 + }, + { + "candidates": [ + "tp4_mns32", + "tp4_mns64", + "tp2_mns16", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp1_mns8" + ], + "expanded_k": 7, + "nominal_k": 7, + "real_regret": 0.25634517766497456, + "selected_cell_after_real_final": "tp4_mns64", + "selected_real_score": 2.441666666666667 + }, + { + "candidates": [ + "tp4_mns32", + "tp4_mns64", + "tp2_mns16", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp1_mns8", + "tp2_mns32", + "tp2_mns64", + "tp2_mns8", + "tp4_mns16", + "tp4_mns8" + ], + "expanded_k": 12, + "nominal_k": 8, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp4_mns32", + "tp4_mns64", + "tp2_mns16", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp1_mns8", + "tp2_mns32", + "tp2_mns64", + "tp2_mns8", + "tp4_mns16", + "tp4_mns8" + ], + "expanded_k": 12, + "nominal_k": 9, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp4_mns32", + "tp4_mns64", + "tp2_mns16", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp1_mns8", + "tp2_mns32", + "tp2_mns64", + "tp2_mns8", + "tp4_mns16", + "tp4_mns8" + ], + "expanded_k": 12, + "nominal_k": 10, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp4_mns32", + "tp4_mns64", + "tp2_mns16", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp1_mns8", + "tp2_mns32", + "tp2_mns64", + "tp2_mns8", + "tp4_mns16", + "tp4_mns8" + ], + "expanded_k": 12, + "nominal_k": 11, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp4_mns32", + "tp4_mns64", + "tp2_mns16", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp1_mns8", + "tp2_mns32", + "tp2_mns64", + "tp2_mns8", + "tp4_mns16", + "tp4_mns8" + ], + "expanded_k": 12, + "nominal_k": 12, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + } + ], + "real_best": 3.283333333333333 + }, + "uncalibrated/throughput-proxy": { + "minimum_k": { + "five_percent": { + "expanded_k": 3, + "nominal_k": 3 + }, + "one_percent": { + "expanded_k": 3, + "nominal_k": 3 + }, + "zero": { + "expanded_k": 5, + "nominal_k": 5 + } + }, + "points": [ + { + "candidates": [ + "tp1_mns64" + ], + "expanded_k": 1, + "nominal_k": 1, + "real_regret": 0.30456852791878175, + "selected_cell_after_real_final": "tp1_mns64", + "selected_real_score": 2.283333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32" + ], + "expanded_k": 2, + "nominal_k": 2, + "real_regret": 0.30456852791878175, + "selected_cell_after_real_final": "tp1_mns64", + "selected_real_score": 2.283333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns64" + ], + "expanded_k": 3, + "nominal_k": 3, + "real_regret": 0.0076142131979695105, + "selected_cell_after_real_final": "tp2_mns64", + "selected_real_score": 3.2583333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns64", + "tp1_mns16" + ], + "expanded_k": 4, + "nominal_k": 4, + "real_regret": 0.0076142131979695105, + "selected_cell_after_real_final": "tp2_mns64", + "selected_real_score": 3.2583333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns64", + "tp1_mns16", + "tp2_mns32" + ], + "expanded_k": 5, + "nominal_k": 5, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns64", + "tp1_mns16", + "tp2_mns32", + "tp4_mns64" + ], + "expanded_k": 6, + "nominal_k": 6, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns64", + "tp1_mns16", + "tp2_mns32", + "tp4_mns64", + "tp1_mns8" + ], + "expanded_k": 7, + "nominal_k": 7, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns64", + "tp1_mns16", + "tp2_mns32", + "tp4_mns64", + "tp1_mns8", + "tp4_mns32" + ], + "expanded_k": 8, + "nominal_k": 8, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns64", + "tp1_mns16", + "tp2_mns32", + "tp4_mns64", + "tp1_mns8", + "tp4_mns32", + "tp2_mns16" + ], + "expanded_k": 9, + "nominal_k": 9, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns64", + "tp1_mns16", + "tp2_mns32", + "tp4_mns64", + "tp1_mns8", + "tp4_mns32", + "tp2_mns16", + "tp2_mns8" + ], + "expanded_k": 10, + "nominal_k": 10, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns64", + "tp1_mns16", + "tp2_mns32", + "tp4_mns64", + "tp1_mns8", + "tp4_mns32", + "tp2_mns16", + "tp2_mns8", + "tp4_mns16" + ], + "expanded_k": 11, + "nominal_k": 11, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + }, + { + "candidates": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns64", + "tp1_mns16", + "tp2_mns32", + "tp4_mns64", + "tp1_mns8", + "tp4_mns32", + "tp2_mns16", + "tp2_mns8", + "tp4_mns16", + "tp4_mns8" + ], + "expanded_k": 12, + "nominal_k": 12, + "real_regret": 0.0, + "selected_cell_after_real_final": "tp2_mns32", + "selected_real_score": 3.283333333333333 + } + ], + "real_best": 3.283333333333333 + } + } +} diff --git a/runs/fidelity-headroom/pilot_controller.py b/runs/fidelity-headroom/pilot_controller.py new file mode 100644 index 0000000..e63c9dd --- /dev/null +++ b/runs/fidelity-headroom/pilot_controller.py @@ -0,0 +1,436 @@ +#!/usr/bin/env python3 +"""Serialized dash0 controller for the exact-timestamp prefix pilot.""" + +from __future__ import annotations + +import argparse +import json +import os +import shlex +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + + +HERE = Path(__file__).resolve().parent +PHASE6 = HERE.parent / "opprof-phase6" +sys.path.insert(0, str(PHASE6)) + +import opprof_phase6_controller as base # noqa: E402 + + +ORDER = ( + "tp1_mns8", + "tp1_mns64", + "tp2_mns8", + "tp2_mns64", + "tp4_mns16", + "tp4_mns64", +) +CELL_ESTIMATE_H20_HOURS = {1: 0.20, 2: 0.40, 4: 0.80} +SAFETY_H20_HOURS = 0.20 + + +def atomic_json(path: Path, payload: Any) -> None: + base.atomic_json(path, payload) + + +def wait_all_idle(timeout_s: float = 30.0) -> None: + deadline = time.monotonic() + timeout_s + last_error: Exception | None = None + while time.monotonic() < deadline: + try: + base.assert_all_idle() + return + except RuntimeError as error: + last_error = error + time.sleep(1.0) + raise last_error or RuntimeError("GPU idle timeout") + + +def configure_base(args: argparse.Namespace, manifest: dict[str, Any]) -> None: + base.WORKDIR = args.run_root.parent + base.RUN_ROOT = args.run_root + base.STATE = args.run_root / "controller-state.json" + base.SOURCE = args.vllm_source + base.VENV = args.venv + base.AITUNER = args.aituner_root + base.MODEL = args.model + base.CLIENT = args.client + base.GPU_LIMIT = float(manifest["execution"]["hard_cap_h20_hours"]) + base.MARKER = "fidelity-prefix-pilot-20260714" + base.CELLS = { + cell: {"tp": int(config["tp"]), "mns": int(config["mns"])} + for cell, config in manifest["cells"].items() + } + + +def load_state(path: Path, hard_cap: float) -> dict[str, Any]: + if path.exists(): + return json.loads(path.read_text(encoding="utf-8")) + return { + "schema": "fidelity-prefix-pilot-state-v1", + "status": "initialized", + "hard_cap_h20_hours": hard_cap, + "gpu_hours_total": 0.0, + "completed_cells": 0, + "cells": {}, + "failures": [], + "started_at": time.time(), + } + + +def save_state(path: Path, state: dict[str, Any]) -> None: + atomic_json(path, state) + + +def append_echo(run_root: Path, line: str) -> None: + run_root.mkdir(parents=True, exist_ok=True) + with (run_root / "launch-echo.log").open("a", encoding="utf-8") as target: + target.write(line + "\n") + print(line, flush=True) + + +def remaining_projection(manifest: dict[str, Any], index: int) -> float: + return sum( + CELL_ESTIMATE_H20_HOURS[int(manifest["cells"][cell]["tp"])] + for cell in ORDER[index:] + ) + SAFETY_H20_HOURS + + +def start_server( + *, + cell: str, + index: int, + run_root: Path, +) -> dict[str, Any]: + config = base.CELLS[cell] + gpus = tuple(range(int(config["tp"]))) + cell_root = run_root / "cells" / cell + cell_root.mkdir(parents=True, exist_ok=True) + port = 8900 + index + command = base.server_command(cell, gpus, port) + with (cell_root / "commands.log").open("a", encoding="utf-8") as log: + log.write(f"SERVER {shlex.join(command)}\n") + server_log = (cell_root / "server.log").open("ab", buffering=0) + environment = os.environ.copy() + environment.update( + { + "CUDA_VISIBLE_DEVICES": ",".join(map(str, gpus)), + "VLLM_OPPROF_DIR": str(cell_root / "opprof"), + "OPPROF_PHASE6_MARKER": base.MARKER, + "AITUNER_ROOT": str(base.AITUNER), + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "PYTHONUNBUFFERED": "1", + } + ) + server = subprocess.Popen( + command, + cwd=base.SOURCE, + env=environment, + stdout=server_log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + base.OWNED_PGIDS.add(server.pid) + return { + "cell": cell, + "gpus": gpus, + "port": port, + "dir": cell_root, + "server": server, + "server_handle": server_log, + "spawned_at": time.time(), + "results": [], + } + + +def selection_for( + manifest: dict[str, Any], cell: str, role: str +) -> tuple[str, dict[str, Any]]: + level = "low" if role == "burnin" or role.startswith("low") else "high" + return level, manifest["cells"][cell]["targets"][level]["selections"][role] + + +def client_command( + entry: dict[str, Any], + *, + role: str, + selection: dict[str, Any], + output: Path, + warmup: bool, +) -> list[str]: + config = base.CELLS[entry["cell"]] + return [ + "taskset", + "-c", + base.cpu_mask(entry["gpus"]), + str(base.VENV / "bin/python"), + str(base.CLIENT), + "warmup" if warmup else "run-anchor", + "--study", + str(selection["study"]), + "--cell", + entry["cell"], + "--anchor", + str(selection["anchor"]), + "--tp", + str(config["tp"]), + "--mns", + str(config["mns"]), + "--base-url", + f"http://127.0.0.1:{entry['port']}", + "--result-dir", + str(output), + ] + + +def run_client( + *, + entry: dict[str, Any], + role: str, + selection: dict[str, Any], + output: Path, + state: dict[str, Any], + warmup: bool = False, +) -> dict[str, Any]: + command = client_command( + entry, role=role, selection=selection, output=output, warmup=warmup + ) + with (entry["dir"] / "commands.log").open("a", encoding="utf-8") as log: + log.write(f"CLIENT role={role} {shlex.join(command)}\n") + handle = (output.parent / f"{output.name}.log").open("ab", buffering=0) + environment = os.environ.copy() + environment.update({"AITUNER_ROOT": str(base.AITUNER), "PYTHONUNBUFFERED": "1"}) + process = subprocess.Popen( + command, + cwd=base.WORKDIR, + env=environment, + stdout=handle, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + deadline = time.monotonic() + 180.0 + try: + while process.poll() is None: + if time.monotonic() > deadline: + process.terminate() + raise TimeoutError(f"client timeout: {entry['cell']} {role}") + if entry["server"].poll() is not None: + raise RuntimeError(f"server exited during {entry['cell']} {role}") + base.assert_no_other_compute() + if state["gpu_hours_total"] + base.live_gpu_hours([entry]) >= base.GPU_LIMIT: + process.terminate() + raise RuntimeError("pilot H20-hour hard cap reached") + time.sleep(1.0) + finally: + handle.close() + if process.returncode: + raise RuntimeError( + f"client failed: cell={entry['cell']} role={role} rc={process.returncode}" + ) + result = json.loads((output / "result.json").read_text(encoding="utf-8")) + if int(result["selection"]["count"]) != int(selection["selected_count"]): + raise RuntimeError(f"selection count mismatch: {entry['cell']} {role}") + for key in ( + "request_id_order_sha256", + "arrival_order_sha256", + "raw_length_order_sha256", + ): + manifest_key = ( + "input_length_order_sha256" if key == "raw_length_order_sha256" else key + ) + if result["selection"][key] != selection[manifest_key]: + raise RuntimeError(f"selection hash mismatch {key}: {entry['cell']} {role}") + entry["results"].append( + {"anchor": float(selection["anchor"]), "dir": str(output), "kind": result["kind"]} + ) + return result + + +def execute_cell( + *, + index: int, + cell: str, + manifest: dict[str, Any], + run_root: Path, + state_path: Path, + state: dict[str, Any], +) -> None: + if state["cells"].get(cell, {}).get("status") == "complete": + return + projection = remaining_projection(manifest, index) + if state["gpu_hours_total"] + projection > base.GPU_LIMIT: + state["status"] = "budget_projection_stop" + state["budget_stop"] = { + "before_cell": cell, + "spent_h20_hours": state["gpu_hours_total"], + "remaining_projection_h20_hours": projection, + "hard_cap_h20_hours": base.GPU_LIMIT, + } + save_state(state_path, state) + raise RuntimeError(f"projected pilot cost exceeds hard cap before {cell}") + + config = manifest["cells"][cell] + echo = ( + f"PILOT_CELL_ECHO cell={cell} tp={config['tp']} mns={config['mns']} " + f"gpus=0-{int(config['tp']) - 1} workload={manifest['source']['window_id']} " + f"roles=burnin+low1/high1/low2/high2/low3/high3 " + f"spent_h20h={state['gpu_hours_total']:.6f} " + f"remaining_projection_h20h={projection:.3f} cap_h20h={base.GPU_LIMIT:.1f} " + f"manifest={run_root / 'pilot-manifest.json'}" + ) + append_echo(run_root, echo) + wait_all_idle() + cell_state = { + "status": "starting", + "tp": int(config["tp"]), + "mns": int(config["mns"]), + "started_at": time.time(), + "runs": [], + } + state["status"] = "running" + state["cells"][cell] = cell_state + save_state(state_path, state) + entry = start_server(cell=cell, index=index, run_root=run_root) + failure: Exception | None = None + try: + base.wait_ready(entry) + _level, burnin = selection_for(manifest, cell, "burnin") + cell_state["status"] = "warmup" + save_state(state_path, state) + warmup = run_client( + entry=entry, + role="burnin", + selection=burnin, + output=entry["dir"] / "warmup", + state=state, + warmup=True, + ) + cell_state["warmup"] = { + "exact_output_count": warmup["exact_output_count"], + "long_gt4096": warmup["selection"]["long_gt4096"], + } + cell_state["status"] = "burnin" + save_state(state_path, state) + burnin_result = run_client( + entry=entry, + role="burnin", + selection=burnin, + output=entry["dir"] / "burnin", + state=state, + ) + cell_state["burnin"] = { + "pass_rate": burnin_result["pass_rate"], + "feasible": burnin_result["feasible"], + } + role_order = manifest["execution"][ + "even_cell_order" if index % 2 == 0 else "odd_cell_order" + ] + cell_state["status"] = "measured" + cell_state["role_order"] = role_order + save_state(state_path, state) + for role in role_order: + level, selection = selection_for(manifest, cell, role) + result = run_client( + entry=entry, + role=role, + selection=selection, + output=entry["dir"] / f"{level}-rep{role[-1]}", + state=state, + ) + cell_state["runs"].append( + { + "role": role, + "level": level, + "anchor": selection["anchor"], + "selected_count": selection["selected_count"], + "pass_rate": result["pass_rate"], + "feasible": result["feasible"], + "elapsed_s": result["interval"]["elapsed_s"], + } + ) + save_state(state_path, state) + cell_state["status"] = "stopping" + save_state(state_path, state) + except Exception as error: # noqa: BLE001 + failure = error + finally: + try: + base.stop_entry(entry) + except Exception as error: # noqa: BLE001 + failure = failure or error + time.sleep(2.0) + try: + wait_all_idle() + except Exception as error: # noqa: BLE001 + failure = failure or error + + cell_hours = base.live_gpu_hours([entry]) + state["gpu_hours_total"] += cell_hours + cell_state["gpu_hours"] = cell_hours + if failure is not None: + cell_state["status"] = "failed" + cell_state["failure"] = repr(failure) + state["status"] = "failed" + state["failures"].append({"cell": cell, "failure": repr(failure)}) + save_state(state_path, state) + raise failure + validation = base.validate_cell(entry) + cell_state["validation"] = validation + cell_state["status"] = "complete" + cell_state["completed_at"] = time.time() + state["completed_cells"] += 1 + save_state(state_path, state) + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser() + result.add_argument("--manifest", type=Path, required=True) + result.add_argument("--run-root", type=Path, required=True) + result.add_argument("--aituner-root", type=Path, required=True) + result.add_argument("--vllm-source", type=Path, required=True) + result.add_argument("--venv", type=Path, required=True) + result.add_argument("--model", type=Path, required=True) + result.add_argument("--client", type=Path, required=True) + return result + + +def main() -> None: + args = parser().parse_args() + manifest = json.loads(args.manifest.read_text(encoding="utf-8")) + if manifest["status"] != "PASS": + raise RuntimeError("pilot manifest did not pass preflight") + args.run_root.mkdir(parents=True, exist_ok=True) + copied_manifest = args.run_root / "pilot-manifest.json" + if not copied_manifest.exists(): + atomic_json(copied_manifest, manifest) + configure_base(args, manifest) + state_path = args.run_root / "controller-state.json" + state = load_state(state_path, base.GPU_LIMIT) + state["status"] = "running" + save_state(state_path, state) + for index, cell in enumerate(ORDER): + execute_cell( + index=index, + cell=cell, + manifest=manifest, + run_root=args.run_root, + state_path=state_path, + state=state, + ) + state["status"] = "complete" + state["completed_at"] = time.time() + save_state(state_path, state) + print(json.dumps({ + "status": state["status"], + "completed_cells": state["completed_cells"], + "gpu_hours_total": state["gpu_hours_total"], + }, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/runs/fidelity-headroom/prefix-metrics.json b/runs/fidelity-headroom/prefix-metrics.json new file mode 100644 index 0000000..eeb6210 --- /dev/null +++ b/runs/fidelity-headroom/prefix-metrics.json @@ -0,0 +1,3696 @@ +{ + "cutoffs": { + "10": { + "cells": 12, + "completion_time_sources": { + "reconstructed_from_latency": 37 + }, + "examples": 37, + "headline": { + "bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.1333333333333333, + 0.05555555555555547 + ], + "point": -0.027027027027027084 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.047874993358699436, + 0.04753434529263568 + ], + "point": -0.0026830647286030934 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "instrumentation_aware": { + "classification": { + "accuracy": 0.8918918918918919, + "balanced_accuracy": 0.8531746031746033, + "brier": 0.07268572313801988, + "confusion": { + "false_negative": 2, + "false_positive": 2, + "true_negative": 7, + "true_positive": 26 + }, + "log_loss": 0.22136936001258795 + }, + "policies": [ + { + "abstain_continue_full": 3, + "correctly_saved_h20_hours": 0.7938369600363888, + "decision_coverage": 0.918918918918919, + "early_accept": 27, + "early_reject": 7, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.8623652659223954, + "remaining_h20_hours": 0.007340132528333333 + } + ], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.11022926282564288, + "remaining_h20_hours": 0.018465968481111112 + } + ], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.025806101009444443, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.8196430610458332, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 8, + "correctly_saved_h20_hours": 0.7340601279138887, + "decision_coverage": 0.7837837837837838, + "early_accept": 24, + "early_reject": 5, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.7340601279138887, + "threshold": 0.9, + "valid_cost_reduction_fraction": 0.6879924922438714, + "valid_zero_error_policy": true + }, + { + "abstain_continue_full": 15, + "correctly_saved_h20_hours": 0.620235621837222, + "decision_coverage": 0.5945945945945946, + "early_accept": 17, + "early_reject": 5, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.620235621837222, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.5813113054633519, + "valid_zero_error_policy": true + } + ] + }, + "outcome_only": { + "classification": { + "accuracy": 0.918918918918919, + "balanced_accuracy": 0.8333333333333333, + "brier": 0.07536878786662297, + "confusion": { + "false_negative": 0, + "false_positive": 3, + "true_negative": 6, + "true_positive": 28 + }, + "log_loss": 0.23513650573912553 + }, + "policies": [ + { + "abstain_continue_full": 5, + "correctly_saved_h20_hours": 0.7625537088033332, + "decision_coverage": 0.8648648648648649, + "early_accept": 27, + "early_reject": 5, + "false_accept": 2, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.8486897200744762, + "remaining_h20_hours": 0.007340132528333333 + }, + { + "anchor": 0.034252608017, + "cell": "tp4_mns8", + "label_feasible": false, + "probability_feasible": 0.8216385310591849, + "remaining_h20_hours": 0.01680367415111111 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.024143806679444442, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.7866975154827778, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 11, + "correctly_saved_h20_hours": 0.7042463331286111, + "decision_coverage": 0.7027027027027027, + "early_accept": 22, + "early_reject": 4, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.7042463331286111, + "threshold": 0.9, + "valid_cost_reduction_fraction": 0.6600497308847137, + "valid_zero_error_policy": true + }, + { + "abstain_continue_full": 19, + "correctly_saved_h20_hours": 0.5876473660130556, + "decision_coverage": 0.4864864864864865, + "early_accept": 15, + "early_reject": 3, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.5876473660130556, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.5507682007641966, + "valid_zero_error_policy": true + } + ] + }, + "paired_correctness": { + "both_correct": 32, + "both_wrong": 2, + "instrumentation_only_correct": 1, + "mcnemar_exact_two_sided_p": 1.0, + "outcome_only_correct": 2 + } + }, + "headline_regularization": 1.0, + "label_sanity": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 37, + "negative": 9, + "positive": 28, + "primary_adjudicated_disagreements": 3 + }, + "regularization_sensitivity": { + "0.1": { + "bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.08571428571428574, + 0.07317073170731703 + ], + "point": 0.0 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.01721313553473963, + 0.0846932121032576 + ], + "point": 0.019153359424323196 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "instrumentation_aware": { + "classification": { + "accuracy": 0.918918918918919, + "balanced_accuracy": 0.871031746031746, + "brier": 0.06720185291989068, + "confusion": { + "false_negative": 1, + "false_positive": 2, + "true_negative": 7, + "true_positive": 27 + }, + "log_loss": 0.22428253113260074 + }, + "policies": [ + { + "abstain_continue_full": 2, + "correctly_saved_h20_hours": 0.802005166048611, + "decision_coverage": 0.9459459459459459, + "early_accept": 27, + "early_reject": 8, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.905553117931728, + "remaining_h20_hours": 0.007340132528333333 + } + ], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.02745974548325657, + "remaining_h20_hours": 0.018465968481111112 + } + ], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.025806101009444443, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.8278112670580555, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 4, + "correctly_saved_h20_hours": 0.7653433791469442, + "decision_coverage": 0.8918918918918919, + "early_accept": 26, + "early_reject": 7, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.905553117931728, + "remaining_h20_hours": 0.007340132528333333 + } + ], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.02745974548325657, + "remaining_h20_hours": 0.018465968481111112 + } + ], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.025806101009444443, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.7911494801563888, + "threshold": 0.9, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 7, + "correctly_saved_h20_hours": 0.7226741330227777, + "decision_coverage": 0.8108108108108109, + "early_accept": 23, + "early_reject": 7, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.02745974548325657, + "remaining_h20_hours": 0.018465968481111112 + } + ], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.018465968481111112, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.7411401015038888, + "threshold": 0.95, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + } + ] + }, + "outcome_only": { + "classification": { + "accuracy": 0.918918918918919, + "balanced_accuracy": 0.8333333333333333, + "brier": 0.04804849349556749, + "confusion": { + "false_negative": 0, + "false_positive": 3, + "true_negative": 6, + "true_positive": 28 + }, + "log_loss": 0.15081259765362642 + }, + "policies": [ + { + "abstain_continue_full": 4, + "correctly_saved_h20_hours": 0.7836433176388888, + "decision_coverage": 0.8918918918918919, + "early_accept": 27, + "early_reject": 6, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.8878008796435344, + "remaining_h20_hours": 0.007340132528333333 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.007340132528333333, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.7909834501672222, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 8, + "correctly_saved_h20_hours": 0.718791861695, + "decision_coverage": 0.7837837837837838, + "early_accept": 24, + "early_reject": 5, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.718791861695, + "threshold": 0.9, + "valid_cost_reduction_fraction": 0.6736824212718535, + "valid_zero_error_policy": true + }, + { + "abstain_continue_full": 10, + "correctly_saved_h20_hours": 0.7046020469899998, + "decision_coverage": 0.7297297297297297, + "early_accept": 23, + "early_reject": 4, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.7046020469899998, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.6603831211026486, + "valid_zero_error_policy": true + } + ] + }, + "paired_correctness": { + "both_correct": 33, + "both_wrong": 2, + "instrumentation_only_correct": 1, + "mcnemar_exact_two_sided_p": 1.0, + "outcome_only_correct": 1 + } + }, + "1.0": { + "bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.1333333333333333, + 0.05555555555555547 + ], + "point": -0.027027027027027084 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.047874993358699436, + 0.04753434529263568 + ], + "point": -0.0026830647286030934 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "instrumentation_aware": { + "classification": { + "accuracy": 0.8918918918918919, + "balanced_accuracy": 0.8531746031746033, + "brier": 0.07268572313801988, + "confusion": { + "false_negative": 2, + "false_positive": 2, + "true_negative": 7, + "true_positive": 26 + }, + "log_loss": 0.22136936001258795 + }, + "policies": [ + { + "abstain_continue_full": 3, + "correctly_saved_h20_hours": 0.7938369600363888, + "decision_coverage": 0.918918918918919, + "early_accept": 27, + "early_reject": 7, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.8623652659223954, + "remaining_h20_hours": 0.007340132528333333 + } + ], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.11022926282564288, + "remaining_h20_hours": 0.018465968481111112 + } + ], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.025806101009444443, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.8196430610458332, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 8, + "correctly_saved_h20_hours": 0.7340601279138887, + "decision_coverage": 0.7837837837837838, + "early_accept": 24, + "early_reject": 5, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.7340601279138887, + "threshold": 0.9, + "valid_cost_reduction_fraction": 0.6879924922438714, + "valid_zero_error_policy": true + }, + { + "abstain_continue_full": 15, + "correctly_saved_h20_hours": 0.620235621837222, + "decision_coverage": 0.5945945945945946, + "early_accept": 17, + "early_reject": 5, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.620235621837222, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.5813113054633519, + "valid_zero_error_policy": true + } + ] + }, + "outcome_only": { + "classification": { + "accuracy": 0.918918918918919, + "balanced_accuracy": 0.8333333333333333, + "brier": 0.07536878786662297, + "confusion": { + "false_negative": 0, + "false_positive": 3, + "true_negative": 6, + "true_positive": 28 + }, + "log_loss": 0.23513650573912553 + }, + "policies": [ + { + "abstain_continue_full": 5, + "correctly_saved_h20_hours": 0.7625537088033332, + "decision_coverage": 0.8648648648648649, + "early_accept": 27, + "early_reject": 5, + "false_accept": 2, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.8486897200744762, + "remaining_h20_hours": 0.007340132528333333 + }, + { + "anchor": 0.034252608017, + "cell": "tp4_mns8", + "label_feasible": false, + "probability_feasible": 0.8216385310591849, + "remaining_h20_hours": 0.01680367415111111 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.024143806679444442, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.7866975154827778, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 11, + "correctly_saved_h20_hours": 0.7042463331286111, + "decision_coverage": 0.7027027027027027, + "early_accept": 22, + "early_reject": 4, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.7042463331286111, + "threshold": 0.9, + "valid_cost_reduction_fraction": 0.6600497308847137, + "valid_zero_error_policy": true + }, + { + "abstain_continue_full": 19, + "correctly_saved_h20_hours": 0.5876473660130556, + "decision_coverage": 0.4864864864864865, + "early_accept": 15, + "early_reject": 3, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.5876473660130556, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.5507682007641966, + "valid_zero_error_policy": true + } + ] + }, + "paired_correctness": { + "both_correct": 32, + "both_wrong": 2, + "instrumentation_only_correct": 1, + "mcnemar_exact_two_sided_p": 1.0, + "outcome_only_correct": 2 + } + }, + "10.0": { + "bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + 0.0, + 0.0 + ], + "point": 0.0 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.026808476987744526, + 0.02087777396654144 + ], + "point": -0.005783107568599871 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "instrumentation_aware": { + "classification": { + "accuracy": 0.8378378378378378, + "balanced_accuracy": 0.7420634920634921, + "brier": 0.08901809268133021, + "confusion": { + "false_negative": 2, + "false_positive": 4, + "true_negative": 5, + "true_positive": 26 + }, + "log_loss": 0.2787387373669301 + }, + "policies": [ + { + "abstain_continue_full": 5, + "correctly_saved_h20_hours": 0.7770332858852778, + "decision_coverage": 0.8648648648648649, + "early_accept": 27, + "early_reject": 5, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.8449371690787663, + "remaining_h20_hours": 0.007340132528333333 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.007340132528333333, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.7843734184136111, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 17, + "correctly_saved_h20_hours": 0.6023881611255555, + "decision_coverage": 0.5405405405405406, + "early_accept": 16, + "early_reject": 4, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.6023881611255555, + "threshold": 0.9, + "valid_cost_reduction_fraction": 0.564583903295168, + "valid_zero_error_policy": true + }, + { + "abstain_continue_full": 30, + "correctly_saved_h20_hours": 0.28353444813305556, + "decision_coverage": 0.1891891891891892, + "early_accept": 6, + "early_reject": 1, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.28353444813305556, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.2657405901644815, + "valid_zero_error_policy": true + } + ] + }, + "outcome_only": { + "classification": { + "accuracy": 0.8378378378378378, + "balanced_accuracy": 0.7420634920634921, + "brier": 0.09480120024993008, + "confusion": { + "false_negative": 2, + "false_positive": 4, + "true_negative": 5, + "true_positive": 26 + }, + "log_loss": 0.31278388302658006 + }, + "policies": [ + { + "abstain_continue_full": 8, + "correctly_saved_h20_hours": 0.7566135917169443, + "decision_coverage": 0.7837837837837838, + "early_accept": 26, + "early_reject": 3, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.8320431084673425, + "remaining_h20_hours": 0.007340132528333333 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.007340132528333333, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.7639537242452776, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 24, + "correctly_saved_h20_hours": 0.5226926016833333, + "decision_coverage": 0.35135135135135137, + "early_accept": 12, + "early_reject": 1, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.5226926016833333, + "threshold": 0.9, + "valid_cost_reduction_fraction": 0.48988982242028906, + "valid_zero_error_policy": true + }, + { + "abstain_continue_full": 34, + "correctly_saved_h20_hours": 0.16934238506333332, + "decision_coverage": 0.08108108108108109, + "early_accept": 3, + "early_reject": 0, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.8475150590230554, + "saved_h20_hours_if_decisions_used": 0.16934238506333332, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.15871491327739184, + "valid_zero_error_policy": true + } + ] + }, + "paired_correctness": { + "both_correct": 31, + "both_wrong": 6, + "instrumentation_only_correct": 0, + "mcnemar_exact_two_sided_p": 1.0, + "outcome_only_correct": 0 + } + } + }, + "remaining_h20_hours": { + "distinct_n": 37, + "max": 0.057199826214444444, + "min": 0.0012684118816666664, + "n": 37 + } + }, + "15": { + "cells": 12, + "completion_time_sources": { + "reconstructed_from_latency": 36 + }, + "examples": 36, + "headline": { + "bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.06896551724137923, + 0.11428571428571421 + ], + "point": 0.02777777777777779 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.07972316141025643, + -0.004101962239149991 + ], + "point": -0.03744665409603411 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "instrumentation_aware": { + "classification": { + "accuracy": 0.9166666666666666, + "balanced_accuracy": 0.8571428571428572, + "brier": 0.03946379437502599, + "confusion": { + "false_negative": 1, + "false_positive": 2, + "true_negative": 6, + "true_positive": 27 + }, + "log_loss": 0.12484297369457635 + }, + "policies": [ + { + "abstain_continue_full": 3, + "correctly_saved_h20_hours": 0.7164359347680556, + "decision_coverage": 0.9166666666666666, + "early_accept": 27, + "early_reject": 6, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.7164359347680556, + "threshold": 0.8, + "valid_cost_reduction_fraction": 0.6740304458143536, + "valid_zero_error_policy": true + }, + { + "abstain_continue_full": 6, + "correctly_saved_h20_hours": 0.6905889710727777, + "decision_coverage": 0.8333333333333334, + "early_accept": 25, + "early_reject": 5, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.6905889710727777, + "threshold": 0.9, + "valid_cost_reduction_fraction": 0.6497133511279798, + "valid_zero_error_policy": true + }, + { + "abstain_continue_full": 12, + "correctly_saved_h20_hours": 0.5895315664361112, + "decision_coverage": 0.6666666666666666, + "early_accept": 20, + "early_reject": 4, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.5895315664361112, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.5546374843344664, + "valid_zero_error_policy": true + } + ] + }, + "outcome_only": { + "classification": { + "accuracy": 0.8888888888888888, + "balanced_accuracy": 0.75, + "brier": 0.0769104484710601, + "confusion": { + "false_negative": 0, + "false_positive": 4, + "true_negative": 4, + "true_positive": 28 + }, + "log_loss": 0.23230947656220863 + }, + "policies": [ + { + "abstain_continue_full": 6, + "correctly_saved_h20_hours": 0.6709655458816667, + "decision_coverage": 0.8333333333333334, + "early_accept": 26, + "early_reject": 4, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.5, + "cell": "tp1_mns64", + "label_feasible": false, + "probability_feasible": 0.8578028168458919, + "remaining_h20_hours": 0.0014445317586111114 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0014445317586111114, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.6724100776402777, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 11, + "correctly_saved_h20_hours": 0.626660078601389, + "decision_coverage": 0.6944444444444444, + "early_accept": 22, + "early_reject": 3, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.626660078601389, + "threshold": 0.9, + "valid_cost_reduction_fraction": 0.5895683782116529, + "valid_zero_error_policy": true + }, + { + "abstain_continue_full": 16, + "correctly_saved_h20_hours": 0.5494646261305556, + "decision_coverage": 0.5555555555555556, + "early_accept": 17, + "early_reject": 3, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.5494646261305556, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.5169420864266074, + "valid_zero_error_policy": true + } + ] + }, + "paired_correctness": { + "both_correct": 31, + "both_wrong": 2, + "instrumentation_only_correct": 2, + "mcnemar_exact_two_sided_p": 1.0, + "outcome_only_correct": 1 + } + }, + "headline_regularization": 1.0, + "label_sanity": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 36, + "negative": 8, + "positive": 28, + "primary_adjudicated_disagreements": 3 + }, + "regularization_sensitivity": { + "0.1": { + "bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + 0.0, + 0.13793103448275856 + ], + "point": 0.05555555555555558 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.08924754562701061, + 0.007148232838160279 + ], + "point": -0.03187976039254431 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "instrumentation_aware": { + "classification": { + "accuracy": 0.9444444444444444, + "balanced_accuracy": 0.875, + "brier": 0.03162164266499348, + "confusion": { + "false_negative": 0, + "false_positive": 2, + "true_negative": 6, + "true_positive": 28 + }, + "log_loss": 0.08770869035960555 + }, + "policies": [ + { + "abstain_continue_full": 3, + "correctly_saved_h20_hours": 0.7164359347680556, + "decision_coverage": 0.9166666666666666, + "early_accept": 27, + "early_reject": 6, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.7164359347680556, + "threshold": 0.8, + "valid_cost_reduction_fraction": 0.6740304458143536, + "valid_zero_error_policy": true + }, + { + "abstain_continue_full": 3, + "correctly_saved_h20_hours": 0.7164359347680556, + "decision_coverage": 0.9166666666666666, + "early_accept": 27, + "early_reject": 6, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.7164359347680556, + "threshold": 0.9, + "valid_cost_reduction_fraction": 0.6740304458143536, + "valid_zero_error_policy": true + }, + { + "abstain_continue_full": 4, + "correctly_saved_h20_hours": 0.6910240435036111, + "decision_coverage": 0.8888888888888888, + "early_accept": 26, + "early_reject": 6, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.6910240435036111, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.6501226718366222, + "valid_zero_error_policy": true + } + ] + }, + "outcome_only": { + "classification": { + "accuracy": 0.8888888888888888, + "balanced_accuracy": 0.7946428571428572, + "brier": 0.06350140305753779, + "confusion": { + "false_negative": 1, + "false_positive": 3, + "true_negative": 5, + "true_positive": 27 + }, + "log_loss": 0.2433253313244278 + }, + "policies": [ + { + "abstain_continue_full": 4, + "correctly_saved_h20_hours": 0.689579511745, + "decision_coverage": 0.8888888888888888, + "early_accept": 27, + "early_reject": 5, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.5, + "cell": "tp1_mns64", + "label_feasible": false, + "probability_feasible": 0.9922655130114992, + "remaining_h20_hours": 0.0014445317586111114 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0014445317586111114, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.6910240435036111, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 5, + "correctly_saved_h20_hours": 0.6783313931494445, + "decision_coverage": 0.8611111111111112, + "early_accept": 27, + "early_reject": 4, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.5, + "cell": "tp1_mns64", + "label_feasible": false, + "probability_feasible": 0.9922655130114992, + "remaining_h20_hours": 0.0014445317586111114 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0014445317586111114, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.6797759249080555, + "threshold": 0.9, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 8, + "correctly_saved_h20_hours": 0.6449625425236111, + "decision_coverage": 0.7777777777777778, + "early_accept": 24, + "early_reject": 4, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.5, + "cell": "tp1_mns64", + "label_feasible": false, + "probability_feasible": 0.9922655130114992, + "remaining_h20_hours": 0.0014445317586111114 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0014445317586111114, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.6464070742822221, + "threshold": 0.95, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + } + ] + }, + "paired_correctness": { + "both_correct": 32, + "both_wrong": 2, + "instrumentation_only_correct": 2, + "mcnemar_exact_two_sided_p": 0.5, + "outcome_only_correct": 0 + } + }, + "1.0": { + "bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.06896551724137923, + 0.11428571428571421 + ], + "point": 0.02777777777777779 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.07972316141025643, + -0.004101962239149991 + ], + "point": -0.03744665409603411 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "instrumentation_aware": { + "classification": { + "accuracy": 0.9166666666666666, + "balanced_accuracy": 0.8571428571428572, + "brier": 0.03946379437502599, + "confusion": { + "false_negative": 1, + "false_positive": 2, + "true_negative": 6, + "true_positive": 27 + }, + "log_loss": 0.12484297369457635 + }, + "policies": [ + { + "abstain_continue_full": 3, + "correctly_saved_h20_hours": 0.7164359347680556, + "decision_coverage": 0.9166666666666666, + "early_accept": 27, + "early_reject": 6, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.7164359347680556, + "threshold": 0.8, + "valid_cost_reduction_fraction": 0.6740304458143536, + "valid_zero_error_policy": true + }, + { + "abstain_continue_full": 6, + "correctly_saved_h20_hours": 0.6905889710727777, + "decision_coverage": 0.8333333333333334, + "early_accept": 25, + "early_reject": 5, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.6905889710727777, + "threshold": 0.9, + "valid_cost_reduction_fraction": 0.6497133511279798, + "valid_zero_error_policy": true + }, + { + "abstain_continue_full": 12, + "correctly_saved_h20_hours": 0.5895315664361112, + "decision_coverage": 0.6666666666666666, + "early_accept": 20, + "early_reject": 4, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.5895315664361112, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.5546374843344664, + "valid_zero_error_policy": true + } + ] + }, + "outcome_only": { + "classification": { + "accuracy": 0.8888888888888888, + "balanced_accuracy": 0.75, + "brier": 0.0769104484710601, + "confusion": { + "false_negative": 0, + "false_positive": 4, + "true_negative": 4, + "true_positive": 28 + }, + "log_loss": 0.23230947656220863 + }, + "policies": [ + { + "abstain_continue_full": 6, + "correctly_saved_h20_hours": 0.6709655458816667, + "decision_coverage": 0.8333333333333334, + "early_accept": 26, + "early_reject": 4, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.5, + "cell": "tp1_mns64", + "label_feasible": false, + "probability_feasible": 0.8578028168458919, + "remaining_h20_hours": 0.0014445317586111114 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0014445317586111114, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.6724100776402777, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 11, + "correctly_saved_h20_hours": 0.626660078601389, + "decision_coverage": 0.6944444444444444, + "early_accept": 22, + "early_reject": 3, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.626660078601389, + "threshold": 0.9, + "valid_cost_reduction_fraction": 0.5895683782116529, + "valid_zero_error_policy": true + }, + { + "abstain_continue_full": 16, + "correctly_saved_h20_hours": 0.5494646261305556, + "decision_coverage": 0.5555555555555556, + "early_accept": 17, + "early_reject": 3, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.5494646261305556, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.5169420864266074, + "valid_zero_error_policy": true + } + ] + }, + "paired_correctness": { + "both_correct": 31, + "both_wrong": 2, + "instrumentation_only_correct": 2, + "mcnemar_exact_two_sided_p": 1.0, + "outcome_only_correct": 1 + } + }, + "10.0": { + "bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + 0.0, + 0.17142857142857137 + ], + "point": 0.08333333333333326 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.051136314558473485, + -0.009942733239401492 + ], + "point": -0.029809575955846893 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "instrumentation_aware": { + "classification": { + "accuracy": 0.8888888888888888, + "balanced_accuracy": 0.8392857142857143, + "brier": 0.07023194254174596, + "confusion": { + "false_negative": 2, + "false_positive": 2, + "true_negative": 6, + "true_positive": 26 + }, + "log_loss": 0.23287532021698182 + }, + "policies": [ + { + "abstain_continue_full": 5, + "correctly_saved_h20_hours": 0.6924315406702778, + "decision_coverage": 0.8611111111111112, + "early_accept": 27, + "early_reject": 4, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.75, + "cell": "tp2_mns64", + "label_feasible": false, + "probability_feasible": 0.801686701254931, + "remaining_h20_hours": 0.004004611363888889 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.004004611363888889, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.6964361520341666, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 14, + "correctly_saved_h20_hours": 0.5771830138255556, + "decision_coverage": 0.6111111111111112, + "early_accept": 19, + "early_reject": 3, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.5771830138255556, + "threshold": 0.9, + "valid_cost_reduction_fraction": 0.5430198364509199, + "valid_zero_error_policy": true + }, + { + "abstain_continue_full": 26, + "correctly_saved_h20_hours": 0.30550869967055555, + "decision_coverage": 0.2777777777777778, + "early_accept": 8, + "early_reject": 2, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.30550869967055555, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.2874257907034979, + "valid_zero_error_policy": true + } + ] + }, + "outcome_only": { + "classification": { + "accuracy": 0.8055555555555556, + "balanced_accuracy": 0.6517857142857143, + "brier": 0.10004151849759285, + "confusion": { + "false_negative": 2, + "false_positive": 5, + "true_negative": 3, + "true_positive": 26 + }, + "log_loss": 0.32055333394171265 + }, + "policies": [ + { + "abstain_continue_full": 7, + "correctly_saved_h20_hours": 0.6877041925958333, + "decision_coverage": 0.8055555555555556, + "early_accept": 27, + "early_reject": 2, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.8140722809073944, + "remaining_h20_hours": 0.004562354750555555 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.004562354750555555, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.6922665473463888, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 24, + "correctly_saved_h20_hours": 0.4570601465275, + "decision_coverage": 0.3333333333333333, + "early_accept": 11, + "early_reject": 1, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.4570601465275, + "threshold": 0.9, + "valid_cost_reduction_fraction": 0.43000698230978923, + "valid_zero_error_policy": true + }, + { + "abstain_continue_full": 33, + "correctly_saved_h20_hours": 0.15267571839666666, + "decision_coverage": 0.08333333333333333, + "early_accept": 3, + "early_reject": 0, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.7379133138080555, + "saved_h20_hours_if_decisions_used": 0.15267571839666666, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.14363891806913368, + "valid_zero_error_policy": true + } + ] + }, + "paired_correctness": { + "both_correct": 29, + "both_wrong": 4, + "instrumentation_only_correct": 3, + "mcnemar_exact_two_sided_p": 0.25, + "outcome_only_correct": 0 + } + } + }, + "remaining_h20_hours": { + "distinct_n": 36, + "max": 0.05164427065888889, + "min": 0.0014445317586111114, + "n": 36 + } + }, + "20": { + "cells": 12, + "completion_time_sources": { + "reconstructed_from_latency": 36 + }, + "examples": 36, + "headline": { + "bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + 0.0, + 0.12195121951219512 + ], + "point": 0.05555555555555547 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.09816256823569453, + -0.009602741441109449 + ], + "point": -0.04807041919163683 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "instrumentation_aware": { + "classification": { + "accuracy": 0.9166666666666666, + "balanced_accuracy": 0.8571428571428572, + "brier": 0.05902138516195827, + "confusion": { + "false_negative": 1, + "false_positive": 2, + "true_negative": 6, + "true_positive": 27 + }, + "log_loss": 0.24132948833387438 + }, + "policies": [ + { + "abstain_continue_full": 2, + "correctly_saved_h20_hours": 0.6191580696761111, + "decision_coverage": 0.9444444444444444, + "early_accept": 28, + "early_reject": 6, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.8399807697230681, + "remaining_h20_hours": 0.0017845769727777771 + } + ], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.008919034978436878, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.009139434342777777, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.6282975040188888, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 5, + "correctly_saved_h20_hours": 0.6101058238716667, + "decision_coverage": 0.8611111111111112, + "early_accept": 27, + "early_reject": 4, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.008919034978436878, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0073548573700000005, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.6174606812416666, + "threshold": 0.9, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 9, + "correctly_saved_h20_hours": 0.5390392875763889, + "decision_coverage": 0.75, + "early_accept": 23, + "early_reject": 4, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.008919034978436878, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0073548573700000005, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.5463941449463888, + "threshold": 0.95, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + } + ] + }, + "outcome_only": { + "classification": { + "accuracy": 0.8611111111111112, + "balanced_accuracy": 0.7321428571428572, + "brier": 0.1070918043535951, + "confusion": { + "false_negative": 1, + "false_positive": 4, + "true_negative": 4, + "true_positive": 27 + }, + "log_loss": 0.437208492871466 + }, + "policies": [ + { + "abstain_continue_full": 4, + "correctly_saved_h20_hours": 0.5864084509805555, + "decision_coverage": 0.8888888888888888, + "early_accept": 28, + "early_reject": 4, + "false_accept": 3, + "false_accept_examples": [ + { + "anchor": 0.5, + "cell": "tp1_mns64", + "label_feasible": false, + "probability_feasible": 0.9419008835359353, + "remaining_h20_hours": 5.564286972222258e-05 + }, + { + "anchor": 0.75, + "cell": "tp2_mns64", + "label_feasible": false, + "probability_feasible": 0.8529942983175343, + "remaining_h20_hours": 0.0012268335861111111 + }, + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.8268198678404454, + "remaining_h20_hours": 0.0017845769727777771 + } + ], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.0017338236710752416, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.01042191079861111, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.5968303617791667, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 8, + "correctly_saved_h20_hours": 0.5682670563286111, + "decision_coverage": 0.7777777777777778, + "early_accept": 25, + "early_reject": 3, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.5, + "cell": "tp1_mns64", + "label_feasible": false, + "probability_feasible": 0.9419008835359353, + "remaining_h20_hours": 5.564286972222258e-05 + } + ], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.0017338236710752416, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.007410500239722223, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.5756775565683333, + "threshold": 0.9, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 17, + "correctly_saved_h20_hours": 0.40106617994777777, + "decision_coverage": 0.5277777777777778, + "early_accept": 16, + "early_reject": 3, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.0017338236710752416, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0073548573700000005, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.4084210373177778, + "threshold": 0.95, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + } + ] + }, + "paired_correctness": { + "both_correct": 31, + "both_wrong": 3, + "instrumentation_only_correct": 2, + "mcnemar_exact_two_sided_p": 0.5, + "outcome_only_correct": 0 + } + }, + "headline_regularization": 1.0, + "label_sanity": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 36, + "negative": 8, + "positive": 28, + "primary_adjudicated_disagreements": 3 + }, + "regularization_sensitivity": { + "0.1": { + "bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + 0.0, + 0.1842105263157895 + ], + "point": 0.08333333333333326 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.1289106028344674, + -0.00039980805027801426 + ], + "point": -0.0571263639718715 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "instrumentation_aware": { + "classification": { + "accuracy": 0.9444444444444444, + "balanced_accuracy": 0.9196428571428572, + "brier": 0.05738846072877349, + "confusion": { + "false_negative": 1, + "false_positive": 1, + "true_negative": 7, + "true_positive": 27 + }, + "log_loss": 0.3180853188579326 + }, + "policies": [ + { + "abstain_continue_full": 2, + "correctly_saved_h20_hours": 0.6191580696761111, + "decision_coverage": 0.9444444444444444, + "early_accept": 28, + "early_reject": 6, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.8655614945720641, + "remaining_h20_hours": 0.0017845769727777771 + } + ], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.00028842593944516384, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.009139434342777777, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.6282975040188888, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 3, + "correctly_saved_h20_hours": 0.6191580696761111, + "decision_coverage": 0.9166666666666666, + "early_accept": 27, + "early_reject": 6, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.00028842593944516384, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0073548573700000005, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.6265129270461112, + "threshold": 0.9, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 4, + "correctly_saved_h20_hours": 0.6074562703719444, + "decision_coverage": 0.8888888888888888, + "early_accept": 26, + "early_reject": 6, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.00028842593944516384, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0073548573700000005, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.6148111277419444, + "threshold": 0.95, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + } + ] + }, + "outcome_only": { + "classification": { + "accuracy": 0.8611111111111112, + "balanced_accuracy": 0.7767857142857143, + "brier": 0.11451482470064499, + "confusion": { + "false_negative": 2, + "false_positive": 3, + "true_negative": 5, + "true_positive": 26 + }, + "log_loss": 0.7747412375287079 + }, + "policies": [ + { + "abstain_continue_full": 3, + "correctly_saved_h20_hours": 0.5848221568852777, + "decision_coverage": 0.9166666666666666, + "early_accept": 27, + "early_reject": 6, + "false_accept": 2, + "false_accept_examples": [ + { + "anchor": 0.5, + "cell": "tp1_mns64", + "label_feasible": false, + "probability_feasible": 0.9999405808371276, + "remaining_h20_hours": 5.564286972222258e-05 + }, + { + "anchor": 0.75, + "cell": "tp2_mns64", + "label_feasible": false, + "probability_feasible": 0.9739951015698483, + "remaining_h20_hours": 0.0012268335861111111 + } + ], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 1.6376668165938014e-05, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.008637333825833335, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.5934594907111111, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 4, + "correctly_saved_h20_hours": 0.5791295938452777, + "decision_coverage": 0.8888888888888888, + "early_accept": 27, + "early_reject": 5, + "false_accept": 2, + "false_accept_examples": [ + { + "anchor": 0.5, + "cell": "tp1_mns64", + "label_feasible": false, + "probability_feasible": 0.9999405808371276, + "remaining_h20_hours": 5.564286972222258e-05 + }, + { + "anchor": 0.75, + "cell": "tp2_mns64", + "label_feasible": false, + "probability_feasible": 0.9739951015698483, + "remaining_h20_hours": 0.0012268335861111111 + } + ], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 1.6376668165938014e-05, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.008637333825833335, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.5877669276711112, + "threshold": 0.9, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 5, + "correctly_saved_h20_hours": 0.5676638636697222, + "decision_coverage": 0.8611111111111112, + "early_accept": 26, + "early_reject": 5, + "false_accept": 2, + "false_accept_examples": [ + { + "anchor": 0.5, + "cell": "tp1_mns64", + "label_feasible": false, + "probability_feasible": 0.9999405808371276, + "remaining_h20_hours": 5.564286972222258e-05 + }, + { + "anchor": 0.75, + "cell": "tp2_mns64", + "label_feasible": false, + "probability_feasible": 0.9739951015698483, + "remaining_h20_hours": 0.0012268335861111111 + } + ], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 1.6376668165938014e-05, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.008637333825833335, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.5763011974955555, + "threshold": 0.95, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + } + ] + }, + "paired_correctness": { + "both_correct": 31, + "both_wrong": 2, + "instrumentation_only_correct": 3, + "mcnemar_exact_two_sided_p": 0.25, + "outcome_only_correct": 0 + } + }, + "1.0": { + "bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + 0.0, + 0.12195121951219512 + ], + "point": 0.05555555555555547 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.09816256823569453, + -0.009602741441109449 + ], + "point": -0.04807041919163683 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "instrumentation_aware": { + "classification": { + "accuracy": 0.9166666666666666, + "balanced_accuracy": 0.8571428571428572, + "brier": 0.05902138516195827, + "confusion": { + "false_negative": 1, + "false_positive": 2, + "true_negative": 6, + "true_positive": 27 + }, + "log_loss": 0.24132948833387438 + }, + "policies": [ + { + "abstain_continue_full": 2, + "correctly_saved_h20_hours": 0.6191580696761111, + "decision_coverage": 0.9444444444444444, + "early_accept": 28, + "early_reject": 6, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.8399807697230681, + "remaining_h20_hours": 0.0017845769727777771 + } + ], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.008919034978436878, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.009139434342777777, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.6282975040188888, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 5, + "correctly_saved_h20_hours": 0.6101058238716667, + "decision_coverage": 0.8611111111111112, + "early_accept": 27, + "early_reject": 4, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.008919034978436878, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0073548573700000005, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.6174606812416666, + "threshold": 0.9, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 9, + "correctly_saved_h20_hours": 0.5390392875763889, + "decision_coverage": 0.75, + "early_accept": 23, + "early_reject": 4, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.008919034978436878, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0073548573700000005, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.5463941449463888, + "threshold": 0.95, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + } + ] + }, + "outcome_only": { + "classification": { + "accuracy": 0.8611111111111112, + "balanced_accuracy": 0.7321428571428572, + "brier": 0.1070918043535951, + "confusion": { + "false_negative": 1, + "false_positive": 4, + "true_negative": 4, + "true_positive": 27 + }, + "log_loss": 0.437208492871466 + }, + "policies": [ + { + "abstain_continue_full": 4, + "correctly_saved_h20_hours": 0.5864084509805555, + "decision_coverage": 0.8888888888888888, + "early_accept": 28, + "early_reject": 4, + "false_accept": 3, + "false_accept_examples": [ + { + "anchor": 0.5, + "cell": "tp1_mns64", + "label_feasible": false, + "probability_feasible": 0.9419008835359353, + "remaining_h20_hours": 5.564286972222258e-05 + }, + { + "anchor": 0.75, + "cell": "tp2_mns64", + "label_feasible": false, + "probability_feasible": 0.8529942983175343, + "remaining_h20_hours": 0.0012268335861111111 + }, + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.8268198678404454, + "remaining_h20_hours": 0.0017845769727777771 + } + ], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.0017338236710752416, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.01042191079861111, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.5968303617791667, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 8, + "correctly_saved_h20_hours": 0.5682670563286111, + "decision_coverage": 0.7777777777777778, + "early_accept": 25, + "early_reject": 3, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.5, + "cell": "tp1_mns64", + "label_feasible": false, + "probability_feasible": 0.9419008835359353, + "remaining_h20_hours": 5.564286972222258e-05 + } + ], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.0017338236710752416, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.007410500239722223, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.5756775565683333, + "threshold": 0.9, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 17, + "correctly_saved_h20_hours": 0.40106617994777777, + "decision_coverage": 0.5277777777777778, + "early_accept": 16, + "early_reject": 3, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.0017338236710752416, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0073548573700000005, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.4084210373177778, + "threshold": 0.95, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + } + ] + }, + "paired_correctness": { + "both_correct": 31, + "both_wrong": 3, + "instrumentation_only_correct": 2, + "mcnemar_exact_two_sided_p": 0.5, + "outcome_only_correct": 0 + } + }, + "10.0": { + "bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + 0.0, + 0.17142857142857137 + ], + "point": 0.08333333333333326 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.047457441513371, + -0.01422543161645023 + ], + "point": -0.029913894255621076 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "instrumentation_aware": { + "classification": { + "accuracy": 0.9166666666666666, + "balanced_accuracy": 0.8571428571428572, + "brier": 0.07346598213380073, + "confusion": { + "false_negative": 1, + "false_positive": 2, + "true_negative": 6, + "true_positive": 27 + }, + "log_loss": 0.24820863982463115 + }, + "policies": [ + { + "abstain_continue_full": 5, + "correctly_saved_h20_hours": 0.6026586119891666, + "decision_coverage": 0.8611111111111112, + "early_accept": 27, + "early_reject": 4, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.8171600256181173, + "remaining_h20_hours": 0.0017845769727777771 + } + ], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.08736122237187642, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.009139434342777777, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.6117980463319445, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 8, + "correctly_saved_h20_hours": 0.5909011698152777, + "decision_coverage": 0.7777777777777778, + "early_accept": 25, + "early_reject": 3, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.08736122237187642, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0073548573700000005, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.5982560271852778, + "threshold": 0.9, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 27, + "correctly_saved_h20_hours": 0.27948134858055557, + "decision_coverage": 0.25, + "early_accept": 9, + "early_reject": 0, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.27948134858055557, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.26293898566315754, + "valid_zero_error_policy": true + } + ] + }, + "outcome_only": { + "classification": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.7142857142857143, + "brier": 0.10337987638942181, + "confusion": { + "false_negative": 2, + "false_positive": 4, + "true_negative": 4, + "true_positive": 26 + }, + "log_loss": 0.34599307331049317 + }, + "policies": [ + { + "abstain_continue_full": 7, + "correctly_saved_h20_hours": 0.6007090416925, + "decision_coverage": 0.8055555555555556, + "early_accept": 27, + "early_reject": 2, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.8140697427804628, + "remaining_h20_hours": 0.0017845769727777771 + } + ], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.0444153615651932, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.009139434342777777, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.6098484760352777, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 12, + "correctly_saved_h20_hours": 0.5199227922302778, + "decision_coverage": 0.6666666666666666, + "early_accept": 23, + "early_reject": 1, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.0444153615651932, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0073548573700000005, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.5272776496002778, + "threshold": 0.9, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 33, + "correctly_saved_h20_hours": 0.08992033662666665, + "decision_coverage": 0.08333333333333333, + "early_accept": 2, + "early_reject": 1, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.033717411016, + "cell": "tp4_mns32", + "label_feasible": true, + "probability_feasible": 0.0444153615651932, + "remaining_h20_hours": 0.0073548573700000005 + } + ], + "full_trial_h20_hours": 1.0629133138080555, + "invalidly_saved_h20_hours": 0.0073548573700000005, + "remaining_h20_hours_at_cutoff": 0.6295799804747222, + "saved_h20_hours_if_decisions_used": 0.09727519399666666, + "threshold": 0.95, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + } + ] + }, + "paired_correctness": { + "both_correct": 30, + "both_wrong": 3, + "instrumentation_only_correct": 3, + "mcnemar_exact_two_sided_p": 0.25, + "outcome_only_correct": 0 + } + } + }, + "remaining_h20_hours": { + "distinct_n": 36, + "max": 0.04608871510333333, + "min": 5.564286972222258e-05, + "n": 36 + } + }, + "5": { + "cells": 12, + "completion_time_sources": { + "none_completed": 2, + "reconstructed_from_latency": 35 + }, + "examples": 37, + "headline": { + "bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + 0.02777777777777779, + 0.19999999999999996 + ], + "point": 0.10810810810810811 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.08739451923629489, + 0.010803161364314168 + ], + "point": -0.03964626050589401 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "instrumentation_aware": { + "classification": { + "accuracy": 0.8918918918918919, + "balanced_accuracy": 0.8154761904761905, + "brier": 0.09006542265758688, + "confusion": { + "false_negative": 1, + "false_positive": 3, + "true_negative": 6, + "true_positive": 27 + }, + "log_loss": 0.2730810945072346 + }, + "policies": [ + { + "abstain_continue_full": 9, + "correctly_saved_h20_hours": 0.7828422774630556, + "decision_coverage": 0.7567567567567568, + "early_accept": 23, + "early_reject": 5, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.918171067470718, + "remaining_h20_hours": 0.010117910306111111 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.010117910306111111, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.7929601877691667, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 12, + "correctly_saved_h20_hours": 0.7136471349655555, + "decision_coverage": 0.6756756756756757, + "early_accept": 21, + "early_reject": 4, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.918171067470718, + "remaining_h20_hours": 0.010117910306111111 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.010117910306111111, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.7237650452716666, + "threshold": 0.9, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 15, + "correctly_saved_h20_hours": 0.6518691238655556, + "decision_coverage": 0.5945945945945946, + "early_accept": 18, + "early_reject": 4, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.6518691238655556, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.6109595741422738, + "valid_zero_error_policy": true + } + ] + }, + "outcome_only": { + "classification": { + "accuracy": 0.7837837837837838, + "balanced_accuracy": 0.7063492063492063, + "brier": 0.1297116831634809, + "confusion": { + "false_negative": 4, + "false_positive": 4, + "true_negative": 5, + "true_positive": 24 + }, + "log_loss": 0.38484417997476916 + }, + "policies": [ + { + "abstain_continue_full": 9, + "correctly_saved_h20_hours": 0.745153891865, + "decision_coverage": 0.7567567567567568, + "early_accept": 24, + "early_reject": 4, + "false_accept": 3, + "false_accept_examples": [ + { + "anchor": 0.75, + "cell": "tp2_mns64", + "label_feasible": false, + "probability_feasible": 0.8051358474368825, + "remaining_h20_hours": 0.009560166919444444 + }, + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.9221804334720108, + "remaining_h20_hours": 0.010117910306111111 + }, + { + "anchor": 0.034252608017, + "cell": "tp4_mns8", + "label_feasible": false, + "probability_feasible": 0.8933807379877649, + "remaining_h20_hours": 0.022359229706666667 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.042037306932222226, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.7871911987972221, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 14, + "correctly_saved_h20_hours": 0.6362291990002777, + "decision_coverage": 0.6216216216216216, + "early_accept": 19, + "early_reject": 4, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.9221804334720108, + "remaining_h20_hours": 0.010117910306111111 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.010117910306111111, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.6463471093063888, + "threshold": 0.9, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 21, + "correctly_saved_h20_hours": 0.3878475556194444, + "decision_coverage": 0.43243243243243246, + "early_accept": 13, + "early_reject": 3, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.3878475556194444, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.3635072881013599, + "valid_zero_error_policy": true + } + ] + }, + "paired_correctness": { + "both_correct": 29, + "both_wrong": 4, + "instrumentation_only_correct": 4, + "mcnemar_exact_two_sided_p": 0.125, + "outcome_only_correct": 0 + } + }, + "headline_regularization": 1.0, + "label_sanity": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 37, + "negative": 9, + "positive": 28, + "primary_adjudicated_disagreements": 3 + }, + "regularization_sensitivity": { + "0.1": { + "bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.06980897009966774, + 0.21875 + ], + "point": 0.08108108108108103 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.1649967051920624, + 0.06447487686406284 + ], + "point": -0.05737762784013967 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "instrumentation_aware": { + "classification": { + "accuracy": 0.8378378378378378, + "balanced_accuracy": 0.7797619047619048, + "brier": 0.13521691422103324, + "confusion": { + "false_negative": 3, + "false_positive": 3, + "true_negative": 6, + "true_positive": 25 + }, + "log_loss": 0.4511157566254769 + }, + "policies": [ + { + "abstain_continue_full": 3, + "correctly_saved_h20_hours": 0.8122652196319445, + "decision_coverage": 0.918918918918919, + "early_accept": 26, + "early_reject": 8, + "false_accept": 3, + "false_accept_examples": [ + { + "anchor": 0.75, + "cell": "tp2_mns64", + "label_feasible": false, + "probability_feasible": 0.9405657729239719, + "remaining_h20_hours": 0.009560166919444444 + }, + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.9783420846880035, + "remaining_h20_hours": 0.010117910306111111 + }, + { + "anchor": 0.033717411016, + "cell": "tp4_mns16", + "label_feasible": false, + "probability_feasible": 0.9635702290378427, + "remaining_h20_hours": 0.023106262014444445 + } + ], + "false_reject": 2, + "false_reject_examples": [ + { + "anchor": 0.21875, + "cell": "tp1_mns8", + "label_feasible": true, + "probability_feasible": 0.07387488532278061, + "remaining_h20_hours": 0.015631309493055554 + }, + { + "anchor": 0.49609375, + "cell": "tp2_mns16", + "label_feasible": true, + "probability_feasible": 0.17417966056624304, + "remaining_h20_hours": 0.03127135866722222 + } + ], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.08968700740027778, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.9019522270322223, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 6, + "correctly_saved_h20_hours": 0.765429306841111, + "decision_coverage": 0.8378378378378378, + "early_accept": 24, + "early_reject": 7, + "false_accept": 3, + "false_accept_examples": [ + { + "anchor": 0.75, + "cell": "tp2_mns64", + "label_feasible": false, + "probability_feasible": 0.9405657729239719, + "remaining_h20_hours": 0.009560166919444444 + }, + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.9783420846880035, + "remaining_h20_hours": 0.010117910306111111 + }, + { + "anchor": 0.033717411016, + "cell": "tp4_mns16", + "label_feasible": false, + "probability_feasible": 0.9635702290378427, + "remaining_h20_hours": 0.023106262014444445 + } + ], + "false_reject": 1, + "false_reject_examples": [ + { + "anchor": 0.21875, + "cell": "tp1_mns8", + "label_feasible": true, + "probability_feasible": 0.07387488532278061, + "remaining_h20_hours": 0.015631309493055554 + } + ], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.05841564873305555, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.8238449555741666, + "threshold": 0.9, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 9, + "correctly_saved_h20_hours": 0.7469523484622221, + "decision_coverage": 0.7567567567567568, + "early_accept": 22, + "early_reject": 6, + "false_accept": 2, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.9783420846880035, + "remaining_h20_hours": 0.010117910306111111 + }, + { + "anchor": 0.033717411016, + "cell": "tp4_mns16", + "label_feasible": false, + "probability_feasible": 0.9635702290378427, + "remaining_h20_hours": 0.023106262014444445 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.03322417232055556, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.7801765207827777, + "threshold": 0.95, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + } + ] + }, + "outcome_only": { + "classification": { + "accuracy": 0.7567567567567568, + "balanced_accuracy": 0.7261904761904762, + "brier": 0.1925945420611729, + "confusion": { + "false_negative": 6, + "false_positive": 3, + "true_negative": 6, + "true_positive": 22 + }, + "log_loss": 0.7361068786964186 + }, + "policies": [ + { + "abstain_continue_full": 4, + "correctly_saved_h20_hours": 0.7745768340338888, + "decision_coverage": 0.8918918918918919, + "early_accept": 25, + "early_reject": 8, + "false_accept": 3, + "false_accept_examples": [ + { + "anchor": 0.75, + "cell": "tp2_mns64", + "label_feasible": false, + "probability_feasible": 0.977871407650825, + "remaining_h20_hours": 0.009560166919444444 + }, + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.9783309849310482, + "remaining_h20_hours": 0.010117910306111111 + }, + { + "anchor": 0.034252608017, + "cell": "tp4_mns8", + "label_feasible": false, + "probability_feasible": 0.9900156648758626, + "remaining_h20_hours": 0.022359229706666667 + } + ], + "false_reject": 3, + "false_reject_examples": [ + { + "anchor": 0.21875, + "cell": "tp1_mns8", + "label_feasible": true, + "probability_feasible": 0.012882187111638552, + "remaining_h20_hours": 0.015631309493055554 + }, + { + "anchor": 0.2265625, + "cell": "tp1_mns8", + "label_feasible": true, + "probability_feasible": 0.016042537020255383, + "remaining_h20_hours": 0.01563306773861111 + }, + { + "anchor": 0.23046875, + "cell": "tp1_mns8", + "label_feasible": true, + "probability_feasible": 0.15321211351284938, + "remaining_h20_hours": 0.015631133334166665 + } + ], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.08893281749805555, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.8635096515319445, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 8, + "correctly_saved_h20_hours": 0.6784465414575, + "decision_coverage": 0.7837837837837838, + "early_accept": 22, + "early_reject": 7, + "false_accept": 3, + "false_accept_examples": [ + { + "anchor": 0.75, + "cell": "tp2_mns64", + "label_feasible": false, + "probability_feasible": 0.977871407650825, + "remaining_h20_hours": 0.009560166919444444 + }, + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.9783309849310482, + "remaining_h20_hours": 0.010117910306111111 + }, + { + "anchor": 0.034252608017, + "cell": "tp4_mns8", + "label_feasible": false, + "probability_feasible": 0.9900156648758626, + "remaining_h20_hours": 0.022359229706666667 + } + ], + "false_reject": 2, + "false_reject_examples": [ + { + "anchor": 0.21875, + "cell": "tp1_mns8", + "label_feasible": true, + "probability_feasible": 0.012882187111638552, + "remaining_h20_hours": 0.015631309493055554 + }, + { + "anchor": 0.2265625, + "cell": "tp1_mns8", + "label_feasible": true, + "probability_feasible": 0.016042537020255383, + "remaining_h20_hours": 0.01563306773861111 + } + ], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0733016841638889, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.7517482256213891, + "threshold": 0.9, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 9, + "correctly_saved_h20_hours": 0.6471751827902779, + "decision_coverage": 0.7567567567567568, + "early_accept": 21, + "early_reject": 7, + "false_accept": 3, + "false_accept_examples": [ + { + "anchor": 0.75, + "cell": "tp2_mns64", + "label_feasible": false, + "probability_feasible": 0.977871407650825, + "remaining_h20_hours": 0.009560166919444444 + }, + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.9783309849310482, + "remaining_h20_hours": 0.010117910306111111 + }, + { + "anchor": 0.034252608017, + "cell": "tp4_mns8", + "label_feasible": false, + "probability_feasible": 0.9900156648758626, + "remaining_h20_hours": 0.022359229706666667 + } + ], + "false_reject": 2, + "false_reject_examples": [ + { + "anchor": 0.21875, + "cell": "tp1_mns8", + "label_feasible": true, + "probability_feasible": 0.012882187111638552, + "remaining_h20_hours": 0.015631309493055554 + }, + { + "anchor": 0.2265625, + "cell": "tp1_mns8", + "label_feasible": true, + "probability_feasible": 0.016042537020255383, + "remaining_h20_hours": 0.01563306773861111 + } + ], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0733016841638889, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.7204768669541667, + "threshold": 0.95, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + } + ] + }, + "paired_correctness": { + "both_correct": 26, + "both_wrong": 4, + "instrumentation_only_correct": 5, + "mcnemar_exact_two_sided_p": 0.453125, + "outcome_only_correct": 2 + } + }, + "1.0": { + "bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + 0.02777777777777779, + 0.19999999999999996 + ], + "point": 0.10810810810810811 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.08739451923629489, + 0.010803161364314168 + ], + "point": -0.03964626050589401 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "instrumentation_aware": { + "classification": { + "accuracy": 0.8918918918918919, + "balanced_accuracy": 0.8154761904761905, + "brier": 0.09006542265758688, + "confusion": { + "false_negative": 1, + "false_positive": 3, + "true_negative": 6, + "true_positive": 27 + }, + "log_loss": 0.2730810945072346 + }, + "policies": [ + { + "abstain_continue_full": 9, + "correctly_saved_h20_hours": 0.7828422774630556, + "decision_coverage": 0.7567567567567568, + "early_accept": 23, + "early_reject": 5, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.918171067470718, + "remaining_h20_hours": 0.010117910306111111 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.010117910306111111, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.7929601877691667, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 12, + "correctly_saved_h20_hours": 0.7136471349655555, + "decision_coverage": 0.6756756756756757, + "early_accept": 21, + "early_reject": 4, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.918171067470718, + "remaining_h20_hours": 0.010117910306111111 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.010117910306111111, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.7237650452716666, + "threshold": 0.9, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 15, + "correctly_saved_h20_hours": 0.6518691238655556, + "decision_coverage": 0.5945945945945946, + "early_accept": 18, + "early_reject": 4, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.6518691238655556, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.6109595741422738, + "valid_zero_error_policy": true + } + ] + }, + "outcome_only": { + "classification": { + "accuracy": 0.7837837837837838, + "balanced_accuracy": 0.7063492063492063, + "brier": 0.1297116831634809, + "confusion": { + "false_negative": 4, + "false_positive": 4, + "true_negative": 5, + "true_positive": 24 + }, + "log_loss": 0.38484417997476916 + }, + "policies": [ + { + "abstain_continue_full": 9, + "correctly_saved_h20_hours": 0.745153891865, + "decision_coverage": 0.7567567567567568, + "early_accept": 24, + "early_reject": 4, + "false_accept": 3, + "false_accept_examples": [ + { + "anchor": 0.75, + "cell": "tp2_mns64", + "label_feasible": false, + "probability_feasible": 0.8051358474368825, + "remaining_h20_hours": 0.009560166919444444 + }, + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.9221804334720108, + "remaining_h20_hours": 0.010117910306111111 + }, + { + "anchor": 0.034252608017, + "cell": "tp4_mns8", + "label_feasible": false, + "probability_feasible": 0.8933807379877649, + "remaining_h20_hours": 0.022359229706666667 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.042037306932222226, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.7871911987972221, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 14, + "correctly_saved_h20_hours": 0.6362291990002777, + "decision_coverage": 0.6216216216216216, + "early_accept": 19, + "early_reject": 4, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.9221804334720108, + "remaining_h20_hours": 0.010117910306111111 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.010117910306111111, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.6463471093063888, + "threshold": 0.9, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 21, + "correctly_saved_h20_hours": 0.3878475556194444, + "decision_coverage": 0.43243243243243246, + "early_accept": 13, + "early_reject": 3, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.3878475556194444, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.3635072881013599, + "valid_zero_error_policy": true + } + ] + }, + "paired_correctness": { + "both_correct": 29, + "both_wrong": 4, + "instrumentation_only_correct": 4, + "mcnemar_exact_two_sided_p": 0.125, + "outcome_only_correct": 0 + } + }, + "10.0": { + "bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.08698849104859337, + 0.07499999999999996 + ], + "point": 0.0 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.0419671863122155, + 0.010410939632453053 + ], + "point": -0.015389088872407489 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "instrumentation_aware": { + "classification": { + "accuracy": 0.8378378378378378, + "balanced_accuracy": 0.7420634920634921, + "brier": 0.09695183324390907, + "confusion": { + "false_negative": 2, + "false_positive": 4, + "true_negative": 5, + "true_positive": 26 + }, + "log_loss": 0.31480154295615903 + }, + "policies": [ + { + "abstain_continue_full": 9, + "correctly_saved_h20_hours": 0.7917544064236111, + "decision_coverage": 0.7567567567567568, + "early_accept": 24, + "early_reject": 4, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.8905819069991598, + "remaining_h20_hours": 0.010117910306111111 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.010117910306111111, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.8018723167297224, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 21, + "correctly_saved_h20_hours": 0.5056768371675, + "decision_coverage": 0.43243243243243246, + "early_accept": 13, + "early_reject": 3, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.5056768371675, + "threshold": 0.9, + "valid_cost_reduction_fraction": 0.4739419214357306, + "valid_zero_error_policy": true + }, + { + "abstain_continue_full": 30, + "correctly_saved_h20_hours": 0.19128570447222223, + "decision_coverage": 0.1891891891891892, + "early_accept": 6, + "early_reject": 1, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.19128570447222223, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.17928112908743485, + "valid_zero_error_policy": true + } + ] + }, + "outcome_only": { + "classification": { + "accuracy": 0.8378378378378378, + "balanced_accuracy": 0.7043650793650793, + "brier": 0.11234092211631656, + "confusion": { + "false_negative": 1, + "false_positive": 5, + "true_negative": 4, + "true_positive": 27 + }, + "log_loss": 0.36773394130409404 + }, + "policies": [ + { + "abstain_continue_full": 15, + "correctly_saved_h20_hours": 0.6981008277897223, + "decision_coverage": 0.5945945945945946, + "early_accept": 21, + "early_reject": 1, + "false_accept": 1, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.8520031544496152, + "remaining_h20_hours": 0.010117910306111111 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.010117910306111111, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.7082187380958334, + "threshold": 0.8, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + }, + { + "abstain_continue_full": 31, + "correctly_saved_h20_hours": 0.23290921672555553, + "decision_coverage": 0.16216216216216217, + "early_accept": 6, + "early_reject": 0, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.23290921672555553, + "threshold": 0.9, + "valid_cost_reduction_fraction": 0.21829246186816503, + "valid_zero_error_policy": true + }, + { + "abstain_continue_full": 34, + "correctly_saved_h20_hours": 0.13888648644666668, + "decision_coverage": 0.08108108108108109, + "early_accept": 3, + "early_reject": 0, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.13888648644666668, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.13017034479312573, + "valid_zero_error_policy": true + } + ] + }, + "paired_correctness": { + "both_correct": 30, + "both_wrong": 5, + "instrumentation_only_correct": 1, + "mcnemar_exact_two_sided_p": 1.0, + "outcome_only_correct": 1 + } + } + }, + "remaining_h20_hours": { + "distinct_n": 37, + "max": 0.06275538177, + "min": 0.0026573007705555556, + "n": 37 + } + } + }, + "cutoffs_s": [ + 5.0, + 10.0, + 15.0, + 20.0 + ], + "decision": { + "contribution_established": false, + "reason": "This dataset contains one workload and reconstructed rather than exact request completion times. Three TP4 primary trials also disagree with their 2-of-3 labels. It can reject a missing-signal premise but cannot establish generalization or a paper-facing cost reduction." + }, + "estimand": "2-of-3 adjudicated anchor feasibility from the first primary trial's identical short real prefix", + "headline_incremental_deltas": { + "10": { + "accuracy": -0.027027027027027084, + "brier": -0.0026830647286030934 + }, + "15": { + "accuracy": 0.02777777777777779, + "brier": -0.03744665409603411 + }, + "20": { + "accuracy": 0.05555555555555547, + "brier": -0.04807041919163683 + }, + "5": { + "accuracy": 0.10810810810810811, + "brier": -0.03964626050589401 + } + }, + "instrumentation_features": [ + "model_steps_per_second", + "waiting_mean", + "waiting_max", + "waiting_nonzero_share", + "running_mean", + "running_max", + "decode_batch_mean", + "decode_batch_max", + "decode_batch_cv", + "kv_usage_mean", + "kv_usage_max", + "kv_usage_end_minus_start", + "graph_none_share", + "graph_full_share", + "padding_fraction", + "prefill_token_fraction", + "preemptions" + ], + "model": "same L2 logistic model and folds; instrumentation model appends Layer-1 features", + "outcome_features": [ + "log_offered_rate_per_gpu", + "log2_tp", + "log2_max_num_seqs", + "admitted_fraction", + "completed_over_admitted", + "completed_pass_rate", + "completed_fail_fraction_of_total", + "outstanding_over_admitted", + "ttft_max_over_slo_max", + "ttft_mean_over_slo_max", + "tpot_max_over_slo", + "tpot_mean_over_slo", + "admitted_input_tokens_mean_over_limit" + ], + "provenance": { + "phase6_metrics": "/home/gahow/phd/aituner/runs/opprof-phase6/phase6/metrics.json", + "phase6_metrics_sha256": "290ba7fcb8727291166de7e4d47afdc84e230052495c81dd087db0ace9f93a16", + "raw_root": "/home/gahow/phd/aituner/runs/opprof-phase6/phase6/solo-authoritative/cells" + }, + "sanity": { + "cutoff_count": 4, + "invariants": { + "cutoffs_positive": true, + "full_trial_label_not_used_as_feature": true, + "paired_same_model_family": true, + "probabilities_checked_in_unit_interval": true, + "records_strictly_prefix_sliced": true + }, + "red_flags": [] + }, + "schema": "fidelity-prefix-v1", + "scope": "retrospective single-workload prefix diagnostic; model selection, threshold choice, and contribution claims require held-out prospective tasks", + "split": "leave-one-configuration-cell-out", + "status": "PASS" +} diff --git a/runs/fidelity-headroom/prepare_pilot.py b/runs/fidelity-headroom/prepare_pilot.py new file mode 100644 index 0000000..bb849c4 --- /dev/null +++ b/runs/fidelity-headroom/prepare_pilot.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +"""Materialize session-disjoint pilot repeats and freeze attainable anchors. + +The private outputs retain prompt text and stay on the experiment host. The +public manifest contains only aggregate counts, hashes, paths, and parameters. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import sys +from pathlib import Path +from typing import Any + + +AITUNER_ROOT = Path(os.environ.get("AITUNER_ROOT", Path(__file__).resolve().parents[2])) +sys.path.insert(0, str(AITUNER_ROOT / "src")) + +from aituner.spec import load_study_spec # noqa: E402 +from aituner.trace import load_trace_requests, select_requests_for_threshold # noqa: E402 + + +ROLES = ("burnin", "low1", "high1", "low2", "high2", "low3", "high3") +CELLS = { + "tp1_mns8": {"tp": 1, "mns": 8, "frontier_req_s_gpu": 2.3833333333333333}, + "tp1_mns64": {"tp": 1, "mns": 64, "frontier_req_s_gpu": 2.3833333333333333}, + "tp2_mns8": {"tp": 2, "mns": 8, "frontier_req_s_gpu": 2.2416666666666667}, + "tp2_mns64": {"tp": 2, "mns": 64, "frontier_req_s_gpu": 2.3}, + "tp4_mns16": {"tp": 4, "mns": 16, "frontier_req_s_gpu": 2.5}, + "tp4_mns64": {"tp": 4, "mns": 64, "frontier_req_s_gpu": 2.5}, +} +TARGET_MULTIPLIERS = {"low": 0.85, "high": 1.25} + + +def atomic_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.replace(tmp, path) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def order_hash(values: list[str]) -> str: + return hashlib.sha256("\n".join(values).encode()).hexdigest() + + +def resolve_source_trace(windows_path: Path, window_id: str) -> tuple[dict[str, Any], Path]: + payload = json.loads(windows_path.read_text(encoding="utf-8")) + for window in payload["windows"]: + if window["window_id"] != window_id: + continue + trace = Path(window["trace_file"]) + if not trace.is_absolute(): + trace = (windows_path.parent / trace).resolve() + return window, trace + raise ValueError(f"window not found: {window_id}") + + +def materialize_bands( + source_trace: Path, + source_window: dict[str, Any], + private_root: Path, +) -> tuple[Path, dict[str, Any]]: + traces_root = private_root / "traces" + traces_root.mkdir(parents=True, exist_ok=True) + temporary = {role: traces_root / f".{role}.jsonl.tmp" for role in ROLES} + final = {role: traces_root / f"{role}.jsonl" for role in ROLES} + handles = {role: temporary[role].open("w", encoding="utf-8") for role in ROLES} + stats = { + role: { + "rows": 0, + "sum_input_tokens": 0, + "min_timestamp": None, + "max_timestamp": None, + } + for role in ROLES + } + try: + with source_trace.open(encoding="utf-8") as source: + for line_number, line in enumerate(source): + row = json.loads(line) + value = float(row["sampling_u"]) + if not 0.0 <= value <= 1.0: + raise ValueError(f"sampling_u outside [0,1] at line {line_number}") + band = min(len(ROLES) - 1, int(value * len(ROLES))) + role = ROLES[band] + remapped = value * len(ROLES) - band + row["sampling_u"] = min(remapped, math.nextafter(1.0, 0.0)) + row["fidelity_pilot_band"] = role + handles[role].write(json.dumps(row, ensure_ascii=False) + "\n") + timestamp = float(row["timestamp"]) + item = stats[role] + item["rows"] += 1 + item["sum_input_tokens"] += int(row.get("input_length") or 0) + item["min_timestamp"] = ( + timestamp if item["min_timestamp"] is None + else min(float(item["min_timestamp"]), timestamp) + ) + item["max_timestamp"] = ( + timestamp if item["max_timestamp"] is None + else max(float(item["max_timestamp"]), timestamp) + ) + finally: + for handle in handles.values(): + handle.close() + for role in ROLES: + os.replace(temporary[role], final[role]) + stats[role]["sha256"] = sha256_file(final[role]) + stats[role]["bytes"] = final[role].stat().st_size + + windows = [] + for role in ROLES: + window = dict(source_window) + window["window_id"] = f"fidelity_pilot_{role}" + window["trace_file"] = f"traces/{role}.jsonl" + window["num_requests"] = stats[role]["rows"] + window["sum_input_length"] = stats[role]["sum_input_tokens"] + window["sampling_strategy"] = "session_uniform_seven_disjoint_bands_remapped" + window["fidelity_pilot_role"] = role + windows.append(window) + private_windows = private_root / "windows.json" + atomic_json( + private_windows, + { + "schema": "fidelity-pilot-private-windows-v1", + "roles": list(ROLES), + "windows": windows, + }, + ) + return private_windows, stats + + +def write_studies( + *, + base_primary: Path, + base_tp4: Path, + private_windows: Path, + private_root: Path, +) -> dict[str, dict[str, Path]]: + bases = { + "primary": json.loads(base_primary.read_text(encoding="utf-8")), + "tp4": json.loads(base_tp4.read_text(encoding="utf-8")), + } + result: dict[str, dict[str, Path]] = {} + for role in ROLES: + result[role] = {} + for tier, base in bases.items(): + payload = json.loads(json.dumps(base)) + payload["study_id"] = f"fidelity-prefix-pilot-{role}-{tier}" + payload["hardware"]["host_candidates"] = ["dash0"] + payload["engine"]["engine_version"] = "0.24.1.dev3+opprof" + payload["trace"]["windows_path"] = str(private_windows) + payload["trace"]["window_id"] = f"fidelity_pilot_{role}" + path = private_root / "studies" / f"{role}-{tier}.json" + atomic_json(path, payload) + result[role][tier] = path + return result + + +def attainable_anchor(requests: list[Any], target_count: int) -> tuple[float, list[Any]]: + ordered = sorted(float(request.sampling_u) for request in requests) + if not ordered: + raise ValueError("no requests after study filtering") + candidate_indices = sorted({ + max(0, min(len(ordered) - 1, target_count - 1)), + max(0, min(len(ordered) - 1, target_count)), + }) + candidates = [] + for index in candidate_indices: + anchor = ordered[index] + selected = select_requests_for_threshold(requests, threshold=anchor) + candidates.append((abs(len(selected) - target_count), len(selected), anchor, selected)) + _error, _count, anchor, selected = min(candidates, key=lambda item: (item[0], item[1])) + return anchor, selected + + +def selected_record(selected: list[Any], *, tp: int, duration_s: float) -> dict[str, Any]: + return { + "anchor": max(float(request.sampling_u) for request in selected), + "selected_count": len(selected), + "offered_req_s": len(selected) / duration_s, + "offered_req_s_per_gpu": len(selected) / duration_s / tp, + "request_id_order_sha256": order_hash([request.row_id for request in selected]), + "arrival_order_sha256": order_hash([f"{request.arrival_s:.12f}" for request in selected]), + "input_length_order_sha256": order_hash( + [str(request.prompt_tokens_hint) for request in selected] + ), + } + + +def build_manifest( + *, + studies: dict[str, dict[str, Path]], + private_windows: Path, + band_stats: dict[str, Any], + source_trace: Path, + source_windows: Path, + source_window_id: str, +) -> dict[str, Any]: + loaded = {} + durations = {} + for role, tiers in studies.items(): + loaded[role] = {} + for tier, path in tiers.items(): + study = load_study_spec(path) + window, requests = load_trace_requests(study, study_spec_path=path) + loaded[role][tier] = requests + durations[role] = float(window.window_end - window.window_start) + + cells = {} + all_hashes = [] + for cell, config in CELLS.items(): + tp = int(config["tp"]) + tier = "tp4" if tp == 4 else "primary" + targets = {} + for level, multiplier in TARGET_MULTIPLIERS.items(): + target_rate = float(config["frontier_req_s_gpu"]) * multiplier + target_count = round(target_rate * durations["low1"] * tp) + roles = [role for role in ROLES if role == "burnin" or role.startswith(level)] + selections = {} + for role in roles: + anchor, selected = attainable_anchor(loaded[role][tier], target_count) + record = selected_record(selected, tp=tp, duration_s=durations[role]) + record["anchor"] = anchor + record["study"] = str(studies[role][tier]) + selections[role] = record + all_hashes.append(record["request_id_order_sha256"]) + targets[level] = { + "multiplier": multiplier, + "target_req_s_per_gpu": target_rate, + "target_count": target_count, + "selections": selections, + } + cells[cell] = {**config, "targets": targets} + + red_flags = [] + for cell, config in cells.items(): + for level, target in config["targets"].items(): + if not target["selections"]: + red_flags.append(f"missing_{cell}_{level}") + for selection in target["selections"].values(): + if selection["selected_count"] <= 0: + red_flags.append(f"empty_{cell}_{level}") + per_cell_distinct = {} + for cell, config in cells.items(): + hashes = [ + selection["request_id_order_sha256"] + for target in config["targets"].values() + for selection in target["selections"].values() + ] + per_cell_distinct[cell] = len(hashes) == len(set(hashes)) + if not per_cell_distinct[cell]: + red_flags.append(f"session_bands_overlap_{cell}") + return { + "schema": "fidelity-prefix-pilot-manifest-v1", + "status": "PASS" if not red_flags else "STOP", + "source": { + "windows": str(source_windows), + "window_id": source_window_id, + "trace": str(source_trace), + "trace_sha256": sha256_file(source_trace), + }, + "private": { + "windows": str(private_windows), + "windows_sha256": sha256_file(private_windows), + "band_stats": band_stats, + "studies": { + role: {tier: str(path) for tier, path in tiers.items()} + for role, tiers in studies.items() + }, + }, + "roles": list(ROLES), + "cells": cells, + "execution": { + "cutoff_s": 5.0, + "replicates_per_level": 3, + "label": "2-of-3 session-disjoint repetitions", + "even_cell_order": ["low1", "high1", "high2", "low2", "low3", "high3"], + "odd_cell_order": ["high1", "low1", "low2", "high2", "high3", "low3"], + "hard_cap_h20_hours": 3.5, + }, + "sanity": { + "red_flags": red_flags, + "n_cells": len(cells), + "n_roles": len(ROLES), + "selected_sets": len(all_hashes), + "distinct_selected_sets": len(set(all_hashes)), + "per_cell_selected_sets_distinct": per_cell_distinct, + "invariants": { + "cells_6": len(cells) == 6, + "roles_7": len(ROLES) == 7, + "band_rows_nonzero": all(stats["rows"] > 0 for stats in band_stats.values()), + "session_bands_disjoint_per_cell": all(per_cell_distinct.values()), + }, + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--source-windows", type=Path, required=True) + parser.add_argument("--source-window-id", default="chat_w20260312_1000") + parser.add_argument("--base-primary-study", type=Path, required=True) + parser.add_argument("--base-tp4-study", type=Path, required=True) + parser.add_argument("--private-root", type=Path, required=True) + parser.add_argument("--public-manifest", type=Path, required=True) + args = parser.parse_args() + + source_window, source_trace = resolve_source_trace( + args.source_windows, args.source_window_id + ) + private_windows, band_stats = materialize_bands( + source_trace, source_window, args.private_root + ) + studies = write_studies( + base_primary=args.base_primary_study, + base_tp4=args.base_tp4_study, + private_windows=private_windows, + private_root=args.private_root, + ) + manifest = build_manifest( + studies=studies, + private_windows=private_windows, + band_stats=band_stats, + source_trace=source_trace, + source_windows=args.source_windows, + source_window_id=args.source_window_id, + ) + atomic_json(args.public_manifest, manifest) + print(json.dumps({ + "status": manifest["status"], + "manifest": str(args.public_manifest), + "sanity": manifest["sanity"], + }, sort_keys=True)) + if manifest["status"] != "PASS": + raise RuntimeError(f"pilot preflight failed: {manifest['sanity']['red_flags']}") + + +if __name__ == "__main__": + main() diff --git a/runs/fidelity-headroom/test_analysis.py b/runs/fidelity-headroom/test_analysis.py new file mode 100644 index 0000000..434cd37 --- /dev/null +++ b/runs/fidelity-headroom/test_analysis.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import importlib.util +import math +import sys +from pathlib import Path + + +HERE = Path(__file__).resolve().parent + + +def load_analysis(): + spec = importlib.util.spec_from_file_location("fidelity_headroom", HERE / "analyze_existing.py") + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def main() -> None: + analysis = load_analysis() + curve = analysis.topk_curve( + {"a": 3.0, "b": 2.0, "c": 1.0}, + {"a": 1.0, "b": 2.0, "c": 2.0}, + 2e-6, + ) + assert curve["points"][0]["expanded_k"] == 2 + assert curve["points"][0]["candidates"] == ["b", "c"] + assert math.isclose(curve["points"][0]["real_regret"], 1.0 / 3.0) + assert curve["points"][2]["real_regret"] == 0.0 + assert curve["minimum_k"]["five_percent"] == {"nominal_k": 3, "expanded_k": 3} + assert analysis._mcnemar_exact_p(0, 1) == 1.0 + assert analysis._mcnemar_exact_p(0, 5) == 0.0625 + print("fidelity headroom analysis: PASS") + + +if __name__ == "__main__": + main() diff --git a/runs/fidelity-headroom/test_pilot_tools.py b/runs/fidelity-headroom/test_pilot_tools.py new file mode 100644 index 0000000..a2e1a32 --- /dev/null +++ b/runs/fidelity-headroom/test_pilot_tools.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import math +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path + + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +import pilot_controller as controller # noqa: E402 +import prepare_pilot as prepare # noqa: E402 + + +@dataclass +class Request: + row_id: str + sampling_u: float + arrival_s: float = 0.0 + prompt_tokens_hint: int = 1 + + +def main() -> None: + requests = [ + Request("a", 0.1), + Request("b", 0.2), + Request("c", 0.2), + Request("d", 0.9), + ] + anchor, selected = prepare.attainable_anchor(requests, target_count=2) + assert anchor == 0.2 + assert [request.row_id for request in selected] == ["a", "b", "c"] + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / "source.jsonl" + rows = [] + for index, role in enumerate(prepare.ROLES): + rows.append( + { + "request_id": role, + "timestamp": float(index), + "sampling_u": (index + 0.5) / len(prepare.ROLES), + "input_length": 16 + index, + "messages": [{"role": "user", "content": role}], + } + ) + source.write_text( + "".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8" + ) + windows, stats = prepare.materialize_bands( + source, + { + "window_id": "source", + "trace_type": "chat", + "window_start": 0.0, + "window_end": 600.0, + }, + root / "private", + ) + assert windows.is_file() + assert all(stats[role]["rows"] == 1 for role in prepare.ROLES) + for role in prepare.ROLES: + row = json.loads((root / "private" / "traces" / f"{role}.jsonl").read_text()) + assert row["fidelity_pilot_band"] == role + assert abs(float(row["sampling_u"]) - 0.5) < 1e-12 + + assert len(controller.ORDER) == 6 + assert set(controller.ORDER) == set(prepare.CELLS) + assert math.isclose( + sum( + controller.CELL_ESTIMATE_H20_HOURS[int(config["tp"])] + for config in prepare.CELLS.values() + ) + controller.SAFETY_H20_HOURS, + 3.0, + ) + print("fidelity pilot tools: PASS") + + +if __name__ == "__main__": + main() diff --git a/runs/fidelity-headroom/test_prefix_analysis.py b/runs/fidelity-headroom/test_prefix_analysis.py new file mode 100644 index 0000000..bc812d6 --- /dev/null +++ b/runs/fidelity-headroom/test_prefix_analysis.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import math +import sys +from pathlib import Path + + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +import analyze_prefixes as analysis # noqa: E402 + + +def main() -> None: + exact, exact_source = analysis.completion_elapsed_s( + {"completed_elapsed_s": 7.25} + ) + assert exact == 7.25 and exact_source == "exact_monotonic" + + reconstructed, reconstructed_source = analysis.completion_elapsed_s( + { + "success": True, + "arrival_s": 2.0, + "ttft_ms": 100.0, + "tpot_ms": 10.0, + "completion_tokens": 11, + } + ) + assert math.isclose(reconstructed or 0.0, 2.2) + assert reconstructed_source == "reconstructed_from_latency" + missing, missing_source = analysis.completion_elapsed_s({"success": False}) + assert missing is None and missing_source == "unobserved_failure" + + examples = [ + analysis.PrefixExample( + cell=f"c{index}", + anchor=float(index), + cutoff_s=5.0, + tp=1, + full_elapsed_s=65.0, + feasible=label, + primary_feasible=label, + outcome=(float(index),), + instrumentation=(float(index % 2),), + completion_time_source="exact_monotonic", + ) + for index, label in enumerate((0, 1, 1)) + ] + labels = analysis.np.asarray([0, 1, 1]) + probabilities = analysis.np.asarray([0.01, 0.99, 0.60]) + policy = analysis.policy_metrics(examples, labels, probabilities, 0.95) + assert policy["early_accept"] == 1 + assert policy["early_reject"] == 1 + assert policy["abstain_continue_full"] == 1 + assert policy["false_accept"] == 0 and policy["false_reject"] == 0 + assert policy["valid_zero_error_policy"] + assert policy["valid_cost_reduction_fraction"] is not None + model = analysis.fit_frozen_model( + examples, + instrumentation_aware=True, + regularization=1.0, + ) + frozen_probability = analysis.predict_frozen_model(model, examples) + assert len(frozen_probability) == len(examples) + assert analysis.np.all(frozen_probability >= 0.0) + assert analysis.np.all(frozen_probability <= 1.0) + print("fidelity prefix analysis: PASS") + + +if __name__ == "__main__": + main() diff --git a/runs/opprof-phase6/opprof_phase6_client.py b/runs/opprof-phase6/opprof_phase6_client.py index 11c5e3c..1dde17b 100644 --- a/runs/opprof-phase6/opprof_phase6_client.py +++ b/runs/opprof-phase6/opprof_phase6_client.py @@ -122,6 +122,11 @@ def run_replay(args: argparse.Namespace, *, warmup: bool) -> dict[str, Any]: "tpot_ms": outcome.tpot_ms, "completion_tokens": outcome.completion_tokens, "completion_tokens_source": outcome.completion_tokens_source, + "completed_mono_ns": outcome.completed_mono_ns, + "completed_elapsed_s": ( + (outcome.completed_mono_ns - interval_start_mono_ns) / 1e9 + if outcome.completed_mono_ns is not None else None + ), "slo_pass": evaluation.passed, "reasons": evaluation.reasons, "error": outcome.error, diff --git a/src/aituner/slo.py b/src/aituner/slo.py index 1a86a8d..fcbb5c5 100644 --- a/src/aituner/slo.py +++ b/src/aituner/slo.py @@ -16,6 +16,7 @@ class RequestOutcome: completion_tokens: int | None error: str = "" completion_tokens_source: str = "" + completed_mono_ns: int | None = None @dataclass(frozen=True) diff --git a/src/aituner/worker.py b/src/aituner/worker.py index d8db0b1..0439537 100644 --- a/src/aituner/worker.py +++ b/src/aituner/worker.py @@ -127,6 +127,7 @@ def _run_one_request( f"actual={actual_completion_tokens}" ), completion_tokens_source=completion_tokens_source, + completed_mono_ns=time.monotonic_ns(), ) if actual_completion_tokens != expected_completion_tokens: return RequestOutcome( @@ -142,6 +143,7 @@ def _run_one_request( f"actual={actual_completion_tokens}" ), completion_tokens_source=completion_tokens_source, + completed_mono_ns=time.monotonic_ns(), ) return RequestOutcome( request_id=request.row_id, @@ -151,6 +153,7 @@ def _run_one_request( prompt_tokens=request.prompt_tokens_hint, completion_tokens=actual_completion_tokens or request.completion_tokens_hint, completion_tokens_source=completion_tokens_source, + completed_mono_ns=time.monotonic_ns(), ) except HttpClientError as exc: return RequestOutcome( @@ -161,6 +164,7 @@ def _run_one_request( prompt_tokens=request.prompt_tokens_hint, completion_tokens=request.completion_tokens_hint, error=str(exc), + completed_mono_ns=time.monotonic_ns(), ) diff --git a/tests/test_core_flow.py b/tests/test_core_flow.py index e665c81..465244a 100644 --- a/tests/test_core_flow.py +++ b/tests/test_core_flow.py @@ -5604,15 +5604,17 @@ class CoreFlowTests(unittest.TestCase): completion_tokens=1, ), ): - outcome = _run_one_request( - request, - base_url="http://127.0.0.1:8000", - timeout_s=1.0, - ) + with mock.patch("aituner.worker.time.monotonic_ns", return_value=123456789): + outcome = _run_one_request( + request, + base_url="http://127.0.0.1:8000", + timeout_s=1.0, + ) self.assertFalse(outcome.success) self.assertEqual(outcome.error, "completion_tokens_mismatch expected=2 actual=1") self.assertEqual(outcome.completion_tokens, 1) + self.assertEqual(outcome.completed_mono_ns, 123456789) def test_build_prompt_mentions_completion_tokens_override(self) -> None: with tempfile.TemporaryDirectory() as tmp: From 8eeba597b3b386f87388f9ad20c6276154dc3d2e Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 12:52:00 +0800 Subject: [PATCH 002/148] Fix pilot burn-in selection manifest --- runs/fidelity-headroom/prepare_pilot.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/runs/fidelity-headroom/prepare_pilot.py b/runs/fidelity-headroom/prepare_pilot.py index bb849c4..168beb6 100644 --- a/runs/fidelity-headroom/prepare_pilot.py +++ b/runs/fidelity-headroom/prepare_pilot.py @@ -227,7 +227,11 @@ def build_manifest( for level, multiplier in TARGET_MULTIPLIERS.items(): target_rate = float(config["frontier_req_s_gpu"]) * multiplier target_count = round(target_rate * durations["low1"] * tp) - roles = [role for role in ROLES if role == "burnin" or role.startswith(level)] + roles = [ + role + for role in ROLES + if role.startswith(level) or (level == "low" and role == "burnin") + ] selections = {} for role in roles: anchor, selected = attainable_anchor(loaded[role][tier], target_count) From 93daf291f6fd14d4ab1f732dcc8625faa51e9d71 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 12:58:01 +0800 Subject: [PATCH 003/148] Fix prefix pilot warmup validation --- runs/fidelity-headroom/pilot_controller.py | 37 +++++++++++++--- runs/fidelity-headroom/test_pilot_tools.py | 51 ++++++++++++++++++++++ 2 files changed, 82 insertions(+), 6 deletions(-) diff --git a/runs/fidelity-headroom/pilot_controller.py b/runs/fidelity-headroom/pilot_controller.py index e63c9dd..6ffb4f1 100644 --- a/runs/fidelity-headroom/pilot_controller.py +++ b/runs/fidelity-headroom/pilot_controller.py @@ -233,8 +233,37 @@ def run_client( f"client failed: cell={entry['cell']} role={role} rc={process.returncode}" ) result = json.loads((output / "result.json").read_text(encoding="utf-8")) + validate_result_selection( + result=result, + selection=selection, + cell=entry["cell"], + role=role, + warmup=warmup, + ) + entry["results"].append( + {"anchor": float(selection["anchor"]), "dir": str(output), "kind": result["kind"]} + ) + return result + + +def validate_result_selection( + *, + result: dict[str, Any], + selection: dict[str, Any], + cell: str, + role: str, + warmup: bool, +) -> None: + if warmup: + if result["kind"] != "warmup" or int(result["selection"]["count"]) != 16: + raise RuntimeError(f"invalid warmup selection: {cell} {role}") + for key in ("warmup_16", "warmup_exact_16", "warmup_long"): + if not result["invariants"].get(key, False): + raise RuntimeError(f"warmup invariant {key} failed: {cell} {role}") + return + if int(result["selection"]["count"]) != int(selection["selected_count"]): - raise RuntimeError(f"selection count mismatch: {entry['cell']} {role}") + raise RuntimeError(f"selection count mismatch: {cell} {role}") for key in ( "request_id_order_sha256", "arrival_order_sha256", @@ -244,11 +273,7 @@ def run_client( "input_length_order_sha256" if key == "raw_length_order_sha256" else key ) if result["selection"][key] != selection[manifest_key]: - raise RuntimeError(f"selection hash mismatch {key}: {entry['cell']} {role}") - entry["results"].append( - {"anchor": float(selection["anchor"]), "dir": str(output), "kind": result["kind"]} - ) - return result + raise RuntimeError(f"selection hash mismatch {key}: {cell} {role}") def execute_cell( diff --git a/runs/fidelity-headroom/test_pilot_tools.py b/runs/fidelity-headroom/test_pilot_tools.py index a2e1a32..f5a674f 100644 --- a/runs/fidelity-headroom/test_pilot_tools.py +++ b/runs/fidelity-headroom/test_pilot_tools.py @@ -78,6 +78,57 @@ def main() -> None: ) + controller.SAFETY_H20_HOURS, 3.0, ) + selection = { + "selected_count": 122, + "request_id_order_sha256": "request-hash", + "arrival_order_sha256": "arrival-hash", + "input_length_order_sha256": "length-hash", + } + warmup = { + "kind": "warmup", + "selection": {"count": 16}, + "invariants": { + "warmup_16": True, + "warmup_exact_16": True, + "warmup_long": True, + }, + } + controller.validate_result_selection( + result=warmup, + selection=selection, + cell="tp1_mns8", + role="burnin", + warmup=True, + ) + measured = { + "kind": "anchor", + "selection": { + "count": 122, + "request_id_order_sha256": "request-hash", + "arrival_order_sha256": "arrival-hash", + "raw_length_order_sha256": "length-hash", + }, + "invariants": {}, + } + controller.validate_result_selection( + result=measured, + selection=selection, + cell="tp1_mns8", + role="low1", + warmup=False, + ) + try: + controller.validate_result_selection( + result=warmup, + selection=selection, + cell="tp1_mns8", + role="low1", + warmup=False, + ) + except RuntimeError as error: + assert "selection count mismatch" in str(error) + else: + raise AssertionError("measured selection accepted a warmup subset") print("fidelity pilot tools: PASS") From 23142aa359e57a7e82be18427419de3ae69ca7f1 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 13:08:45 +0800 Subject: [PATCH 004/148] Strengthen fidelity calibration baseline --- ...idelity-aware-harness-headroom-20260714.md | 34 ++ ...idelity-aware-harness-protocol-20260714.md | 41 +- .../analyze_strong_baseline.py | 298 +++++++++++ .../strong-baseline-metrics.json | 477 ++++++++++++++++++ .../fidelity-headroom/test_strong_baseline.py | 37 ++ 5 files changed, 881 insertions(+), 6 deletions(-) create mode 100644 runs/fidelity-headroom/analyze_strong_baseline.py create mode 100644 runs/fidelity-headroom/strong-baseline-metrics.json create mode 100644 runs/fidelity-headroom/test_strong_baseline.py diff --git a/docs/fidelity-aware-harness-headroom-20260714.md b/docs/fidelity-aware-harness-headroom-20260714.md index a3cb832..8f11346 100644 --- a/docs/fidelity-aware-harness-headroom-20260714.md +++ b/docs/fidelity-aware-harness-headroom-20260714.md @@ -46,6 +46,35 @@ at 15 seconds it is 88.89% versus 91.67%; at 20 seconds it is 86.11% versus 91.67%, but both 0.95 policies make one false reject. Five seconds is therefore a training-selected operating point, not a test result. +## Strong simulator-aware calibration baseline + +The original nested comparison used the same simulator shortlist but did not +put Frontier's per-anchor prediction in either model. A stronger retrospective +audit now gives both models frozen-calibrated simulated throughput, simulated +SLO pass rate, and simulated feasibility. Under the same leave-one-cell-out +folds, 5-second cutoff, L2 logistic family, regularization 1.0, and threshold +0.95: + +| Metric | Sim + outcome | Sim + outcome + instrumentation | Delta | +|---|---:|---:|---:| +| Accuracy | 81.08% | 89.19% | +8.11 pp | +| Balanced accuracy | 72.42% | 81.55% | +9.13 pp | +| Brier score | 0.1058 | 0.0957 | -0.0101 | +| Safe early decisions | 20/37 | 25/37 | +5 | +| Valid full-trial cost reduction | 50.89% | 68.98% | +18.09 pp | +| Residual verification H20-hours | 0.5240 | 0.3310 | -36.84% | + +Both 0.95 policies have zero false accept and zero false reject on this +retrospective task. Only three 0.5-threshold classifications differ in favor +of instrumentation and none in favor of the strong baseline; McNemar's exact +two-sided p-value is 0.25. The cell-bootstrap accuracy-delta interval is +`[0.00,+18.18]` percentage points. The result is not robust to regularization: +at 0.1 the strong baseline is more accurate and the instrumentation policy +makes two unsafe decisions; at 10.0 the strong baseline is also more accurate. +Thus the stronger comparison still has enough point-estimate headroom for a +held-out test, but it materially weakens the evidence and makes a prospective +task-level result mandatory. + ## Interpretation There is enough headroom to run a held-out pilot, but not enough evidence to @@ -73,6 +102,9 @@ with three full repetitions. The registered protocol is - `runs/fidelity-headroom/prefix-metrics.json` - `runs/fidelity-headroom/test_analysis.py` - `runs/fidelity-headroom/test_prefix_analysis.py` +- `runs/fidelity-headroom/analyze_strong_baseline.py` +- `runs/fidelity-headroom/strong-baseline-metrics.json` +- `runs/fidelity-headroom/test_strong_baseline.py` ## Sanity block @@ -86,6 +118,8 @@ with three full repetitions. The registered protocol is | Outcome probability | 37 | in `[0,1]` | in `[0,1]` | >1 | Checked before metrics | | Instrumentation probability | 37 | in `[0,1]` | in `[0,1]` | >1 | Checked before metrics | | Layer-1 streams | 12 | 14,174 records | 58,725 records | 12 | Contiguous, zero drops | +| Matched frozen simulator anchors | 37 | pass rate 0.0688 | pass rate 1.0 | 12 pass-rate values | Every prefix matched exactly once | +| Frozen simulator anchor corpus | 92 | positive throughput | positive throughput | >1 | No duplicate cell/anchor run | Checked invariants: same folds/model family and cutoff; no full verdict in a feature; prefix-only Layer-1 slicing; non-negative costs/counters; bounded diff --git a/docs/fidelity-aware-harness-protocol-20260714.md b/docs/fidelity-aware-harness-protocol-20260714.md index 1bdb1b2..8bf05f0 100644 --- a/docs/fidelity-aware-harness-protocol-20260714.md +++ b/docs/fidelity-aware-harness-protocol-20260714.md @@ -52,6 +52,35 @@ difference is Z. The initial family is intentionally simple: a positive result then demonstrates value in the engine signal rather than capacity in a larger learner. A sequence model is admissible only as a later, paired ablation. +### Amendment A1: strengthen the calibration baseline before P2 + +Frozen 2026-07-14 13:08 Asia/Singapore, after P1 launch but before P1 +completion or analysis. A baseline audit found that the first frozen P1 +models use the simulator only to define candidate order; their feature vectors +do not contain the simulator's per-anchor prediction. This is insufficient +for the stronger term **outcome-only calibration**. P1 therefore remains a +prospective test of the originally frozen cross-workload predictor, but cannot +by itself open a contribution claim. + +For P2/P3, both nested models must additionally receive the identical frozen +simulator outputs available at that decision: predicted completed throughput +per GPU, predicted SLO pass rate, and predicted feasibility. The comparison +is consequently `sim + config + workload + real outcome prefix` versus that +exact vector plus real engine state. Simulator features, regularization, +cutoff, and thresholds are frozen before any P2 task. If telemetry does not +improve this stronger baseline, the harness has no independent contribution. + +The same audit also separates algorithm cost from benchmark-oracle cost. +Headline method cost includes every action the method would execute online: +simulator profiling/calibration, model onboarding, server startup, warm-up, +real prefix, continuation after abstention, method-requested confirmation, +logging overhead, failures, and cleanup. Exhaustive real-oracle runs and the +extra repetitions used only to construct 2-of-3 evaluation labels are common +benchmark annotation cost; they are reported separately and charged to no +method. A second, deliberately conservative table adds that common cost to +all methods. This prevents both hiding real method cost and making the +percentage gate mathematically depend on offline ground-truth annotation. + The frozen first policy uses a 5-second prefix, L2 regularization 1.0, and a two-sided abstaining threshold of 0.95: accept at `p(feasible)>=0.95`, reject at `p(feasible)<=0.05`, otherwise continue the exact same trial to completion. @@ -64,15 +93,15 @@ therefore not evidence; all claims come from subsequent held-out tasks. |---|---:|---:|---:|---:|---:| | Real-only oracle | no | no | full | optional diagnostic | every candidate/anchor | | Sim top-k + real final | yes | included in full run | full | no decision use | every shortlisted candidate/anchor | -| Outcome-only calibration | yes | yes | yes | no | only on abstention | -| Instrumentation-aware | yes | yes | yes | yes | only on abstention | +| Outcome-only calibration | yes, including its prediction features | yes | yes | no | only on abstention | +| Instrumentation-aware | same prediction features | yes | yes | yes | only on abstention | Tie buckets are expanded before top-k. `k` is selected on training tasks and is fixed on held-out tasks; an oracle per-task k is forbidden. Outcome-only -receives all information available outside the engine, including config and -workload features. Instrumentation cannot use any record submitted after the -cutoff. The full label, confirmation votes, simulator error, and later -requests are never model features. +receives all information available outside the engine, including config, +workload, and frozen simulator-prediction features. Instrumentation cannot use +any record submitted after the cutoff. The full label, confirmation votes, +realized simulator error, and later requests are never model features. ## Staged experiment diff --git a/runs/fidelity-headroom/analyze_strong_baseline.py b/runs/fidelity-headroom/analyze_strong_baseline.py new file mode 100644 index 0000000..4880afd --- /dev/null +++ b/runs/fidelity-headroom/analyze_strong_baseline.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +"""Audit telemetry against a simulator-aware outcome calibration baseline. + +This is a retrospective headroom check. It strengthens the earlier +outcome-only baseline by giving both nested models the same per-anchor +Frontier throughput and SLO predictions. The only additional inputs to the +larger model are real engine Layer-1 features. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from pathlib import Path +from typing import Any + +import numpy as np + +from analyze_existing import ( + DEFAULT_REGULARIZATION, + REGULARIZATION_SENSITIVITY, + _classification_metrics, + _fit_logistic, + _group_bootstrap_delta, + _mcnemar_exact_p, + _sigmoid, +) +from analyze_prefixes import ( + INSTRUMENTATION_FEATURES, + OUTCOME_FEATURES, + PrefixExample, + build_examples, + numeric, + policy_metrics, + sha256_file, +) + + +SIMULATOR_FEATURES = ( + "log_sim_completed_throughput_per_gpu", + "sim_slo_pass_rate", + "sim_slo_feasible", +) + + +def load_simulator_features(raw_root: Path) -> tuple[dict[tuple[str, float], tuple[float, ...]], str]: + features: dict[tuple[str, float], tuple[float, ...]] = {} + digest = hashlib.sha256() + paths = sorted(raw_root.glob("*/trial-0001/run_manifest.json")) + for manifest_path in paths: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + run = manifest["run"] + if run["mode"] != "frozen-calibrated": + continue + scorer_path = manifest_path.parent / "scorer_output.json" + scorer = json.loads(scorer_path.read_text(encoding="utf-8")) + key = (str(run["cell_id"]), float(run["sampling_u"])) + if key in features: + raise ValueError(f"duplicate frozen simulator run: {key}") + throughput = float(scorer["throughput_requests_per_second_per_gpu"]) + pass_rate = float(scorer["slo"]["pass_rate"]) + if throughput <= 0 or not 0.0 <= pass_rate <= 1.0: + raise ValueError(f"invalid simulator output: {key}") + features[key] = ( + math.log(throughput), + pass_rate, + float(bool(scorer["slo"]["feasible"])), + ) + for path in (manifest_path, scorer_path): + digest.update(str(path.relative_to(raw_root)).encode()) + digest.update(path.read_bytes()) + return features, digest.hexdigest() + + +def simulator_row( + example: PrefixExample, + features: dict[tuple[str, float], tuple[float, ...]], +) -> tuple[float, ...]: + matches = [ + values + for (cell, anchor), values in features.items() + if cell == example.cell + and math.isclose(anchor, example.anchor, rel_tol=0.0, abs_tol=1e-12) + ] + if len(matches) != 1: + raise ValueError( + f"expected one simulator match for {example.cell}/{example.anchor}: {len(matches)}" + ) + return matches[0] + + +def grouped_predictions( + examples: list[PrefixExample], + simulator: dict[tuple[str, float], tuple[float, ...]], + *, + instrumentation_aware: bool, + regularization: float, +) -> tuple[np.ndarray, np.ndarray, list[str]]: + probabilities: list[float] = [] + labels: list[int] = [] + groups: list[str] = [] + for held_out in sorted({example.cell for example in examples}): + train = [example for example in examples if example.cell != held_out] + test = [example for example in examples if example.cell == held_out] + + def row(example: PrefixExample) -> np.ndarray: + values = example.outcome + simulator_row(example, simulator) + if instrumentation_aware: + values += example.instrumentation + return np.asarray((1.0, *values), dtype=np.float64) + + x_train = np.stack([row(example) for example in train]) + x_test = np.stack([row(example) for example in test]) + y_train = np.asarray([example.feasible for example in train], dtype=np.float64) + mean = x_train[:, 1:].mean(axis=0) + standard_deviation = x_train[:, 1:].std(axis=0) + standard_deviation[standard_deviation < 1e-8] = 1.0 + x_train[:, 1:] = (x_train[:, 1:] - mean) / standard_deviation + x_test[:, 1:] = (x_test[:, 1:] - mean) / standard_deviation + weights = _fit_logistic(x_train, y_train, regularization) + probabilities.extend(_sigmoid(x_test @ weights).tolist()) + labels.extend(example.feasible for example in test) + groups.extend(held_out for _ in test) + return ( + np.asarray(labels, dtype=np.int64), + np.asarray(probabilities, dtype=np.float64), + groups, + ) + + +def analyze( + phase6_path: Path, + phase6_raw_root: Path, + simulator_raw_root: Path, + simulator_metrics_path: Path, +) -> dict[str, Any]: + phase6 = json.loads(phase6_path.read_text(encoding="utf-8")) + examples = build_examples(phase6, phase6_raw_root, 5.0) + simulator, simulator_raw_sha256 = load_simulator_features(simulator_raw_root) + red_flags = [] + try: + matched = [simulator_row(example, simulator) for example in examples] + except ValueError as error: + matched = [] + red_flags.append(str(error)) + + sensitivity = {} + if matched: + for regularization in REGULARIZATION_SENSITIVITY: + labels, baseline_probability, groups = grouped_predictions( + examples, + simulator, + instrumentation_aware=False, + regularization=regularization, + ) + instrument_labels, instrument_probability, instrument_groups = grouped_predictions( + examples, + simulator, + instrumentation_aware=True, + regularization=regularization, + ) + if not np.array_equal(labels, instrument_labels) or groups != instrument_groups: + raise AssertionError("nested baseline folds differ") + baseline_correct = (baseline_probability >= 0.5) == labels + instrument_correct = (instrument_probability >= 0.5) == labels + paired = { + "both_correct": int(np.sum(baseline_correct & instrument_correct)), + "sim_outcome_only_correct": int( + np.sum(baseline_correct & ~instrument_correct) + ), + "instrumentation_only_correct": int( + np.sum(~baseline_correct & instrument_correct) + ), + "both_wrong": int(np.sum(~baseline_correct & ~instrument_correct)), + } + paired["mcnemar_exact_two_sided_p"] = _mcnemar_exact_p( + paired["sim_outcome_only_correct"], + paired["instrumentation_only_correct"], + ) + sensitivity[str(regularization)] = { + "sim_plus_outcome": { + "classification": _classification_metrics(labels, baseline_probability), + "policy_0p95": policy_metrics( + examples, labels, baseline_probability, 0.95 + ), + }, + "sim_plus_outcome_plus_instrumentation": { + "classification": _classification_metrics(labels, instrument_probability), + "policy_0p95": policy_metrics( + examples, labels, instrument_probability, 0.95 + ), + }, + "paired_correctness": paired, + "group_bootstrap": _group_bootstrap_delta( + labels, + baseline_probability, + instrument_probability, + groups, + ), + } + + headline = sensitivity.get(str(DEFAULT_REGULARIZATION)) + simulator_pass_rates = [row[1] for row in matched] + labels = [example.feasible for example in examples] + if len(examples) != 37: + red_flags.append("examples_not_37") + if len(simulator) != 92: + red_flags.append("frozen_simulator_runs_not_92") + if len(set(labels)) != 2: + red_flags.append("single_label") + if matched and not all(0.0 <= value <= 1.0 for value in simulator_pass_rates): + red_flags.append("simulator_pass_rate_out_of_range") + + return { + "schema": "fidelity-strong-baseline-v1", + "status": "PASS" if not red_flags else "STOP", + "scope": "retrospective one-task headroom audit; not contribution evidence", + "comparison": ( + "same 5-second prefix, folds, logistic family, regularization, and frozen " + "Frontier outputs; the only nested difference is real Layer-1 engine state" + ), + "features": { + "shared_outcome": list(OUTCOME_FEATURES), + "shared_simulator": list(SIMULATOR_FEATURES), + "instrumentation_only": list(INSTRUMENTATION_FEATURES), + }, + "headline_regularization": DEFAULT_REGULARIZATION, + "headline": headline, + "regularization_sensitivity": sensitivity, + "provenance": { + "phase6_metrics": str(phase6_path.resolve()), + "phase6_metrics_sha256": sha256_file(phase6_path), + "phase6_raw_root": str(phase6_raw_root.resolve()), + "simulator_metrics": str(simulator_metrics_path.resolve()), + "simulator_metrics_sha256": sha256_file(simulator_metrics_path), + "simulator_raw_root": str(simulator_raw_root.resolve()), + "frozen_simulator_manifest_scorer_set_sha256": simulator_raw_sha256, + }, + "decision": { + "contribution_established": False, + "prospective_requirement": ( + "repeat sim+outcome versus sim+outcome+instrumentation on complete held-out tasks" + ), + }, + "sanity": { + "red_flags": red_flags, + "examples": numeric([1 for _ in examples]), + "labels": { + **numeric(labels), + "positive": sum(labels), + "negative": len(labels) - sum(labels), + }, + "matched_simulator_pass_rate": numeric(simulator_pass_rates), + "frozen_simulator_runs": len(simulator), + "invariants": { + "all_examples_matched_once": len(matched) == len(examples), + "same_nested_folds": True, + "simulator_ratios_bounded": all( + 0.0 <= value <= 1.0 for value in simulator_pass_rates + ), + "labels_not_identical": len(set(labels)) == 2, + "per_config_results_not_all_identical": len(set(simulator_pass_rates)) > 1, + }, + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--phase6-metrics", type=Path, required=True) + parser.add_argument("--phase6-raw-root", type=Path, required=True) + parser.add_argument("--simulator-raw-root", type=Path, required=True) + parser.add_argument("--simulator-metrics", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + result = analyze( + args.phase6_metrics, + args.phase6_raw_root, + args.simulator_raw_root, + args.simulator_metrics, + ) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") + print( + json.dumps( + { + "status": result["status"], + "output": str(args.output), + "red_flags": result["sanity"]["red_flags"], + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/fidelity-headroom/strong-baseline-metrics.json b/runs/fidelity-headroom/strong-baseline-metrics.json new file mode 100644 index 0000000..bf681ad --- /dev/null +++ b/runs/fidelity-headroom/strong-baseline-metrics.json @@ -0,0 +1,477 @@ +{ + "comparison": "same 5-second prefix, folds, logistic family, regularization, and frozen Frontier outputs; the only nested difference is real Layer-1 engine state", + "decision": { + "contribution_established": false, + "prospective_requirement": "repeat sim+outcome versus sim+outcome+instrumentation on complete held-out tasks" + }, + "features": { + "instrumentation_only": [ + "model_steps_per_second", + "waiting_mean", + "waiting_max", + "waiting_nonzero_share", + "running_mean", + "running_max", + "decode_batch_mean", + "decode_batch_max", + "decode_batch_cv", + "kv_usage_mean", + "kv_usage_max", + "kv_usage_end_minus_start", + "graph_none_share", + "graph_full_share", + "padding_fraction", + "prefill_token_fraction", + "preemptions" + ], + "shared_outcome": [ + "log_offered_rate_per_gpu", + "log2_tp", + "log2_max_num_seqs", + "admitted_fraction", + "completed_over_admitted", + "completed_pass_rate", + "completed_fail_fraction_of_total", + "outstanding_over_admitted", + "ttft_max_over_slo_max", + "ttft_mean_over_slo_max", + "tpot_max_over_slo", + "tpot_mean_over_slo", + "admitted_input_tokens_mean_over_limit" + ], + "shared_simulator": [ + "log_sim_completed_throughput_per_gpu", + "sim_slo_pass_rate", + "sim_slo_feasible" + ] + }, + "headline": { + "group_bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + 0.0, + 0.18181818181818188 + ], + "point": 0.08108108108108103 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.04292727744470806, + 0.019924730979981074 + ], + "point": -0.010145365131402809 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "paired_correctness": { + "both_correct": 30, + "both_wrong": 4, + "instrumentation_only_correct": 3, + "mcnemar_exact_two_sided_p": 0.25, + "sim_outcome_only_correct": 0 + }, + "sim_plus_outcome": { + "classification": { + "accuracy": 0.8108108108108109, + "balanced_accuracy": 0.7242063492063493, + "brier": 0.1058226346682949, + "confusion": { + "false_negative": 3, + "false_positive": 4, + "true_negative": 5, + "true_positive": 25 + }, + "log_loss": 0.3011048455679668 + }, + "policy_0p95": { + "abstain_continue_full": 17, + "correctly_saved_h20_hours": 0.5429431818208333, + "decision_coverage": 0.5405405405405406, + "early_accept": 16, + "early_reject": 4, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.5429431818208333, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.5088695307144538, + "valid_zero_error_policy": true + } + }, + "sim_plus_outcome_plus_instrumentation": { + "classification": { + "accuracy": 0.8918918918918919, + "balanced_accuracy": 0.8154761904761905, + "brier": 0.0956772695368921, + "confusion": { + "false_negative": 1, + "false_positive": 3, + "true_negative": 6, + "true_positive": 27 + }, + "log_loss": 0.288823031828762 + }, + "policy_0p95": { + "abstain_continue_full": 12, + "correctly_saved_h20_hours": 0.7360063646722222, + "decision_coverage": 0.6756756756756757, + "early_accept": 20, + "early_reject": 5, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.7360063646722222, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.6898165884274738, + "valid_zero_error_policy": true + } + } + }, + "headline_regularization": 1.0, + "provenance": { + "frozen_simulator_manifest_scorer_set_sha256": "833842d96ecaa0b059ef99852621752f7989e63d100118b6025425fb119b7a55", + "phase6_metrics": "/home/gahow/phd/aituner/runs/opprof-phase6/phase6/metrics.json", + "phase6_metrics_sha256": "290ba7fcb8727291166de7e4d47afdc84e230052495c81dd087db0ace9f93a16", + "phase6_raw_root": "/home/gahow/phd/aituner/runs/opprof-phase6/phase6/solo-authoritative/cells", + "simulator_metrics": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/metrics.json", + "simulator_metrics_sha256": "55edb37d5692e979ab6f6dc6c65913a9db0aa0a836c350e4c05d9c38eee78206", + "simulator_raw_root": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/raw" + }, + "regularization_sensitivity": { + "0.1": { + "group_bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.17500000000000004, + 0.0 + ], + "point": -0.08108108108108103 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.026383192545085435, + 0.0607951286646285 + ], + "point": 0.019228316404518567 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "paired_correctness": { + "both_correct": 30, + "both_wrong": 4, + "instrumentation_only_correct": 0, + "mcnemar_exact_two_sided_p": 0.25, + "sim_outcome_only_correct": 3 + }, + "sim_plus_outcome": { + "classification": { + "accuracy": 0.8918918918918919, + "balanced_accuracy": 0.8154761904761905, + "brier": 0.10990776306815446, + "confusion": { + "false_negative": 1, + "false_positive": 3, + "true_negative": 6, + "true_positive": 27 + }, + "log_loss": 0.328357763455984 + }, + "policy_0p95": { + "abstain_continue_full": 12, + "correctly_saved_h20_hours": 0.7402314096841667, + "decision_coverage": 0.6756756756756757, + "early_accept": 20, + "early_reject": 5, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.7402314096841667, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.6937764809990414, + "valid_zero_error_policy": true + } + }, + "sim_plus_outcome_plus_instrumentation": { + "classification": { + "accuracy": 0.8108108108108109, + "balanced_accuracy": 0.7619047619047619, + "brier": 0.12913607947267303, + "confusion": { + "false_negative": 4, + "false_positive": 3, + "true_negative": 6, + "true_positive": 24 + }, + "log_loss": 0.4373556318820343 + }, + "policy_0p95": { + "abstain_continue_full": 9, + "correctly_saved_h20_hours": 0.7469523484622221, + "decision_coverage": 0.7567567567567568, + "early_accept": 22, + "early_reject": 6, + "false_accept": 2, + "false_accept_examples": [ + { + "anchor": 0.49609375, + "cell": "tp2_mns8", + "label_feasible": false, + "probability_feasible": 0.9869795738005246, + "remaining_h20_hours": 0.010117910306111111 + }, + { + "anchor": 0.033717411016, + "cell": "tp4_mns16", + "label_feasible": false, + "probability_feasible": 0.9855364057197005, + "remaining_h20_hours": 0.023106262014444445 + } + ], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.03322417232055556, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.7801765207827777, + "threshold": 0.95, + "valid_cost_reduction_fraction": null, + "valid_zero_error_policy": false + } + } + }, + "1.0": { + "group_bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + 0.0, + 0.18181818181818188 + ], + "point": 0.08108108108108103 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.04292727744470806, + 0.019924730979981074 + ], + "point": -0.010145365131402809 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "paired_correctness": { + "both_correct": 30, + "both_wrong": 4, + "instrumentation_only_correct": 3, + "mcnemar_exact_two_sided_p": 0.25, + "sim_outcome_only_correct": 0 + }, + "sim_plus_outcome": { + "classification": { + "accuracy": 0.8108108108108109, + "balanced_accuracy": 0.7242063492063493, + "brier": 0.1058226346682949, + "confusion": { + "false_negative": 3, + "false_positive": 4, + "true_negative": 5, + "true_positive": 25 + }, + "log_loss": 0.3011048455679668 + }, + "policy_0p95": { + "abstain_continue_full": 17, + "correctly_saved_h20_hours": 0.5429431818208333, + "decision_coverage": 0.5405405405405406, + "early_accept": 16, + "early_reject": 4, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.5429431818208333, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.5088695307144538, + "valid_zero_error_policy": true + } + }, + "sim_plus_outcome_plus_instrumentation": { + "classification": { + "accuracy": 0.8918918918918919, + "balanced_accuracy": 0.8154761904761905, + "brier": 0.0956772695368921, + "confusion": { + "false_negative": 1, + "false_positive": 3, + "true_negative": 6, + "true_positive": 27 + }, + "log_loss": 0.288823031828762 + }, + "policy_0p95": { + "abstain_continue_full": 12, + "correctly_saved_h20_hours": 0.7360063646722222, + "decision_coverage": 0.6756756756756757, + "early_accept": 20, + "early_reject": 5, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.7360063646722222, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.6898165884274738, + "valid_zero_error_policy": true + } + } + }, + "10.0": { + "group_bootstrap": { + "accuracy_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.13333333333333341, + 0.05555555555555558 + ], + "point": -0.027027027027027084 + }, + "brier_delta_instrumentation_minus_outcome": { + "ci95": [ + -0.03091105649870874, + 0.01684192005239855 + ], + "point": -0.007318433328714388 + }, + "replicates": 10000, + "seed": 20260714, + "semantics": "group bootstrap over cells; diagnostic confidence interval" + }, + "paired_correctness": { + "both_correct": 30, + "both_wrong": 4, + "instrumentation_only_correct": 1, + "mcnemar_exact_two_sided_p": 1.0, + "sim_outcome_only_correct": 2 + }, + "sim_plus_outcome": { + "classification": { + "accuracy": 0.8648648648648649, + "balanced_accuracy": 0.7222222222222222, + "brier": 0.10613344425735322, + "confusion": { + "false_negative": 0, + "false_positive": 5, + "true_negative": 4, + "true_positive": 28 + }, + "log_loss": 0.3404203142465075 + }, + "policy_0p95": { + "abstain_continue_full": 32, + "correctly_saved_h20_hours": 0.21727432337249997, + "decision_coverage": 0.13513513513513514, + "early_accept": 5, + "early_reject": 0, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.21727432337249997, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.20363877229302757, + "valid_zero_error_policy": true + } + }, + "sim_plus_outcome_plus_instrumentation": { + "classification": { + "accuracy": 0.8378378378378378, + "balanced_accuracy": 0.7420634920634921, + "brier": 0.09881501092863883, + "confusion": { + "false_negative": 2, + "false_positive": 4, + "true_negative": 5, + "true_positive": 26 + }, + "log_loss": 0.312914193285738 + }, + "policy_0p95": { + "abstain_continue_full": 30, + "correctly_saved_h20_hours": 0.2384080185036111, + "decision_coverage": 0.1891891891891892, + "early_accept": 6, + "early_reject": 1, + "false_accept": 0, + "false_accept_examples": [], + "false_reject": 0, + "false_reject_examples": [], + "full_trial_h20_hours": 1.0669595034675, + "invalidly_saved_h20_hours": 0.0, + "remaining_h20_hours_at_cutoff": 0.957237281245278, + "saved_h20_hours_if_decisions_used": 0.2384080185036111, + "threshold": 0.95, + "valid_cost_reduction_fraction": 0.22344617366339725, + "valid_zero_error_policy": true + } + } + } + }, + "sanity": { + "examples": { + "distinct_n": 1, + "max": 1.0, + "min": 1.0, + "n": 37 + }, + "frozen_simulator_runs": 92, + "invariants": { + "all_examples_matched_once": true, + "labels_not_identical": true, + "per_config_results_not_all_identical": true, + "same_nested_folds": true, + "simulator_ratios_bounded": true + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 37, + "negative": 9, + "positive": 28 + }, + "matched_simulator_pass_rate": { + "distinct_n": 12, + "max": 1.0, + "min": 0.06884057971014493, + "n": 37 + }, + "red_flags": [] + }, + "schema": "fidelity-strong-baseline-v1", + "scope": "retrospective one-task headroom audit; not contribution evidence", + "status": "PASS" +} diff --git a/runs/fidelity-headroom/test_strong_baseline.py b/runs/fidelity-headroom/test_strong_baseline.py new file mode 100644 index 0000000..5c1e9f1 --- /dev/null +++ b/runs/fidelity-headroom/test_strong_baseline.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +from pathlib import Path + +from analyze_strong_baseline import analyze + + +ROOT = Path(__file__).resolve().parents[2] +REPLAYSERVE = ROOT.parent / "replayserve" + + +def main() -> None: + result = analyze( + ROOT / "runs/opprof-phase6/phase6/metrics.json", + ROOT / "runs/opprof-phase6/phase6/solo-authoritative/cells", + REPLAYSERVE / "runs/simfid_s2rb/results/raw", + REPLAYSERVE / "runs/simfid_s2rb/results/metrics.json", + ) + assert result["status"] == "PASS", json.dumps(result["sanity"], indent=2) + assert result["sanity"]["frozen_simulator_runs"] == 92 + assert result["sanity"]["labels"]["n"] == 37 + headline = result["headline"] + assert headline["sim_plus_outcome"]["policy_0p95"]["false_accept"] == 0 + assert headline["sim_plus_outcome"]["policy_0p95"]["false_reject"] == 0 + assert ( + headline["sim_plus_outcome_plus_instrumentation"]["policy_0p95"][ + "false_accept" + ] + == 0 + ) + print("fidelity strong baseline: PASS") + + +if __name__ == "__main__": + main() From a3b25f4a92c88d338a3437e91fdec8fc88ea8a9c Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 13:28:16 +0800 Subject: [PATCH 005/148] Add simulator-aware fidelity pilot audit --- .../fidelity-headroom/analyze_strong_pilot.py | 398 ++++++++++++++++++ .../prepare_pilot_simulator.py | 331 +++++++++++++++ runs/fidelity-headroom/run_pilot_simulator.py | 292 +++++++++++++ runs/fidelity-headroom/test_strong_pilot.py | 75 ++++ 4 files changed, 1096 insertions(+) create mode 100644 runs/fidelity-headroom/analyze_strong_pilot.py create mode 100644 runs/fidelity-headroom/prepare_pilot_simulator.py create mode 100644 runs/fidelity-headroom/run_pilot_simulator.py create mode 100644 runs/fidelity-headroom/test_strong_pilot.py diff --git a/runs/fidelity-headroom/analyze_strong_pilot.py b/runs/fidelity-headroom/analyze_strong_pilot.py new file mode 100644 index 0000000..57e79c3 --- /dev/null +++ b/runs/fidelity-headroom/analyze_strong_pilot.py @@ -0,0 +1,398 @@ +#!/usr/bin/env python3 +"""Exploratory P1 audit against the strengthened simulator-aware baseline. + +P1 was already running when the strong baseline was added, so this script is +not paper-facing prospective evidence. It trains only on the historical +Phase-6 task and evaluates the exact P1 primary probes. Both nested models +receive identical Frontier predictions; engine telemetry is the sole feature +difference. +""" + +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +from typing import Any + +import numpy as np + +from analyze_existing import ( + DEFAULT_REGULARIZATION, + REGULARIZATION_SENSITIVITY, + _classification_metrics, + _fit_logistic, + _mcnemar_exact_p, + _sigmoid, +) +from analyze_pilot import build_pilot_examples +from analyze_prefixes import ( + INSTRUMENTATION_FEATURES, + OUTCOME_FEATURES, + PrefixExample, + build_examples, + numeric, + policy_metrics, + sha256_file, +) +from analyze_strong_baseline import ( + SIMULATOR_FEATURES, + load_simulator_features, + simulator_row, +) + + +def load_pilot_simulator( + path: Path, +) -> tuple[dict[tuple[str, str], tuple[float, ...]], list[str]]: + payload = json.loads(path.read_text(encoding="utf-8")) + red_flags = [] + if payload.get("status") != "PASS": + red_flags.append("pilot_simulator_not_pass") + features: dict[tuple[str, str], tuple[float, ...]] = {} + for item in payload.get("results", []): + key = (str(item["cell"]), str(item["role"])) + if key in features: + red_flags.append(f"duplicate_pilot_simulator_{key[0]}_{key[1]}") + continue + scorer = item["scorer"] + throughput = float(scorer["throughput_requests_per_second_per_gpu"]) + pass_rate = float(scorer["slo"]["pass_rate"]) + if throughput <= 0: + red_flags.append(f"nonpositive_pilot_simulator_throughput_{key[0]}_{key[1]}") + if not 0.0 <= pass_rate <= 1.0: + red_flags.append(f"pilot_simulator_ratio_out_of_range_{key[0]}_{key[1]}") + features[key] = ( + math.log(throughput), + pass_rate, + float(bool(scorer["slo"]["feasible"])), + ) + if len(features) != 12: + red_flags.append("pilot_simulator_entries_not_12") + return features, red_flags + + +def fit_model( + examples: list[PrefixExample], + simulator: list[tuple[float, ...]], + *, + instrumentation_aware: bool, + regularization: float, +) -> dict[str, Any]: + rows = [] + for example, simulator_features in zip(examples, simulator): + values = example.outcome + simulator_features + if instrumentation_aware: + values += example.instrumentation + rows.append((1.0, *values)) + matrix = np.asarray(rows, dtype=np.float64) + labels = np.asarray([example.feasible for example in examples], dtype=np.float64) + mean = matrix[:, 1:].mean(axis=0) + standard_deviation = matrix[:, 1:].std(axis=0) + standard_deviation[standard_deviation < 1e-8] = 1.0 + standardized = matrix.copy() + standardized[:, 1:] = (standardized[:, 1:] - mean) / standard_deviation + weights = _fit_logistic(standardized, labels, regularization) + return { + "instrumentation_aware": instrumentation_aware, + "regularization": regularization, + "feature_mean": mean, + "feature_standard_deviation": standard_deviation, + "weights": weights, + } + + +def predict_model( + model: dict[str, Any], + examples: list[PrefixExample], + simulator: list[tuple[float, ...]], +) -> np.ndarray: + rows = [] + for example, simulator_features in zip(examples, simulator): + values = example.outcome + simulator_features + if model["instrumentation_aware"]: + values += example.instrumentation + rows.append((1.0, *values)) + matrix = np.asarray(rows, dtype=np.float64) + matrix[:, 1:] = ( + matrix[:, 1:] - model["feature_mean"] + ) / model["feature_standard_deviation"] + return _sigmoid(matrix @ model["weights"]) + + +def comparison( + training_examples: list[PrefixExample], + training_simulator: list[tuple[float, ...]], + pilot_examples: list[PrefixExample], + pilot_simulator: list[tuple[float, ...]], + regularization: float, +) -> dict[str, Any]: + labels = np.asarray([example.feasible for example in pilot_examples], dtype=np.int64) + baseline_model = fit_model( + training_examples, + training_simulator, + instrumentation_aware=False, + regularization=regularization, + ) + instrument_model = fit_model( + training_examples, + training_simulator, + instrumentation_aware=True, + regularization=regularization, + ) + baseline_probability = predict_model( + baseline_model, pilot_examples, pilot_simulator + ) + instrument_probability = predict_model( + instrument_model, pilot_examples, pilot_simulator + ) + baseline_correct = (baseline_probability >= 0.5) == labels + instrument_correct = (instrument_probability >= 0.5) == labels + paired = { + "both_correct": int(np.sum(baseline_correct & instrument_correct)), + "sim_outcome_only_correct": int( + np.sum(baseline_correct & ~instrument_correct) + ), + "instrumentation_only_correct": int( + np.sum(~baseline_correct & instrument_correct) + ), + "both_wrong": int(np.sum(~baseline_correct & ~instrument_correct)), + } + paired["mcnemar_exact_two_sided_p"] = _mcnemar_exact_p( + paired["sim_outcome_only_correct"], paired["instrumentation_only_correct"] + ) + return { + "sim_plus_outcome": { + "classification": _classification_metrics(labels, baseline_probability), + "policy_0p95": policy_metrics( + pilot_examples, labels, baseline_probability, 0.95 + ), + "probability": baseline_probability.tolist(), + }, + "sim_plus_outcome_plus_instrumentation": { + "classification": _classification_metrics(labels, instrument_probability), + "policy_0p95": policy_metrics( + pilot_examples, labels, instrument_probability, 0.95 + ), + "probability": instrument_probability.tolist(), + }, + "paired_correctness": paired, + } + + +def analyze( + phase6_path: Path, + phase6_raw_root: Path, + training_simulator_root: Path, + pilot_manifest_path: Path, + pilot_run_root: Path, + pilot_simulator_path: Path, +) -> dict[str, Any]: + phase6 = json.loads(phase6_path.read_text(encoding="utf-8")) + pilot_manifest = json.loads(pilot_manifest_path.read_text(encoding="utf-8")) + training_examples = build_examples(phase6, phase6_raw_root, 5.0) + training_simulator_map, training_simulator_sha256 = load_simulator_features( + training_simulator_root + ) + training_simulator = [ + simulator_row(example, training_simulator_map) + for example in training_examples + ] + pilot_examples, pilot_details, red_flags = build_pilot_examples( + pilot_manifest, pilot_run_root, 5.0 + ) + pilot_simulator_map, simulator_red_flags = load_pilot_simulator( + pilot_simulator_path + ) + red_flags.extend(simulator_red_flags) + pilot_simulator = [] + for example, detail in zip(pilot_examples, pilot_details): + role = f"{detail['level']}1" + key = (example.cell, role) + if key not in pilot_simulator_map: + red_flags.append(f"missing_pilot_simulator_{example.cell}_{role}") + pilot_simulator.append((0.0, 0.0, 0.0)) + else: + pilot_simulator.append(pilot_simulator_map[key]) + + sensitivity = {} + if not red_flags: + for regularization in REGULARIZATION_SENSITIVITY: + sensitivity[str(regularization)] = comparison( + training_examples, + training_simulator, + pilot_examples, + pilot_simulator, + regularization, + ) + headline = sensitivity.get(str(DEFAULT_REGULARIZATION)) + labels = [example.feasible for example in pilot_examples] + simulator_pass_rates = [row[1] for row in pilot_simulator] + simulator_labels = [int(row[2]) for row in pilot_simulator] + if len(training_examples) != 37: + red_flags.append("training_examples_not_37") + if len(pilot_examples) != 12: + red_flags.append("pilot_examples_not_12") + if len(set(labels)) != 2: + red_flags.append("pilot_single_label") + if len(set(simulator_pass_rates)) <= 1: + red_flags.append("pilot_simulator_results_identical") + + if headline is None: + decision = { + "strong_incremental_gate": False, + "reason": "analysis red flag prevented nested comparison", + } + else: + baseline_policy = headline["sim_plus_outcome"]["policy_0p95"] + instrument_policy = headline[ + "sim_plus_outcome_plus_instrumentation" + ]["policy_0p95"] + baseline_errors = baseline_policy["false_accept"] + baseline_policy["false_reject"] + instrument_errors = ( + instrument_policy["false_accept"] + instrument_policy["false_reject"] + ) + baseline_reduction = baseline_policy["valid_cost_reduction_fraction"] + instrument_reduction = instrument_policy["valid_cost_reduction_fraction"] + reduction_delta = ( + instrument_reduction - baseline_reduction + if baseline_reduction is not None and instrument_reduction is not None + else None + ) + per_lambda_safe_and_better = [] + for item in sensitivity.values(): + baseline = item["sim_plus_outcome"]["policy_0p95"] + instrument = item["sim_plus_outcome_plus_instrumentation"]["policy_0p95"] + base_errors = baseline["false_accept"] + baseline["false_reject"] + inst_errors = instrument["false_accept"] + instrument["false_reject"] + base_reduction = baseline["valid_cost_reduction_fraction"] + inst_reduction = instrument["valid_cost_reduction_fraction"] + per_lambda_safe_and_better.append( + inst_errors == 0 + and inst_errors <= base_errors + and base_reduction is not None + and inst_reduction is not None + and inst_reduction > base_reduction + ) + decision = { + "strong_incremental_gate": bool( + not red_flags + and instrument_errors == 0 + and instrument_errors <= baseline_errors + and reduction_delta is not None + and reduction_delta >= 0.15 + ), + "regularization_robust": all(per_lambda_safe_and_better), + "valid_cost_reduction_fraction_delta": reduction_delta, + "scope": "exploratory task; may choose P2 design but cannot establish contribution", + } + + return { + "schema": "fidelity-strong-pilot-v1", + "status": "PASS" if not red_flags else "STOP", + "scope": ( + "post-amendment exploratory P1 audit; strong model was not frozen before " + "partial P1 outcomes, so this is not prospective contribution evidence" + ), + "features": { + "shared_outcome": list(OUTCOME_FEATURES), + "shared_simulator": list(SIMULATOR_FEATURES), + "instrumentation_only": list(INSTRUMENTATION_FEATURES), + }, + "headline_regularization": DEFAULT_REGULARIZATION, + "headline": headline, + "regularization_sensitivity": sensitivity, + "simulator_only": { + "classification": _classification_metrics( + np.asarray(labels, dtype=np.int64), + np.asarray(simulator_labels, dtype=np.float64), + ) + if labels + else None, + "predicted_feasible": simulator_labels, + }, + "pilot_examples": [ + { + **detail, + "sim_completed_throughput_per_gpu": math.exp(simulator[0]), + "sim_slo_pass_rate": simulator[1], + "sim_slo_feasible": bool(simulator[2]), + } + for detail, simulator in zip(pilot_details, pilot_simulator) + ], + "decision": decision, + "provenance": { + "phase6_metrics": str(phase6_path.resolve()), + "phase6_metrics_sha256": sha256_file(phase6_path), + "phase6_raw_root": str(phase6_raw_root.resolve()), + "training_simulator_root": str(training_simulator_root.resolve()), + "training_simulator_manifest_scorer_set_sha256": training_simulator_sha256, + "pilot_manifest": str(pilot_manifest_path.resolve()), + "pilot_manifest_sha256": sha256_file(pilot_manifest_path), + "pilot_run_root": str(pilot_run_root.resolve()), + "pilot_simulator": str(pilot_simulator_path.resolve()), + "pilot_simulator_sha256": sha256_file(pilot_simulator_path), + }, + "sanity": { + "red_flags": red_flags, + "training_examples": numeric([1 for _ in training_examples]), + "pilot_labels": { + **numeric(labels), + "positive": sum(labels), + "negative": len(labels) - sum(labels), + }, + "pilot_simulator_pass_rate": numeric(simulator_pass_rates), + "invariants": { + "training_examples_37": len(training_examples) == 37, + "pilot_examples_12": len(pilot_examples) == 12, + "pilot_cells_6": len({example.cell for example in pilot_examples}) == 6, + "pilot_both_labels": len(set(labels)) == 2, + "simulator_ratios_bounded": all( + 0.0 <= value <= 1.0 for value in simulator_pass_rates + ), + "per_config_not_all_identical": len(set(simulator_pass_rates)) > 1, + "all_prefixes_exact_monotonic": all( + example.completion_time_source in {"exact_monotonic", "none_completed"} + for example in pilot_examples + ), + }, + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--phase6-metrics", type=Path, required=True) + parser.add_argument("--phase6-raw-root", type=Path, required=True) + parser.add_argument("--training-simulator-root", type=Path, required=True) + parser.add_argument("--pilot-manifest", type=Path, required=True) + parser.add_argument("--pilot-run-root", type=Path, required=True) + parser.add_argument("--pilot-simulator", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + result = analyze( + args.phase6_metrics, + args.phase6_raw_root, + args.training_simulator_root, + args.pilot_manifest, + args.pilot_run_root, + args.pilot_simulator, + ) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") + print( + json.dumps( + { + "status": result["status"], + "red_flags": result["sanity"]["red_flags"], + "decision": result["decision"], + }, + sort_keys=True, + ) + ) + if result["status"] != "PASS": + raise RuntimeError(result["sanity"]["red_flags"]) + + +if __name__ == "__main__": + main() diff --git a/runs/fidelity-headroom/prepare_pilot_simulator.py b/runs/fidelity-headroom/prepare_pilot_simulator.py new file mode 100644 index 0000000..c3d77fb --- /dev/null +++ b/runs/fidelity-headroom/prepare_pilot_simulator.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python3 +"""Prepare exact Frontier fixtures for the P1 primary low/high probes. + +Prompt-bearing band traces remain under ``--private-root``. The emitted +fixtures and public manifest contain token IDs, block IDs, hashes, and +aggregate metadata, but no prompt text. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import math +import subprocess +import sys +from pathlib import Path +from typing import Any + +from transformers import AutoTokenizer + + +HERE = Path(__file__).resolve().parent +AITUNER_ROOT = HERE.parents[1] +sys.path.insert(0, str(HERE)) + +import prepare_pilot as pilot # noqa: E402 + + +PRIMARY_ROLES = ("low1", "high1") + + +def load_module(path: Path): + spec = importlib.util.spec_from_file_location("simfid_s2rb_prepare", path) + if spec is None or spec.loader is None: + raise ImportError(path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def order_hash(values: list[str]) -> str: + return hashlib.sha256("\n".join(values).encode()).hexdigest() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def git_capture(root: Path, *arguments: str) -> str: + return subprocess.run( + ["git", "-C", str(root), *arguments], + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ).stdout + + +def raw_rows(path: Path) -> dict[int, dict[str, Any]]: + result = {} + with path.open(encoding="utf-8") as source: + for index, line in enumerate(source): + if line.strip(): + result[index] = json.loads(line) + return result + + +def selected_hashes( + selected: list[Any], rows: dict[int, dict[str, Any]] +) -> dict[str, str]: + identifiers = [] + arrivals = [] + lengths = [] + for item in selected: + row = rows[item.row_index] + identifiers.append(str(row.get("request_id") or row.get("id") or item.row_index)) + arrivals.append(f"{float(item.timestamp) * 0.1:.12f}") + lengths.append(str(int(item.input_length))) + return { + "request_id_order_sha256": order_hash(identifiers), + "arrival_order_sha256": order_hash(arrivals), + "input_length_order_sha256": order_hash(lengths), + } + + +def kv_blocks(raw_root: Path, cell: str) -> int: + stream = next((raw_root / cell / "opprof").glob("*.jsonl")) + with stream.open(encoding="utf-8") as source: + for line in source: + record = json.loads(line) + if "step_index" in record: + return int(record["kv"]["total_blocks"]) + raise ValueError(f"no Layer-1 record for {cell}") + + +def source_window(windows_path: Path, window_id: str) -> tuple[dict[str, Any], Path]: + return pilot.resolve_source_trace(windows_path, window_id) + + +def prepare(args: argparse.Namespace) -> dict[str, Any]: + simulator = load_module(args.replayserve_root / "tools/simfid_s2rb_prepare.py") + manifest = json.loads(args.pilot_manifest.read_text(encoding="utf-8")) + window, trace = source_window(args.source_windows, args.source_window_id) + if args.band_root is not None: + role_paths = { + role: (args.band_root / f"{role}.jsonl").resolve() + for role in PRIMARY_ROLES + } + band_stats = { + role: manifest["private"]["band_stats"][role] + for role in PRIMARY_ROLES + } + for role, path in role_paths.items(): + if sha256_file(path) != band_stats[role]["sha256"]: + raise ValueError(f"pre-materialized band hash mismatch: {role}") + private_windows = None + else: + private_windows, all_band_stats = pilot.materialize_bands( + trace, window, args.private_root + ) + private_payload = json.loads(private_windows.read_text(encoding="utf-8")) + role_paths = { + item["fidelity_pilot_role"]: ( + private_windows.parent / item["trace_file"] + ).resolve() + for item in private_payload["windows"] + } + band_stats = { + role: all_band_stats[role] + for role in PRIMARY_ROLES + } + + tokenizer = AutoTokenizer.from_pretrained( + args.tokenizer, local_files_only=True, use_fast=True + ) + fixture_root = args.output / "fixtures" + config_root = args.output / "configs" + fixture_root.mkdir(parents=True, exist_ok=True) + config_root.mkdir(parents=True, exist_ok=True) + entries = [] + red_flags = [] + for role in PRIMARY_ROLES: + trace_path = role_paths[role] + retained, trace_stats = simulator.scan_trace(trace_path) + rows = raw_rows(trace_path) + primary_pool = [retained[(index * len(retained)) // 512] for index in range(512)] + selections: dict[str, list[Any]] = {} + selected_union: set[int] = set() + for cell, cell_manifest in sorted(manifest["cells"].items()): + level = "low" if role.startswith("low") else "high" + expected = cell_manifest["targets"][level]["selections"][role] + pool = retained if int(cell_manifest["tp"]) == 4 else primary_pool + selected = [item for item in pool if item.sampling_u <= float(expected["anchor"])] + selections[cell] = selected + selected_union.update(item.row_index for item in selected) + hashes = selected_hashes(selected, rows) + if len(selected) != int(expected["selected_count"]): + red_flags.append(f"selection_count_{cell}_{role}") + for key, value in hashes.items(): + if value != expected[key]: + red_flags.append(f"selection_hash_{cell}_{role}_{key}") + + token_gates, selected_records, block_stats = simulator.tokenize_and_hash( + trace=trace_path, + tokenizer=tokenizer, + retained=retained, + selected_union=selected_union, + ) + if any(gate["status"] != "pass" for gate in token_gates.values()): + red_flags.append(f"token_gate_{role}") + for cell, selected in selections.items(): + cell_manifest = manifest["cells"][cell] + level = "low" if role.startswith("low") else "high" + expected = cell_manifest["targets"][level]["selections"][role] + fixture_id = f"fidelity_p1_{cell}_{role}" + cell_record = { + "cell_id": cell, + "tensor_parallel_size": int(cell_manifest["tp"]), + "max_num_seqs": int(cell_manifest["mns"]), + "store_role": "companion" if int(cell_manifest["tp"]) == 4 else "primary", + "kv_capacity": { + "block_size_tokens": 16, + "num_blocks": kv_blocks(args.phase6_raw_root, cell), + }, + } + probe = { + "probe_index": 0 if role == "low1" else 1, + "sampling_u": float(expected["anchor"]), + } + fixture = simulator.create_fixture( + fixture_root=fixture_root, + fixture_id=fixture_id, + cell=cell_record, + probe=probe, + row_indexes=[item.row_index for item in selected], + meta_by_index={item.row_index: item for item in retained}, + selected_records=selected_records, + ) + config_path = config_root / f"{fixture_id}.json" + config = simulator.build_config( + path=config_path, + cell=cell_record, + mode="frozen-calibrated", + fixture_ids=[fixture_id], + frontier_root=args.frontier_root, + cache_dir=args.cache_dir, + ) + entries.append( + { + "cell": cell, + "role": role, + "level": level, + "anchor": expected["anchor"], + "selected_count": len(selected), + "fixture_id": fixture_id, + "fixture_manifest": str( + (fixture_root / fixture_id / "fixture_manifest.json").resolve() + ), + "frontier_csv": fixture["frontier_csv"]["path"], + "sidecar": fixture["sidecar_jsonl"]["path"], + "config": str(config_path.resolve()), + "calibration_scale": config["calibration"]["a_tp"], + } + ) + if block_stats["selected_union_records"] != len(selected_union): + red_flags.append(f"selected_union_{role}") + if trace_stats["retained_inclusive_0_8192"] < 512: + red_flags.append(f"retained_too_small_{role}") + + selected_counts = [int(entry["selected_count"]) for entry in entries] + calibration = [float(entry["calibration_scale"]) for entry in entries] + result = { + "schema": "fidelity-p1-frontier-prepared-v1", + "status": "PASS" if not red_flags else "STOP", + "source": { + "pilot_manifest": str(args.pilot_manifest.resolve()), + "source_windows": str(args.source_windows.resolve()), + "source_window_id": args.source_window_id, + "source_trace": str(trace.resolve()), + "private_windows": ( + str(private_windows.resolve()) if private_windows is not None else None + ), + "pre_materialized_band_root": ( + str(args.band_root.resolve()) if args.band_root is not None else None + ), + "band_stats": band_stats, + }, + "simulator": { + "replayserve_root": str(args.replayserve_root.resolve()), + "frontier_root": str(args.frontier_root.resolve()), + "tokenizer": str(args.tokenizer.resolve()), + "mode": "frozen-calibrated", + }, + "generator": { + "script": str(Path(__file__).resolve()), + "script_sha256": sha256_file(Path(__file__).resolve()), + "aituner_git_head": git_capture(AITUNER_ROOT, "rev-parse", "HEAD").strip(), + "aituner_git_status_short": git_capture(AITUNER_ROOT, "status", "--short"), + }, + "entries": entries, + "sanity": { + "red_flags": red_flags, + "n": len(entries), + "selected_count": { + "n": len(selected_counts), + "min": min(selected_counts), + "max": max(selected_counts), + "distinct_n": len(set(selected_counts)), + }, + "calibration_scale": { + "n": len(calibration), + "min": min(calibration), + "max": max(calibration), + "distinct_n": len(set(calibration)), + }, + "invariants": { + "entries_12": len(entries) == 12, + "roles_2": {entry["role"] for entry in entries} == set(PRIMARY_ROLES), + "cells_6": len({entry["cell"] for entry in entries}) == 6, + "selected_nonnegative": all(value > 0 for value in selected_counts), + "per_config_not_identical": len(set(selected_counts)) > 1, + }, + }, + } + args.public_manifest.parent.mkdir(parents=True, exist_ok=True) + args.public_manifest.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") + return result + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser() + result.add_argument("--pilot-manifest", type=Path, required=True) + result.add_argument("--source-windows", type=Path, required=True) + result.add_argument("--source-window-id", required=True) + result.add_argument("--private-root", type=Path, required=True) + result.add_argument("--band-root", type=Path) + result.add_argument("--output", type=Path, required=True) + result.add_argument("--public-manifest", type=Path, required=True) + result.add_argument("--phase6-raw-root", type=Path, required=True) + result.add_argument("--replayserve-root", type=Path, required=True) + result.add_argument("--frontier-root", type=Path, required=True) + result.add_argument("--cache-dir", type=Path, required=True) + result.add_argument("--tokenizer", type=Path, required=True) + return result + + +def main() -> None: + result = prepare(parser().parse_args()) + print( + json.dumps( + { + "status": result["status"], + "entries": len(result["entries"]), + "red_flags": result["sanity"]["red_flags"], + }, + sort_keys=True, + ) + ) + if result["status"] != "PASS": + raise RuntimeError(result["sanity"]["red_flags"]) + + +if __name__ == "__main__": + main() diff --git a/runs/fidelity-headroom/run_pilot_simulator.py b/runs/fidelity-headroom/run_pilot_simulator.py new file mode 100644 index 0000000..d393916 --- /dev/null +++ b/runs/fidelity-headroom/run_pilot_simulator.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +"""Run and score the 12 frozen Frontier P1 primary probes, CPU only.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + + +def load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise ImportError(path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def atomic_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + os.replace(temporary, path) + + +def git_capture(root: Path, *arguments: str) -> str: + return subprocess.run( + ["git", "-C", str(root), *arguments], + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ).stdout + + +def execute(args: argparse.Namespace) -> dict[str, Any]: + prepared = json.loads(args.prepared_manifest.read_text(encoding="utf-8")) + if prepared["status"] != "PASS": + raise RuntimeError("prepared simulator manifest did not pass") + driver = load_module( + "simfid_execution_driver", + args.replayserve_root + / "runs/simfid_s2rb/results/execution_driver.py", + ) + head = git_capture(args.frontier_root, "rev-parse", "HEAD").strip() + status_short = git_capture(args.frontier_root, "status", "--short") + aituner_root = Path(__file__).resolve().parents[2] + aituner_head = git_capture(aituner_root, "rev-parse", "HEAD").strip() + aituner_status_short = git_capture(aituner_root, "status", "--short") + results = [] + failures = [] + gpu_visibility_disabled = True + for sequence, entry in enumerate(prepared["entries"]): + run_root = args.output / f"{sequence:02d}_{entry['fixture_id']}" + scorer_path = run_root / "scorer_output.json" + if scorer_path.is_file() and args.resume: + scorer = json.loads(scorer_path.read_text(encoding="utf-8")) + results.append({**entry, "sequence": sequence, "scorer": scorer, "resumed": True}) + continue + run_root.mkdir(parents=True, exist_ok=True) + config_path = Path(entry["config"]) + config = json.loads(config_path.read_text(encoding="utf-8")) + fixture_manifest_path = Path(entry["fixture_manifest"]) + fixture = json.loads(fixture_manifest_path.read_text(encoding="utf-8")) + trace_path = Path(entry["frontier_csv"]) + sidecar_path = Path(entry["sidecar"]) + metrics_root = run_root / "frontier_metrics" + run_id = f"fidelity_p1_frontier_{sequence:02d}_{entry['cell']}_{entry['role']}" + knobs = config["frontier"]["knobs"] + command = driver.build_command( + trace_path=trace_path, + metrics_root=metrics_root, + run_id=run_id, + knobs=knobs, + ) + driver.audit_command(command, knobs) + row = { + "hook_path": config["calibration"]["hook_path"], + "applied_a_tp": config["calibration"]["a_tp"], + "sidecar_path": str(sidecar_path), + "request_count": int(fixture["request_count"]), + "tensor_parallel_size": int(fixture["tensor_parallel_size"]), + } + environment = driver.environment_for(row) + gpu_visibility_disabled = gpu_visibility_disabled and ( + environment.get("CUDA_VISIBLE_DEVICES") == "" + and environment.get("NVIDIA_VISIBLE_DEVICES") == "void" + ) + run_manifest = { + "schema": "fidelity-p1-frontier-run-v1", + "sequence": sequence, + "cell": entry["cell"], + "role": entry["role"], + "anchor": entry["anchor"], + "request_count": entry["selected_count"], + "frontier": { + "root": str(args.frontier_root.resolve()), + "git_head": head, + "git_status_short": status_short, + }, + "runner": { + "script": str(Path(__file__).resolve()), + "script_sha256": sha256_file(Path(__file__).resolve()), + "aituner_git_head": aituner_head, + "aituner_git_status_short": aituner_status_short, + }, + "inputs": { + "config": str(config_path), + "config_sha256": sha256_file(config_path), + "fixture_manifest": str(fixture_manifest_path), + "fixture_manifest_sha256": sha256_file(fixture_manifest_path), + "frontier_csv": str(trace_path), + "frontier_csv_sha256": sha256_file(trace_path), + "sidecar": str(sidecar_path), + "sidecar_sha256": sha256_file(sidecar_path), + }, + "environment": { + key: environment[key] + for key in ( + "PYTHONPATH", + "FRONTIER_EXECUTION_TIME_SCALE", + "CUDA_VISIBLE_DEVICES", + "NVIDIA_VISIBLE_DEVICES", + "FRONTIER_LOG_LEVEL", + ) + }, + "command": command, + "contains_prompt_text": False, + } + atomic_json(run_root / "run_manifest.json", run_manifest) + start = time.time() + with (run_root / "stdout.log").open("w", encoding="utf-8") as stdout, ( + run_root / "stderr.log" + ).open("w", encoding="utf-8") as stderr: + try: + process = subprocess.run( + command, + cwd=args.frontier_root, + env=environment, + stdout=stdout, + stderr=stderr, + timeout=args.timeout_s, + ) + return_code = int(process.returncode) + except subprocess.TimeoutExpired: + return_code = 124 + runtime = time.time() - start + if return_code != 0: + failure = { + "sequence": sequence, + "cell": entry["cell"], + "role": entry["role"], + "return_code": return_code, + "runtime_s": runtime, + } + failures.append(failure) + atomic_json(run_root / "failure.json", failure) + break + system_path, request_path = driver.find_metrics(run_root) + scorer = driver.score_trial(row, system_path, request_path) + scorer["runtime_s"] = runtime + atomic_json(scorer_path, scorer) + results.append({**entry, "sequence": sequence, "scorer": scorer, "resumed": False}) + print( + json.dumps( + { + "sequence": sequence, + "cell": entry["cell"], + "role": entry["role"], + "runtime_s": runtime, + "sim_pass_rate": scorer["slo"]["pass_rate"], + "sim_feasible": scorer["slo"]["feasible"], + }, + sort_keys=True, + ), + flush=True, + ) + + pass_rates = [float(item["scorer"]["slo"]["pass_rate"]) for item in results] + throughputs = [ + float(item["scorer"]["throughput_requests_per_second_per_gpu"]) + for item in results + ] + runtimes = [float(item["scorer"]["runtime_s"]) for item in results] + red_flags = [] + if failures: + red_flags.append("frontier_run_failure") + if len(results) != 12: + red_flags.append("runs_not_12") + if any(not 0.0 <= value <= 1.0 for value in pass_rates): + red_flags.append("pass_rate_out_of_range") + if any(value <= 0 for value in throughputs): + red_flags.append("nonpositive_throughput") + result = { + "schema": "fidelity-p1-frontier-result-v1", + "status": "PASS" if not red_flags else "STOP", + "prepared_manifest": str(args.prepared_manifest.resolve()), + "prepared_manifest_sha256": sha256_file(args.prepared_manifest), + "frontier": { + "root": str(args.frontier_root.resolve()), + "git_head": head, + "git_status_short": status_short, + }, + "runner": { + "script": str(Path(__file__).resolve()), + "script_sha256": sha256_file(Path(__file__).resolve()), + "aituner_git_head": aituner_head, + "aituner_git_status_short": aituner_status_short, + }, + "results": results, + "failures": failures, + "sanity": { + "red_flags": red_flags, + "n": len(results), + "pass_rate": { + "n": len(pass_rates), + "min": min(pass_rates) if pass_rates else None, + "max": max(pass_rates) if pass_rates else None, + "distinct_n": len(set(pass_rates)), + }, + "throughput_per_gpu": { + "n": len(throughputs), + "min": min(throughputs) if throughputs else None, + "max": max(throughputs) if throughputs else None, + "distinct_n": len(set(throughputs)), + }, + "runtime_s": { + "n": len(runtimes), + "min": min(runtimes) if runtimes else None, + "max": max(runtimes) if runtimes else None, + "distinct_n": len(set(runtimes)), + }, + "invariants": { + "runs_12": len(results) == 12, + "zero_failures": not failures, + "ratios_bounded": all(0.0 <= value <= 1.0 for value in pass_rates), + "nonnegative_metrics": all(value > 0 for value in throughputs), + "per_config_not_identical": len(set(pass_rates)) > 1, + "gpu_visibility_disabled": gpu_visibility_disabled, + }, + }, + } + atomic_json(args.output / "metrics.json", result) + return result + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser() + result.add_argument("--prepared-manifest", type=Path, required=True) + result.add_argument("--output", type=Path, required=True) + result.add_argument("--replayserve-root", type=Path, required=True) + result.add_argument("--frontier-root", type=Path, required=True) + result.add_argument("--timeout-s", type=float, default=900.0) + result.add_argument("--resume", action="store_true") + return result + + +def main() -> None: + result = execute(parser().parse_args()) + print( + json.dumps( + { + "status": result["status"], + "runs": len(result["results"]), + "red_flags": result["sanity"]["red_flags"], + }, + sort_keys=True, + ) + ) + if result["status"] != "PASS": + raise RuntimeError(result["sanity"]["red_flags"]) + + +if __name__ == "__main__": + main() diff --git a/runs/fidelity-headroom/test_strong_pilot.py b/runs/fidelity-headroom/test_strong_pilot.py new file mode 100644 index 0000000..48f6089 --- /dev/null +++ b/runs/fidelity-headroom/test_strong_pilot.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import tempfile +from pathlib import Path + +import numpy as np + +from analyze_prefixes import PrefixExample +from analyze_strong_pilot import ( + fit_model, + load_pilot_simulator, + predict_model, +) + + +def example(index: int) -> PrefixExample: + label = int(index >= 4) + return PrefixExample( + cell=f"cell-{index // 2}", + anchor=float(index), + cutoff_s=5.0, + tp=1, + full_elapsed_s=10.0, + feasible=label, + primary_feasible=label, + outcome=tuple(float(index + offset) for offset in range(13)), + instrumentation=tuple(float(index * offset + 1) for offset in range(17)), + completion_time_source="exact_monotonic", + ) + + +def main() -> None: + examples = [example(index) for index in range(8)] + simulator = [(float(index), index / 10.0, float(index >= 4)) for index in range(8)] + for instrumentation_aware in (False, True): + model = fit_model( + examples, + simulator, + instrumentation_aware=instrumentation_aware, + regularization=1.0, + ) + probability = predict_model(model, examples, simulator) + assert probability.shape == (8,) + assert np.all((probability >= 0.0) & (probability <= 1.0)) + + payload = { + "status": "PASS", + "results": [ + { + "cell": f"cell-{index // 2}", + "role": "low1" if index % 2 == 0 else "high1", + "scorer": { + "throughput_requests_per_second_per_gpu": 1.0 + index, + "slo": { + "pass_rate": index / 12.0, + "feasible": index % 2 == 0, + }, + }, + } + for index in range(12) + ], + } + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "metrics.json" + path.write_text(json.dumps(payload), encoding="utf-8") + features, red_flags = load_pilot_simulator(path) + assert len(features) == 12 + assert red_flags == [] + print("fidelity strong pilot: PASS") + + +if __name__ == "__main__": + main() From 12d1d4ad022361c2ea95e0126df170ea1ece66ad Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 13:29:38 +0800 Subject: [PATCH 006/148] Make fidelity simulator tools self-contained --- runs/fidelity-headroom/prepare_pilot_simulator.py | 3 +++ runs/fidelity-headroom/run_pilot_simulator.py | 3 +++ runs/fidelity-headroom/test_strong_pilot.py | 15 +++++++++++++++ 3 files changed, 21 insertions(+) diff --git a/runs/fidelity-headroom/prepare_pilot_simulator.py b/runs/fidelity-headroom/prepare_pilot_simulator.py index c3d77fb..fc84bde 100644 --- a/runs/fidelity-headroom/prepare_pilot_simulator.py +++ b/runs/fidelity-headroom/prepare_pilot_simulator.py @@ -32,6 +32,9 @@ PRIMARY_ROLES = ("low1", "high1") def load_module(path: Path): + module_root = str(path.parent.resolve()) + if module_root not in sys.path: + sys.path.insert(0, module_root) spec = importlib.util.spec_from_file_location("simfid_s2rb_prepare", path) if spec is None or spec.loader is None: raise ImportError(path) diff --git a/runs/fidelity-headroom/run_pilot_simulator.py b/runs/fidelity-headroom/run_pilot_simulator.py index d393916..6db18b4 100644 --- a/runs/fidelity-headroom/run_pilot_simulator.py +++ b/runs/fidelity-headroom/run_pilot_simulator.py @@ -16,6 +16,9 @@ from typing import Any def load_module(name: str, path: Path): + module_root = str(path.parent.resolve()) + if module_root not in sys.path: + sys.path.insert(0, module_root) spec = importlib.util.spec_from_file_location(name, path) if spec is None or spec.loader is None: raise ImportError(path) diff --git a/runs/fidelity-headroom/test_strong_pilot.py b/runs/fidelity-headroom/test_strong_pilot.py index 48f6089..921c2b3 100644 --- a/runs/fidelity-headroom/test_strong_pilot.py +++ b/runs/fidelity-headroom/test_strong_pilot.py @@ -8,6 +8,8 @@ from pathlib import Path import numpy as np from analyze_prefixes import PrefixExample +from prepare_pilot_simulator import load_module as load_prepare_module +from run_pilot_simulator import load_module as load_run_module from analyze_strong_pilot import ( fit_model, load_pilot_simulator, @@ -68,6 +70,19 @@ def main() -> None: features, red_flags = load_pilot_simulator(path) assert len(features) == 12 assert red_flags == [] + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "prepare_dependency.py").write_text("VALUE = 17\n", encoding="utf-8") + (root / "prepare_target.py").write_text( + "from prepare_dependency import VALUE\n", encoding="utf-8" + ) + assert load_prepare_module(root / "prepare_target.py").VALUE == 17 + (root / "run_dependency.py").write_text("VALUE = 23\n", encoding="utf-8") + (root / "run_target.py").write_text( + "from run_dependency import VALUE\n", encoding="utf-8" + ) + assert load_run_module("run_target", root / "run_target.py").VALUE == 23 print("fidelity strong pilot: PASS") From 4ad699ef9709943fcd01c5c3f6022997279f62ba Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 13:38:32 +0800 Subject: [PATCH 007/148] Report fidelity pilot covariate shift --- .../fidelity-headroom/analyze_strong_pilot.py | 56 +++++++++++++++++++ runs/fidelity-headroom/test_strong_pilot.py | 10 ++++ 2 files changed, 66 insertions(+) diff --git a/runs/fidelity-headroom/analyze_strong_pilot.py b/runs/fidelity-headroom/analyze_strong_pilot.py index 57e79c3..de19dce 100644 --- a/runs/fidelity-headroom/analyze_strong_pilot.py +++ b/runs/fidelity-headroom/analyze_strong_pilot.py @@ -121,6 +121,45 @@ def predict_model( return _sigmoid(matrix @ model["weights"]) +def covariate_shift( + training_examples: list[PrefixExample], + training_simulator: list[tuple[float, ...]], + pilot_examples: list[PrefixExample], + pilot_simulator: list[tuple[float, ...]], + *, + instrumentation_aware: bool, +) -> dict[str, Any]: + def matrix( + examples: list[PrefixExample], simulator: list[tuple[float, ...]] + ) -> np.ndarray: + rows = [] + for example, simulator_features in zip(examples, simulator): + values = example.outcome + simulator_features + if instrumentation_aware: + values += example.instrumentation + rows.append(values) + return np.asarray(rows, dtype=np.float64) + + training = matrix(training_examples, training_simulator) + pilot = matrix(pilot_examples, pilot_simulator) + mean = training.mean(axis=0) + standard_deviation = training.std(axis=0) + standard_deviation[standard_deviation < 1e-8] = 1.0 + absolute_z = np.abs((pilot - mean) / standard_deviation) + names = [*OUTCOME_FEATURES, *SIMULATOR_FEATURES] + if instrumentation_aware: + names.extend(INSTRUMENTATION_FEATURES) + return { + "values": numeric(absolute_z.ravel().tolist()), + "count_gt_3": int(np.sum(absolute_z > 3.0)), + "count_gt_5": int(np.sum(absolute_z > 5.0)), + "total_feature_values": int(absolute_z.size), + "per_feature_max_abs_z": { + name: float(value) for name, value in zip(names, absolute_z.max(axis=0)) + }, + } + + def comparison( training_examples: list[PrefixExample], training_simulator: list[tuple[float, ...]], @@ -238,6 +277,22 @@ def analyze( red_flags.append("pilot_single_label") if len(set(simulator_pass_rates)) <= 1: red_flags.append("pilot_simulator_results_identical") + covariate_diagnostics = { + "sim_plus_outcome": covariate_shift( + training_examples, + training_simulator, + pilot_examples, + pilot_simulator, + instrumentation_aware=False, + ), + "sim_plus_outcome_plus_instrumentation": covariate_shift( + training_examples, + training_simulator, + pilot_examples, + pilot_simulator, + instrumentation_aware=True, + ), + } if headline is None: decision = { @@ -321,6 +376,7 @@ def analyze( } for detail, simulator in zip(pilot_details, pilot_simulator) ], + "covariate_shift_diagnostic": covariate_diagnostics, "decision": decision, "provenance": { "phase6_metrics": str(phase6_path.resolve()), diff --git a/runs/fidelity-headroom/test_strong_pilot.py b/runs/fidelity-headroom/test_strong_pilot.py index 921c2b3..c888438 100644 --- a/runs/fidelity-headroom/test_strong_pilot.py +++ b/runs/fidelity-headroom/test_strong_pilot.py @@ -11,6 +11,7 @@ from analyze_prefixes import PrefixExample from prepare_pilot_simulator import load_module as load_prepare_module from run_pilot_simulator import load_module as load_run_module from analyze_strong_pilot import ( + covariate_shift, fit_model, load_pilot_simulator, predict_model, @@ -46,6 +47,15 @@ def main() -> None: probability = predict_model(model, examples, simulator) assert probability.shape == (8,) assert np.all((probability >= 0.0) & (probability <= 1.0)) + shift = covariate_shift( + examples, + simulator, + examples, + simulator, + instrumentation_aware=instrumentation_aware, + ) + assert shift["values"]["min"] >= 0.0 + assert shift["count_gt_3"] == 0 payload = { "status": "PASS", From 1f32ae217e06adf477b24d7cf606b899e7ea66b0 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 13:39:38 +0800 Subject: [PATCH 008/148] Harden strong fidelity pilot validation --- .../fidelity-headroom/analyze_strong_pilot.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/runs/fidelity-headroom/analyze_strong_pilot.py b/runs/fidelity-headroom/analyze_strong_pilot.py index de19dce..19752a7 100644 --- a/runs/fidelity-headroom/analyze_strong_pilot.py +++ b/runs/fidelity-headroom/analyze_strong_pilot.py @@ -13,6 +13,7 @@ from __future__ import annotations import argparse import json import math +import subprocess from pathlib import Path from typing import Any @@ -43,6 +44,19 @@ from analyze_strong_baseline import ( ) +AITUNER_ROOT = Path(__file__).resolve().parents[2] + + +def git_capture(*arguments: str) -> str: + return subprocess.run( + ["git", "-C", str(AITUNER_ROOT), *arguments], + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ).stdout + + def load_pilot_simulator( path: Path, ) -> tuple[dict[tuple[str, str], tuple[float, ...]], list[str]]: @@ -230,6 +244,8 @@ def analyze( ) -> dict[str, Any]: phase6 = json.loads(phase6_path.read_text(encoding="utf-8")) pilot_manifest = json.loads(pilot_manifest_path.read_text(encoding="utf-8")) + pilot_state_path = pilot_run_root / "controller-state.json" + pilot_state = json.loads(pilot_state_path.read_text(encoding="utf-8")) training_examples = build_examples(phase6, phase6_raw_root, 5.0) training_simulator_map, training_simulator_sha256 = load_simulator_features( training_simulator_root @@ -277,6 +293,23 @@ def analyze( red_flags.append("pilot_single_label") if len(set(simulator_pass_rates)) <= 1: red_flags.append("pilot_simulator_results_identical") + if pilot_state.get("status") != "complete" or int( + pilot_state.get("completed_cells", 0) + ) != 6: + red_flags.append("pilot_campaign_incomplete") + if any(detail["actual_timestamped_outcomes"] == 0 for detail in pilot_details): + red_flags.append("pilot_no_exact_request_timestamps") + all_cell_validations = all( + cell.get("validation") is not None + and all(cell["validation"]["invariants"].values()) + for cell in pilot_state.get("cells", {}).values() + ) + if not all_cell_validations: + red_flags.append("pilot_cell_validation_failed") + if float(pilot_state.get("gpu_hours_total", math.inf)) >= float( + pilot_state.get("hard_cap_h20_hours", -math.inf) + ): + red_flags.append("pilot_hard_cap_exceeded") covariate_diagnostics = { "sim_plus_outcome": covariate_shift( training_examples, @@ -378,6 +411,12 @@ def analyze( ], "covariate_shift_diagnostic": covariate_diagnostics, "decision": decision, + "analysis": { + "script": str(Path(__file__).resolve()), + "script_sha256": sha256_file(Path(__file__).resolve()), + "aituner_git_head": git_capture("rev-parse", "HEAD").strip(), + "aituner_git_status_short": git_capture("status", "--short"), + }, "provenance": { "phase6_metrics": str(phase6_path.resolve()), "phase6_metrics_sha256": sha256_file(phase6_path), @@ -387,6 +426,8 @@ def analyze( "pilot_manifest": str(pilot_manifest_path.resolve()), "pilot_manifest_sha256": sha256_file(pilot_manifest_path), "pilot_run_root": str(pilot_run_root.resolve()), + "pilot_controller_state": str(pilot_state_path.resolve()), + "pilot_controller_state_sha256": sha256_file(pilot_state_path), "pilot_simulator": str(pilot_simulator_path.resolve()), "pilot_simulator_sha256": sha256_file(pilot_simulator_path), }, @@ -412,6 +453,11 @@ def analyze( example.completion_time_source in {"exact_monotonic", "none_completed"} for example in pilot_examples ), + "all_cell_validations": all_cell_validations, + "gpu_cost_nonnegative_below_cap": ( + 0.0 <= float(pilot_state["gpu_hours_total"]) + < float(pilot_state["hard_cap_h20_hours"]) + ), }, }, } From 2261818994a2e3a7960fc2d6c30187a953333ac4 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 13:41:46 +0800 Subject: [PATCH 009/148] Account for failed fidelity pilot attempts --- runs/fidelity-headroom/analyze_pilot.py | 51 +++++++++++++++++-- .../fidelity-headroom/analyze_strong_pilot.py | 19 ++++--- runs/fidelity-headroom/test_pilot_tools.py | 27 ++++++++++ 3 files changed, 87 insertions(+), 10 deletions(-) diff --git a/runs/fidelity-headroom/analyze_pilot.py b/runs/fidelity-headroom/analyze_pilot.py index 3797e18..b1293b3 100644 --- a/runs/fidelity-headroom/analyze_pilot.py +++ b/runs/fidelity-headroom/analyze_pilot.py @@ -37,6 +37,41 @@ def selection_for( return manifest["cells"][cell]["targets"][level]["selections"][role] +def campaign_gpu_accounting( + primary_state_path: Path, prior_state_paths: tuple[Path, ...] = () +) -> dict[str, Any]: + attempts = [] + for role, path in ( + [("prior_failure", path) for path in prior_state_paths] + + [("primary", primary_state_path)] + ): + state = json.loads(path.read_text(encoding="utf-8")) + gpu_hours = float(state["gpu_hours_total"]) + attempts.append( + { + "role": role, + "path": str(path.resolve()), + "sha256": sha256_file(path), + "status": state["status"], + "h20_hours": gpu_hours, + } + ) + total = sum(attempt["h20_hours"] for attempt in attempts) + primary = json.loads(primary_state_path.read_text(encoding="utf-8")) + hard_cap = float(primary["hard_cap_h20_hours"]) + return { + "attempts": attempts, + "aggregate_h20_hours": total, + "hard_cap_h20_hours": hard_cap, + "invariants": { + "costs_nonnegative": all( + attempt["h20_hours"] >= 0.0 for attempt in attempts + ), + "aggregate_below_cap": 0.0 <= total < hard_cap, + }, + } + + def build_pilot_examples( manifest: dict[str, Any], run_root: Path, cutoff_s: float ) -> tuple[list[PrefixExample], list[dict[str, Any]], list[str]]: @@ -125,11 +160,13 @@ def analyze( manifest_path: Path, model_path: Path, run_root: Path, + prior_state_paths: tuple[Path, ...] = (), ) -> dict[str, Any]: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) models = json.loads(model_path.read_text(encoding="utf-8")) state_path = run_root / "controller-state.json" state = json.loads(state_path.read_text(encoding="utf-8")) + gpu_accounting = campaign_gpu_accounting(state_path, prior_state_paths) cutoff_s = float(models["cutoff_s"]) threshold = float(models["accept_probability"]) examples, details, red_flags = build_pilot_examples(manifest, run_root, cutoff_s) @@ -171,7 +208,7 @@ def analyze( detail["actual_timestamped_outcomes"] == 0 for detail in details ): red_flags.append("no_exact_request_timestamps") - if float(state["gpu_hours_total"]) >= float(state["hard_cap_h20_hours"]): + if not all(gpu_accounting["invariants"].values()): red_flags.append("hard_cap_exceeded") outcome_errors = outcome_policy["false_accept"] + outcome_policy["false_reject"] @@ -232,8 +269,8 @@ def analyze( "opens_expanded_p2": pilot_pass, }, "gpu": { - "actual_h20_hours": state["gpu_hours_total"], - "hard_cap_h20_hours": state["hard_cap_h20_hours"], + "primary_attempt_h20_hours": state["gpu_hours_total"], + **gpu_accounting, }, "sanity": { "red_flags": red_flags, @@ -277,9 +314,15 @@ def main() -> None: parser.add_argument("--manifest", type=Path, required=True) parser.add_argument("--frozen-models", type=Path, required=True) parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("--prior-state", type=Path, action="append", default=[]) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() - result = analyze(args.manifest, args.frozen_models, args.run_root) + result = analyze( + args.manifest, + args.frozen_models, + args.run_root, + tuple(args.prior_state), + ) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") print(json.dumps({ diff --git a/runs/fidelity-headroom/analyze_strong_pilot.py b/runs/fidelity-headroom/analyze_strong_pilot.py index 19752a7..05cebca 100644 --- a/runs/fidelity-headroom/analyze_strong_pilot.py +++ b/runs/fidelity-headroom/analyze_strong_pilot.py @@ -27,7 +27,7 @@ from analyze_existing import ( _mcnemar_exact_p, _sigmoid, ) -from analyze_pilot import build_pilot_examples +from analyze_pilot import build_pilot_examples, campaign_gpu_accounting from analyze_prefixes import ( INSTRUMENTATION_FEATURES, OUTCOME_FEATURES, @@ -241,11 +241,15 @@ def analyze( pilot_manifest_path: Path, pilot_run_root: Path, pilot_simulator_path: Path, + prior_state_paths: tuple[Path, ...] = (), ) -> dict[str, Any]: phase6 = json.loads(phase6_path.read_text(encoding="utf-8")) pilot_manifest = json.loads(pilot_manifest_path.read_text(encoding="utf-8")) pilot_state_path = pilot_run_root / "controller-state.json" pilot_state = json.loads(pilot_state_path.read_text(encoding="utf-8")) + gpu_accounting = campaign_gpu_accounting( + pilot_state_path, prior_state_paths + ) training_examples = build_examples(phase6, phase6_raw_root, 5.0) training_simulator_map, training_simulator_sha256 = load_simulator_features( training_simulator_root @@ -306,9 +310,7 @@ def analyze( ) if not all_cell_validations: red_flags.append("pilot_cell_validation_failed") - if float(pilot_state.get("gpu_hours_total", math.inf)) >= float( - pilot_state.get("hard_cap_h20_hours", -math.inf) - ): + if not all(gpu_accounting["invariants"].values()): red_flags.append("pilot_hard_cap_exceeded") covariate_diagnostics = { "sim_plus_outcome": covariate_shift( @@ -411,6 +413,10 @@ def analyze( ], "covariate_shift_diagnostic": covariate_diagnostics, "decision": decision, + "gpu": { + "primary_attempt_h20_hours": pilot_state["gpu_hours_total"], + **gpu_accounting, + }, "analysis": { "script": str(Path(__file__).resolve()), "script_sha256": sha256_file(Path(__file__).resolve()), @@ -455,8 +461,7 @@ def analyze( ), "all_cell_validations": all_cell_validations, "gpu_cost_nonnegative_below_cap": ( - 0.0 <= float(pilot_state["gpu_hours_total"]) - < float(pilot_state["hard_cap_h20_hours"]) + all(gpu_accounting["invariants"].values()) ), }, }, @@ -471,6 +476,7 @@ def main() -> None: parser.add_argument("--pilot-manifest", type=Path, required=True) parser.add_argument("--pilot-run-root", type=Path, required=True) parser.add_argument("--pilot-simulator", type=Path, required=True) + parser.add_argument("--prior-state", type=Path, action="append", default=[]) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() result = analyze( @@ -480,6 +486,7 @@ def main() -> None: args.pilot_manifest, args.pilot_run_root, args.pilot_simulator, + tuple(args.prior_state), ) args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") print( diff --git a/runs/fidelity-headroom/test_pilot_tools.py b/runs/fidelity-headroom/test_pilot_tools.py index f5a674f..46d9cba 100644 --- a/runs/fidelity-headroom/test_pilot_tools.py +++ b/runs/fidelity-headroom/test_pilot_tools.py @@ -14,6 +14,7 @@ sys.path.insert(0, str(HERE)) import pilot_controller as controller # noqa: E402 import prepare_pilot as prepare # noqa: E402 +from analyze_pilot import campaign_gpu_accounting # noqa: E402 @dataclass @@ -69,6 +70,32 @@ def main() -> None: assert row["fidelity_pilot_band"] == role assert abs(float(row["sampling_u"]) - 0.5) < 1e-12 + prior = root / "prior-state.json" + primary = root / "primary-state.json" + prior.write_text( + json.dumps( + { + "status": "failed", + "gpu_hours_total": 0.02, + "hard_cap_h20_hours": 3.5, + } + ), + encoding="utf-8", + ) + primary.write_text( + json.dumps( + { + "status": "complete", + "gpu_hours_total": 1.5, + "hard_cap_h20_hours": 3.5, + } + ), + encoding="utf-8", + ) + accounting = campaign_gpu_accounting(primary, (prior,)) + assert math.isclose(accounting["aggregate_h20_hours"], 1.52) + assert all(accounting["invariants"].values()) + assert len(controller.ORDER) == 6 assert set(controller.ORDER) == set(prepare.CELLS) assert math.isclose( From 24a9c27b10f1190b20346bfab10bebf6e5b38294 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 13:53:14 +0800 Subject: [PATCH 010/148] Add fidelity pilot shortlist replay --- runs/fidelity-headroom/analyze_pilot_e2e.py | 429 ++++++++++++++++++++ runs/fidelity-headroom/test_pilot_e2e.py | 50 +++ 2 files changed, 479 insertions(+) create mode 100644 runs/fidelity-headroom/analyze_pilot_e2e.py create mode 100644 runs/fidelity-headroom/test_pilot_e2e.py diff --git a/runs/fidelity-headroom/analyze_pilot_e2e.py b/runs/fidelity-headroom/analyze_pilot_e2e.py new file mode 100644 index 0000000..09a6581 --- /dev/null +++ b/runs/fidelity-headroom/analyze_pilot_e2e.py @@ -0,0 +1,429 @@ +#!/usr/bin/env python3 +"""Replay the P1 simulator shortlist under full and prefix policies.""" + +from __future__ import annotations + +import argparse +import json +import math +import subprocess +from pathlib import Path +from typing import Any + +from analyze_prefixes import numeric, sha256_file + + +AITUNER_ROOT = Path(__file__).resolve().parents[2] +FROZEN_K = 2 +CUTOFF_S = 5.0 +THRESHOLD = 0.95 + + +def git_capture(*arguments: str) -> str: + return subprocess.run( + ["git", "-C", str(AITUNER_ROOT), *arguments], + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ).stdout + + +def setup_costs(state: dict[str, Any]) -> dict[str, float]: + result = {} + for cell, payload in state["cells"].items(): + tp = int(payload["tp"]) + annotation_intervals = sum( + float(run["elapsed_s"]) * tp / 3600.0 + for run in payload["runs"] + if run["role"] not in {"low1", "high1"} + ) + primary_intervals = sum( + float(run["elapsed_s"]) * tp / 3600.0 + for run in payload["runs"] + if run["role"] in {"low1", "high1"} + ) + setup = float(payload["gpu_hours"]) - annotation_intervals - primary_intervals + if setup < -1e-12: + raise ValueError(f"negative inferred setup cost: {cell}={setup}") + result[cell] = max(0.0, setup) + return result + + +def build_candidates( + manifest: dict[str, Any], + state: dict[str, Any], + strong: dict[str, Any], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + baseline_probability = strong["headline"]["sim_plus_outcome"]["probability"] + instrument_probability = strong["headline"][ + "sim_plus_outcome_plus_instrumentation" + ]["probability"] + setup = setup_costs(state) + anchors = [] + for detail, baseline_p, instrument_p in zip( + strong["pilot_examples"], baseline_probability, instrument_probability + ): + cell = str(detail["cell"]) + level = str(detail["level"]) + role = f"{level}1" + selection = manifest["cells"][cell]["targets"][level]["selections"][role] + run = next( + item for item in state["cells"][cell]["runs"] if item["role"] == role + ) + tp = int(state["cells"][cell]["tp"]) + full_cost = float(run["elapsed_s"]) * tp / 3600.0 + prefix_cost = min(CUTOFF_S, float(run["elapsed_s"])) * tp / 3600.0 + anchors.append( + { + "cell": cell, + "level": level, + "role": role, + "tp": tp, + "real_feasible": bool(detail["adjudicated_feasible"]), + "real_goodput_req_s_per_gpu": float( + selection["offered_req_s_per_gpu"] + ), + "sim_feasible": bool(detail["sim_slo_feasible"]), + "sim_pass_rate": float(detail["sim_slo_pass_rate"]), + "sim_throughput_req_s_per_gpu": float( + detail["sim_completed_throughput_per_gpu"] + ), + "baseline_probability": float(baseline_p), + "instrument_probability": float(instrument_p), + "setup_h20_hours": setup[cell], + "full_trial_h20_hours": full_cost, + "prefix_h20_hours": prefix_cost, + } + ) + + candidates = [] + for cell in sorted(manifest["cells"]): + feasible = [ + anchor for anchor in anchors if anchor["cell"] == cell and anchor["sim_feasible"] + ] + if not feasible: + continue + candidates.append( + max(feasible, key=lambda anchor: anchor["sim_throughput_req_s_per_gpu"]) + ) + candidates.sort( + key=lambda anchor: ( + -anchor["sim_throughput_req_s_per_gpu"], + anchor["cell"], + ) + ) + return anchors, candidates + + +def expanded_top_k(candidates: list[dict[str, Any]], k: int) -> list[dict[str, Any]]: + if not candidates or k <= 0: + return [] + boundary = candidates[min(k, len(candidates)) - 1][ + "sim_throughput_req_s_per_gpu" + ] + return [ + candidate + for candidate in candidates + if candidate["sim_throughput_req_s_per_gpu"] >= boundary - 1e-12 + ] + + +def selected_result( + evaluated: list[dict[str, Any]], feasible_key: str +) -> tuple[str | None, float | None]: + feasible = [candidate for candidate in evaluated if candidate[feasible_key]] + if not feasible: + return None, None + best = max(feasible, key=lambda candidate: candidate["real_goodput_req_s_per_gpu"]) + return str(best["cell"]), float(best["real_goodput_req_s_per_gpu"]) + + +def replay( + shortlist: list[dict[str, Any]], + *, + probability_key: str | None, + oracle_goodput: float, + common_failure_h20_hours: float, +) -> dict[str, Any]: + evaluated = [] + online_cost = 0.0 + early_accept = 0 + early_reject = 0 + false_accept = 0 + false_reject = 0 + for candidate in shortlist: + current = dict(candidate) + online_cost += current["setup_h20_hours"] + if probability_key is None: + predicted_feasible = current["real_feasible"] + online_cost += current["full_trial_h20_hours"] + action = "full" + else: + probability = float(current[probability_key]) + if probability >= THRESHOLD: + predicted_feasible = True + early_accept += 1 + online_cost += current["prefix_h20_hours"] + action = "early_accept" + false_accept += int(not current["real_feasible"]) + elif probability <= 1.0 - THRESHOLD: + predicted_feasible = False + early_reject += 1 + online_cost += current["prefix_h20_hours"] + action = "early_reject" + false_reject += int(current["real_feasible"]) + else: + predicted_feasible = current["real_feasible"] + online_cost += current["full_trial_h20_hours"] + action = "continue_full" + current["policy_feasible"] = predicted_feasible + current["action"] = action + evaluated.append(current) + selected_cell, selected_goodput = selected_result(evaluated, "policy_feasible") + regret = ( + 1.0 - selected_goodput / oracle_goodput + if selected_goodput is not None and oracle_goodput > 0 + else None + ) + return { + "selected_cell": selected_cell, + "selected_real_goodput_req_s_per_gpu": selected_goodput, + "real_regret": regret, + "online_h20_hours": online_cost, + "conservative_h20_hours_with_prior_failure": ( + online_cost + common_failure_h20_hours + ), + "early_accept": early_accept, + "early_reject": early_reject, + "false_accept": false_accept, + "false_reject": false_reject, + "evaluated": [ + { + "cell": item["cell"], + "level": item["level"], + "action": item["action"], + "real_feasible": item["real_feasible"], + "policy_feasible": item["policy_feasible"], + } + for item in evaluated + ], + } + + +def analyze( + manifest_path: Path, + state_path: Path, + prior_state_path: Path, + strong_path: Path, +) -> dict[str, Any]: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + state = json.loads(state_path.read_text(encoding="utf-8")) + prior = json.loads(prior_state_path.read_text(encoding="utf-8")) + strong = json.loads(strong_path.read_text(encoding="utf-8")) + anchors, candidates = build_candidates(manifest, state, strong) + oracle_anchor = max( + (anchor for anchor in anchors if anchor["real_feasible"]), + key=lambda anchor: anchor["real_goodput_req_s_per_gpu"], + ) + oracle_goodput = float(oracle_anchor["real_goodput_req_s_per_gpu"]) + common_failure = float(prior["gpu_hours_total"]) + by_k = {} + for k in (1, 2, 3, 6): + shortlist = expanded_top_k(candidates, k) + full = replay( + shortlist, + probability_key=None, + oracle_goodput=oracle_goodput, + common_failure_h20_hours=common_failure, + ) + baseline = replay( + shortlist, + probability_key="baseline_probability", + oracle_goodput=oracle_goodput, + common_failure_h20_hours=common_failure, + ) + instrument = replay( + shortlist, + probability_key="instrument_probability", + oracle_goodput=oracle_goodput, + common_failure_h20_hours=common_failure, + ) + for result in (baseline, instrument): + result["online_cost_reduction_vs_full"] = ( + 1.0 - result["online_h20_hours"] / full["online_h20_hours"] + ) + result["conservative_cost_reduction_vs_full"] = 1.0 - ( + result["conservative_h20_hours_with_prior_failure"] + / full["conservative_h20_hours_with_prior_failure"] + ) + by_k[str(k)] = { + "actual_shortlist_size": len(shortlist), + "shortlist": [candidate["cell"] for candidate in shortlist], + "sim_top_k_plus_real_final": full, + "sim_plus_outcome": baseline, + "sim_plus_outcome_plus_instrumentation": instrument, + } + + frozen = by_k[str(FROZEN_K)] + full = frozen["sim_top_k_plus_real_final"] + baseline = frozen["sim_plus_outcome"] + instrument = frozen["sim_plus_outcome_plus_instrumentation"] + baseline_safe = baseline["false_accept"] == 0 and baseline["false_reject"] == 0 + instrument_safe = ( + instrument["false_accept"] == 0 and instrument["false_reject"] == 0 + ) + incremental_reduction = ( + 1.0 - instrument["online_h20_hours"] / baseline["online_h20_hours"] + if baseline_safe and instrument_safe and baseline["online_h20_hours"] > 0 + else None + ) + contribution_gate = { + "frozen_k": FROZEN_K, + "instrument_safe": instrument_safe, + "outcome_baseline_safe": baseline_safe, + "instrument_regret_at_most_5pct": ( + instrument["real_regret"] is not None + and instrument["real_regret"] <= 0.05 + ), + "instrument_cost_reduction_vs_full_at_least_30pct": ( + instrument["online_cost_reduction_vs_full"] >= 0.30 + ), + "instrument_cost_reduction_vs_outcome_at_least_20pct": ( + incremental_reduction is not None and incremental_reduction >= 0.20 + ), + "incremental_reduction_vs_outcome": incremental_reduction, + } + contribution_gate["passes"] = all( + contribution_gate[key] + for key in ( + "instrument_safe", + "outcome_baseline_safe", + "instrument_regret_at_most_5pct", + "instrument_cost_reduction_vs_full_at_least_30pct", + "instrument_cost_reduction_vs_outcome_at_least_20pct", + ) + ) + + red_flags = [] + if state["status"] != "complete" or int(state["completed_cells"]) != 6: + red_flags.append("pilot_incomplete") + if strong["status"] != "PASS" or strong["sanity"]["red_flags"]: + red_flags.append("strong_input_invalid") + if len(anchors) != 12 or len(candidates) != 6: + red_flags.append("unexpected_surface_size") + probabilities = [ + value + for anchor in anchors + for value in (anchor["baseline_probability"], anchor["instrument_probability"]) + ] + costs = [ + value + for anchor in anchors + for value in ( + anchor["setup_h20_hours"], + anchor["full_trial_h20_hours"], + anchor["prefix_h20_hours"], + ) + ] + if not all(0.0 <= value <= 1.0 for value in probabilities): + red_flags.append("probability_out_of_range") + if not all(value >= 0.0 and math.isfinite(value) for value in costs): + red_flags.append("invalid_cost") + + return { + "schema": "fidelity-pilot-e2e-v1", + "status": "PASS" if not red_flags else "STOP", + "scope": "held-out P1 replay; gate diagnostic, not paper-facing evidence", + "ranking": [ + { + "rank": rank, + "cell": candidate["cell"], + "level": candidate["level"], + "sim_throughput_req_s_per_gpu": candidate[ + "sim_throughput_req_s_per_gpu" + ], + "real_feasible": candidate["real_feasible"], + "real_goodput_req_s_per_gpu": candidate[ + "real_goodput_req_s_per_gpu" + ], + } + for rank, candidate in enumerate(candidates, start=1) + ], + "real_oracle": { + "cell": oracle_anchor["cell"], + "level": oracle_anchor["level"], + "goodput_req_s_per_gpu": oracle_goodput, + }, + "by_k": by_k, + "contribution_gate": contribution_gate, + "analysis": { + "script": str(Path(__file__).resolve()), + "script_sha256": sha256_file(Path(__file__).resolve()), + "aituner_git_head": git_capture("rev-parse", "HEAD").strip(), + "aituner_git_status_short": git_capture("status", "--short"), + }, + "provenance": { + "manifest": str(manifest_path.resolve()), + "manifest_sha256": sha256_file(manifest_path), + "controller_state": str(state_path.resolve()), + "controller_state_sha256": sha256_file(state_path), + "prior_state": str(prior_state_path.resolve()), + "prior_state_sha256": sha256_file(prior_state_path), + "strong_metrics": str(strong_path.resolve()), + "strong_metrics_sha256": sha256_file(strong_path), + }, + "sanity": { + "red_flags": red_flags, + "anchors": numeric([1 for _ in anchors]), + "candidates": numeric([1 for _ in candidates]), + "probabilities": numeric(probabilities), + "costs_h20_hours": numeric(costs), + "invariants": { + "anchors_12": len(anchors) == 12, + "candidates_6": len(candidates) == 6, + "probabilities_bounded": all( + 0.0 <= value <= 1.0 for value in probabilities + ), + "costs_nonnegative": all(value >= 0.0 for value in costs), + "per_config_not_all_identical": len( + {candidate["sim_throughput_req_s_per_gpu"] for candidate in candidates} + ) + > 1, + "tie_expansion_applied": True, + }, + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--controller-state", type=Path, required=True) + parser.add_argument("--prior-state", type=Path, required=True) + parser.add_argument("--strong-metrics", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + result = analyze( + args.manifest, + args.controller_state, + args.prior_state, + args.strong_metrics, + ) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") + print( + json.dumps( + { + "status": result["status"], + "red_flags": result["sanity"]["red_flags"], + "contribution_gate": result["contribution_gate"], + }, + sort_keys=True, + ) + ) + if result["status"] != "PASS": + raise RuntimeError(result["sanity"]["red_flags"]) + + +if __name__ == "__main__": + main() diff --git a/runs/fidelity-headroom/test_pilot_e2e.py b/runs/fidelity-headroom/test_pilot_e2e.py new file mode 100644 index 0000000..e837b42 --- /dev/null +++ b/runs/fidelity-headroom/test_pilot_e2e.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +from analyze_pilot_e2e import expanded_top_k, replay + + +def candidate( + cell: str, + sim_score: float, + real_feasible: bool, + probability: float, +) -> dict[str, object]: + return { + "cell": cell, + "level": "high", + "sim_throughput_req_s_per_gpu": sim_score, + "real_goodput_req_s_per_gpu": sim_score, + "real_feasible": real_feasible, + "setup_h20_hours": 0.1, + "full_trial_h20_hours": 0.05, + "prefix_h20_hours": 0.01, + "instrument_probability": probability, + } + + +def main() -> None: + candidates = [ + candidate("a", 3.0, True, 0.5), + candidate("b", 2.0, False, 0.01), + candidate("c", 2.0, True, 0.99), + ] + shortlist = expanded_top_k(candidates, 2) + assert [item["cell"] for item in shortlist] == ["a", "b", "c"] + result = replay( + shortlist, + probability_key="instrument_probability", + oracle_goodput=3.0, + common_failure_h20_hours=0.02, + ) + assert result["selected_cell"] == "a" + assert result["false_accept"] == 0 + assert result["false_reject"] == 0 + assert result["early_accept"] == 1 + assert result["early_reject"] == 1 + assert result["online_h20_hours"] > 0 + print("fidelity pilot e2e: PASS") + + +if __name__ == "__main__": + main() From f01819680d28e647d3ca1634c68693139b05dfe0 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 13:57:12 +0800 Subject: [PATCH 011/148] Report failed fidelity pilot gate --- ...idelity-aware-harness-headroom-20260714.md | 8 +- ...delity-aware-harness-p1-report-20260714.md | 242 ++++++++++++++++++ ...idelity-aware-harness-protocol-20260714.md | 11 +- 3 files changed, 259 insertions(+), 2 deletions(-) create mode 100644 docs/fidelity-aware-harness-p1-report-20260714.md diff --git a/docs/fidelity-aware-harness-headroom-20260714.md b/docs/fidelity-aware-harness-headroom-20260714.md index 8f11346..6de9328 100644 --- a/docs/fidelity-aware-harness-headroom-20260714.md +++ b/docs/fidelity-aware-harness-headroom-20260714.md @@ -1,10 +1,16 @@ # Fidelity-aware harness headroom audit -Status: **PROMISING PREMISE, NO CONTRIBUTION CLAIM**. +Status: **HISTORICAL PREMISE DID NOT PASS PROSPECTIVE P1; NO CONTRIBUTION CLAIM**. The audit answers whether engine instrumentation has enough incremental signal to justify a prospective experiment. It does not establish generalization. +Post-run update: the exact-timestamp held-out P1 completed and failed the +registered gate. Under the stronger simulator-aware `k=2` end-to-end replay, +telemetry preserved zero regret but saved only 1.426% online H20-hours versus +sim top-k + real final. The current route is closed; see +`docs/fidelity-aware-harness-p1-report-20260714.md`. + ## Simulator shortlist lower bound On the frozen 12-cell SimFid task, the strongest calibrated SLO simulator diff --git a/docs/fidelity-aware-harness-p1-report-20260714.md b/docs/fidelity-aware-harness-p1-report-20260714.md new file mode 100644 index 0000000..b996dca --- /dev/null +++ b/docs/fidelity-aware-harness-p1-report-20260714.md @@ -0,0 +1,242 @@ +# Fidelity-aware harness P1 result + +Status: **REGISTERED ROUTE REJECTED; DO NOT OPEN P2/P3 FOR THE CURRENT METHOD**. + +Date: 2026-07-14 (Asia/Singapore). + +## Outcome + +The registered five-second instrumentation-aware verifier did not pass P1. +The stronger simulator-aware comparison also failed the independent +contribution bar. On the frozen `k=2` end-to-end replay: + +- `sim top-k + real final` selected the real oracle with zero regret; +- instrumentation-aware also selected the oracle, but reduced online H20-hours + by only **1.426%** (1.329% when the prior failed attempt is added to both); +- the required reduction was 30% versus full real final and 20% versus a safe + outcome-only calibrator; +- the outcome-only calibrator was not safe: it rejected the true best cell, so + its apparent cost saving is not a deployable comparison. + +This rejects the claim that the **current joint logistic verifier**, trained on +one historical workload, gives the harness an independent tuning contribution. +It does not prove that engine telemetry contains no useful signal. Telemetry +improved held-out classification and removed unsafe decisions, but did not turn +that signal into meaningful end-to-end tuning-cost reduction. + +## Frozen setup + +- Host: `dash0`, 8 NVIDIA H20 GPUs; cells were serialized and used TP1, TP2, + or TP4 without co-resident serving jobs. +- Engine/model: patched vLLM 0.24.1.dev3, Qwen3-30B-A3B BF16. +- Workload: held-out `chat_w20260312_1000`, seven disjoint repeat bands, + 60-second replay after 0.1 time scaling, input `[0,8192]`, exactly 128 output + tokens. +- SLO: stepped TTFT 2/4/6 seconds, TPOT 50 ms, request pass rate at least 0.95. +- Cells: TP1/MNS8, TP1/MNS64, TP2/MNS8, TP2/MNS64, TP4/MNS16, TP4/MNS64. +- Per cell: burn-in, three low-rate repeats, and three high-rate repeats. The + first repeat supplied the five-second prefix; 2-of-3 supplied its label. +- Models: the registered pair used config/workload/outcome versus the same + vector plus Layer-1 engine telemetry. The strengthened pair additionally + gave both models identical frozen Frontier throughput, SLO pass-rate, and + feasibility predictions. +- Policy: accept at `p>=0.95`, reject at `p<=0.05`, otherwise continue the same + trial. Model, cutoff, threshold, role order, request hashes, and cap were + frozen before their applicable evaluation. + +The first launch failed its warm-up input-count validation before a measured +anchor. It cost 0.020552 H20-hours. The corrected primary attempt cost +1.722112 H20-hours, so aggregate campaign cost was **1.742664 H20-hours**, below +the 3.5 cap. The fix changed only warm-up validation; formal request counts and +hash checks were unchanged. + +## P1 labels are not an artificial easy split + +The 12 adjudicated anchor labels contain 7 feasible and 5 infeasible examples. +They are not simply “low feasible, high infeasible”: + +- TP2/MNS64 high was feasible in all three repeats; +- TP4/MNS64 low and high were feasible in all six repeats; +- TP4/MNS16 low and high were infeasible in all six repeats. + +That last pair creates a large real MNS interaction under an otherwise matched +TP4 configuration. Frontier correctly predicted TP4/MNS64 high as feasible, +but incorrectly predicted TP4/MNS16 low as feasible. It also incorrectly +predicted TP1/MNS64 high as feasible. Overall simulator-only feasibility was +10/12 correct: 83.33% accuracy, with two false-feasible predictions and no +false-infeasible prediction. + +The two false-feasible cases expose the intended latent-state problem. At five +seconds, all 26 completed TP4/MNS16-low requests and all 9 completed +TP1/MNS64-high requests still passed their SLO, although both full anchors were +infeasible. External outcomes had not yet exposed the future failure; queue, +running-batch, and scheduler state existed before the tail outcome. This is +mechanistic evidence that instrumentation can be useful, not evidence that the +current learned policy uses it well enough. + +## Registered and strengthened prefix results + +At the frozen 0.95 policy threshold: + +| Comparison | Accuracy | Balanced acc. | Early decisions | False accept | False reject | Valid primary-trial saving | +|---|---:|---:|---:|---:|---:|---:| +| Registered outcome-only | 41.67% | 50.00% | 6/12 | 0 | 2 | invalid | +| Registered + telemetry | 66.67% | 71.43% | 4/12 | 0 | 0 | 11.44% | +| Strong sim + outcome | 66.67% | 68.57% | 5/12 | 0 | 1 | invalid | +| Strong sim + outcome + telemetry | 83.33% | 85.71% | 4/12 | 0 | 0 | 11.44% | + +For the strong pair, telemetry was correct on two examples where the baseline +was wrong and lost none; McNemar's exact two-sided p-value is 0.5 at `n=12`. +This is a safety/classification improvement, not a cost contribution. The +registered instrumentation policy made two fewer early decisions than its +baseline, so it failed the registered `+3 decisions or +15 percentage points` +incremental gate. + +The result is not robust to the frozen regularization sensitivity: + +| L2 lambda | Sim+outcome acc. | +telemetry acc. | Base policy errors | Telemetry policy errors | Base saving | Telemetry saving | +|---:|---:|---:|---:|---:|---:|---:| +| 0.1 | 41.67% | 75.00% | 4 | 2 | invalid | invalid | +| 1.0 | 66.67% | 83.33% | 1 | 0 | invalid | 11.44% | +| 10.0 | 83.33% | 83.33% | 0 | 0 | 0.00% | 5.98% | + +Consequently the positive classification delta is neither statistically nor +hyperparameter robust. + +## End-to-end shortlist result + +Frontier's simulator-feasible ranking on the tested P1 surface was: + +| Rank | Cell / anchor | Sim throughput/GPU | Real feasible | Real offered goodput/GPU | +|---:|---|---:|---:|---:| +| 1 | TP4/MNS64 high | 3.0718 | yes | 3.1250 | +| 2 | TP1/MNS64 high | 2.8823 | no | 2.9833 | +| 3 | TP2/MNS64 high | 2.8096 | yes | 2.8750 | +| 4 | TP4/MNS16 low | 2.0866 | no | 2.1250 | +| 5 | TP1/MNS8 low | 1.9806 | yes | 2.0333 | +| 6 | TP2/MNS8 low | 1.8637 | yes | 1.9083 | + +The real oracle was TP4/MNS64 high at 3.125 req/s/GPU. Cost includes an +inferred per-cell startup/warm-up/burn-in/cleanup component and the selected +anchor; benchmark-only 2-of-3 annotation intervals are removed. Gaps around +annotation runs remain in the shared setup term, making this a conservative +method-cost estimate. + +| Frozen policy (`k=2`) | Online H20-hours | + prior failure | Real regret | Safety | Saving vs full | +|---|---:|---:|---:|---|---:| +| Sim top-2 + real final | 0.281383 | 0.301935 | 0.00% | valid | — | +| Sim + outcome prefix | 0.214664 | 0.235216 | no selected cell | 1 false reject | invalid | +| Sim + outcome + telemetry | 0.277370 | 0.297922 | 0.00% | valid | **1.426%** | + +For `k=1`, the simulator already picked the oracle and telemetry abstained, so +its saving was exactly 0%. For `k=3`, telemetry remained correct but saved +1.007%. The result therefore does not depend on an unfavorable choice of `k`. + +Across all six cells, a generous replay that removes benchmark-only repeat +intervals estimates 0.944244 H20-hours for the full online workflow. The +instrumentation policy saved 0.043467 H20-hours: 11.44% of primary trial time, +but only **4.603%** of end-to-end online cost. Even a post-hoc oracle symmetric +threshold cannot make the current telemetry model reach the contribution bar; +its best zero-error envelope saves at most 5.69% of inferred online cost. A +strong outcome-only model at another post-hoc regularization/threshold can save +16.13%, which further prevents attributing a unique advantage to telemetry. +These oracle-threshold numbers are diagnostics only and are not test evidence. + +## Why the learned verifier did not generalize + +The training corpus has only 37 anchors from one workload/SLO task. P1 shows +large covariate shift: + +- sim+outcome: 12/192 feature values exceed 3 training standard deviations and + 4 exceed 5; maximum absolute z-score is 10.36; +- sim+outcome+telemetry: 19/396 exceed 3 and 9 exceed 5; +- the largest shifts include admitted input-length mean (10.36), waiting state + (7.77), running maximum (6.38), and decode-batch maximum (6.08). + +Coefficient attribution shows that the input-length feature dominates several +wrong feasible-anchor logits. Because all training examples share one task, +the joint classifier can learn incidental within-task correlation and override +a correct simulator prior on TP2/MNS64-high and TP4/MNS64-high. This is a +supported diagnosis of model/data insufficiency; it is not a causal proof that +one feature alone caused the P1 failure. + +More importantly, retuning lambda, threshold, features, or cutoff on P1 and +then calling P1 a held-out result would violate calibration/evaluation +separation. P1 may now be used only as development data. + +## Decision and the only defensible reopening condition + +Do not run registered P2/P3 with the current model. It failed the predeclared +gate on the favorable primary-trial denominator and is even farther from the +bar under end-to-end cost. Spending six-task headline GPU budget on the same +method would be metric shopping, not replication. + +A new route may be opened only as a new hypothesis: + +1. Replace the joint classifier with a **simulator-residual verifier**. The + simulator prediction remains an explicit prior; nested outcome-only and + telemetry models learn when that prior is wrong, rather than freely + relearning feasibility and overriding it under workload shift. +2. Train on multiple complete workload/SLO tasks. SLO thresholds and target + pass rate must be explicit inputs; splits are by complete task. +3. Calibrate abstention with task-level risk control. No threshold is selected + on a headline task, and “never early decide” is included as the safe + outcome-only baseline. +4. Treat Phase 6 and P1 as development only, freeze the residual architecture, + features, cutoff, threshold, simulator reading, and `k`, then use entirely + new trace windows for a new gate. + +This reopening is justified only if development data show both (a) the +simulator's errors are predictable from pre-outcome engine state and (b) a +simulator-preserving residual model does not corrupt correct simulator +predictions. It is a new project decision, not a continuation automatically +authorized by P1. + +## Benchmark audit + +| Audit item | Verdict | Severity | Evidence / disposition | +|---|---|---|---| +| Calibration set separate from P1 | PASS | — | Phase 6/0311 trained; P1/0312 tested | +| Strong simulator-aware baseline | PASS | — | Identical Frontier features in both nested models | +| Sim top-k + real-final E2E baseline | PASS | — | Frozen `k=2`, tie expansion, measured setup/continuation cost | +| Multiple independent headline tasks | NEEDS EVIDENCE | Blocking for a positive claim | P1 gate failed; P2 correctly not opened | +| Statistical significance | NEEDS EVIDENCE | Blocking for a positive claim | n=12 anchors from one task; McNemar p=0.5 | +| Hyperparameter robustness | FAIL | Blocking | Lambda sensitivity changes safety and relative result | +| Full resource accounting | PASS for P1 | — | Failures, startup/warm-up/burn-in, continuation and annotation separated | +| Avoid post-test retuning | PASS only if route stops | Blocking if violated | P1 is now development-only | +| Selective winning-workload reporting | PASS | — | Negative P1 and TP/MNS losing cases retained | + +Overall recommendation: **Block the current independent harness contribution +claim.** + +## Artifacts + +- Registered protocol: `docs/fidelity-aware-harness-protocol-20260714.md` +- Historical headroom: `docs/fidelity-aware-harness-headroom-20260714.md` +- Registered P1 analysis: `runs/fidelity-headroom/analyze_pilot.py` +- Strong P1 analysis: `runs/fidelity-headroom/analyze_strong_pilot.py` +- E2E shortlist replay: `runs/fidelity-headroom/analyze_pilot_e2e.py` +- External immutable result root: + `/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714` + +## Data sanity block + +| Data | n | Min | Max | Distinct | Invariant | +|---|---:|---:|---:|---:|---| +| P1 labels | 12 | 0 | 1 | 2 | 7 feasible / 5 infeasible | +| Primary elapsed seconds | 12 | 19.448 | 61.435 | 12 | Every five-second prefix is in range | +| Prefix Layer-1 records | 12 | 332 | 557 | 12 | Contiguous; zero drops | +| Exact timestamped outcomes | 12 anchors | 54 | 750 | 11 | Monotonic completion timestamps | +| Simulator pass rate | 12 | 0.1548 | 1.0 | 7 | Ratios in `[0,1]` | +| Strong nested probabilities | 24 | 0.000208 | 0.809422 | 24 | Ratios in `[0,1]` | +| E2E cost components | 36 | 0.001389 | 0.169653 H20-h | 21 | Non-negative | +| GPU attempts | 2 | 0.020552 | 1.722112 H20-h | 2 | Aggregate 1.742664 < 3.5 | +| Copied raw files | 191 | — | 153,093,348 bytes total | — | Remote/local aggregate SHA identical | + +Checked invariants: six cells and twelve anchors; exact request count and +request-ID/arrival/length hashes; all cell validation flags true; both labels +present; probabilities bounded; costs and counters non-negative; simulator +results not all identical; committed simulator rerun 12/12 numerically +identical to the exploratory run; no prompt text in public simulator fixtures; +no co-resident serving process; final eight GPUs at 0 MiB and 0% utilization. +No red flag remains. diff --git a/docs/fidelity-aware-harness-protocol-20260714.md b/docs/fidelity-aware-harness-protocol-20260714.md index 8bf05f0..b6176f5 100644 --- a/docs/fidelity-aware-harness-protocol-20260714.md +++ b/docs/fidelity-aware-harness-protocol-20260714.md @@ -1,9 +1,18 @@ # Fidelity-aware real-verification harness protocol -Status: **PRE-REGISTERED STAGED EVALUATION; CONTRIBUTION NOT YET ESTABLISHED**. +Status: **P1 FAILED; P2/P3 CLOSED FOR THE REGISTERED METHOD; CONTRIBUTION NOT ESTABLISHED**. Date frozen: 2026-07-14 (Asia/Singapore). +Post-run disposition (2026-07-14): P1 completed with valid data but failed its +registered incremental gate. The strengthened simulator-aware comparison and +end-to-end `k=2` replay also failed: instrumentation was safe and retained zero +regret, but reduced online H20-hours by only 1.426% versus sim top-k + real +final, against the 30% bar. Outcome-only was unsafe. P2/P3 are therefore not +opened for this model. Full results and the permitted reopening condition are +in `docs/fidelity-aware-harness-p1-report-20260714.md`; the protocol below is +retained unchanged as the pre-run record. + ## Research question and contribution bar The harness has an independent systems contribution only if engine-internal From 7a3631b528965bc2a5f20bb66eb138a8a0a83e88 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 15:44:37 +0800 Subject: [PATCH 012/148] Audit telemetry residual tuning premise --- ...emetry-residual-tuning-roadmap-20260714.md | 286 +++++++++++ runs/telemetry-residual/analyze_p1_state.py | 174 +++++++ runs/telemetry-residual/analyze_r0_gate.py | 292 +++++++++++ .../analyze_residual_transfer.py | 480 ++++++++++++++++++ runs/telemetry-residual/common_state.py | 400 +++++++++++++++ runs/telemetry-residual/run_frontier_state.py | 316 ++++++++++++ .../run_frontier_state_campaign.py | 169 ++++++ runs/telemetry-residual/test_common_state.py | 183 +++++++ .../test_residual_transfer.py | 67 +++ 9 files changed, 2367 insertions(+) create mode 100644 docs/telemetry-residual-tuning-roadmap-20260714.md create mode 100644 runs/telemetry-residual/analyze_p1_state.py create mode 100644 runs/telemetry-residual/analyze_r0_gate.py create mode 100644 runs/telemetry-residual/analyze_residual_transfer.py create mode 100644 runs/telemetry-residual/common_state.py create mode 100644 runs/telemetry-residual/run_frontier_state.py create mode 100644 runs/telemetry-residual/run_frontier_state_campaign.py create mode 100644 runs/telemetry-residual/test_common_state.py create mode 100644 runs/telemetry-residual/test_residual_transfer.py diff --git a/docs/telemetry-residual-tuning-roadmap-20260714.md b/docs/telemetry-residual-tuning-roadmap-20260714.md new file mode 100644 index 0000000..ba6ceab --- /dev/null +++ b/docs/telemetry-residual-tuning-roadmap-20260714.md @@ -0,0 +1,286 @@ +# Telemetry-conditioned residual tuning roadmap + +Status: **R0 COMPLETE / FAILED; R1 AND R2 CLOSED FOR THIS MODEL**. + +Date: 2026-07-14 (Asia/Singapore). + +## Research question and claim boundary + +The question is whether a small number of real engine observations can correct +a simulator's task-specific error over **unmeasured configurations**, and +whether that correction reduces the real-GPU cost of finding a high +SLO-goodput serving configuration. + +The intended headline claim, if the evidence supports it, is: + +> An engine-state-conditioned residual model turns a simulator prediction into +> a task-specific posterior over unmeasured serving configurations, allowing a +> sequential tuner to reach near-oracle SLO-goodput with materially fewer +> H20-hours than simulator-only and outcome-only tuning. + +Classification accuracy, simulator-error diagnosis, and telemetry overhead are +supporting evidence. None is an end-to-end tuning contribution by itself. + +The following method is closed and will not be revived under another name: +per-candidate five-second accept/reject as the headline contribution. The P1 +result showed only 1.426% cost reduction in the frozen `k=2` workflow. + +## Two models, one evaluation + +Both branches use the same legal candidate set, real measurements, task split, +cost accounting, and acquisition function. + +### Simulator-residual branch (primary) + +For measured anchor `c_t` and unmeasured candidate `c'`: + +```text +y_hat(c') = y_real(c_t) + + [y_sim(c') - y_sim(c_t)] + + f(state_real(c_t) - state_sim(c_t), c' - c_t, workload, SLO) +``` + +The simulator delta is the prior. The learned model may correct it only with +training-supported state/config transitions; uncertainty or distribution shift +must shrink the correction back toward the simulator prior. + +### Telemetry-only branch (mandatory) + +```text +y_hat(c') = y_real(c_t) + + g(state_real(c_t), c' - c_t, workload, SLO) +``` + +This branch tests whether the simulator is actually necessary. It does not +use a hand-authored bottleneck-to-knob rule. + +### Search policy + +Legal configurations are enumerated independently of telemetry. A generic +cost-aware acquisition rule ranks candidates from predicted improvement, +uncertainty, and measured H20 cost. The current production harness's +bottleneck scores, topology-first ordering, and hand-set relief constants are +not consumed by either branch. The validator may enforce legality, +full-config no-repeat, failure accounting, and resource caps only. + +## Hypotheses + +| ID | Hypothesis | Direct test | Failure meaning | +|---|---|---|---| +| H0 | Existing artifacts can express a common, direct-measurement state without heuristic labels. | Engine/simulator extractor coverage and invariants. | Route is not currently implementable. | +| H1 | Simulator errors are predictable from engine/simulator state discrepancy at measured anchors. | Task-held-out pairwise inversion correction and new-inversion rate. | Telemetry is diagnostic but cannot correct the surface. | +| H2 | Telemetry alone predicts useful config transitions beyond outcome-only history. | Telemetry-only versus real-outcome-only sequential replay. | Direct telemetry-guided tuning has no independent value. | +| H3 | Residual correction changes actual tuning decisions and cost. | H20-hours to 95% oracle and regret AUC against the strongest safe baseline. | No system contribution even if H1/H2 prediction metrics improve. | + +## Common-state contract + +Only directly observed or exactly reconstructed quantities are admitted. + +| Quantity | vLLM Layer-1 | Frontier | R0 status | +|---|---|---|---| +| Scheduled requests / batch size | Per scheduler step | Existing per-batch metric, disabled in P1 output | Common after CPU replay | +| Scheduled prefill/decode tokens | Per scheduler step | Existing per-batch metrics | Common after CPU replay | +| Scheduler/batch rate | Monotonic step timestamps | Batch count / simulated duration | Common after CPU replay | +| Waiting queue area | Time-weighted queue gauge | Sum of request waiting times | Common aggregate | +| Running request area | Time-weighted running gauge | Sum of E2E minus waiting time | Common aggregate, semantics audited | +| Preemption count | Per step | Per request | Common | +| KV usage/headroom | Exact blocks and ratio | Not in committed output | Engine-only until exact reconstruction exists | +| CUDA graph mode/padding | Exact per step | Not modeled | Engine-only omitted-mechanism signal | +| Request TTFT/TPOT/pass rate | Exact real outcomes | Exact simulated request metrics | Common outcome, not state | + +Unavailable fields remain null. They cannot be imputed from a human +`prefill/decode/queueing` label. + +Frontier already contains the required detailed batch and timestamped +stage-batch ledger output. P1 disabled it for artifact size. R0 replays the +same immutable fixtures with the existing output flags enabled; it does not +change the simulator model or calibration. + +## Data separation + +- Phase 6 / `chat_w20260311_1000`: development only. +- P1 / `chat_w20260312_1000`: development only. +- R1 / `chat_w20260313_1000`: new development surface. +- R2: trace windows not used for feature, model, threshold, candidate-space, + cutoff, or acquisition decisions. +- Splits are by complete workload/SLO task. Anchor- or pair-level random + splits are prohibited. +- Sequential-policy seeds measure algorithmic variability; they are not + counted as independent system tasks. + +The two existing development tasks have an important limitation: the now- +available SLO-gated simulator reading already retains the real oracle at its +top rank/tie. They therefore cannot establish a positive end-to-end ranking +claim. They are used for plumbing, known false-feasible cases, and negative +evidence. R1 must be run as an unbiased complete surface, not selected after +observing simulator success or failure. + +## Step-by-step roadmap + +### R0.1 — Inventory and roadmap + +Deliverables: + +- this roadmap; +- rolling untracked `ONGOING.md`; +- exact engine/simulator field and artifact inventory. + +Gate: every claimed input has an authoritative file path and provenance. + +### R0.2 — Common-state plumbing + +Deliverables: + +- `runs/telemetry-residual/common_state.py`; +- synthetic correctness tests; +- one exact P1 Frontier replay with individual batch metrics and the full + stage-batch ledger enabled; +- paired engine/simulator state summary for the same fixture. + +Gate: + +- replay request count and SLO scorer exactly agree with the committed replay; +- batch/ledger outputs are non-empty; +- all counters are non-negative, ratios bounded, times monotonic; +- no GPU is visible to Frontier; +- output volume is practical before expanding to twelve replays. + +### R0.3 — Development residual/headroom audit + +Use all frozen P1 primary fixtures and corresponding engine intervals. Produce: + +- common-state residuals per anchor; +- simulator-error labels and continuous SLO/goodput residuals; +- ordered source/target diagnostic that removes both config identities from + both roles in every training fold; +- oracle upper bound for cross-candidate correction; +- explicit comparison with simulator+outcome and telemetry-only features. + +R0 is a feasibility gate, not headline evidence. Proceed to R1 only if: + +1. state features are collected with the measured source anchor, vary across + cells, and are available before any target config is evaluated; +2. at least one known simulator error has a state discrepancy not exposed by + the matched external prefix outcome; +3. a prior-preserving model can correct development errors without introducing + a larger number of new errors under regularization sensitivity; +4. an oracle cross-candidate correction has at least 15% sequential tuning-cost + headroom under full startup/warm-up accounting. + +### R0 result and decision + +R0 completed without a data-validity red flag, but failed condition 3. The +decision is **STOP_BEFORE_R1**; no H20 job was launched for this route. + +- All 12 detailed Frontier CPU replays exactly reproduced their committed SLO + scorers. Runtime was 23.943--54.786 seconds per replay, detailed artifacts + were 4.12--13.53 MB, CUDA visibility was empty, and there were zero failures. +- The paired surface contains 12 real/sim anchors, two known simulator + false-feasible anchors, and 120 legal cross-config ordered transitions. A + fold removes both the source and target TP/MNS identity from source and + target roles; the two offered-load anchors remain part of the same task. +- Raw Frontier feasibility is 83.33% on the repeated transition view. The + structurally correct hybrid model uses + `r_target = r_source + delta_r`; the direct model uses + `y_target = y_source + delta_y` and never reads simulator fields. +- Direct telemetry is not robust relative to real-outcome-only: its accuracy + delta over L2 `{0.1,1,10,100}` is `{-0.83,+1.67,0,-4.17}` percentage points, + and its best absolute accuracy is 54.17%, below the raw simulator's 83.33%. +- Hybrid telemetry raises classification accuracy over the corresponding + simulator+outcome transition regression by 1.67--4.17 percentage points, + but worsens pass-rate RMSE by 0.141--0.201 and MAE by 0.084--0.125. Its full + correction reaches only 46.67--53.33% absolute accuracy. +- Across 24 nonzero `(L2, raw-simulator-prior weight)` combinations, no model + both corrects an existing simulator error without more new errors and avoids + worsening RMSE/MAE. Whenever a correction fixes at least one error, it + corrupts at least 11 previously correct transitions. +- A perfect correction could skip the frozen simulator rank-2 real final and + save 0.043469 H20-hours: 15.45% of the prospective online `k=2` cost, or + 14.40% when the prior failed launch is charged. On this development task the + simulator top-1 already is the real oracle with zero regret, so headroom + versus the observed-safe top-1 baseline is 0%. + +The result does not prove that engine telemetry is useless. It shows that the +current one-task anchor-transition evidence cannot support either a safe +simulator-residual tuner or a simulator-free telemetry tuner. A larger model +or an R1 run would add capacity/data after a failed gate and is therefore not +authorized under this roadmap. + +### R1 — New development surface + +Status: **NOT LAUNCHED; CLOSED BY R0**. + +Frozen starting setup: + +- host: dash0, eight NVIDIA H20 GPUs; +- cells run solo; no co-location for SLO verdicts; +- patched vLLM 0.24.1.dev3, Qwen3-30B-A3B BF16; +- trace: `chat_w20260313_1000`; +- output tokens: exactly 128; +- SLO: stepped TTFT 2/4/6 seconds, TPOT 50 ms, pass rate at least 0.95; +- config surface: TP `{1,2,4}` × MNS `{8,16,32,64}`; +- hard campaign cap: 4 H20-hours. + +The load ladder, repetitions, randomized order, exact commands, expected wall +time, and artifact paths are frozen only after R0. A resolved echo is required +before launch. + +R1 passes only if a frozen sequential replay shows at least 15% E2E H20-hour +headroom over the strongest safe baseline with final regret at most 5%. R1 is +development evidence and cannot be reported as the held-out result. + +### R2 — Held-out sequential tuning + +Status: **NOT LAUNCHED; CLOSED BY R0**. + +Required baselines: + +1. random search; +2. real-outcome-only Bayesian/sequential search; +3. Frontier ranking plus real top-k final; +4. simulator plus real-outcome residual; +5. telemetry-only transition tuner; +6. simulator plus telemetry residual tuner; +7. complete real surface as oracle, not as a cost competitor. + +Primary metric: end-to-end H20-hours to first reach 95% of the real full-surface +SLO-goodput oracle. Secondary metrics are cost-normalized regret AUC, final +regret at fixed budgets, oracle false-prune, wall time, and per-task regressions. + +The route is successful only if the winning telemetry method reduces the +primary cost by at least 20% versus the strongest safe baseline and ends within +5% regret on every headline task. If hybrid beats telemetry-only by at least +10%, simulator residual correction is the primary method. If telemetry-only +is within 5% or better, the simulator dependency is removed. If neither clears +the contribution bar, the route is closed and telemetry remains a diagnostic +facility only. + +## Cost discipline + +- R0 simulator work is CPU-only and must set empty CUDA visibility. +- R1 cannot exceed 4 H20-hours. +- R2 receives no budget until R1 passes. +- Startup, warm-up, burn-in, failed launches, real probes, continuation, and + final validation are charged. Benchmark-only annotation repeats are + reported separately and cannot disappear from campaign accounting. + +## Final R0 sanity block + +| Data | n | Min | Max | Distinct | Checked invariant | +|---|---:|---:|---:|---:|---| +| Phase 6 cells | 12 | TP1/MNS8 | TP4/MNS64 | 12 | Surface not identical; solo SLO tier authoritative | +| Phase 6 Layer-1 primary steps | 37 streams | 343 | 12,103 | 37 | Contiguous; zero drops | +| P1 primary anchors | 12 | infeasible | feasible | 2 labels | 7 feasible / 5 infeasible | +| P1 Frontier runtime | 12 | 24.093 s | 54.575 s | 12 | CPU-only; zero failures | +| Detailed Frontier replay runtime | 12 | 23.943 s | 54.786 s | 12 | Exact committed scorers; CUDA hidden | +| Detailed artifact bytes | 12 | 4,123,724 | 13,527,776 | 12 | Non-negative; practical CPU replay size | +| Cross-config transitions | 120 | real pass 0.1067 | real pass 1.0 | 6 outcomes | Both endpoint config identities held out | +| State residual vectors | 12 | 16 fields | 16 fields | 12 vectors | Finite; no missing common field | +| R0 E2E cost values | 4 | 0.237914 | 0.301935 H20-h | 4 | Non-negative; `k=1/2`, online/conservative | + +Checked invariants: non-negative counts and costs; pass rates in `[0,1]`; +simulator results not all identical; exact request count/hash agreement; Layer-1 +step continuity and zero drops; no co-resident SLO measurements; no calibration +or evaluation split reuse for a future headline claim. No current red flag +invalidates R0 plumbing. The R0 tuning gate itself failed because safe +prior-preserving correction was absent. diff --git a/runs/telemetry-residual/analyze_p1_state.py b/runs/telemetry-residual/analyze_p1_state.py new file mode 100644 index 0000000..2c2e7b0 --- /dev/null +++ b/runs/telemetry-residual/analyze_p1_state.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Pair P1 engine intervals with detailed Frontier state summaries.""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from pathlib import Path +from typing import Any + + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +from common_state import load_jsonl, numeric, residual, summarize_engine # noqa: E402 + + +def atomic_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(path) + + +def adjudicated_labels(path: Path) -> dict[tuple[str, str], bool]: + payload = json.loads(path.read_text(encoding="utf-8")) + return { + (row["cell"], row["level"]): bool(row["adjudicated_feasible"]) + for row in payload["examples"] + } + + +def state_runs(root: Path) -> list[Path]: + candidates = list(root.glob("*/result.json")) + if (root / "result.json").is_file(): + candidates.append(root / "result.json") + return sorted( + path + for path in candidates + if json.loads(path.read_text(encoding="utf-8")).get("schema") + == "telemetry-residual-frontier-state-result-v1" + ) + + +def execute(args: argparse.Namespace) -> dict[str, Any]: + labels = adjudicated_labels(args.pilot_metrics) + examples = [] + red_flags = [] + for state_result_path in state_runs(args.sim_state_root): + run_root = state_result_path.parent + manifest = json.loads((run_root / "run_manifest.json").read_text(encoding="utf-8")) + cell = manifest["entry"]["cell"] + role = manifest["entry"]["role"] + level = "low" if role.startswith("low") else "high" + real_run = args.real_root / "cells" / cell / f"{level}-rep1" + real_result = json.loads((real_run / "result.json").read_text(encoding="utf-8")) + stream_paths = sorted((args.real_root / "cells" / cell / "opprof").glob("*.jsonl")) + if len(stream_paths) != 1: + raise ValueError(f"expected one engine stream for {cell}, found {len(stream_paths)}") + engine = summarize_engine( + load_jsonl(stream_paths[0]), + start_ns=int(real_result["interval"]["start_mono_ns"]), + end_ns=int(real_result["interval"]["end_mono_ns"]), + request_count=int(real_result["selection"]["count"]), + ) + simulator = json.loads((run_root / "common-state.json").read_text(encoding="utf-8")) + scorer = json.loads((run_root / "scorer_output.json").read_text(encoding="utf-8")) + if engine["interval"]["request_count"] != simulator["interval"]["request_count"]: + red_flags.append(f"request_count_mismatch:{cell}:{role}") + difference = residual(engine, simulator) + if any(not math.isfinite(float(value)) for value in difference["values"].values()): + red_flags.append(f"nonfinite_residual:{cell}:{role}") + real_feasible = labels[(cell, level)] + sim_feasible = bool(scorer["slo"]["feasible"]) + examples.append( + { + "cell": cell, + "role": role, + "level": level, + "tp": int(cell.split("_")[0][2:]), + "mns": int(cell.split("_")[1][3:]), + "request_count": engine["interval"]["request_count"], + "real_feasible": real_feasible, + "sim_feasible": sim_feasible, + "simulator_error": sim_feasible != real_feasible, + "simulator_false_feasible": sim_feasible and not real_feasible, + "real_pass_rate_rep1": float(real_result["pass_rate"]), + "offered_req_s": float(real_result["selection"]["offered_req_s"]), + "offered_req_s_per_gpu": float( + real_result["selection"]["offered_req_s_per_gpu"] + ), + "sim_pass_rate": float(scorer["slo"]["pass_rate"]), + "pass_rate_residual": float(real_result["pass_rate"]) + - float(scorer["slo"]["pass_rate"]), + "engine": engine, + "simulator": simulator, + "state_residual": difference, + "paths": { + "real_result": str((real_run / "result.json").resolve()), + "engine_stream": str(stream_paths[0].resolve()), + "simulator_result": str(state_result_path.resolve()), + }, + } + ) + if not examples: + red_flags.append("no_state_examples") + request_counts = [int(row["request_count"]) for row in examples] + pass_rate_residuals = [float(row["pass_rate_residual"]) for row in examples] + result = { + "schema": "telemetry-residual-p1-state-pairs-v1", + "status": "PASS" if not red_flags else "STOP", + "scope": "P1 development plumbing; not held-out contribution evidence", + "examples": examples, + "red_flags": red_flags, + "sanity": { + "n": len(examples), + "request_count": numeric(request_counts) if request_counts else None, + "pass_rate_residual": numeric(pass_rate_residuals) + if pass_rate_residuals + else None, + "simulator_errors": sum(bool(row["simulator_error"]) for row in examples), + "invariants": { + "request_counts_match": not any( + flag.startswith("request_count_mismatch") for flag in red_flags + ), + "finite_residuals": not any( + flag.startswith("nonfinite_residual") for flag in red_flags + ), + "ratios_bounded": all( + 0.0 <= row["real_pass_rate_rep1"] <= 1.0 + and 0.0 <= row["sim_pass_rate"] <= 1.0 + for row in examples + ), + "per_config_not_identical": len(set(pass_rate_residuals)) > 1 + if len(pass_rate_residuals) > 1 + else None, + }, + }, + } + atomic_json(args.output, result) + if result["status"] != "PASS": + raise RuntimeError(red_flags) + return result + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser() + result.add_argument("--real-root", type=Path, required=True) + result.add_argument("--sim-state-root", type=Path, required=True) + result.add_argument("--pilot-metrics", type=Path, required=True) + result.add_argument("--output", type=Path, required=True) + return result + + +def main() -> None: + result = execute(parser().parse_args()) + print( + json.dumps( + { + "status": result["status"], + "examples": len(result["examples"]), + "simulator_errors": result["sanity"]["simulator_errors"], + "sanity": result["sanity"], + "red_flags": result["red_flags"], + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/telemetry-residual/analyze_r0_gate.py b/runs/telemetry-residual/analyze_r0_gate.py new file mode 100644 index 0000000..d9912e5 --- /dev/null +++ b/runs/telemetry-residual/analyze_r0_gate.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +"""Make the registered R0 go/no-go decision from development artifacts.""" + +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +from typing import Any + + +def atomic_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(path) + + +def load_pass(path: Path, name: str) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if payload.get("status") != "PASS": + raise RuntimeError(f"{name} is not a valid PASS artifact: {path}") + return payload + + +def reduction(reference: float, candidate: float) -> float: + if reference <= 0.0 or candidate < 0.0 or not math.isfinite(candidate): + raise ValueError("costs must be finite and non-negative with positive reference") + return 1.0 - candidate / reference + + +def execute(args: argparse.Namespace) -> dict[str, Any]: + paired = load_pass(args.paired_state, "paired state") + transfer = load_pass(args.transfer, "transfer diagnostic") + e2e = load_pass(args.pilot_e2e, "P1 E2E replay") + red_flags = [] + + if len(paired.get("examples", [])) != 12: + red_flags.append("paired_state_not_12_anchors") + if transfer.get("sanity", {}).get("transitions") != 120: + red_flags.append("transfer_not_120_cross_config_transitions") + if transfer.get("red_flags"): + red_flags.append("transfer_has_red_flags") + + examples = paired["examples"] + state_available = all( + row["state_residual"]["coverage"]["missing"] == 0 + and row["state_residual"]["coverage"]["available"] > 0 + for row in examples + ) + state_vectors = { + tuple(sorted(row["state_residual"]["values"].items())) for row in examples + } + state_varies = len(state_vectors) > 1 + error_examples = [row for row in examples if row["simulator_error"]] + simulator_errors = len(error_examples) + error_state_discrepancy = any( + any( + abs(float(value)) > 1e-12 + for value in row["state_residual"]["values"].values() + ) + for row in error_examples + ) + + simulator = transfer["simulator"] + prior_safe = [] + direct_sensitivity = [] + hybrid_incremental = [] + for regularization, detail in transfer["regularization_sensitivity"].items(): + for weight, models in detail["hybrid"]["prior_shrinkage"].items(): + if float(weight) == 0.0: + continue + telemetry = models["raw_simulator_prior"][ + "sim_plus_outcome_plus_telemetry" + ] + decision_safe = ( + telemetry["simulator_errors_corrected"] >= 1 + and telemetry["simulator_errors_corrected"] + >= telemetry["simulator_correct_corrupted"] + ) + continuous_safe = ( + telemetry["rmse"] <= simulator["rmse"] + 1e-12 + and telemetry["mae"] <= simulator["mae"] + 1e-12 + ) + if decision_safe and continuous_safe: + prior_safe.append( + { + "regularization": float(regularization), + "prior_weight": float(weight), + "simulator_errors_corrected": telemetry[ + "simulator_errors_corrected" + ], + "simulator_correct_corrupted": telemetry[ + "simulator_correct_corrupted" + ], + "rmse": telemetry["rmse"], + "mae": telemetry["mae"], + } + ) + + direct = detail["direct"] + direct_cmp = direct["comparison"] + direct_sensitivity.append( + { + "regularization": float(regularization), + "accuracy_delta": direct_cmp["delta_telemetry_minus_baseline"][ + "feasibility_accuracy" + ], + "rmse_delta": direct_cmp["delta_telemetry_minus_baseline"]["rmse"], + "mae_delta": direct_cmp["delta_telemetry_minus_baseline"]["mae"], + "errors_corrected": direct_cmp["baseline_errors_corrected"], + "correct_corrupted": direct_cmp["baseline_correct_corrupted"], + "telemetry_accuracy": direct["telemetry_only"][ + "feasibility_accuracy" + ], + } + ) + hybrid_cmp = detail["hybrid"]["comparison"] + hybrid_incremental.append( + { + "regularization": float(regularization), + "accuracy_delta": hybrid_cmp["delta_telemetry_minus_baseline"][ + "feasibility_accuracy" + ], + "rmse_delta": hybrid_cmp["delta_telemetry_minus_baseline"]["rmse"], + "mae_delta": hybrid_cmp["delta_telemetry_minus_baseline"]["mae"], + "errors_corrected": hybrid_cmp["baseline_errors_corrected"], + "correct_corrupted": hybrid_cmp["baseline_correct_corrupted"], + } + ) + + k1 = e2e["by_k"]["1"]["sim_top_k_plus_real_final"] + k2 = e2e["by_k"]["2"]["sim_top_k_plus_real_final"] + headroom = { + "interpretation": ( + "oracle correction stops after the simulator top-1 real final instead " + "of evaluating the frozen safety top-2" + ), + "online": { + "reference_k2_h20_hours": k2["online_h20_hours"], + "oracle_k1_h20_hours": k1["online_h20_hours"], + "absolute_h20_hours": k2["online_h20_hours"] - k1["online_h20_hours"], + "fraction": reduction(k2["online_h20_hours"], k1["online_h20_hours"]), + }, + "with_prior_failure": { + "reference_k2_h20_hours": k2["conservative_h20_hours_with_prior_failure"], + "oracle_k1_h20_hours": k1["conservative_h20_hours_with_prior_failure"], + "absolute_h20_hours": k2["conservative_h20_hours_with_prior_failure"] + - k1["conservative_h20_hours_with_prior_failure"], + "fraction": reduction( + k2["conservative_h20_hours_with_prior_failure"], + k1["conservative_h20_hours_with_prior_failure"], + ), + }, + "versus_observed_safe_k1_fraction": 0.0, + "k1_zero_regret": k1["real_regret"] == 0.0, + "k2_zero_regret": k2["real_regret"] == 0.0, + } + + condition_1 = state_available and state_varies + condition_2 = simulator_errors >= 1 and error_state_discrepancy + condition_3 = bool(prior_safe) + condition_4 = headroom["online"]["fraction"] >= 0.15 + conditions = { + "state_available_and_varies": condition_1, + "known_simulator_error_has_state_discrepancy": condition_2, + "prior_preserving_safe_correction_exists": condition_3, + "oracle_online_headroom_at_least_15pct": condition_4, + } + gate_pass = not red_flags and all(conditions.values()) + direct_incremental = all( + row["accuracy_delta"] >= -1e-12 + and row["errors_corrected"] >= row["correct_corrupted"] + for row in direct_sensitivity + ) + result = { + "schema": "telemetry-residual-r0-gate-v1", + "status": "STOP" if red_flags else "PASS", + "scope": "P1 development premise/headroom audit; not headline evidence", + "decision": "PROCEED_TO_R1" if gate_pass else "STOP_BEFORE_R1", + "r0_gate_pass": gate_pass, + "conditions": conditions, + "route_findings": { + "hybrid_prior_safe_candidates": prior_safe, + "hybrid_incremental_regularization": hybrid_incremental, + "direct_incremental_regularization": direct_sensitivity, + "direct_incremental_decision_signal_all_lambdas": direct_incremental, + "direct_best_absolute_accuracy": max( + row["telemetry_accuracy"] for row in direct_sensitivity + ), + "raw_simulator_accuracy": simulator["feasibility_accuracy"], + }, + "headroom": headroom, + "red_flags": red_flags, + "sanity": { + "anchors": { + "n": len(examples), + "min": min(row["real_pass_rate_rep1"] for row in examples), + "max": max(row["real_pass_rate_rep1"] for row in examples), + "distinct_n": len( + {row["real_pass_rate_rep1"] for row in examples} + ), + }, + "state_vectors": { + "n": len(examples), + "min": min(len(row["state_residual"]["values"]) for row in examples), + "max": max(len(row["state_residual"]["values"]) for row in examples), + "distinct_n": len(state_vectors), + }, + "costs_h20_hours": { + "n": 4, + "min": min( + k1["online_h20_hours"], + k2["online_h20_hours"], + k1["conservative_h20_hours_with_prior_failure"], + k2["conservative_h20_hours_with_prior_failure"], + ), + "max": max( + k1["online_h20_hours"], + k2["online_h20_hours"], + k1["conservative_h20_hours_with_prior_failure"], + k2["conservative_h20_hours_with_prior_failure"], + ), + "distinct_n": len( + { + k1["online_h20_hours"], + k2["online_h20_hours"], + k1["conservative_h20_hours_with_prior_failure"], + k2["conservative_h20_hours_with_prior_failure"], + } + ), + }, + "invariants": { + "no_data_red_flags": not red_flags, + "state_nonempty_and_varied": condition_1, + "pass_rates_bounded": all( + 0.0 <= row["real_pass_rate_rep1"] <= 1.0 + and 0.0 <= row["sim_pass_rate"] <= 1.0 + for row in examples + ), + "costs_nonnegative": all( + value >= 0.0 + for value in ( + k1["online_h20_hours"], + k2["online_h20_hours"], + k1["conservative_h20_hours_with_prior_failure"], + k2["conservative_h20_hours_with_prior_failure"], + ) + ), + "per_config_not_identical": len( + {row["real_pass_rate_rep1"] for row in examples} + ) + > 1, + }, + }, + } + atomic_json(args.output, result) + if result["status"] != "PASS": + raise RuntimeError(red_flags) + return result + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser() + result.add_argument("--paired-state", type=Path, required=True) + result.add_argument("--transfer", type=Path, required=True) + result.add_argument("--pilot-e2e", type=Path, required=True) + result.add_argument("--output", type=Path, required=True) + return result + + +def main() -> None: + result = execute(parser().parse_args()) + print( + json.dumps( + { + "status": result["status"], + "decision": result["decision"], + "r0_gate_pass": result["r0_gate_pass"], + "conditions": result["conditions"], + "headroom": result["headroom"], + "sanity": result["sanity"], + "red_flags": result["red_flags"], + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/telemetry-residual/analyze_residual_transfer.py b/runs/telemetry-residual/analyze_residual_transfer.py new file mode 100644 index 0000000..a07283a --- /dev/null +++ b/runs/telemetry-residual/analyze_residual_transfer.py @@ -0,0 +1,480 @@ +#!/usr/bin/env python3 +"""Development-only cross-config telemetry transfer diagnostic for P1. + +Each example asks whether state observed at one source anchor helps predict the +pass rate at a different target config. The hybrid branch predicts the +real-minus-simulator residual; the direct branch never reads simulator state or +outcomes. Folds exclude both the source and target config identities. The two +offered-load anchors belong to one trace/SLO task, so this is a premise check +rather than generalization evidence. +""" + +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +from typing import Any, Sequence + +import numpy as np + + +REGULARIZATION = (0.1, 1.0, 10.0, 100.0) +PRIOR_SHRINKAGE = (0.0, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0) + + +def atomic_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + temporary.replace(path) + + +def finite(value: Any, *, name: str) -> float: + result = float(value) + if not math.isfinite(result): + raise ValueError(f"non-finite feature {name}={value!r}") + return result + + +def flatten_residual_state(example: dict[str, Any]) -> tuple[float, ...]: + residual = example["state_residual"]["values"] + values = [finite(residual[name], name=name) for name in sorted(residual)] + engine_only = example["engine"]["engine_only"] + values.extend( + finite(engine_only[name], name=name) + for name in sorted(engine_only) + ) + return tuple(values) + + +def flatten_engine_state(example: dict[str, Any]) -> tuple[float, ...]: + values = [] + for name in sorted(example["engine"]["common"]): + value = example["engine"]["common"][name] + if isinstance(value, dict): + values.extend( + finite(value[statistic], name=f"{name}.{statistic}") + for statistic in ("mean", "max", "cv") + ) + elif value is not None: + values.append(finite(value, name=name)) + engine_only = example["engine"]["engine_only"] + values.extend( + finite(engine_only[name], name=name) + for name in sorted(engine_only) + ) + return tuple(values) + + +def config_transition_features( + source: dict[str, Any], target: dict[str, Any] +) -> tuple[float, ...]: + source_rate = finite( + source["offered_req_s_per_gpu"], name="source_offered_req_s_per_gpu" + ) + target_rate = finite( + target["offered_req_s_per_gpu"], name="target_offered_req_s_per_gpu" + ) + return ( + math.log2(float(source["tp"])), + math.log2(float(source["mns"])), + math.log2(float(target["tp"])), + math.log2(float(target["mns"])), + math.log2(float(target["tp"]) / float(source["tp"])), + math.log2(float(target["mns"]) / float(source["mns"])), + math.log2(source_rate), + math.log2(target_rate), + math.log2(target_rate / source_rate), + ) + + +def hybrid_base_features( + source: dict[str, Any], target: dict[str, Any] +) -> tuple[float, ...]: + return ( + finite(source["pass_rate_residual"], name="source_pass_rate_residual"), + finite(source["sim_pass_rate"], name="source_sim_pass_rate"), + finite(target["sim_pass_rate"], name="target_sim_pass_rate"), + ) + config_transition_features(source, target) + + +def direct_base_features( + source: dict[str, Any], target: dict[str, Any] +) -> tuple[float, ...]: + return ( + finite(source["real_pass_rate_rep1"], name="source_real_pass_rate"), + ) + config_transition_features(source, target) + + +def transitions(examples: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + rows = [] + for source in examples: + residual_state = flatten_residual_state(source) + engine_state = flatten_engine_state(source) + for target in examples: + # Low/high are offered-load anchors inside one workload/SLO task. + # Cross-load transitions are legal; same-cell transitions are + # excluded because R0 asks about transfer to a new configuration. + if source["cell"] == target["cell"]: + continue + hybrid_base = hybrid_base_features(source, target) + direct_base = direct_base_features(source, target) + rows.append( + { + "source_cell": source["cell"], + "source_level": source["level"], + "target_cell": target["cell"], + "target_level": target["level"], + "hybrid_base": hybrid_base, + "hybrid_telemetry": hybrid_base + residual_state, + "direct_base": direct_base, + "direct_telemetry": direct_base + engine_state, + "target_residual": finite( + target["pass_rate_residual"], name="target_residual" + ), + "target_residual_delta": finite( + target["pass_rate_residual"], name="target_residual" + ) + - finite(source["pass_rate_residual"], name="source_residual"), + "target_real_pass_rate": finite( + target["real_pass_rate_rep1"], name="target_real_pass_rate" + ), + "target_real_pass_rate_delta": finite( + target["real_pass_rate_rep1"], name="target_real_pass_rate" + ) + - finite( + source["real_pass_rate_rep1"], name="source_real_pass_rate" + ), + "source_residual": finite( + source["pass_rate_residual"], name="source_residual" + ), + "source_real_pass_rate": finite( + source["real_pass_rate_rep1"], name="source_real_pass_rate" + ), + "target_sim_pass_rate": finite( + target["sim_pass_rate"], name="target_sim_pass_rate" + ), + "target_real_feasible": bool(target["real_feasible"]), + "target_sim_feasible": bool(target["sim_feasible"]), + } + ) + return rows + + +def fit_predict( + train_x: np.ndarray, + train_y: np.ndarray, + test_x: np.ndarray, + regularization: float, +) -> np.ndarray: + mean = train_x.mean(axis=0) + std = train_x.std(axis=0) + std[std == 0.0] = 1.0 + train = (train_x - mean) / std + test = (test_x - mean) / std + train = np.column_stack((np.ones(len(train)), train)) + test = np.column_stack((np.ones(len(test)), test)) + penalty = np.eye(train.shape[1], dtype=np.float64) + penalty[0, 0] = 0.0 + weights = np.linalg.lstsq( + train.T @ train + regularization * penalty, + train.T @ train_y, + rcond=None, + )[0] + return test @ weights + + +def grouped_predictions( + rows: Sequence[dict[str, Any]], + *, + feature_name: str, + target_name: str, + regularization: float, +) -> np.ndarray: + predictions = np.zeros(len(rows), dtype=np.float64) + groups = sorted({(row["source_cell"], row["target_cell"]) for row in rows}) + for source_cell, target_cell in groups: + held_out_cells = {source_cell, target_cell} + test_indexes = [ + index + for index, row in enumerate(rows) + if row["source_cell"] == source_cell and row["target_cell"] == target_cell + ] + train_indexes = [ + index + for index, row in enumerate(rows) + if row["source_cell"] not in held_out_cells + and row["target_cell"] not in held_out_cells + ] + if not test_indexes or not train_indexes: + raise ValueError(f"empty grouped fold for {source_cell}->{target_cell}") + train_x = np.asarray([rows[index][feature_name] for index in train_indexes]) + train_y = np.asarray([rows[index][target_name] for index in train_indexes]) + test_x = np.asarray([rows[index][feature_name] for index in test_indexes]) + predictions[test_indexes] = fit_predict( + train_x, train_y, test_x, regularization + ) + return predictions + + +def metrics(rows: Sequence[dict[str, Any]], predicted_pass: np.ndarray) -> dict[str, Any]: + truth = np.asarray( + [row["target_real_pass_rate"] for row in rows], dtype=np.float64 + ) + real_feasible = np.asarray( + [row["target_real_feasible"] for row in rows], dtype=bool + ) + sim_feasible = np.asarray( + [row["target_sim_feasible"] for row in rows], dtype=bool + ) + clipped_pass = np.clip(predicted_pass, 0.0, 1.0) + predicted_feasible = clipped_pass >= 0.95 + baseline_correct = sim_feasible == real_feasible + model_correct = predicted_feasible == real_feasible + return { + "rmse": float(np.sqrt(np.mean((predicted_pass - truth) ** 2))), + "mae": float(np.mean(np.abs(predicted_pass - truth))), + "feasibility_accuracy": float(np.mean(model_correct)), + "false_feasible": int(np.sum(predicted_feasible & ~real_feasible)), + "false_infeasible": int(np.sum(~predicted_feasible & real_feasible)), + "simulator_errors_corrected": int(np.sum(~baseline_correct & model_correct)), + "simulator_correct_corrupted": int(np.sum(baseline_correct & ~model_correct)), + "predicted_pass_rate": { + "n": len(clipped_pass), + "min": float(clipped_pass.min()), + "max": float(clipped_pass.max()), + "distinct_n": len(set(float(value) for value in clipped_pass)), + }, + } + + +def compare( + rows: Sequence[dict[str, Any]], + baseline_prediction: np.ndarray, + telemetry_prediction: np.ndarray, + baseline: dict[str, Any], + telemetry: dict[str, Any], +) -> dict[str, Any]: + truth = np.asarray([row["target_real_feasible"] for row in rows], dtype=bool) + baseline_feasible = np.clip(baseline_prediction, 0.0, 1.0) >= 0.95 + telemetry_feasible = np.clip(telemetry_prediction, 0.0, 1.0) >= 0.95 + baseline_correct = baseline_feasible == truth + telemetry_correct = telemetry_feasible == truth + return { + "delta_telemetry_minus_baseline": { + "rmse": telemetry["rmse"] - baseline["rmse"], + "mae": telemetry["mae"] - baseline["mae"], + "feasibility_accuracy": telemetry["feasibility_accuracy"] + - baseline["feasibility_accuracy"], + }, + "baseline_errors_corrected": int( + np.sum(~baseline_correct & telemetry_correct) + ), + "baseline_correct_corrupted": int( + np.sum(baseline_correct & ~telemetry_correct) + ), + } + + +def execute(args: argparse.Namespace) -> dict[str, Any]: + paired = json.loads(args.paired_state.read_text(encoding="utf-8")) + if paired.get("status") != "PASS" or len(paired["examples"]) != 12: + raise RuntimeError("paired P1 state evidence is incomplete") + rows = transitions(paired["examples"]) + residual_truth = np.asarray( + [row["target_residual"] for row in rows], dtype=np.float64 + ) + pass_truth = np.asarray( + [row["target_real_pass_rate"] for row in rows], dtype=np.float64 + ) + sim_pass = np.asarray( + [row["target_sim_pass_rate"] for row in rows], dtype=np.float64 + ) + simulator = metrics(rows, sim_pass) + sensitivity = {} + for regularization in REGULARIZATION: + hybrid_base_delta = grouped_predictions( + rows, + feature_name="hybrid_base", + target_name="target_residual_delta", + regularization=regularization, + ) + hybrid_telemetry_delta = grouped_predictions( + rows, + feature_name="hybrid_telemetry", + target_name="target_residual_delta", + regularization=regularization, + ) + source_residual = np.asarray( + [row["source_residual"] for row in rows], dtype=np.float64 + ) + hybrid_base_correction = source_residual + hybrid_base_delta + hybrid_telemetry_correction = source_residual + hybrid_telemetry_delta + hybrid_base_prediction = sim_pass + hybrid_base_correction + hybrid_telemetry_prediction = sim_pass + hybrid_telemetry_correction + hybrid_base = metrics(rows, hybrid_base_prediction) + hybrid_telemetry = metrics(rows, hybrid_telemetry_prediction) + prior_shrinkage = {} + for weight in PRIOR_SHRINKAGE: + prior_shrinkage[str(weight)] = { + "raw_simulator_prior": { + "sim_plus_outcome": metrics( + rows, sim_pass + weight * hybrid_base_correction + ), + "sim_plus_outcome_plus_telemetry": metrics( + rows, sim_pass + weight * hybrid_telemetry_correction + ), + }, + "anchor_offset_prior": { + "sim_plus_outcome": metrics( + rows, + sim_pass + source_residual + weight * hybrid_base_delta, + ), + "sim_plus_outcome_plus_telemetry": metrics( + rows, + sim_pass + source_residual + weight * hybrid_telemetry_delta, + ), + }, + } + + direct_base_prediction = grouped_predictions( + rows, + feature_name="direct_base", + target_name="target_real_pass_rate_delta", + regularization=regularization, + ) + direct_telemetry_prediction = grouped_predictions( + rows, + feature_name="direct_telemetry", + target_name="target_real_pass_rate_delta", + regularization=regularization, + ) + source_real_pass = np.asarray( + [row["source_real_pass_rate"] for row in rows], dtype=np.float64 + ) + direct_base_prediction = source_real_pass + direct_base_prediction + direct_telemetry_prediction = source_real_pass + direct_telemetry_prediction + direct_base = metrics(rows, direct_base_prediction) + direct_telemetry = metrics(rows, direct_telemetry_prediction) + sensitivity[str(regularization)] = { + "hybrid": { + "sim_plus_outcome": hybrid_base, + "sim_plus_outcome_plus_telemetry": hybrid_telemetry, + "comparison": compare( + rows, + hybrid_base_prediction, + hybrid_telemetry_prediction, + hybrid_base, + hybrid_telemetry, + ), + "prior_shrinkage": prior_shrinkage, + }, + "direct": { + "real_outcome_only": direct_base, + "telemetry_only": direct_telemetry, + "comparison": compare( + rows, + direct_base_prediction, + direct_telemetry_prediction, + direct_base, + direct_telemetry, + ), + }, + } + red_flags = [] + if any(not math.isfinite(value) for value in residual_truth): + red_flags.append("nonfinite_residual_target") + if any(not math.isfinite(value) for value in pass_truth): + red_flags.append("nonfinite_pass_rate_target") + result = { + "schema": "telemetry-residual-cross-config-diagnostic-v1", + "status": "PASS" if not red_flags else "STOP", + "scope": ( + "single P1 trace/SLO-task development diagnostic; ordered transitions " + "are not independent tasks and cannot support a generalization claim" + ), + "split": ( + "hold out both ordered source config and target config identities; " + "each fold contains both offered-load anchors" + ), + "features": { + "sim_plus_outcome": len(rows[0]["hybrid_base"]), + "sim_plus_outcome_plus_telemetry": len(rows[0]["hybrid_telemetry"]), + "real_outcome_only": len(rows[0]["direct_base"]), + "telemetry_only": len(rows[0]["direct_telemetry"]), + }, + "simulator": simulator, + "regularization_sensitivity": sensitivity, + "red_flags": red_flags, + "sanity": { + "transitions": len(rows), + "load_levels": len( + {row["source_level"] for row in rows} + | {row["target_level"] for row in rows} + ), + "cross_load_transitions": sum( + row["source_level"] != row["target_level"] for row in rows + ), + "ordered_cell_pairs": len( + {(row["source_cell"], row["target_cell"]) for row in rows} + ), + "target_residual": { + "n": len(residual_truth), + "min": float(residual_truth.min()), + "max": float(residual_truth.max()), + "distinct_n": len(set(float(value) for value in residual_truth)), + }, + "target_real_pass_rate": { + "n": len(pass_truth), + "min": float(pass_truth.min()), + "max": float(pass_truth.max()), + "distinct_n": len(set(float(value) for value in pass_truth)), + }, + "invariants": { + "finite_targets": not red_flags, + "ratios_bounded": all( + 0.0 <= row["target_sim_pass_rate"] <= 1.0 for row in rows + ), + "source_differs_from_target": all( + row["source_cell"] != row["target_cell"] for row in rows + ), + "per_config_not_identical": len( + set(float(value) for value in pass_truth) + ) + > 1, + }, + }, + } + atomic_json(args.output, result) + if result["status"] != "PASS": + raise RuntimeError(red_flags) + return result + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser() + result.add_argument("--paired-state", type=Path, required=True) + result.add_argument("--output", type=Path, required=True) + return result + + +def main() -> None: + result = execute(parser().parse_args()) + print( + json.dumps( + { + "status": result["status"], + "transitions": result["sanity"]["transitions"], + "sensitivity": result["regularization_sensitivity"], + "sanity": result["sanity"], + "red_flags": result["red_flags"], + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/telemetry-residual/common_state.py b/runs/telemetry-residual/common_state.py new file mode 100644 index 0000000..9850e1c --- /dev/null +++ b/runs/telemetry-residual/common_state.py @@ -0,0 +1,400 @@ +#!/usr/bin/env python3 +"""Extract directly measurable engine/simulator state into one schema. + +The schema deliberately keeps common, engine-only, and simulator-only fields +separate. Missing simulator mechanisms must remain missing; they must not be +filled by a heuristic bottleneck label. +""" + +from __future__ import annotations + +import csv +import json +import math +from pathlib import Path +from statistics import fmean +from typing import Any, Iterable, Mapping, Sequence + + +SCHEMA = "telemetry-common-state-v1" + + +def numeric(values: Iterable[float | int]) -> dict[str, Any]: + finite = [float(value) for value in values] + if not finite: + raise ValueError("numeric summary requires at least one value") + if any(not math.isfinite(value) for value in finite): + raise ValueError("numeric summary received a non-finite value") + mean = fmean(finite) + variance = fmean((value - mean) ** 2 for value in finite) + return { + "n": len(finite), + "min": min(finite), + "max": max(finite), + "mean": mean, + "cv": math.sqrt(variance) / abs(mean) if mean else 0.0, + "distinct_n": len(set(finite)), + } + + +def load_jsonl(path: Path) -> list[dict[str, Any]]: + rows = [] + with path.open(encoding="utf-8") as source: + for line_number, line in enumerate(source, start=1): + if not line.strip(): + continue + row = json.loads(line) + if not isinstance(row, dict): + raise ValueError(f"{path}:{line_number}: expected a JSON object") + rows.append(row) + if not rows: + raise ValueError(f"{path}: no JSONL records") + return rows + + +def _as_number(value: Any, *, name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be numeric, got {value!r}") + result = float(value) + if not math.isfinite(result): + raise ValueError(f"{name} must be finite, got {value!r}") + return result + + +def _time_weighted_mean( + records: Sequence[Mapping[str, Any]], + *, + start_ns: int, + end_ns: int, + value, +) -> float: + if end_ns <= start_ns: + raise ValueError("time-weighted interval must be positive") + selected = [ + record + for record in records + if start_ns <= int(record["submit_mono_ns"]) <= end_ns + ] + if not selected: + raise ValueError("time-weighted interval contains no records") + selected.sort(key=lambda record: int(record["submit_mono_ns"])) + cursor = start_ns + total = 0.0 + current = _as_number(value(selected[0]), name="time-weighted value") + for record in selected[1:]: + timestamp = int(record["submit_mono_ns"]) + if timestamp < cursor: + raise ValueError("telemetry timestamps are not monotonic") + total += current * (timestamp - cursor) + cursor = timestamp + current = _as_number(value(record), name="time-weighted value") + total += current * (end_ns - cursor) + return total / (end_ns - start_ns) + + +def summarize_engine( + records: Sequence[Mapping[str, Any]], + *, + start_ns: int, + end_ns: int, + request_count: int, +) -> dict[str, Any]: + """Summarize a measured engine interval from Layer-1 records.""" + + if request_count <= 0: + raise ValueError("request_count must be positive") + layer1 = [record for record in records if "step_index" in record] + if not layer1: + raise ValueError("engine stream has no Layer-1 records") + step_indexes = [int(record["step_index"]) for record in layer1] + if len(step_indexes) != len(set(step_indexes)): + raise ValueError("engine Layer-1 step indexes are not unique") + if step_indexes != sorted(step_indexes): + raise ValueError("engine Layer-1 step indexes are not ordered") + if any(int(record.get("dropped_records_before", 0)) != 0 for record in layer1): + raise ValueError("engine Layer-1 stream reports dropped records") + + interval = [ + record + for record in layer1 + if start_ns <= int(record["submit_mono_ns"]) <= end_ns + ] + if not interval: + raise ValueError("engine interval has no Layer-1 records") + executed = [record for record in interval if bool(record["model_executed"])] + if not executed: + raise ValueError("engine interval has no executed model steps") + duration_s = (end_ns - start_ns) / 1e9 + if duration_s <= 0: + raise ValueError("engine interval duration must be positive") + + batch_sizes = [int(record["scheduled_requests"]) for record in executed] + prefill_tokens = [int(record["prefill_tokens"]) for record in executed] + decode_tokens = [int(record["decode_tokens"]) for record in executed] + batch_tokens = [ + prefill + decode + for prefill, decode in zip(prefill_tokens, decode_tokens, strict=True) + ] + decode_batches = [int(record["decode_batch_size"]) for record in executed] + if any(value < 0 for value in batch_sizes + batch_tokens + decode_batches): + raise ValueError("engine batch counters must be non-negative") + if any( + int(record["prefill_tokens"]) + int(record["decode_tokens"]) + <= 0 + for record in executed + ): + raise ValueError("executed engine step has no scheduled tokens") + + waiting_mean = _time_weighted_mean( + interval, + start_ns=start_ns, + end_ns=end_ns, + value=lambda record: record["queues"]["waiting"], + ) + running_mean = _time_weighted_mean( + interval, + start_ns=start_ns, + end_ns=end_ns, + value=lambda record: record["queues"]["running"], + ) + kv_mean = _time_weighted_mean( + interval, + start_ns=start_ns, + end_ns=end_ns, + value=lambda record: record["kv"]["usage"], + ) + kv_values = [float(record["kv"]["usage"]) for record in interval] + if any(not 0.0 <= value <= 1.0 for value in kv_values): + raise ValueError("engine KV usage must be in [0, 1]") + total_prefill = sum(prefill_tokens) + total_decode = sum(decode_tokens) + graph_modes = [str(record["cudagraph"]["runtime_mode"]) for record in executed] + bucket_tokens = sum(int(record["cudagraph"]["bucket_tokens"]) for record in executed) + padding_tokens = sum(int(record["cudagraph"]["padding_tokens"]) for record in executed) + + common = { + "scheduler_steps_per_s": len(executed) / duration_s, + "batch_size": numeric(batch_sizes), + "batch_tokens": numeric(batch_tokens), + "decode_batch_size": numeric(decode_batches), + "prefill_token_fraction": total_prefill / (total_prefill + total_decode), + "queue_waiting_mean": waiting_mean, + "queue_running_mean": running_mean, + "queue_waiting_time_per_request_ms": waiting_mean * duration_s * 1000.0 / request_count, + "queue_running_time_per_request_ms": running_mean * duration_s * 1000.0 / request_count, + "preemptions": sum(int(record["preemptions"]) for record in executed), + } + result = { + "schema": SCHEMA, + "source": "engine_layer1", + "interval": { + "start_ns": start_ns, + "end_ns": end_ns, + "duration_s": duration_s, + "request_count": request_count, + }, + "common": common, + "engine_only": { + "kv_usage_mean": kv_mean, + "kv_usage_max": max(kv_values), + "kv_usage_end_minus_start": kv_values[-1] - kv_values[0], + "graph_none_share": graph_modes.count("NONE") / len(graph_modes), + "graph_full_share": graph_modes.count("FULL") / len(graph_modes), + "graph_padding_fraction": padding_tokens / max(1, bucket_tokens), + }, + "simulator_only": {}, + "sanity": { + "records": len(interval), + "executed_steps": len(executed), + "step_index_min": min(int(record["step_index"]) for record in interval), + "step_index_max": max(int(record["step_index"]) for record in interval), + "invariants": { + "positive_duration": duration_s > 0, + "positive_request_count": request_count > 0, + "zero_drops": True, + "nonnegative_counters": True, + "kv_bounded": True, + "batch_values_not_all_identical": any( + summary["distinct_n"] > 1 + for summary in ( + common["batch_size"], + common["batch_tokens"], + common["decode_batch_size"], + ) + ), + }, + }, + } + return result + + +def _csv_rows(path: Path) -> list[dict[str, str]]: + with path.open(encoding="utf-8", newline="") as source: + rows = list(csv.DictReader(source)) + if not rows: + raise ValueError(f"{path}: CSV contains no rows") + return rows + + +def _column(rows: Sequence[Mapping[str, str]], name: str) -> list[float]: + if name not in rows[0]: + raise ValueError(f"CSV is missing required column {name!r}") + values = [] + for row in rows: + text = row.get(name, "") + if text == "": + raise ValueError(f"CSV column {name!r} contains an empty value") + values.append(_as_number(float(text), name=name)) + return values + + +def summarize_frontier( + *, + system_metrics_path: Path, + request_metrics_path: Path, + batch_metrics_path: Path | None = None, + ledger_path: Path | None = None, +) -> dict[str, Any]: + """Summarize a Frontier run, retaining unavailable state as null.""" + + system = json.loads(system_metrics_path.read_text(encoding="utf-8")) + throughput = system["throughput_metrics"] + duration_s = _as_number( + throughput["total_duration_seconds"], name="total_duration_seconds" + ) + if duration_s <= 0: + raise ValueError("Frontier duration must be positive") + request_rows = _csv_rows(request_metrics_path) + waiting_ms = _column(request_rows, "request_waiting_time_total") + e2e_ms = _column(request_rows, "request_e2e_time") + running_ms = [max(0.0, e2e - waiting) for e2e, waiting in zip(e2e_ms, waiting_ms, strict=True)] + duration_ms = duration_s * 1000.0 + request_count = len(request_rows) + + common: dict[str, Any] = { + "scheduler_steps_per_s": None, + "batch_size": None, + "batch_tokens": None, + "decode_batch_size": None, + "prefill_token_fraction": None, + "queue_waiting_mean": sum(waiting_ms) / duration_ms, + "queue_running_mean": sum(running_ms) / duration_ms, + "queue_waiting_time_per_request_ms": fmean(waiting_ms), + "queue_running_time_per_request_ms": fmean(running_ms), + "preemptions": sum( + int(float(row.get("request_total_preemption_count") or 0)) + for row in request_rows + ), + } + batch_rows: list[dict[str, str]] = [] + if batch_metrics_path is not None: + batch_rows = _csv_rows(batch_metrics_path) + batch_sizes = _column(batch_rows, "batch_size") + batch_tokens = _column(batch_rows, "batch_num_tokens") + prefill_tokens = _column(batch_rows, "batch_num_prefill_tokens") + decode_tokens = _column(batch_rows, "batch_num_decode_tokens") + if any(value < 0 for value in batch_sizes + batch_tokens + prefill_tokens + decode_tokens): + raise ValueError("Frontier batch counters must be non-negative") + common.update( + { + "scheduler_steps_per_s": len(batch_rows) / duration_s, + "batch_size": numeric(batch_sizes), + "batch_tokens": numeric(batch_tokens), + "decode_batch_size": numeric(decode_tokens), + "prefill_token_fraction": sum(prefill_tokens) + / max(1.0, sum(prefill_tokens) + sum(decode_tokens)), + } + ) + + ledger_rows: list[dict[str, Any]] = [] + if ledger_path is not None: + ledger_rows = load_jsonl(ledger_path) + for row in ledger_rows: + start = _as_number(row["stage_start_ts"], name="stage_start_ts") + end = _as_number(row["stage_end_ts"], name="stage_end_ts") + if end < start: + raise ValueError("Frontier ledger has a negative stage duration") + + batch_distinct = ( + max( + summary["distinct_n"] + for summary in ( + common["batch_size"], + common["batch_tokens"], + common["decode_batch_size"], + ) + ) + if batch_rows + else None + ) + return { + "schema": SCHEMA, + "source": "frontier", + "interval": { + "duration_s": duration_s, + "request_count": request_count, + }, + "common": common, + "engine_only": { + "kv_usage_mean": None, + "kv_usage_max": None, + "kv_usage_end_minus_start": None, + "graph_none_share": None, + "graph_full_share": None, + "graph_padding_fraction": None, + }, + "simulator_only": { + "request_waiting_time_ms": numeric(waiting_ms), + "request_running_time_ms": numeric(running_ms), + "ledger_rows": len(ledger_rows) if ledger_path is not None else None, + }, + "sanity": { + "request_rows": request_count, + "batch_rows": len(batch_rows), + "ledger_rows": len(ledger_rows), + "invariants": { + "positive_duration": duration_s > 0, + "positive_request_count": request_count > 0, + "nonnegative_counters": True, + "request_values_not_all_identical": max( + numeric(waiting_ms)["distinct_n"], + numeric(running_ms)["distinct_n"], + ) + > 1, + "batch_values_not_all_identical": ( + batch_distinct > 1 if batch_distinct is not None else None + ), + }, + }, + } + + +def residual(real: Mapping[str, Any], simulated: Mapping[str, Any]) -> dict[str, Any]: + if real.get("schema") != SCHEMA or simulated.get("schema") != SCHEMA: + raise ValueError("residual inputs must use the common-state schema") + values = {} + missing = [] + for name, real_value in real["common"].items(): + sim_value = simulated["common"].get(name) + if isinstance(real_value, dict): + if not isinstance(sim_value, dict): + missing.append(name) + continue + for statistic in ("mean", "max", "cv"): + key = f"{name}.{statistic}" + values[key] = float(real_value[statistic]) - float(sim_value[statistic]) + continue + if real_value is None or sim_value is None: + missing.append(name) + continue + values[name] = float(real_value) - float(sim_value) + return { + "schema": "telemetry-state-residual-v1", + "values": values, + "missing_common_fields": sorted(missing), + "coverage": { + "available": len(values), + "missing": len(missing), + "common_field_count": len(real["common"]), + }, + } diff --git a/runs/telemetry-residual/run_frontier_state.py b/runs/telemetry-residual/run_frontier_state.py new file mode 100644 index 0000000..2fde9c8 --- /dev/null +++ b/runs/telemetry-residual/run_frontier_state.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +"""Replay frozen Frontier fixtures with state-detail outputs enabled. + +This runner does not modify the frozen fixture/config inputs. It reuses the +audited SimFid command and only turns on existing Frontier batch/ledger output +flags in a separate result root. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + + +HERE = Path(__file__).resolve().parent +AITUNER_ROOT = HERE.parents[1] +sys.path.insert(0, str(HERE)) + +from common_state import numeric, summarize_frontier # noqa: E402, F401 + + +def load_module(name: str, path: Path): + module_root = str(path.parent.resolve()) + if module_root not in sys.path: + sys.path.insert(0, module_root) + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise ImportError(path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def git_capture(root: Path, *arguments: str) -> str: + return subprocess.run( + ["git", "-C", str(root), *arguments], + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ).stdout + + +def atomic_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + os.replace(temporary, path) + + +def enable_state_outputs(command: list[str]) -> list[str]: + result = list(command) + disabled_ledger = "--no-metrics_config_store_frontier_stage_batch_ledger" + enabled_ledger = "--metrics_config_store_frontier_stage_batch_ledger" + enabled_batches = "--metrics_config_keep_individual_batch_metrics" + disabled_batches = "--no-metrics_config_keep_individual_batch_metrics" + if result.count(disabled_ledger) != 1 or enabled_ledger in result: + raise ValueError("base command must explicitly disable one full Frontier ledger") + result[result.index(disabled_ledger)] = enabled_ledger + if disabled_batches in result: + if result.count(disabled_batches) != 1: + raise ValueError("duplicate disabled individual-batch flag") + result[result.index(disabled_batches)] = enabled_batches + elif enabled_batches not in result: + result.append(enabled_batches) + if result.count(enabled_ledger) != 1 or result.count(enabled_batches) != 1: + raise ValueError("state-detail flags were not enabled exactly once") + return result + + +def find_state_metrics(run_root: Path) -> dict[str, Path]: + patterns = { + "system": "frontier_metrics/**/system_metrics.json", + "requests": "frontier_metrics/**/request_metrics.csv", + "batches": "frontier_metrics/**/monolithic_batch_metrics.csv", + "ledger": "frontier_metrics/**/frontier_stage_batch_ledger.jsonl", + } + result = {} + for name, pattern in patterns.items(): + matches = sorted(run_root.glob(pattern)) + if len(matches) != 1: + raise ValueError( + f"{run_root}: expected exactly one {name} artifact, found {len(matches)}" + ) + result[name] = matches[0] + if len({path.parent for path in result.values()}) != 1: + raise ValueError("Frontier state artifacts are not colocated") + return result + + +def execute(args: argparse.Namespace) -> dict[str, Any]: + prepared = json.loads(args.prepared_manifest.read_text(encoding="utf-8")) + if prepared.get("status") != "PASS": + raise RuntimeError("prepared simulator manifest did not pass") + matches = [ + entry + for entry in prepared["entries"] + if entry["cell"] == args.cell and entry["role"] == args.role + ] + if len(matches) != 1: + raise ValueError( + f"expected one prepared entry for {args.cell}/{args.role}, found {len(matches)}" + ) + entry = matches[0] + if args.output.exists() and any(args.output.iterdir()): + raise FileExistsError(f"non-empty output already exists: {args.output}") + args.output.mkdir(parents=True, exist_ok=True) + + driver = load_module( + "telemetry_residual_execution_driver", + args.replayserve_root / "runs/simfid_s2rb/results/execution_driver.py", + ) + config_path = Path(entry["config"]) + config = json.loads(config_path.read_text(encoding="utf-8")) + fixture_manifest_path = Path(entry["fixture_manifest"]) + fixture = json.loads(fixture_manifest_path.read_text(encoding="utf-8")) + trace_path = Path(entry["frontier_csv"]) + sidecar_path = Path(entry["sidecar"]) + metrics_root = args.output / "frontier_metrics" + run_id = f"telemetry_residual_r0_{args.cell}_{args.role}" + knobs = config["frontier"]["knobs"] + base_command = driver.build_command( + trace_path=trace_path, + metrics_root=metrics_root, + run_id=run_id, + knobs=knobs, + ) + driver.audit_command(base_command, knobs) + command = enable_state_outputs(base_command) + row = { + "hook_path": config["calibration"]["hook_path"], + "applied_a_tp": config["calibration"]["a_tp"], + "sidecar_path": str(sidecar_path), + "request_count": int(fixture["request_count"]), + "tensor_parallel_size": int(fixture["tensor_parallel_size"]), + } + environment = driver.environment_for(row) + if environment.get("CUDA_VISIBLE_DEVICES") != "": + raise ValueError("Frontier state replay must hide CUDA devices") + manifest = { + "schema": "telemetry-residual-frontier-state-run-v1", + "entry": { + "cell": args.cell, + "role": args.role, + "anchor": entry["anchor"], + "request_count": entry["selected_count"], + }, + "inputs": { + "prepared_manifest": str(args.prepared_manifest.resolve()), + "prepared_manifest_sha256": sha256_file(args.prepared_manifest), + "config": str(config_path.resolve()), + "config_sha256": sha256_file(config_path), + "fixture_manifest": str(fixture_manifest_path.resolve()), + "fixture_manifest_sha256": sha256_file(fixture_manifest_path), + "frontier_csv": str(trace_path.resolve()), + "frontier_csv_sha256": sha256_file(trace_path), + "sidecar": str(sidecar_path.resolve()), + "sidecar_sha256": sha256_file(sidecar_path), + }, + "frontier": { + "root": str(args.frontier_root.resolve()), + "git_head": git_capture(args.frontier_root, "rev-parse", "HEAD").strip(), + "git_status_short": git_capture(args.frontier_root, "status", "--short"), + }, + "runner": { + "script": str(Path(__file__).resolve()), + "script_sha256": sha256_file(Path(__file__).resolve()), + "aituner_git_head": git_capture(AITUNER_ROOT, "rev-parse", "HEAD").strip(), + "aituner_git_status_short": git_capture(AITUNER_ROOT, "status", "--short"), + }, + "environment": { + key: environment[key] + for key in ( + "PYTHONPATH", + "FRONTIER_EXECUTION_TIME_SCALE", + "CUDA_VISIBLE_DEVICES", + "NVIDIA_VISIBLE_DEVICES", + "FRONTIER_LOG_LEVEL", + ) + }, + "command": command, + "state_outputs": { + "individual_batch_metrics": True, + "full_stage_batch_ledger": True, + }, + "contains_prompt_text": False, + } + atomic_json(args.output / "run_manifest.json", manifest) + start = time.monotonic() + with (args.output / "stdout.log").open("w", encoding="utf-8") as stdout, ( + args.output / "stderr.log" + ).open("w", encoding="utf-8") as stderr: + try: + process = subprocess.run( + command, + cwd=args.frontier_root, + env=environment, + stdout=stdout, + stderr=stderr, + timeout=args.timeout_s, + ) + return_code = int(process.returncode) + except subprocess.TimeoutExpired: + return_code = 124 + runtime_s = time.monotonic() - start + if return_code != 0: + failure = { + "status": "STOP", + "return_code": return_code, + "runtime_s": runtime_s, + } + atomic_json(args.output / "failure.json", failure) + raise RuntimeError(f"Frontier state replay failed: {failure}") + + paths = find_state_metrics(args.output) + summary = summarize_frontier( + system_metrics_path=paths["system"], + request_metrics_path=paths["requests"], + batch_metrics_path=paths["batches"], + ledger_path=paths["ledger"], + ) + atomic_json(args.output / "common-state.json", summary) + scorer = driver.score_trial(row, paths["system"], paths["requests"]) + atomic_json(args.output / "scorer_output.json", scorer) + sizes = {name: path.stat().st_size for name, path in paths.items()} + red_flags = [] + if summary["interval"]["request_count"] != int(entry["selected_count"]): + red_flags.append("request_count_mismatch") + if summary["sanity"]["batch_rows"] <= 0: + red_flags.append("no_batch_rows") + if summary["sanity"]["ledger_rows"] <= 0: + red_flags.append("no_ledger_rows") + result = { + "schema": "telemetry-residual-frontier-state-result-v1", + "status": "PASS" if not red_flags else "STOP", + "runtime_s": runtime_s, + "return_code": return_code, + "paths": {name: str(path.resolve()) for name, path in paths.items()}, + "bytes": sizes, + "common_state": str((args.output / "common-state.json").resolve()), + "red_flags": red_flags, + "sanity": { + "request_rows": summary["sanity"]["request_rows"], + "batch_rows": summary["sanity"]["batch_rows"], + "ledger_rows": summary["sanity"]["ledger_rows"], + "artifact_bytes": { + "n": len(sizes), + "min": min(sizes.values()), + "max": max(sizes.values()), + "distinct_n": len(set(sizes.values())), + }, + "invariants": { + "zero_failures": return_code == 0, + "gpu_visibility_disabled": True, + "request_count_match": not red_flags, + "nonnegative_file_sizes": all(size >= 0 for size in sizes.values()), + "state_values_not_all_identical": summary["sanity"]["invariants"][ + "batch_values_not_all_identical" + ], + }, + }, + } + atomic_json(args.output / "result.json", result) + if result["status"] != "PASS": + raise RuntimeError(red_flags) + return result + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser() + result.add_argument("--prepared-manifest", type=Path, required=True) + result.add_argument("--output", type=Path, required=True) + result.add_argument("--replayserve-root", type=Path, required=True) + result.add_argument("--frontier-root", type=Path, required=True) + result.add_argument("--cell", required=True) + result.add_argument("--role", choices=("low1", "high1"), required=True) + result.add_argument("--timeout-s", type=float, default=300.0) + return result + + +def main() -> None: + result = execute(parser().parse_args()) + print( + json.dumps( + { + "status": result["status"], + "runtime_s": result["runtime_s"], + "request_rows": result["sanity"]["request_rows"], + "batch_rows": result["sanity"]["batch_rows"], + "ledger_rows": result["sanity"]["ledger_rows"], + "red_flags": result["red_flags"], + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/telemetry-residual/run_frontier_state_campaign.py b/runs/telemetry-residual/run_frontier_state_campaign.py new file mode 100644 index 0000000..7b9ae0e --- /dev/null +++ b/runs/telemetry-residual/run_frontier_state_campaign.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Run the frozen 12-fixture P1 Frontier state-detail campaign, CPU only.""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path +from types import SimpleNamespace +from typing import Any + + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +import run_frontier_state as state_runner # noqa: E402 + + +def atomic_json(path: Path, payload: Any) -> None: + state_runner.atomic_json(path, payload) + + +def scorer_without_runtime(value: dict[str, Any]) -> dict[str, Any]: + result = dict(value) + result.pop("runtime_s", None) + return result + + +def execute(args: argparse.Namespace) -> dict[str, Any]: + prepared = json.loads(args.prepared_manifest.read_text(encoding="utf-8")) + committed = json.loads(args.committed_results.read_text(encoding="utf-8")) + if prepared.get("status") != "PASS" or committed.get("status") != "PASS": + raise RuntimeError("prepared or committed simulator evidence did not pass") + committed_by_key = { + (row["cell"], row["role"]): scorer_without_runtime(row["scorer"]) + for row in committed["results"] + } + entries = prepared["entries"] + if len(entries) != 12 or len(committed_by_key) != 12: + raise ValueError("P1 state campaign requires exactly 12 fixtures") + args.output.mkdir(parents=True, exist_ok=True) + results = [] + campaign_start = time.monotonic() + for index, entry in enumerate(entries, start=1): + key = (entry["cell"], entry["role"]) + output = args.output / f"{entry['cell']}_{entry['role']}" + result_path = output / "result.json" + if args.resume and result_path.is_file(): + result = json.loads(result_path.read_text(encoding="utf-8")) + if result.get("status") != "PASS": + raise RuntimeError(f"cannot resume failed state replay: {result_path}") + resumed = True + else: + if output.exists() and any(output.iterdir()): + raise FileExistsError(f"non-empty state replay output: {output}") + print( + f"RUN {index:02d}/12 {entry['cell']}/{entry['role']}", + flush=True, + ) + result = state_runner.execute( + SimpleNamespace( + prepared_manifest=args.prepared_manifest, + output=output, + replayserve_root=args.replayserve_root, + frontier_root=args.frontier_root, + cell=entry["cell"], + role=entry["role"], + timeout_s=args.timeout_s, + ) + ) + resumed = False + observed_scorer = json.loads( + (output / "scorer_output.json").read_text(encoding="utf-8") + ) + exact_scorer_match = scorer_without_runtime(observed_scorer) == committed_by_key[key] + if not exact_scorer_match: + raise ValueError(f"state-output replay changed the committed scorer: {key}") + results.append( + { + "cell": entry["cell"], + "role": entry["role"], + "runtime_s": result["runtime_s"], + "request_rows": result["sanity"]["request_rows"], + "batch_rows": result["sanity"]["batch_rows"], + "ledger_rows": result["sanity"]["ledger_rows"], + "artifact_bytes": sum(result["bytes"].values()), + "exact_committed_scorer_match": exact_scorer_match, + "resumed": resumed, + "result": str(result_path.resolve()), + } + ) + print( + f"DONE {index:02d}/12 {entry['cell']}/{entry['role']} " + f"runtime={result['runtime_s']:.3f}s batches={result['sanity']['batch_rows']}", + flush=True, + ) + runtimes = [float(row["runtime_s"]) for row in results] + batches = [int(row["batch_rows"]) for row in results] + bytes_values = [int(row["artifact_bytes"]) for row in results] + red_flags = [] + if len(results) != 12: + red_flags.append("runs_not_12") + if not all(row["exact_committed_scorer_match"] for row in results): + red_flags.append("committed_scorer_mismatch") + if any(value <= 0 for value in batches): + red_flags.append("empty_batch_output") + result = { + "schema": "telemetry-residual-frontier-state-campaign-v1", + "status": "PASS" if not red_flags else "STOP", + "prepared_manifest": str(args.prepared_manifest.resolve()), + "committed_results": str(args.committed_results.resolve()), + "campaign_elapsed_s": time.monotonic() - campaign_start, + "results": results, + "red_flags": red_flags, + "sanity": { + "n": len(results), + "runtime_s": state_runner.numeric(runtimes), + "batch_rows": state_runner.numeric(batches), + "artifact_bytes": state_runner.numeric(bytes_values), + "invariants": { + "runs_12": len(results) == 12, + "zero_failures": not red_flags, + "exact_committed_scorers": all( + row["exact_committed_scorer_match"] for row in results + ), + "nonnegative_counts": all(value > 0 for value in batches), + "per_config_not_identical": len(set(batches)) > 1, + "gpu_visibility_disabled": True, + }, + }, + } + atomic_json(args.output / "campaign-metrics.json", result) + if result["status"] != "PASS": + raise RuntimeError(red_flags) + return result + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser() + result.add_argument("--prepared-manifest", type=Path, required=True) + result.add_argument("--committed-results", type=Path, required=True) + result.add_argument("--output", type=Path, required=True) + result.add_argument("--replayserve-root", type=Path, required=True) + result.add_argument("--frontier-root", type=Path, required=True) + result.add_argument("--timeout-s", type=float, default=300.0) + result.add_argument("--resume", action="store_true") + return result + + +def main() -> None: + result = execute(parser().parse_args()) + print( + json.dumps( + { + "status": result["status"], + "runs": len(result["results"]), + "elapsed_s": result["campaign_elapsed_s"], + "sanity": result["sanity"], + "red_flags": result["red_flags"], + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/telemetry-residual/test_common_state.py b/runs/telemetry-residual/test_common_state.py new file mode 100644 index 0000000..175f1c8 --- /dev/null +++ b/runs/telemetry-residual/test_common_state.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import csv +import json +import math +import tempfile +from pathlib import Path + +from common_state import numeric, residual, summarize_engine, summarize_frontier +from run_frontier_state import enable_state_outputs + + +def engine_record( + step: int, + timestamp_ns: int, + *, + batch: int, + prefill: int, + decode: int, + waiting: int, + running: int, + kv: float, +) -> dict[str, object]: + return { + "step_index": step, + "submit_mono_ns": timestamp_ns, + "model_executed": True, + "scheduled_requests": batch, + "decode_batch_size": decode, + "prefill_tokens": prefill, + "decode_tokens": decode, + "preemptions": 0, + "queues": {"waiting": waiting, "running": running}, + "kv": {"usage": kv}, + "cudagraph": { + "runtime_mode": "NONE" if step == 0 else "FULL", + "bucket_tokens": prefill + decode + 1, + "padding_tokens": 1, + }, + "dropped_records_before": 0, + } + + +def main() -> None: + assert numeric((-2.0, -1.0))["cv"] >= 0.0 + command = [ + "python3", + "-m", + "frontier.main", + "--no-metrics_config_store_frontier_stage_batch_ledger", + ] + state_command = enable_state_outputs(command) + assert "--metrics_config_store_frontier_stage_batch_ledger" in state_command + assert "--metrics_config_keep_individual_batch_metrics" in state_command + assert "--no-metrics_config_store_frontier_stage_batch_ledger" not in state_command + + engine = summarize_engine( + [ + engine_record( + 0, + 0, + batch=2, + prefill=6, + decode=0, + waiting=1, + running=2, + kv=0.1, + ), + engine_record( + 1, + 1_000_000_000, + batch=4, + prefill=0, + decode=4, + waiting=3, + running=4, + kv=0.3, + ), + ], + start_ns=0, + end_ns=2_000_000_000, + request_count=2, + ) + assert math.isclose(engine["common"]["queue_waiting_mean"], 2.0) + assert math.isclose(engine["common"]["queue_running_mean"], 3.0) + assert math.isclose( + engine["common"]["queue_waiting_time_per_request_ms"], 2000.0 + ) + assert engine["common"]["batch_size"]["distinct_n"] == 2 + assert math.isclose(engine["common"]["prefill_token_fraction"], 0.6) + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + system = root / "system.json" + requests = root / "requests.csv" + batches = root / "batches.csv" + ledger = root / "ledger.jsonl" + system.write_text( + json.dumps( + { + "throughput_metrics": { + "total_duration_seconds": 2.0, + } + } + ), + encoding="utf-8", + ) + with requests.open("w", encoding="utf-8", newline="") as output: + writer = csv.DictWriter( + output, + fieldnames=[ + "request_e2e_time", + "request_waiting_time_total", + "request_total_preemption_count", + ], + ) + writer.writeheader() + writer.writerow( + { + "request_e2e_time": 2000, + "request_waiting_time_total": 1000, + "request_total_preemption_count": 0, + } + ) + writer.writerow( + { + "request_e2e_time": 3000, + "request_waiting_time_total": 3000, + "request_total_preemption_count": 0, + } + ) + with batches.open("w", encoding="utf-8", newline="") as output: + writer = csv.DictWriter( + output, + fieldnames=[ + "batch_size", + "batch_num_tokens", + "batch_num_prefill_tokens", + "batch_num_decode_tokens", + ], + ) + writer.writeheader() + writer.writerow( + { + "batch_size": 2, + "batch_num_tokens": 6, + "batch_num_prefill_tokens": 6, + "batch_num_decode_tokens": 0, + } + ) + writer.writerow( + { + "batch_size": 4, + "batch_num_tokens": 4, + "batch_num_prefill_tokens": 0, + "batch_num_decode_tokens": 4, + } + ) + ledger.write_text( + json.dumps({"stage_start_ts": 0.0, "stage_end_ts": 1.0}) + "\n" + + json.dumps({"stage_start_ts": 1.0, "stage_end_ts": 2.0}) + + "\n", + encoding="utf-8", + ) + simulator = summarize_frontier( + system_metrics_path=system, + request_metrics_path=requests, + batch_metrics_path=batches, + ledger_path=ledger, + ) + assert math.isclose(simulator["common"]["queue_waiting_mean"], 2.0) + assert math.isclose(simulator["common"]["queue_running_mean"], 0.5) + assert simulator["common"]["batch_size"]["distinct_n"] == 2 + difference = residual(engine, simulator) + assert difference["coverage"]["missing"] == 0 + assert difference["coverage"]["available"] == 16 + assert math.isclose(difference["values"]["queue_waiting_mean"], 0.0) + print("telemetry common state: PASS") + + +if __name__ == "__main__": + main() diff --git a/runs/telemetry-residual/test_residual_transfer.py b/runs/telemetry-residual/test_residual_transfer.py new file mode 100644 index 0000000..1944f6a --- /dev/null +++ b/runs/telemetry-residual/test_residual_transfer.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Small structural tests for the cross-config transfer diagnostic.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +HERE = Path(__file__).resolve().parent +SPEC = importlib.util.spec_from_file_location( + "analyze_residual_transfer", HERE / "analyze_residual_transfer.py" +) +assert SPEC is not None and SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def example(cell: str, level: str, value: float) -> dict: + tp_text, mns_text = cell.split("_") + return { + "cell": cell, + "level": level, + "tp": int(tp_text[2:]), + "mns": int(mns_text[3:]), + "pass_rate_residual": value - 0.5, + "real_pass_rate_rep1": value, + "sim_pass_rate": 0.5, + "offered_req_s_per_gpu": 1.0 if level == "low" else 2.0, + "real_feasible": value >= 0.95, + "sim_feasible": False, + "state_residual": {"values": {"batch.mean": value}}, + "engine": { + "common": {"batch": {"mean": value, "max": value, "cv": 0.0}}, + "engine_only": {"kv": value}, + }, + } + + +def main() -> None: + cells = ("tp1_mns8", "tp1_mns16", "tp2_mns8", "tp2_mns16") + examples = [ + example(cell, level, 0.1 + index / 10.0) + for level in ("low", "high") + for index, cell in enumerate(cells) + ] + rows = MODULE.transitions(examples) + assert len(rows) == 48 + assert any(row["source_level"] != row["target_level"] for row in rows) + assert all(row["source_cell"] != row["target_cell"] for row in rows) + assert len(rows[0]["hybrid_telemetry"]) > len(rows[0]["hybrid_base"]) + assert len(rows[0]["direct_telemetry"]) > len(rows[0]["direct_base"]) + for feature, target in ( + ("hybrid_base", "target_residual_delta"), + ("hybrid_telemetry", "target_residual_delta"), + ("direct_base", "target_real_pass_rate_delta"), + ("direct_telemetry", "target_real_pass_rate_delta"), + ): + prediction = MODULE.grouped_predictions( + rows, feature_name=feature, target_name=target, regularization=1.0 + ) + assert len(prediction) == len(rows) + print("telemetry residual transfer: PASS") + + +if __name__ == "__main__": + main() From 791f7a8889cb43c057fd74bd64bc2ab3d5dcd62d Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 16:39:38 +0800 Subject: [PATCH 013/148] Audit telemetry intervention response for tuning --- ...ervention-response-v0-protocol-20260714.md | 71 + ...tervention-response-v0-results-20260714.md | 157 + ...ention-response-v1-p1-protocol-20260714.md | 58 + runs/intervention-response-v0/analyze_p1.py | 691 ++ .../analyze_phase6.py | 520 ++ runs/intervention-response-v0/p1-audit.json | 6818 +++++++++++++++++ .../phase6-audit.json | 5121 +++++++++++++ .../intervention-response-v0/test_analysis.py | 57 + .../test_p1_analysis.py | 35 + 9 files changed, 13528 insertions(+) create mode 100644 docs/intervention-response-v0-protocol-20260714.md create mode 100644 docs/intervention-response-v0-results-20260714.md create mode 100644 docs/intervention-response-v1-p1-protocol-20260714.md create mode 100644 runs/intervention-response-v0/analyze_p1.py create mode 100644 runs/intervention-response-v0/analyze_phase6.py create mode 100644 runs/intervention-response-v0/p1-audit.json create mode 100644 runs/intervention-response-v0/phase6-audit.json create mode 100644 runs/intervention-response-v0/test_analysis.py create mode 100644 runs/intervention-response-v0/test_p1_analysis.py diff --git a/docs/intervention-response-v0-protocol-20260714.md b/docs/intervention-response-v0-protocol-20260714.md new file mode 100644 index 0000000..47742f6 --- /dev/null +++ b/docs/intervention-response-v0-protocol-20260714.md @@ -0,0 +1,71 @@ +# Telemetry intervention-response v0 protocol + +Status: **FROZEN BEFORE V0 ANALYSIS**. + +Date: 2026-07-14 (Asia/Singapore). + +## Claim boundary + +The closed residual route asked whether one absolute engine-state snapshot can +predict unmeasured configurations. V0 asks a different, narrower question: + +> Does an adjacent, controlled MNS intervention produce an early engine-state +> response that is distinguishable from same-config repeat noise? + +Passing this gate only authorizes a matched real-GPU pilot. It does not prove +that telemetry improves tuning, that any metric is a causal mediator, or that +the response transfers to a new workload, topology, or knob family. + +## Data and estimand + +- Source: Phase 6 solo-authoritative Qwen3-30B-A3B/vLLM 0.24 Layer-1 streams. +- Action pairs: primary runs at identical study hash, TP, sampling anchor, and + request-order hash, with adjacent `MNS={8,16,32,64}` values. +- Noise pairs: primary versus confirmation at the same complete config, + anchor, and request-order hash. Only primary-to-confirmation pairs are used; + confirmations are not combined into pseudo-independent all-pairs. +- Fixed early windows: 5 seconds and 10 seconds from the measured interval + start. All runs exceed 10 seconds, so early-stop censoring cannot change the + telemetry window. +- Full-run pass rate and feasibility are descriptive only because an early + stop can make full elapsed durations differ. + +The statistical unit is a run pair. Scheduler steps are summarized within a +run and are never counted as independent trials. + +## Frozen response gate + +The directly measured gate features are scheduler-step rate, decode-batch +mean, prefill-token fraction, waiting/running queue mean, KV-usage mean, and +CUDA-graph padding fraction. + +A feature qualifies at one horizon only if: + +1. at least 75% of nonzero action deltas have the same sign; +2. median absolute action delta is at least 2x the median absolute repeat + delta; and +3. at least 50% of action deltas exceed the repeat-noise absolute p95. + +V0 opens a GPU pilot only if: + +- there are exactly 17 frozen adjacent-MNS action pairs; +- there are at least 20 primary/confirmation repeat pairs; +- all identity, finite-value, counter, and ratio invariants pass; and +- at least two gate features qualify at both 5 and 10 seconds. + +Any data red flag stops the analysis before interpreting the response. + +## If V0 passes + +Register a dash0 pilot around a known scaling knee. The pilot must use the +same request sequence and arrival times, one serving job at a time, one changed +knob, randomized `A/B` versus `B/A` order, common non-censored measurement +windows, and trial-level repetitions. It must compare a response-aware next +action against an outcome-only policy under complete startup, warm-up, and +H20-hour accounting. + +## If V0 fails + +Do not add telemetry fields or train a larger model. The current Layer-1 state +does not identify even an MNS intervention above repeat noise on this task, so +the telemetry-guided tuning route remains diagnostic only. diff --git a/docs/intervention-response-v0-results-20260714.md b/docs/intervention-response-v0-results-20260714.md new file mode 100644 index 0000000..7bc2996 --- /dev/null +++ b/docs/intervention-response-v0-results-20260714.md @@ -0,0 +1,157 @@ +# Telemetry intervention-response v0/v1 results + +Date: 2026-07-14 (Asia/Singapore). + +## Decision + +**STOP before a new H20 pilot.** The current Layer-1 aggregate telemetry does +not identify a sufficiently general early response to an MNS intervention, +and it does not improve action-efficacy prediction over exact external prefix +outcomes on the available development tasks. + +This is a negative result about the present state representation and +experiment design. It does not establish that engine telemetry is useless for +tuning, and it is not held-out evidence. + +## Hypothesis and frozen test + +The tested hypothesis was: + +> With the workload and all non-MNS settings held fixed, increasing MNS causes +> a 5--10 second engine-state response that is larger than same-config repeat +> noise and that predicts whether the action makes the full run feasible. + +A response feature had to satisfy all three frozen conditions at both 5 and +10 seconds: at least 0.75 sign consistency, median absolute action effect at +least 2x the repeat median, and at least 0.50 of action deltas above the repeat +absolute p95. At least two features had to pass. A telemetry feature was +decision-relevant only if its leave-one-repeat-out balanced accuracy was at +least 0.75 and at least 0.15 above the best exact external prefix outcome. + +## What was implemented + +- A common-window analyzer over the existing per-scheduler-step Layer-1 stream. +- Exact action pairing with request-order hash, offered load, TP, load role, + and repetition held fixed. +- Same-config repeat-noise estimation without treating scheduler steps as + independent samples. +- Exact 5/10-second request-prefix outcomes using monotonic completion times. +- A one-feature leave-one-repeat-out efficacy audit; no multivariate model was + fitted to the 12 examples. +- Input hashes, stream hashes, frozen thresholds, pair-level deltas, and sanity + invariants in machine-readable audit artifacts. +- Trial-by-trial validation against the P1 manifest, plus content hashes for + every result, request file, and Layer-1 stream. + +## Experiment A: Phase-6 retrospective audit + +Phase 6 supplied 17 adjacent-MNS actions and 29 same-config +primary/confirmation pairs. No feature passed at either horizon, producing +`STOP_NO_IDENTIFIABLE_RESPONSE`. + +The confirmation sample is not a clean replication distribution: confirmations +were selectively run after disputed primary outcomes. Several same-config +pairs consequently followed radically different trajectories. This result +therefore remains a valid failure of the frozen v0 gate, but it cannot by itself +separate normal run variance from confirmation-selection bias. + +## Experiment B: prospective-repeat confirmation + +P1 supplied three pre-arranged, disjoint request bands for every cell/load. +Exact matched actions exist for TP1 `MNS 8 -> 64` and TP4 `MNS 16 -> 64`, at +low/high load and repetitions 1/2/3. This yields 12 action pairs and 24 +same-config consecutive-repeat pairs. + +The 24 adjacent repeat differences share their middle repetition within each +three-run group. They define a conservative empirical noise reference; they +are not used as 24 independent samples in an inferential test. + +The result is `STOP_NO_PROSPECTIVE_RESPONSE`: zero features passed the response +gate at either horizon. + +The strongest response was mean waiting-queue occupancy: + +| Horizon | Sign consistency | Action/repeat median | Action above repeat p95 | Gate | +|---|---:|---:|---:|---| +| 5 s | 1.000 | 1.292x | 0.167 | fail | +| 10 s | 1.000 | 2.611x | 0.250 | fail | + +The direction is real enough to merit diagnosis, but the effect is not broad +enough to guide a general action. It is large for TP4/high-load trials and +small or absent in other regimes. + +Full-run transitions contain six beneficial actions (`false -> true`) and six +non-beneficial actions (three `false -> false`, three `true -> true`). The +beneficial label is also perfectly confounded with TP4 in this small dataset, +so it cannot support a topology-general claim. + +| Horizon | Best telemetry delta | Balanced accuracy | Best external prefix delta | Balanced accuracy | Telemetry advantage | +|---|---|---:|---|---:|---:| +| 5 s | waiting queue | 0.750 | max TPOT / SLO | 0.833 | -0.083 | +| 10 s | waiting queue | 0.750 | outstanding / admitted | 0.750 | 0.000 | + +No telemetry feature reaches the preregistered `+0.15` incremental threshold. + +## What this rules out + +It rules out using the current vector of 5/10-second global means as a solid +mechanism for choosing the next config. In particular, adding these aggregates +to an LLM prompt or fitting a larger predictor would currently hide, rather +than solve, the identifiability problem. + +It does not rule out an instrumentation-aware tuner built around a deliberately +excited local system. The existing runs were designed for endpoint/fidelity +evaluation, not system identification: the MNS action is large, efficacy is +confounded with TP, repeat bands contain different requests, and global means +erase when queue buildup or service-rate changes occur. + +## Required redesign before spending H20-hours + +The next admissible experiment is a randomized, local A/B system-identification +pilot around one fixed TP and one load knee: + +1. Replay the exact same request sequence and arrival times for both endpoints. +2. Use small adjacent actions and randomized `A/B` versus `B/A` order. +3. Record event-aligned response curves, including queue growth/drain rate, + prefill/decode service rate, and per-step service time, rather than only one + global mean. +4. Separate a mechanism gate (repeatable response) from the end-to-end gate: + fewer trials or H20-hours to select a feasible near-optimal config than an + outcome-only tuner. +5. Hold out a second load/workload for the final policy comparison. + +Until that design is frozen, a wider sweep would only generate more correlated +observations and is not justified by the evidence above. + +## Reproduction + +```bash +python3 runs/intervention-response-v0/test_analysis.py +python3 runs/intervention-response-v0/test_p1_analysis.py + +python3 runs/intervention-response-v0/analyze_phase6.py \ + --metrics runs/opprof-phase6/phase6/metrics.json \ + --raw-root runs/opprof-phase6/phase6/solo-authoritative/cells \ + --output runs/intervention-response-v0/phase6-audit.json + +python3 runs/intervention-response-v0/analyze_p1.py \ + --run-root /home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b \ + --manifest /home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/pilot-manifest.json \ + --output runs/intervention-response-v0/p1-audit.json +``` + +## Data sanity + +- Phase 6: action pairs `n=17`, repeat pairs `n=29`, trials `n=66`; MNS + action size min/max `8/32`, `3` distinct; action-state vectors `n=17`, `17` + distinct; streams `n=12`, bytes min/max `12,745,297/52,957,710`, `12` + distinct. +- P1: action pairs `n=12`, repeat pairs `n=24`, trials `n=36`; MNS action + size min/max `48/56`, `2` distinct; efficacy labels `n=12`, min/max `0/1`, + `2` distinct; streams `n=6`, bytes min/max `17,449,143/29,431,988`, `6` + distinct. +- Checked invariants: exact action request hashes and offered loads match; + all `36/36` P1 trials match the manifest; expected pair counts hold; all + deltas are finite; non-negative counters and bounded ratios hold; per-config + state vectors are not all identical; both efficacy classes are present. No + red flags were observed. diff --git a/docs/intervention-response-v1-p1-protocol-20260714.md b/docs/intervention-response-v1-p1-protocol-20260714.md new file mode 100644 index 0000000..9a3e413 --- /dev/null +++ b/docs/intervention-response-v1-p1-protocol-20260714.md @@ -0,0 +1,58 @@ +# Intervention-response v1 prospective-repeat confirmation + +Status: **FROZEN AFTER PHASE-6 V0 FAILURE AND BEFORE P1 RESPONSE ANALYSIS**. + +Date: 2026-07-14 (Asia/Singapore). + +## Why this is a new confirmation, not a relaxed V0 + +Phase-6 V0 failed its frozen global response gate. Its 29 same-config +confirmations were triggered after disputed outcomes, and the resulting noise +sample contains extreme trajectory divergence by construction. V0 remains +failed and its thresholds are unchanged. + +The already-completed P1 campaign supplies a distinct test: three +prospectively scheduled, disjoint repeat bands for every cell/load. TP1 and +TP4 use identical offered loads and exact request-order hashes across their MNS +endpoints. V1 asks whether an MNS response is identifiable against this +prospective workload-repeat noise, and whether that response predicts action +efficacy beyond exact external prefix outcomes. + +P1 is now development data. No result here is held-out or paper-facing. + +## Frozen pairs + +- Action pairs: TP1 `MNS 8 -> 64` and TP4 `MNS 16 -> 64`, at low/high load and + repeat 1/2/3. Endpoints must have identical TP, offered rate, repeat role, + and request-order hash. Expected `n=12`. +- Repeat-noise pairs: consecutive pre-arranged repeat bands within each of six + cells and low/high load: `rep1 -> rep2`, `rep2 -> rep3`. Expected `n=24`. + Repeat bands intentionally contain different requests and therefore include + workload-sampling noise rather than pretending to be identical trials. + Adjacent differences share the middle run; the gate uses their empirical + magnitude only and does not treat the 24 differences as independent samples + for a p-value or confidence interval. +- Prefix horizons: 5 and 10 seconds. Exact monotonic request completion times + and the same Layer-1 intervals are used. + +## Frozen gates + +The response-identifiability thresholds are exactly the Phase-6 V0 thresholds: +75% sign consistency, 2x median effect/repeat noise, and at least 50% of action +deltas above repeat absolute p95. At least two response features must qualify +at both horizons. + +Action efficacy is one only for an infeasible-to-feasible full-run transition. +The 12 action pairs must contain at least four examples of each class. + +For decision relevance, each individual external-outcome response feature and +each individual telemetry-response feature is evaluated by leave-one-repeat- +band-out threshold fitting. This intentionally avoids a multivariate model on +12 examples. At least one telemetry feature must, at both horizons: + +1. reach balanced accuracy at least 0.75; and +2. exceed the best external-outcome response feature by at least 0.15. + +Only if data validity, response identifiability, and incremental decision +relevance all pass does V1 open a newly registered matched GPU pilot. No +threshold or feature is changed after observing V1. diff --git a/runs/intervention-response-v0/analyze_p1.py b/runs/intervention-response-v0/analyze_p1.py new file mode 100644 index 0000000..90f44ab --- /dev/null +++ b/runs/intervention-response-v0/analyze_p1.py @@ -0,0 +1,691 @@ +#!/usr/bin/env python3 +"""Prospective-repeat confirmation of the intervention-response hypothesis. + +P1 contains three pre-arranged, disjoint request bands per cell/load. TP1 and +TP4 use matched offered loads and request sequences across their MNS endpoints. +This script asks both whether the MNS response exceeds prospective repeat noise +and whether an early telemetry delta predicts full-run action efficacy beyond +the corresponding external-outcome delta. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import math +import re +import sys +from collections import defaultdict +from pathlib import Path +from statistics import fmean +from typing import Any, Iterable, Mapping + + +HERE = Path(__file__).resolve().parent +COMMON_STATE_DIR = HERE.parent / "telemetry-residual" +sys.path.insert(0, str(COMMON_STATE_DIR)) + +from common_state import load_jsonl, summarize_engine # noqa: E402 + + +def _load_v0(): + spec = importlib.util.spec_from_file_location( + "intervention_response_phase6_v0", HERE / "analyze_phase6.py" + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +V0 = _load_v0() +SCHEMA = "intervention-response-p1-confirmation-v1" +HORIZONS_S = V0.HORIZONS_S +EXPECTED_ACTION_PAIRS = 12 +EXPECTED_REPEAT_PAIRS = 24 +MIN_EFFICACY_CLASS = 4 +MIN_EFFICACY_BALANCED_ACCURACY = 0.75 +MIN_EFFICACY_DELTA_OVER_OUTCOME = 0.15 +OUTCOME_FEATURES = ( + "admitted_fraction", + "completed_over_admitted", + "completed_pass_rate", + "completed_fail_fraction_of_total", + "outstanding_over_admitted", + "ttft_max_over_slo_max", + "ttft_mean_over_slo_max", + "tpot_max_over_slo", + "tpot_mean_over_slo", + "admitted_input_tokens_mean_over_limit", +) +RUN_PATTERN = re.compile(r"^(low|high)-rep([123])$") + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _prefix_outcome( + result: Mapping[str, Any], + requests: list[dict[str, Any]], + horizon_s: float, +) -> dict[str, float]: + admitted = [request for request in requests if float(request["arrival_s"]) <= horizon_s] + completed = [ + request + for request in requests + if request.get("completed_elapsed_s") is not None + and float(request["completed_elapsed_s"]) <= horizon_s + ] + if not admitted: + raise ValueError("prefix contains no admitted request") + admitted_ids = {str(request["request_id"]) for request in admitted} + if any(str(request["request_id"]) not in admitted_ids for request in completed): + raise ValueError("completed request was not admitted in the prefix") + passed = sum(bool(request["slo_pass"]) for request in completed) + ttft = [float(request["ttft_ms"]) for request in completed] + tpot = [float(request["tpot_ms"]) for request in completed] + total = int(result["selection"]["count"]) + if total != len(requests): + raise ValueError("request JSONL count does not match the result") + return { + "admitted_fraction": len(admitted) / total, + "completed_over_admitted": len(completed) / len(admitted), + "completed_pass_rate": passed / max(1, len(completed)), + "completed_fail_fraction_of_total": (len(completed) - passed) / total, + "outstanding_over_admitted": (len(admitted) - len(completed)) / len(admitted), + "ttft_max_over_slo_max": max(ttft, default=0.0) / 6000.0, + "ttft_mean_over_slo_max": fmean(ttft) / 6000.0 if ttft else 0.0, + "tpot_max_over_slo": max(tpot, default=0.0) / 50.0, + "tpot_mean_over_slo": fmean(tpot) / 50.0 if tpot else 0.0, + "admitted_input_tokens_mean_over_limit": fmean( + float(request["raw_input_tokens"]) for request in admitted + ) + / 8192.0, + } + + +def load_trials( + run_root: Path, + *, + horizons_s: tuple[float, ...] = HORIZONS_S, +) -> tuple[dict[float, list[dict[str, Any]]], list[dict[str, Any]]]: + by_horizon = {horizon: [] for horizon in horizons_s} + streams = [] + for cell_dir in sorted((run_root / "cells").iterdir()): + if not cell_dir.is_dir(): + continue + stream_paths = sorted((cell_dir / "opprof").glob("*.jsonl")) + if len(stream_paths) != 1: + raise ValueError(f"{cell_dir}: expected one Layer-1 stream") + stream_path = stream_paths[0] + stream = load_jsonl(stream_path) + streams.append( + { + "path": str(stream_path.resolve()), + "sha256": sha256_file(stream_path), + "bytes": stream_path.stat().st_size, + } + ) + for run_dir in sorted(cell_dir.iterdir()): + match = RUN_PATTERN.match(run_dir.name) + if match is None: + continue + level, replicate_text = match.groups() + replicate = int(replicate_text) + result_path = run_dir / "result.json" + requests_path = run_dir / "requests.jsonl" + result = json.loads(result_path.read_text(encoding="utf-8")) + requests = load_jsonl(requests_path) + elapsed_s = float(result["interval"]["elapsed_s"]) + start_ns = int(result["interval"]["start_mono_ns"]) + for horizon_s in horizons_s: + if elapsed_s < horizon_s: + raise ValueError( + f"{result_path}: elapsed {elapsed_s} shorter than {horizon_s}s" + ) + state = V0.flatten_state( + summarize_engine( + stream, + start_ns=start_ns, + end_ns=start_ns + int(horizon_s * 1e9), + request_count=int(result["selection"]["count"]), + ) + ) + by_horizon[horizon_s].append( + { + "trial_id": str(result_path.relative_to(run_root)), + "cell": str(result["cell"]), + "tp": int(result["tp"]), + "mns": int(result["mns"]), + "level": level, + "replicate": replicate, + "offered_rate_per_gpu": float( + result["selection"]["offered_req_s_per_gpu"] + ), + "request_hash": str( + result["selection"]["request_id_order_sha256"] + ), + "request_count": int(result["selection"]["count"]), + "result_sha256": sha256_file(result_path), + "requests_sha256": sha256_file(requests_path), + "full_pass_rate": float(result["pass_rate"]), + "full_feasible": bool(result["feasible"]), + "early_stopped": bool(result["early_stopped"]), + "state": state, + "outcome": _prefix_outcome(result, requests, horizon_s), + } + ) + return by_horizon, streams + + +def validate_manifest( + trials: list[dict[str, Any]], manifest_path: Path +) -> dict[str, Any]: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest.get("schema") != "fidelity-prefix-pilot-manifest-v1": + raise ValueError("unexpected P1 manifest schema") + cells = manifest.get("cells") + if not isinstance(cells, dict): + raise ValueError("P1 manifest has no cell mapping") + seen = set() + for trial in trials: + key = (trial["cell"], trial["level"], trial["replicate"]) + if key in seen: + raise ValueError(f"duplicate P1 trial identity: {key}") + seen.add(key) + try: + cell = cells[trial["cell"]] + selection = cell["targets"][trial["level"]]["selections"][ + f"{trial['level']}{trial['replicate']}" + ] + except (KeyError, TypeError) as error: + raise ValueError(f"trial is absent from P1 manifest: {key}") from error + if int(cell["tp"]) != trial["tp"] or int(cell["mns"]) != trial["mns"]: + raise ValueError(f"trial config disagrees with P1 manifest: {key}") + if str(selection["request_id_order_sha256"]) != trial["request_hash"]: + raise ValueError(f"trial request hash disagrees with P1 manifest: {key}") + if int(selection["selected_count"]) != trial["request_count"]: + raise ValueError(f"trial request count disagrees with P1 manifest: {key}") + if not math.isclose( + float(selection["offered_req_s_per_gpu"]), + trial["offered_rate_per_gpu"], + rel_tol=0.0, + abs_tol=1e-12, + ): + raise ValueError(f"trial offered load disagrees with P1 manifest: {key}") + expected = { + (cell_name, level, replicate) + for cell_name in cells + for level in ("low", "high") + for replicate in (1, 2, 3) + } + if seen != expected: + missing = sorted(expected - seen) + unexpected = sorted(seen - expected) + raise ValueError( + f"P1 trial/manifest coverage mismatch: missing={missing}, " + f"unexpected={unexpected}" + ) + return { + "schema": str(manifest["schema"]), + "expected_trials": len(expected), + "matched_trials": len(seen), + } + + +def _delta( + source: Mapping[str, Any], + target: Mapping[str, Any], + features: Iterable[str], +) -> dict[str, float]: + return { + feature: float(target[feature]) - float(source[feature]) + for feature in features + } + + +def _action_pair(source: Mapping[str, Any], target: Mapping[str, Any]) -> dict[str, Any]: + if source["tp"] != target["tp"]: + raise ValueError("action endpoints changed TP") + if source["level"] != target["level"] or source["replicate"] != target["replicate"]: + raise ValueError("action endpoints changed load role or repeat") + if source["request_hash"] != target["request_hash"]: + raise ValueError("action endpoints changed request sequence") + if not math.isclose( + source["offered_rate_per_gpu"], + target["offered_rate_per_gpu"], + rel_tol=0.0, + abs_tol=1e-12, + ): + raise ValueError("action endpoints changed offered load") + if source["mns"] >= target["mns"]: + raise ValueError("action must increase MNS") + beneficial = target["full_feasible"] and not source["full_feasible"] + return { + "kind": "matched_mns_increase", + "group": { + "tp": source["tp"], + "level": source["level"], + "replicate": source["replicate"], + "request_hash": source["request_hash"], + "offered_rate_per_gpu": source["offered_rate_per_gpu"], + }, + "source": { + key: source[key] + for key in ( + "trial_id", + "result_sha256", + "requests_sha256", + "cell", + "mns", + "full_pass_rate", + "full_feasible", + "early_stopped", + ) + }, + "target": { + key: target[key] + for key in ( + "trial_id", + "result_sha256", + "requests_sha256", + "cell", + "mns", + "full_pass_rate", + "full_feasible", + "early_stopped", + ) + }, + "delta_state": _delta(source["state"], target["state"], V0.ALL_FEATURES), + "delta_outcome": _delta(source["outcome"], target["outcome"], OUTCOME_FEATURES), + "full_action_efficacy": int(beneficial), + "full_feasibility_transition": ( + f"{str(source['full_feasible']).lower()}->" + f"{str(target['full_feasible']).lower()}" + ), + } + + +def _repeat_pair(source: Mapping[str, Any], target: Mapping[str, Any]) -> dict[str, Any]: + if source["cell"] != target["cell"] or source["level"] != target["level"]: + raise ValueError("repeat endpoints changed config or load role") + if target["replicate"] != source["replicate"] + 1: + raise ValueError("repeat endpoints are not consecutive pre-arranged bands") + if not math.isclose( + source["offered_rate_per_gpu"], + target["offered_rate_per_gpu"], + rel_tol=0.0, + abs_tol=1e-12, + ): + raise ValueError("repeat endpoints changed offered load") + return { + "kind": "same_config_workload_repeat", + "group": { + "cell": source["cell"], + "tp": source["tp"], + "mns": source["mns"], + "level": source["level"], + "source_replicate": source["replicate"], + "target_replicate": target["replicate"], + }, + "source": { + key: source[key] + for key in ("trial_id", "result_sha256", "requests_sha256") + }, + "target": { + key: target[key] + for key in ("trial_id", "result_sha256", "requests_sha256") + }, + "delta_state": _delta(source["state"], target["state"], V0.ALL_FEATURES), + "delta_outcome": _delta(source["outcome"], target["outcome"], OUTCOME_FEATURES), + } + + +def build_pairs( + trials: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + action_groups: dict[tuple[Any, ...], list[dict[str, Any]]] = defaultdict(list) + repeat_groups: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for trial in trials: + action_groups[ + ( + trial["tp"], + trial["level"], + trial["replicate"], + trial["request_hash"], + trial["offered_rate_per_gpu"], + ) + ].append(trial) + repeat_groups[(trial["cell"], trial["level"])].append(trial) + + actions = [] + for group in action_groups.values(): + if len(group) != 2: + continue + source, target = sorted(group, key=lambda trial: trial["mns"]) + actions.append(_action_pair(source, target)) + + repeats = [] + for group in repeat_groups.values(): + ordered = sorted(group, key=lambda trial: trial["replicate"]) + if len(ordered) != 3: + raise ValueError("each prospective repeat group must contain three runs") + repeats.extend( + _repeat_pair(source, target) + for source, target in zip(ordered, ordered[1:], strict=False) + ) + return actions, repeats + + +def _balanced_accuracy(labels: list[int], predictions: list[int]) -> float: + positive = [prediction for label, prediction in zip(labels, predictions) if label == 1] + negative = [prediction for label, prediction in zip(labels, predictions) if label == 0] + if not positive or not negative: + raise ValueError("balanced accuracy requires both classes") + sensitivity = sum(prediction == 1 for prediction in positive) / len(positive) + specificity = sum(prediction == 0 for prediction in negative) / len(negative) + return (sensitivity + specificity) / 2.0 + + +def _threshold_candidates(values: list[float]) -> list[float]: + unique = sorted(set(values)) + if len(unique) == 1: + return [unique[0] - 1.0, unique[0], unique[0] + 1.0] + scale = max(1.0, max(abs(value) for value in unique)) + candidates = [unique[0] - scale * 1e-6] + candidates.extend( + (left + right) / 2.0 + for left, right in zip(unique, unique[1:], strict=False) + ) + candidates.append(unique[-1] + scale * 1e-6) + return candidates + + +def _fit_threshold(values: list[float], labels: list[int]) -> tuple[float, int, float]: + best: tuple[float, int, float, float] | None = None + for threshold in _threshold_candidates(values): + for direction in (-1, 1): + predictions = [int(direction * (value - threshold) >= 0.0) for value in values] + balanced = _balanced_accuracy(labels, predictions) + accuracy = sum( + prediction == label + for prediction, label in zip(predictions, labels, strict=True) + ) / len(labels) + candidate = (balanced, accuracy, -abs(threshold), float(direction)) + if best is None or candidate > best: + best = candidate + selected_threshold = threshold + selected_direction = direction + assert best is not None + return selected_threshold, selected_direction, best[0] + + +def one_feature_leave_repeat_out( + actions: list[dict[str, Any]], + *, + delta_key: str, + features: tuple[str, ...], +) -> dict[str, Any]: + labels = [int(pair["full_action_efficacy"]) for pair in actions] + results = {} + for feature in features: + predictions = [] + held_out_labels = [] + folds = [] + for held_out in (1, 2, 3): + train = [pair for pair in actions if pair["group"]["replicate"] != held_out] + test = [pair for pair in actions if pair["group"]["replicate"] == held_out] + train_values = [float(pair[delta_key][feature]) for pair in train] + train_labels = [int(pair["full_action_efficacy"]) for pair in train] + threshold, direction, train_balanced = _fit_threshold( + train_values, train_labels + ) + test_values = [float(pair[delta_key][feature]) for pair in test] + test_predictions = [ + int(direction * (value - threshold) >= 0.0) for value in test_values + ] + test_labels = [int(pair["full_action_efficacy"]) for pair in test] + predictions.extend(test_predictions) + held_out_labels.extend(test_labels) + folds.append( + { + "held_out_replicate": held_out, + "threshold": threshold, + "direction": direction, + "train_balanced_accuracy": train_balanced, + "test_labels": test_labels, + "test_predictions": test_predictions, + } + ) + balanced = _balanced_accuracy(held_out_labels, predictions) + accuracy = sum( + prediction == label + for prediction, label in zip(predictions, held_out_labels, strict=True) + ) / len(held_out_labels) + results[feature] = { + "balanced_accuracy": balanced, + "accuracy": accuracy, + "folds": folds, + } + best_feature = max( + results, + key=lambda feature: ( + results[feature]["balanced_accuracy"], + results[feature]["accuracy"], + feature, + ), + ) + return { + "labels": V0.numeric(labels), + "positive": sum(labels), + "negative": len(labels) - sum(labels), + "features": results, + "best_feature": best_feature, + "best_balanced_accuracy": results[best_feature]["balanced_accuracy"], + "best_accuracy": results[best_feature]["accuracy"], + } + + +def analyze_horizon(trials: list[dict[str, Any]], horizon_s: float) -> dict[str, Any]: + actions, repeats = build_pairs(trials) + response = V0.response_statistics(actions, repeats) + qualifying_response = sorted( + feature for feature, item in response.items() if item["qualifies"] + ) + outcome_cv = one_feature_leave_repeat_out( + actions, + delta_key="delta_outcome", + features=OUTCOME_FEATURES, + ) + telemetry_cv = one_feature_leave_repeat_out( + actions, + delta_key="delta_state", + features=V0.GATE_FEATURES, + ) + outcome_best = float(outcome_cv["best_balanced_accuracy"]) + efficacy_qualifying = sorted( + feature + for feature, item in telemetry_cv["features"].items() + if item["balanced_accuracy"] >= MIN_EFFICACY_BALANCED_ACCURACY + and item["balanced_accuracy"] + >= outcome_best + MIN_EFFICACY_DELTA_OVER_OUTCOME + ) + action_hashes_match = all( + pair["group"]["request_hash"] for pair in actions + ) + labels = [int(pair["full_action_efficacy"]) for pair in actions] + invariants = { + "expected_action_pair_count": len(actions) == EXPECTED_ACTION_PAIRS, + "expected_repeat_pair_count": len(repeats) == EXPECTED_REPEAT_PAIRS, + "matched_action_request_hashes": action_hashes_match, + "efficacy_label_balance": ( + sum(labels) >= MIN_EFFICACY_CLASS + and len(labels) - sum(labels) >= MIN_EFFICACY_CLASS + ), + "finite_deltas": all( + math.isfinite(value) + for pair in [*actions, *repeats] + for values in (pair["delta_state"], pair["delta_outcome"]) + for value in values.values() + ), + "probabilities_bounded": all( + 0.0 <= trial["outcome"][feature] <= 1.0 + for trial in trials + for feature in ( + "admitted_fraction", + "completed_over_admitted", + "completed_pass_rate", + "completed_fail_fraction_of_total", + "outstanding_over_admitted", + "admitted_input_tokens_mean_over_limit", + ) + ), + } + red_flags = [name for name, passed in invariants.items() if not passed] + transitions = defaultdict(int) + for pair in actions: + transitions[pair["full_feasibility_transition"]] += 1 + return { + "horizon_s": horizon_s, + "actions": actions, + "repeats": repeats, + "response_statistics": response, + "qualifying_response_features": qualifying_response, + "efficacy": { + "outcome_delta": outcome_cv, + "telemetry_delta": telemetry_cv, + "telemetry_qualifying_features": efficacy_qualifying, + "minimum_balanced_accuracy": MIN_EFFICACY_BALANCED_ACCURACY, + "minimum_delta_over_best_outcome": MIN_EFFICACY_DELTA_OVER_OUTCOME, + "feasibility_transitions": dict(sorted(transitions.items())), + }, + "sanity": { + "trials": len(trials), + "action_pairs": len(actions), + "repeat_pairs": len(repeats), + "invariants": invariants, + "red_flags": red_flags, + }, + } + + +def audit(*, run_root: Path, manifest_path: Path, output_path: Path) -> dict[str, Any]: + trials_by_horizon, streams = load_trials(run_root) + manifest_validation = validate_manifest( + trials_by_horizon[min(trials_by_horizon)], manifest_path + ) + horizons = { + str(int(horizon)): analyze_horizon(trials, horizon) + for horizon, trials in sorted(trials_by_horizon.items()) + } + red_flags = sorted( + { + flag + for horizon in horizons.values() + for flag in horizon["sanity"]["red_flags"] + } + ) + stable_response = sorted( + set.intersection( + *( + set(horizon["qualifying_response_features"]) + for horizon in horizons.values() + ) + ) + ) + stable_efficacy = sorted( + set.intersection( + *( + set(horizon["efficacy"]["telemetry_qualifying_features"]) + for horizon in horizons.values() + ) + ) + ) + if red_flags: + decision = "STOP_DATA_INVALID" + elif len(stable_response) < V0.MIN_STABLE_FEATURES: + decision = "STOP_NO_PROSPECTIVE_RESPONSE" + elif not stable_efficacy: + decision = "STOP_NO_INCREMENTAL_TUNING_SIGNAL" + else: + decision = "OPEN_MATCHED_GPU_PILOT" + payload = { + "schema": SCHEMA, + "status": "COMPLETE", + "decision": decision, + "claim_boundary": ( + "Development-only confirmation on an already-consumed P1 task. " + "Passing can open a newly registered matched pilot but cannot be " + "reported as held-out tuning evidence." + ), + "frozen_gate": { + "response_thresholds_identical_to_phase6_v0": True, + "expected_action_pairs": EXPECTED_ACTION_PAIRS, + "expected_repeat_pairs": EXPECTED_REPEAT_PAIRS, + "minimum_stable_response_features": V0.MIN_STABLE_FEATURES, + "minimum_efficacy_class": MIN_EFFICACY_CLASS, + "minimum_efficacy_balanced_accuracy": MIN_EFFICACY_BALANCED_ACCURACY, + "minimum_efficacy_delta_over_best_outcome": ( + MIN_EFFICACY_DELTA_OVER_OUTCOME + ), + }, + "stable_response_features": stable_response, + "stable_incremental_efficacy_features": stable_efficacy, + "horizons": horizons, + "provenance": { + "analysis_script": str(Path(__file__).resolve()), + "analysis_script_sha256": sha256_file(Path(__file__).resolve()), + "phase6_v0_script_sha256": sha256_file(HERE / "analyze_phase6.py"), + "run_root": str(run_root.resolve()), + "manifest": str(manifest_path.resolve()), + "manifest_sha256": sha256_file(manifest_path), + "manifest_validation": manifest_validation, + "streams": streams, + }, + "sanity": { + "stream_count": len(streams), + "stream_bytes": V0.numeric(item["bytes"] for item in streams), + "red_flags": red_flags, + }, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + return payload + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + payload = audit( + run_root=args.run_root, + manifest_path=args.manifest, + output_path=args.output, + ) + print( + json.dumps( + { + "decision": payload["decision"], + "stable_response_features": payload["stable_response_features"], + "stable_incremental_efficacy_features": payload[ + "stable_incremental_efficacy_features" + ], + "sanity": payload["sanity"], + }, + indent=2, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/intervention-response-v0/analyze_phase6.py b/runs/intervention-response-v0/analyze_phase6.py new file mode 100644 index 0000000..900bdb5 --- /dev/null +++ b/runs/intervention-response-v0/analyze_phase6.py @@ -0,0 +1,520 @@ +#!/usr/bin/env python3 +"""Audit whether a controlled knob change produces identifiable telemetry deltas. + +This is a development-only feasibility audit. It compares adjacent MNS +interventions at an identical TP, offered-load anchor, and request sequence +against same-config primary/confirmation repeat noise. It does not claim that +the observed response is causal or that it improves an end-to-end tuner. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import sys +from collections import defaultdict +from pathlib import Path +from statistics import median +from typing import Any, Iterable, Mapping + + +HERE = Path(__file__).resolve().parent +COMMON_STATE_DIR = HERE.parent / "telemetry-residual" +sys.path.insert(0, str(COMMON_STATE_DIR)) + +from common_state import load_jsonl, summarize_engine # noqa: E402 + + +SCHEMA = "intervention-response-audit-v0" +HORIZONS_S = (5.0, 10.0) +GATE_FEATURES = ( + "scheduler_steps_per_s", + "decode_batch_size.mean", + "prefill_token_fraction", + "queue_waiting_mean", + "queue_running_mean", + "kv_usage_mean", + "graph_padding_fraction", +) +ALL_FEATURES = ( + "scheduler_steps_per_s", + "batch_size.mean", + "batch_tokens.mean", + "decode_batch_size.mean", + "prefill_token_fraction", + "queue_waiting_mean", + "queue_running_mean", + "preemptions", + "kv_usage_mean", + "kv_usage_max", + "kv_usage_end_minus_start", + "graph_none_share", + "graph_full_share", + "graph_padding_fraction", +) +EXPECTED_ACTION_PAIRS = 17 +MIN_REPEAT_PAIRS = 20 +MIN_STABLE_FEATURES = 2 +MIN_SIGN_CONSISTENCY = 0.75 +MIN_EFFECT_TO_NOISE = 2.0 +MIN_ABOVE_NOISE_P95_FRACTION = 0.5 + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def numeric(values: Iterable[float]) -> dict[str, Any]: + finite = [float(value) for value in values] + if not finite: + raise ValueError("numeric summary requires at least one value") + if any(not math.isfinite(value) for value in finite): + raise ValueError("numeric summary received a non-finite value") + return { + "n": len(finite), + "min": min(finite), + "max": max(finite), + "distinct_n": len(set(finite)), + } + + +def quantile(values: Iterable[float], probability: float) -> float: + ordered = sorted(float(value) for value in values) + if not ordered: + raise ValueError("quantile requires at least one value") + if not 0.0 <= probability <= 1.0: + raise ValueError("quantile probability must be in [0, 1]") + position = probability * (len(ordered) - 1) + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + weight = position - lower + return ordered[lower] * (1.0 - weight) + ordered[upper] * weight + + +def flatten_state(summary: Mapping[str, Any]) -> dict[str, float]: + common = summary["common"] + engine = summary["engine_only"] + state = { + "scheduler_steps_per_s": float(common["scheduler_steps_per_s"]), + "batch_size.mean": float(common["batch_size"]["mean"]), + "batch_tokens.mean": float(common["batch_tokens"]["mean"]), + "decode_batch_size.mean": float(common["decode_batch_size"]["mean"]), + "prefill_token_fraction": float(common["prefill_token_fraction"]), + "queue_waiting_mean": float(common["queue_waiting_mean"]), + "queue_running_mean": float(common["queue_running_mean"]), + "preemptions": float(common["preemptions"]), + "kv_usage_mean": float(engine["kv_usage_mean"]), + "kv_usage_max": float(engine["kv_usage_max"]), + "kv_usage_end_minus_start": float(engine["kv_usage_end_minus_start"]), + "graph_none_share": float(engine["graph_none_share"]), + "graph_full_share": float(engine["graph_full_share"]), + "graph_padding_fraction": float(engine["graph_padding_fraction"]), + } + if set(state) != set(ALL_FEATURES): + raise ValueError("flattened state does not match the frozen feature set") + if any(not math.isfinite(value) for value in state.values()): + raise ValueError("flattened state contains a non-finite value") + return state + + +def _trial_role(path: Path) -> str: + return "confirmation" if path.parent.name.startswith("confirm-") else "primary" + + +def load_trials( + raw_root: Path, + *, + horizons_s: tuple[float, ...] = HORIZONS_S, +) -> tuple[dict[float, list[dict[str, Any]]], list[dict[str, Any]]]: + by_horizon = {horizon: [] for horizon in horizons_s} + stream_provenance = [] + for cell_dir in sorted(path for path in raw_root.iterdir() if path.is_dir()): + streams = sorted((cell_dir / "opprof").glob("*.jsonl")) + if len(streams) != 1: + raise ValueError(f"{cell_dir}: expected exactly one Layer-1 stream") + stream = streams[0] + records = load_jsonl(stream) + stream_provenance.append( + { + "path": str(stream), + "sha256": sha256_file(stream), + "bytes": stream.stat().st_size, + } + ) + result_paths = sorted(cell_dir.glob("anchor-*/result.json")) + result_paths.extend(sorted(cell_dir.glob("confirm-*-anchor-*/result.json"))) + for result_path in result_paths: + result = json.loads(result_path.read_text(encoding="utf-8")) + start_ns = int(result["interval"]["start_mono_ns"]) + elapsed_s = float(result["interval"]["elapsed_s"]) + for horizon_s in horizons_s: + if elapsed_s < horizon_s: + raise ValueError( + f"{result_path}: elapsed {elapsed_s} is shorter than {horizon_s}s" + ) + state = flatten_state( + summarize_engine( + records, + start_ns=start_ns, + end_ns=start_ns + int(horizon_s * 1e9), + request_count=int(result["selection"]["count"]), + ) + ) + by_horizon[horizon_s].append( + { + "trial_id": str(result_path.relative_to(raw_root)), + "result_sha256": sha256_file(result_path), + "role": _trial_role(result_path), + "cell": str(result["cell"]), + "study_sha256": str(result["study_sha256"]), + "tp": int(result["tp"]), + "mns": int(result["mns"]), + "anchor": float(result["anchor"]), + "request_hash": str( + result["selection"]["request_id_order_sha256"] + ), + "request_count": int(result["selection"]["count"]), + "early_stopped": bool(result["early_stopped"]), + "full_pass_rate": float(result["pass_rate"]), + "full_feasible": bool(result["feasible"]), + "state": state, + } + ) + return by_horizon, stream_provenance + + +def _group_key(trial: Mapping[str, Any]) -> tuple[Any, ...]: + return ( + trial["study_sha256"], + trial["tp"], + trial["anchor"], + trial["request_hash"], + ) + + +def _delta(source: Mapping[str, Any], target: Mapping[str, Any]) -> dict[str, float]: + return { + feature: float(target["state"][feature]) - float(source["state"][feature]) + for feature in ALL_FEATURES + } + + +def _pair(source: Mapping[str, Any], target: Mapping[str, Any], kind: str) -> dict[str, Any]: + if _group_key(source) != _group_key(target): + raise ValueError("pair endpoints do not share workload identity") + return { + "kind": kind, + "group": { + "study_sha256": source["study_sha256"], + "tp": source["tp"], + "anchor": source["anchor"], + "request_hash": source["request_hash"], + }, + "source": { + "trial_id": source["trial_id"], + "cell": source["cell"], + "mns": source["mns"], + "early_stopped": source["early_stopped"], + "full_pass_rate": source["full_pass_rate"], + "full_feasible": source["full_feasible"], + }, + "target": { + "trial_id": target["trial_id"], + "cell": target["cell"], + "mns": target["mns"], + "early_stopped": target["early_stopped"], + "full_pass_rate": target["full_pass_rate"], + "full_feasible": target["full_feasible"], + }, + "delta_state": _delta(source, target), + "descriptive_full_outcome": { + "delta_pass_rate": target["full_pass_rate"] - source["full_pass_rate"], + "feasibility_transition": ( + f"{str(source['full_feasible']).lower()}->" + f"{str(target['full_feasible']).lower()}" + ), + }, + } + + +def build_pairs( + trials: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + primary_groups: dict[tuple[Any, ...], list[dict[str, Any]]] = defaultdict(list) + primary_by_cell_anchor: dict[tuple[Any, ...], dict[str, Any]] = {} + confirmations = [] + for trial in trials: + if trial["role"] == "primary": + primary_groups[_group_key(trial)].append(trial) + primary_by_cell_anchor[ + (trial["cell"], trial["anchor"], trial["request_hash"]) + ] = trial + else: + confirmations.append(trial) + + actions = [] + for group in primary_groups.values(): + ordered = sorted(group, key=lambda item: item["mns"]) + for source, target in zip(ordered, ordered[1:], strict=False): + if target["mns"] == source["mns"] * 2: + actions.append(_pair(source, target, "mns_increase")) + + repeats = [] + for confirmation in confirmations: + key = ( + confirmation["cell"], + confirmation["anchor"], + confirmation["request_hash"], + ) + primary = primary_by_cell_anchor.get(key) + if primary is None: + raise ValueError(f"{confirmation['trial_id']}: missing matched primary") + if primary["mns"] != confirmation["mns"]: + raise ValueError("repeat endpoints changed MNS") + repeats.append(_pair(primary, confirmation, "same_config_repeat")) + return actions, repeats + + +def response_statistics( + actions: list[dict[str, Any]], + repeats: list[dict[str, Any]], +) -> dict[str, Any]: + statistics = {} + for feature in ALL_FEATURES: + action = [float(pair["delta_state"][feature]) for pair in actions] + noise = [float(pair["delta_state"][feature]) for pair in repeats] + action_abs = [abs(value) for value in action] + noise_abs = [abs(value) for value in noise] + positive = sum(value > 1e-12 for value in action) + negative = sum(value < -1e-12 for value in action) + zero = len(action) - positive - negative + nonzero = positive + negative + sign_consistency = max(positive, negative) / nonzero if nonzero else 0.0 + action_median = median(action_abs) + noise_median = median(noise_abs) + noise_p95 = quantile(noise_abs, 0.95) + effect_to_noise = ( + action_median / noise_median + if noise_median > 0 + else (math.inf if action_median > 0 else 0.0) + ) + above_noise = sum(value > noise_p95 for value in action_abs) / len(action_abs) + qualifies = ( + feature in GATE_FEATURES + and sign_consistency >= MIN_SIGN_CONSISTENCY + and effect_to_noise >= MIN_EFFECT_TO_NOISE + and above_noise >= MIN_ABOVE_NOISE_P95_FRACTION + ) + statistics[feature] = { + "action_delta": numeric(action), + "repeat_delta": numeric(noise), + "action_abs_median": action_median, + "repeat_abs_median": noise_median, + "repeat_abs_p95": noise_p95, + "effect_to_repeat_median": ( + effect_to_noise if math.isfinite(effect_to_noise) else None + ), + "effect_to_repeat_median_is_infinite": math.isinf(effect_to_noise), + "action_signs": { + "positive": positive, + "negative": negative, + "zero": zero, + "consistency": sign_consistency, + }, + "action_above_repeat_p95_fraction": above_noise, + "gate_feature": feature in GATE_FEATURES, + "qualifies": qualifies, + } + return statistics + + +def analyze_horizon(trials: list[dict[str, Any]], horizon_s: float) -> dict[str, Any]: + actions, repeats = build_pairs(trials) + feature_statistics = response_statistics(actions, repeats) + qualifying = sorted( + feature for feature, item in feature_statistics.items() if item["qualifies"] + ) + all_values = [ + value + for trial in trials + for value in trial["state"].values() + ] + action_vectors = { + tuple(round(float(pair["delta_state"][feature]), 12) for feature in ALL_FEATURES) + for pair in actions + } + pair_invariants = { + "expected_action_pair_count": len(actions) == EXPECTED_ACTION_PAIRS, + "sufficient_repeat_pair_count": len(repeats) >= MIN_REPEAT_PAIRS, + "all_pair_hashes_match": all( + pair["group"]["request_hash"] for pair in [*actions, *repeats] + ), + "all_values_finite": all(math.isfinite(value) for value in all_values), + "state_vectors_not_all_identical": len(action_vectors) > 1, + "ratios_bounded": all( + 0.0 <= trial["state"][feature] <= 1.0 + for trial in trials + for feature in ( + "prefill_token_fraction", + "kv_usage_mean", + "kv_usage_max", + "graph_none_share", + "graph_full_share", + "graph_padding_fraction", + ) + ), + "nonnegative_counters": all( + trial["state"][feature] >= 0.0 + for trial in trials + for feature in ( + "scheduler_steps_per_s", + "batch_size.mean", + "batch_tokens.mean", + "decode_batch_size.mean", + "queue_waiting_mean", + "queue_running_mean", + "preemptions", + ) + ), + } + red_flags = [name for name, passed in pair_invariants.items() if not passed] + pass_deltas = [ + pair["descriptive_full_outcome"]["delta_pass_rate"] for pair in actions + ] + transitions = defaultdict(int) + for pair in actions: + transitions[pair["descriptive_full_outcome"]["feasibility_transition"]] += 1 + return { + "horizon_s": horizon_s, + "actions": actions, + "repeats": repeats, + "feature_statistics": feature_statistics, + "qualifying_features": qualifying, + "descriptive_full_outcome": { + "delta_pass_rate": numeric(pass_deltas), + "positive": sum(value > 1e-12 for value in pass_deltas), + "negative": sum(value < -1e-12 for value in pass_deltas), + "zero": sum(abs(value) <= 1e-12 for value in pass_deltas), + "feasibility_transitions": dict(sorted(transitions.items())), + "limitation": ( + "Full outcomes may use different elapsed durations when a trial " + "early-stopped; they are descriptive and are not a gate input." + ), + }, + "sanity": { + "trials": len(trials), + "action_pairs": len(actions), + "repeat_pairs": len(repeats), + "distinct_action_vectors": len(action_vectors), + "invariants": pair_invariants, + "red_flags": red_flags, + }, + } + + +def audit( + *, + metrics_path: Path, + raw_root: Path, + output_path: Path, +) -> dict[str, Any]: + trials_by_horizon, streams = load_trials(raw_root) + horizons = { + str(int(horizon)): analyze_horizon(trials, horizon) + for horizon, trials in sorted(trials_by_horizon.items()) + } + red_flags = sorted( + { + red_flag + for horizon in horizons.values() + for red_flag in horizon["sanity"]["red_flags"] + } + ) + stable_features = sorted( + set.intersection( + *(set(horizon["qualifying_features"]) for horizon in horizons.values()) + ) + ) + if red_flags: + decision = "STOP_DATA_INVALID" + elif len(stable_features) < MIN_STABLE_FEATURES: + decision = "STOP_NO_IDENTIFIABLE_RESPONSE" + else: + decision = "OPEN_MATCHED_PILOT" + payload = { + "schema": SCHEMA, + "status": "COMPLETE", + "decision": decision, + "claim_boundary": ( + "Development-only identifiability gate. Passing opens a controlled " + "real-GPU pilot; it does not establish tuning benefit or causality." + ), + "frozen_gate": { + "horizons_s": list(HORIZONS_S), + "expected_action_pairs": EXPECTED_ACTION_PAIRS, + "minimum_repeat_pairs": MIN_REPEAT_PAIRS, + "minimum_stable_features": MIN_STABLE_FEATURES, + "minimum_sign_consistency": MIN_SIGN_CONSISTENCY, + "minimum_effect_to_repeat_median": MIN_EFFECT_TO_NOISE, + "minimum_action_above_repeat_p95_fraction": ( + MIN_ABOVE_NOISE_P95_FRACTION + ), + "gate_features": list(GATE_FEATURES), + }, + "stable_qualifying_features": stable_features, + "horizons": horizons, + "provenance": { + "analysis_script": str(Path(__file__).resolve()), + "analysis_script_sha256": sha256_file(Path(__file__).resolve()), + "phase6_metrics": str(metrics_path.resolve()), + "phase6_metrics_sha256": sha256_file(metrics_path), + "raw_root": str(raw_root.resolve()), + "streams": streams, + }, + "sanity": { + "stream_count": len(streams), + "stream_bytes": numeric(item["bytes"] for item in streams), + "red_flags": red_flags, + }, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + return payload + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--metrics", type=Path, required=True) + parser.add_argument("--raw-root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + payload = audit( + metrics_path=args.metrics, + raw_root=args.raw_root, + output_path=args.output, + ) + print( + json.dumps( + { + "decision": payload["decision"], + "stable_qualifying_features": payload[ + "stable_qualifying_features" + ], + "sanity": payload["sanity"], + }, + indent=2, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/intervention-response-v0/p1-audit.json b/runs/intervention-response-v0/p1-audit.json new file mode 100644 index 0000000..7164d08 --- /dev/null +++ b/runs/intervention-response-v0/p1-audit.json @@ -0,0 +1,6818 @@ +{ + "claim_boundary": "Development-only confirmation on an already-consumed P1 task. Passing can open a newly registered matched pilot but cannot be reported as held-out tuning evidence.", + "decision": "STOP_NO_PROSPECTIVE_RESPONSE", + "frozen_gate": { + "expected_action_pairs": 12, + "expected_repeat_pairs": 24, + "minimum_efficacy_balanced_accuracy": 0.75, + "minimum_efficacy_class": 4, + "minimum_efficacy_delta_over_best_outcome": 0.15, + "minimum_stable_response_features": 2, + "response_thresholds_identical_to_phase6_v0": true + }, + "horizons": { + "10": { + "actions": [ + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.04424092126093565, + "tpot_mean_over_slo": 0.008821253352839065, + "ttft_max_over_slo_max": 0.001164793997304514, + "ttft_mean_over_slo_max": -0.0007446444380963049 + }, + "delta_state": { + "batch_size.mean": 0.028437757317164802, + "batch_tokens.mean": -0.16003443371509718, + "decode_batch_size.mean": 0.02853132719514928, + "graph_full_share": -0.0012762931357137575, + "graph_none_share": -0.0069204281757616545, + "graph_padding_fraction": 0.006029111786335234, + "kv_usage_end_minus_start": 0.00036424504378218536, + "kv_usage_max": 0.009254174352690048, + "kv_usage_mean": 0.00277340001356393, + "preemptions": 0.0, + "prefill_token_fraction": -0.0003954091503666213, + "queue_running_mean": 0.21264213909999974, + "queue_waiting_mean": -0.073977916, + "scheduler_steps_per_s": 0.20000000000000284 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "false->false", + "group": { + "level": "high", + "offered_rate_per_gpu": 2.9833333333333334, + "replicate": 1, + "request_hash": "3f0ab9be34dbee884f5dbe2135b983d31a47578a43f606a5f390b0528dbedbe3", + "tp": 1 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp1_mns8", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.20670391061452514, + "mns": 8, + "requests_sha256": "5f5bfa4a16846190f488d3369eed89ae895804145cea59197e6b544b703223fc", + "result_sha256": "f65e3dc91d17f5caab1f4aecf97d2c464b99f107231af2cfd96f5484880136ee", + "trial_id": "cells/tp1_mns8/high-rep1/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.24581005586592178, + "mns": 64, + "requests_sha256": "e970dcac20144cb1e13a35997377f6695a9c57d788a88bad7f18cf6c64e35cec", + "result_sha256": "3233686b60b920dcd6cc2f45193a81f3af2d46f781ef42c4d117630612084feb", + "trial_id": "cells/tp1_mns64/high-rep1/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.032258064516129004, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.03225806451612906, + "tpot_max_over_slo": 0.33050636819556467, + "tpot_mean_over_slo": 0.2980276034718437, + "ttft_max_over_slo_max": -0.14499169250484553, + "ttft_mean_over_slo_max": -0.08870032973580096 + }, + "delta_state": { + "batch_size.mean": 4.5469521321594915, + "batch_tokens.mean": 177.93791776288268, + "decode_batch_size.mean": 4.490206104102559, + "graph_full_share": -0.049141465319289424, + "graph_none_share": 0.04533918395046808, + "graph_padding_fraction": 0.004209922294522723, + "kv_usage_end_minus_start": 0.05537535994708087, + "kv_usage_max": 0.08922043547542868, + "kv_usage_mean": 0.04639671324585863, + "preemptions": 0.0, + "prefill_token_fraction": 0.003967110353963799, + "queue_running_mean": 4.6810500217, + "queue_waiting_mean": -2.3262808059, + "scheduler_steps_per_s": -14.599999999999998 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "false->false", + "group": { + "level": "high", + "offered_rate_per_gpu": 2.9833333333333334, + "replicate": 2, + "request_hash": "b19c91876a8417fe06cb2775f65a1111b202f88ce10ec33576af3d84c059f02a", + "tp": 1 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp1_mns8", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.7988826815642458, + "mns": 8, + "requests_sha256": "e802d09057de349e5164c18314fcad09dc88c9673834fd6e5b100a57dd12fbf4", + "result_sha256": "4835a985dd6bb8dc50b53737baa980ac95a49e22f0b8a7326e1fb416c9b4e798", + "trial_id": "cells/tp1_mns8/high-rep2/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.88268156424581, + "mns": 64, + "requests_sha256": "ec06878e2aefdc4853d972197cedc29ab94df7147abb2701e0d88b5dabf29728", + "result_sha256": "8215477a014f061c55e94d879a25bbf1676ee2ef1c5e81efadd23d20d337e07a", + "trial_id": "cells/tp1_mns64/high-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.018804627718169264, + "tpot_mean_over_slo": 0.011575705913082518, + "ttft_max_over_slo_max": 0.0012841941691779934, + "ttft_mean_over_slo_max": -0.0028507176665395206 + }, + "delta_state": { + "batch_size.mean": 0.028484697519924218, + "batch_tokens.mean": 0.386581442994256, + "decode_batch_size.mean": 0.02831307423779572, + "graph_full_share": -0.00017162328212849687, + "graph_none_share": -0.003114247473621861, + "graph_padding_fraction": 0.003200143463807169, + "kv_usage_end_minus_start": 8.07298706054782e-05, + "kv_usage_max": 0.0034809583857736692, + "kv_usage_mean": 0.0010653256527363222, + "preemptions": 0.0, + "prefill_token_fraction": -0.00013286942159473902, + "queue_running_mean": 0.1416205103000001, + "queue_waiting_mean": -0.028138744, + "scheduler_steps_per_s": -0.6000000000000085 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "false->false", + "group": { + "level": "high", + "offered_rate_per_gpu": 2.9833333333333334, + "replicate": 3, + "request_hash": "a85849f1f057a67c82523cff57a52af22d2eb9514a953e076990aa6c25c1ef7e", + "tp": 1 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp1_mns8", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.6424581005586593, + "mns": 8, + "requests_sha256": "cf1226185845cd8aa4e7ead0cb5c95c77eda62b84851a2ab579f984bbb88e8ac", + "result_sha256": "95d5103f2678eeb99c258870324566932e2f2853c3b6e0ee94bc6c538f6ed865", + "trial_id": "cells/tp1_mns8/high-rep3/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.9273743016759777, + "mns": 64, + "requests_sha256": "18d72d1fe16136359e28a1909e395ffd553dbe3edd2c0fd93a9cc270a5ec6d60", + "result_sha256": "480a3b50acaf9b9c1902e2669cf3aa7723a6703a00778da269b5b9757990ceef", + "trial_id": "cells/tp1_mns64/high-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": -0.0040397713397167245, + "tpot_mean_over_slo": -0.007906587129236364, + "ttft_max_over_slo_max": -0.00039426567188155515, + "ttft_mean_over_slo_max": -0.0011763758957386017 + }, + "delta_state": { + "batch_size.mean": -0.08160779537149798, + "batch_tokens.mean": -1.7609001853017006, + "decode_batch_size.mean": -0.08094394190416265, + "graph_full_share": 0.0006306607939687847, + "graph_none_share": -0.005369997286859742, + "graph_padding_fraction": 0.0002696603812660705, + "kv_usage_end_minus_start": 9.361443848254769e-05, + "kv_usage_max": 0.0003780815386457137, + "kv_usage_mean": 2.4214752986723154e-05, + "preemptions": 0.0, + "prefill_token_fraction": -6.638371684375421e-05, + "queue_running_mean": -0.0994468485000004, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 2.3000000000000114 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 2.033333333333333, + "replicate": 1, + "request_hash": "c8ea9278f180ebb23cee595e167627cbed711032e70c9ca5ddc7205320cd75c0", + "tp": 1 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 8, + "requests_sha256": "0edc6095b16e5361a88b2765cc5d0f267ed16ff1d883e02fc3c58f57e59c1018", + "result_sha256": "8bc433109d148c21b7d158df95ec5d1fa4fa619243114563068e1e98c4241a29", + "trial_id": "cells/tp1_mns8/low-rep1/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "44d729c1ace9b09fe2ea4c4e110bd0f95fee69f83edc78ee0f65256922dcd2c2", + "result_sha256": "d2c08e71c5d4e0dd99662bab8457bd1ad0a4aba6bda57794dbfce62e36720f62", + "trial_id": "cells/tp1_mns64/low-rep1/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.08836641039618653, + "tpot_mean_over_slo": 0.05249869848340616, + "ttft_max_over_slo_max": -0.03703611367382109, + "ttft_mean_over_slo_max": -0.011082230367607901 + }, + "delta_state": { + "batch_size.mean": 0.2702547536010833, + "batch_tokens.mean": 7.3674632893748395, + "decode_batch_size.mean": 0.2679754243751127, + "graph_full_share": -0.000425589040939367, + "graph_none_share": -0.003064812355221179, + "graph_padding_fraction": 0.008784696835623326, + "kv_usage_end_minus_start": 2.2346672411965862e-05, + "kv_usage_max": 0.018441793350708946, + "kv_usage_mean": 0.009225433346662204, + "preemptions": 0.0, + "prefill_token_fraction": 2.292768411660262e-05, + "queue_running_mean": 0.7380613438000001, + "queue_waiting_mean": -0.2325565183, + "scheduler_steps_per_s": -3.8000000000000043 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 2.033333333333333, + "replicate": 2, + "request_hash": "1f27fe31ca32fd67f2f70a54eee83146329a7838ff2a33be48ce4372640a858c", + "tp": 1 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 8, + "requests_sha256": "ca58a878401ac44cca0a9a595eab41659dafec2e923f7f2474e4f82ea30b2f3e", + "result_sha256": "42a56af136d1d3bdfc16d9cea092c2db0eaa4db2c85dceb21958c5e418122900", + "trial_id": "cells/tp1_mns8/low-rep2/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "97de3cced64fd88a5c2dcd087ca6c29d8b09a0a43d7bf0e4e844ef246462465b", + "result_sha256": "f194db2f7abb995171cd24626e4e515c000481f81283c929b9a5b63f110f1506", + "trial_id": "cells/tp1_mns64/low-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": -0.0020900686586818318, + "tpot_mean_over_slo": -0.0010018654846283281, + "ttft_max_over_slo_max": 0.0005588701654535982, + "ttft_mean_over_slo_max": 6.347352852269228e-05 + }, + "delta_state": { + "batch_size.mean": -0.012547944623296114, + "batch_tokens.mean": -0.15241343405688212, + "decode_batch_size.mean": -0.012475695661875008, + "graph_full_share": 7.224896142110637e-05, + "graph_none_share": -0.0058946652641868255, + "graph_padding_fraction": 0.00029565151617282764, + "kv_usage_end_minus_start": 0.00011133071931368477, + "kv_usage_max": 0.0002683613903166071, + "kv_usage_mean": -1.5294343296619328e-05, + "preemptions": 0.0, + "prefill_token_fraction": 9.183515573107925e-05, + "queue_running_mean": -0.008641511400000024, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.20000000000000284 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 2.033333333333333, + "replicate": 3, + "request_hash": "a46c62ebd8b7df5534831feb70aed45d23b1a5e7f8bf3d4cd82a79f182fbc69e", + "tp": 1 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 8, + "requests_sha256": "8a72240f84bbeff9eabd63e7d1f7a4c19910660fc5e4335013a1deaf89408127", + "result_sha256": "0214f32cabbcdc1bad6c331c3945c505cad136fcc9a8336aeb6314b27474e279", + "trial_id": "cells/tp1_mns8/low-rep3/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "30f4623100c58580cfa4efe9a1998c63581c8e44511c556ef39cf5bd777883bf", + "result_sha256": "0be457aa2fc9ce95bdb92a6eb25544f5f61a9739932ee5afa40535d26456e348", + "trial_id": "cells/tp1_mns64/low-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.008, + "completed_over_admitted": -0.1610169491525424, + "completed_pass_rate": 0.08823529411764708, + "outstanding_over_admitted": 0.1610169491525424, + "tpot_max_over_slo": 0.29320454629189446, + "tpot_mean_over_slo": 0.2535434351882483, + "ttft_max_over_slo_max": -0.2690165124949999, + "ttft_mean_over_slo_max": -0.13813761862324808 + }, + "delta_state": { + "batch_size.mean": 10.093788314560394, + "batch_tokens.mean": 434.02936828958076, + "decode_batch_size.mean": 9.924284699384675, + "graph_full_share": -0.09005855324219958, + "graph_none_share": 0.07778357071650377, + "graph_padding_fraction": 0.0017247330951379231, + "kv_usage_end_minus_start": 0.05233364294717768, + "kv_usage_max": 0.049202174029810086, + "kv_usage_mean": 0.019156054903148284, + "preemptions": 0.0, + "prefill_token_fraction": 0.013365962658000052, + "queue_running_mean": 20.948292951200003, + "queue_waiting_mean": -13.1590807183, + "scheduler_steps_per_s": -34.5 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "high", + "offered_rate_per_gpu": 3.125, + "replicate": 1, + "request_hash": "5240182174ec5e25247be01c017cbefe2f1e59426bb24a57746dc40be4a82208", + "tp": 4 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.10666666666666667, + "mns": 16, + "requests_sha256": "8695422a0f1e0e1c097a08a7f4c082f1ba2e3bb26b58caadae0afc7438aeca90", + "result_sha256": "c54a2bb5c1b78bdef3fa09c39f5974215370d6fc7c2b6d403a3d376d53a1d9eb", + "trial_id": "cells/tp4_mns16/high-rep1/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "b4aeaf5b3175bd85fdac08a2ef657fb2278c6c4b36ea32ea417275ab6d55812d", + "result_sha256": "111a1042b614d05772888665a7cfbc86925b2fbeb52ffe19edb62a5bcfe2f513", + "trial_id": "cells/tp4_mns64/high-rep1/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.012, + "completed_over_admitted": -0.1578947368421053, + "completed_pass_rate": 0.13636363636363635, + "outstanding_over_admitted": 0.1578947368421053, + "tpot_max_over_slo": 0.4737703299230537, + "tpot_mean_over_slo": 0.3241796650751651, + "ttft_max_over_slo_max": -0.3133456981643879, + "ttft_mean_over_slo_max": -0.1578698176905678 + }, + "delta_state": { + "batch_size.mean": 19.42960244648318, + "batch_tokens.mean": 656.7778499176663, + "decode_batch_size.mean": 19.184206069160197, + "graph_full_share": -0.13147494707127738, + "graph_none_share": 0.11760056457304163, + "graph_padding_fraction": 0.0010721073978083185, + "kv_usage_end_minus_start": 0.04855346265723404, + "kv_usage_max": 0.042070962023699754, + "kv_usage_mean": 0.022662705875080867, + "preemptions": 0.0, + "prefill_token_fraction": 0.007938598543200115, + "queue_running_mean": 24.0049637595, + "queue_waiting_mean": -15.5760687653, + "scheduler_steps_per_s": -32.900000000000006 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "high", + "offered_rate_per_gpu": 3.125, + "replicate": 2, + "request_hash": "51be592ba9a59c8d66eb1bcc70c3d2b011c87c66128aff36574973b40389ad96", + "tp": 4 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.092, + "mns": 16, + "requests_sha256": "aa0482b10414d911af227e2df172c19ecca0add2e0024dc88f85ffbd063854bb", + "result_sha256": "91e2491de4ca0868b667e7d126b755f03bdd78378681b8070a2bbc0c38fb9055", + "trial_id": "cells/tp4_mns16/high-rep2/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "5d82b6885f5c654ae1ae126696df4bafe04f5be1c1109db24c7856b0456d9443", + "result_sha256": "6ed574c18723e265ab1327c5e43d3c76d37b3fc5711d06c77a51718e4ba16601", + "trial_id": "cells/tp4_mns64/high-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.12844036697247707, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.12844036697247707, + "tpot_max_over_slo": 0.2467615478745419, + "tpot_mean_over_slo": 0.024430165887946775, + "ttft_max_over_slo_max": -0.08022667251255673, + "ttft_mean_over_slo_max": -0.04385035177332515 + }, + "delta_state": { + "batch_size.mean": 2.1439298581773514, + "batch_tokens.mean": 116.96904413675719, + "decode_batch_size.mean": 2.0885971125866956, + "graph_full_share": -0.028466450255056452, + "graph_none_share": 0.016098373855303175, + "graph_padding_fraction": 0.004743554546834421, + "kv_usage_end_minus_start": 0.0262231865121626, + "kv_usage_max": 0.020094834170883313, + "kv_usage_mean": 0.005378376045436206, + "preemptions": 0.0, + "prefill_token_fraction": 0.005752175072525123, + "queue_running_mean": 4.9523955869, + "queue_waiting_mean": -4.365988725199999, + "scheduler_steps_per_s": -10.100000000000009 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "high", + "offered_rate_per_gpu": 3.125, + "replicate": 3, + "request_hash": "604e338cf576d187157293922dada2bceb311567268ddae74eb7a089debf0242", + "tp": 4 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.14666666666666667, + "mns": 16, + "requests_sha256": "3a23ffbe0e724648226152ef5433cc5825ad640c37cf47997c2f091210eceeec", + "result_sha256": "136dde0451f806841ff3996b081fe0bffac0a76f60c3d78b7516f2689e106008", + "trial_id": "cells/tp4_mns16/high-rep3/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "5c5861d716181f06dc3f951ae0d6ee479b1b61fc21ebf24541c5b45053fcc167", + "result_sha256": "35b8f0465f6b0c197f71b5ecaf4e771e6613f86998ae44f8701ab9e4e1e440b4", + "trial_id": "cells/tp4_mns64/high-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.03209297007177109, + "tpot_mean_over_slo": -0.01184639150074901, + "ttft_max_over_slo_max": -0.04185720401195188, + "ttft_mean_over_slo_max": -0.005481651257700743 + }, + "delta_state": { + "batch_size.mean": -0.9669213868166988, + "batch_tokens.mean": -22.564692860834526, + "decode_batch_size.mean": -0.9550554240524107, + "graph_full_share": 0.009900102186990445, + "graph_none_share": -0.015869663207727508, + "graph_padding_fraction": -0.0007445684074620391, + "kv_usage_end_minus_start": 5.549832249451825e-05, + "kv_usage_max": 0.004800695185764425, + "kv_usage_mean": -0.0003621004636635179, + "preemptions": 0.0, + "prefill_token_fraction": -0.00013559912520566542, + "queue_running_mean": -0.46276202019999957, + "queue_waiting_mean": -0.0902122614, + "scheduler_steps_per_s": 11.900000000000006 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 2.125, + "replicate": 1, + "request_hash": "6fb170f366b68a4e4e0fc976d370cb1f5b7b915d4dc22ebe5db3c6b9615afe68", + "tp": 4 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.6196078431372549, + "mns": 16, + "requests_sha256": "d7eaaefd35cc906f759dcaa7ac3c8543249a1a02f28572bec953e41e4379a29f", + "result_sha256": "11d9ec9d2a3358aed988f7b4b1014af087d61c4b5a72eb9076ae19bb5cc481cd", + "trial_id": "cells/tp4_mns16/low-rep1/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "931d372ded1e3aaa9c67f06653fac8f79b4ee4f369028c1148163580386681ce", + "result_sha256": "325ce57f33133670a6761b5dffb831ed3b2f464755ff743b56976b4a974c1215", + "trial_id": "cells/tp4_mns64/low-rep1/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.05128205128205132, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.051282051282051294, + "tpot_max_over_slo": 0.03807669731890995, + "tpot_mean_over_slo": -0.014506958738293885, + "ttft_max_over_slo_max": -0.12963614600206105, + "ttft_mean_over_slo_max": -0.048670619592321884 + }, + "delta_state": { + "batch_size.mean": -1.4718645603306726, + "batch_tokens.mean": -31.86018426395526, + "decode_batch_size.mean": -1.4608669627184998, + "graph_full_share": 0.02356281394840931, + "graph_none_share": -0.027804605789997328, + "graph_padding_fraction": 0.0035831460549140947, + "kv_usage_end_minus_start": -0.002621229743826836, + "kv_usage_max": 0.0030741209471074615, + "kv_usage_mean": -8.774771567193577e-05, + "preemptions": 0.0, + "prefill_token_fraction": 0.0001064077287293852, + "queue_running_mean": -0.40493394709999997, + "queue_waiting_mean": -2.0223754996, + "scheduler_steps_per_s": 10.599999999999994 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 2.125, + "replicate": 2, + "request_hash": "eb4c0445556d23d73d27d728b45e63fec44e17e84a4f2f341787108b7f7a4901", + "tp": 4 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.6803921568627451, + "mns": 16, + "requests_sha256": "7fdb90f679bba1747fbacc92fac2045c820e09965c755b20f6dafd4fd9e26a7a", + "result_sha256": "800e2aa6e9f9efef4be12ee778be6817364914d63ae0624775d02729089768d6", + "trial_id": "cells/tp4_mns16/low-rep2/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "72c8f70a240f79edcaa4eb0a2fe9bade8ac07093339c5b2d09c49ddd1cf27d16", + "result_sha256": "ff8b2283d02d5967893ca6225b561ae11011da6149d1f6cff7b4275c00170e1d", + "trial_id": "cells/tp4_mns64/low-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.04088999417529798, + "tpot_mean_over_slo": -0.0032529423795618906, + "ttft_max_over_slo_max": 0.0002604483355147097, + "ttft_mean_over_slo_max": -0.001620827191598239 + }, + "delta_state": { + "batch_size.mean": 0.14852247907791494, + "batch_tokens.mean": 11.553519812007579, + "decode_batch_size.mean": 0.14728805801238387, + "graph_full_share": 0.0005649503708670123, + "graph_none_share": -0.0023940560373101466, + "graph_padding_fraction": -0.00035714669425012356, + "kv_usage_end_minus_start": 0.0014602804352583654, + "kv_usage_max": 0.003991863602817669, + "kv_usage_mean": 4.656051020407101e-06, + "preemptions": 0.0, + "prefill_token_fraction": 0.0018350834191880327, + "queue_running_mean": 0.06592412640000056, + "queue_waiting_mean": -0.2173940436, + "scheduler_steps_per_s": 1.1000000000000085 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 2.125, + "replicate": 3, + "request_hash": "a660c44d1f8b9c0ada5a19285fa579871bcfcae7c6fa71649546efda9aa3baef", + "tp": 4 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.6294117647058823, + "mns": 16, + "requests_sha256": "b592fe254476957265fc9139aefdea77be4b5540f3c1b8a429c10ea6ccab9a6a", + "result_sha256": "cc1b1f7f5280c2892d0c6deac9df9f34ce126b0284e6ad893ffbf0ddbf27969f", + "trial_id": "cells/tp4_mns16/low-rep3/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "8b2c779d2097c5e4038e94e1ac9b986e29f55d9acfd67eeb366c4e0e9270f6e0", + "result_sha256": "f026b69356701894fab184e0f2f7504a30b537982e823373899c9ab123a33891", + "trial_id": "cells/tp4_mns64/low-rep3/result.json" + } + } + ], + "efficacy": { + "feasibility_transitions": { + "false->false": 3, + "false->true": 6, + "true->true": 3 + }, + "minimum_balanced_accuracy": 0.75, + "minimum_delta_over_best_outcome": 0.15, + "outcome_delta": { + "best_accuracy": 0.75, + "best_balanced_accuracy": 0.75, + "best_feature": "outstanding_over_admitted", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.006, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.004, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.004, + "train_balanced_accuracy": 0.75 + } + ] + }, + "completed_over_admitted": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.08034921574430304, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.06422018348623854, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.09507640067911716, + "train_balanced_accuracy": 0.75 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.06818181818181818, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.04411764705882354, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.04411764705882354, + "train_balanced_accuracy": 0.75 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.08034921574430307, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.06422018348623854, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.09507640067911718, + "train_balanced_accuracy": 0.75 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": 0.028440662518539606, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.025448798894970176, + "train_balanced_accuracy": 0.875 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": 0.014026599366027181, + "train_balanced_accuracy": 0.625 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.41666666666666663, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 1 + ], + "threshold": -0.0021274039320951094, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 0 + ], + "threshold": 0.018002935900514647, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.009876489314992687, + "train_balanced_accuracy": 0.75 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 1 + ], + "threshold": 0.00040965925048415397, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.00040965925048415397, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.039446658842886485, + "train_balanced_accuracy": 0.875 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.027466291070466525, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.0013986015436684204, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.0033290135767196725, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_delta": { + "best_accuracy": 0.75, + "best_balanced_accuracy": 0.75, + "best_feature": "queue_waiting_mean", + "features": { + "decode_batch_size.mean": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.0878005661250898, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 0 + ], + "threshold": 0.08790969260376658, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.5179996829782867, + "train_balanced_accuracy": 0.75 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -3.074758903864796e-05, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -4.3743156492026536e-05, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 0, + 1 + ], + "threshold": 0.0038965341747184087, + "train_balanced_accuracy": 0.875 + } + ] + }, + "kv_usage_mean": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -5.152102948427755e-05, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 0 + ], + "threshold": 0.004075888029500068, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -3.176648134260631e-05, + "train_balanced_accuracy": 0.75 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 9.912144223023223e-05, + "train_balanced_accuracy": 0.875 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 0.000963459287459556, + "train_balanced_accuracy": 0.875 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 1 + ], + "threshold": 6.466770642299391e-05, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_running_mean": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 4.816722804299999, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 2.5825188629999998, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.2521903978000002, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.1227663938, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.0820950887, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.0820950887, + "train_balanced_accuracy": 0.75 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 1 + ], + "threshold": 0.6500000000000057, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": -5.3500000000000085, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 6.450000000000003, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_qualifying_features": [] + }, + "horizon_s": 10.0, + "qualifying_response_features": [], + "repeats": [ + { + "delta_outcome": { + "admitted_fraction": 0.027932960893854747, + "admitted_input_tokens_mean_over_limit": 0.06722636258335918, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.21464019851116622, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.21464019851116628, + "tpot_max_over_slo": 0.3907463973292743, + "tpot_mean_over_slo": 0.45304077139972754, + "ttft_max_over_slo_max": -0.026048107499567152, + "ttft_mean_over_slo_max": 0.02040584175783646 + }, + "delta_state": { + "batch_size.mean": 8.048982941677576, + "batch_tokens.mean": 298.1121465228864, + "decode_batch_size.mean": 7.965265224708595, + "graph_full_share": -0.07611315423133669, + "graph_none_share": 0.08050759417399073, + "graph_padding_fraction": -0.006831997792023563, + "kv_usage_end_minus_start": 0.08232590668398432, + "kv_usage_max": 0.11409731241882304, + "kv_usage_mean": 0.08571139047521727, + "preemptions": 0.0, + "prefill_token_fraction": 0.010675483227336025, + "queue_running_mean": 7.2005321570000005, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -46.900000000000006 + }, + "group": { + "cell": "tp1_mns64", + "level": "high", + "mns": 64, + "source_replicate": 1, + "target_replicate": 2, + "tp": 1 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "e970dcac20144cb1e13a35997377f6695a9c57d788a88bad7f18cf6c64e35cec", + "result_sha256": "3233686b60b920dcd6cc2f45193a81f3af2d46f781ef42c4d117630612084feb", + "trial_id": "cells/tp1_mns64/high-rep1/result.json" + }, + "target": { + "requests_sha256": "ec06878e2aefdc4853d972197cedc29ab94df7147abb2701e0d88b5dabf29728", + "result_sha256": "8215477a014f061c55e94d879a25bbf1676ee2ef1c5e81efadd23d20d337e07a", + "trial_id": "cells/tp1_mns64/high-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.03910614525139666, + "admitted_input_tokens_mean_over_limit": -0.15364353631132394, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.31720430107526887, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.31720430107526887, + "tpot_max_over_slo": -0.4318866070898938, + "tpot_mean_over_slo": -0.4648576303649653, + "ttft_max_over_slo_max": -0.0451326228309578, + "ttft_mean_over_slo_max": -0.03512127695333522 + }, + "delta_state": { + "batch_size.mean": -8.639628684110797, + "batch_tokens.mean": -331.31332381027744, + "decode_batch_size.mean": -8.548044927723337, + "graph_full_share": 0.08397919364981532, + "graph_none_share": -0.08346278303674427, + "graph_padding_fraction": 0.0049766447911436746, + "kv_usage_end_minus_start": -0.12798481366769898, + "kv_usage_max": -0.12688580277750028, + "kv_usage_mean": -0.09580544440945148, + "preemptions": 0.0, + "prefill_token_fraction": -0.024511024899494305, + "queue_running_mean": -7.6525189056, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 65.0 + }, + "group": { + "cell": "tp1_mns64", + "level": "high", + "mns": 64, + "source_replicate": 2, + "target_replicate": 3, + "tp": 1 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "ec06878e2aefdc4853d972197cedc29ab94df7147abb2701e0d88b5dabf29728", + "result_sha256": "8215477a014f061c55e94d879a25bbf1676ee2ef1c5e81efadd23d20d337e07a", + "trial_id": "cells/tp1_mns64/high-rep2/result.json" + }, + "target": { + "requests_sha256": "18d72d1fe16136359e28a1909e395ffd553dbe3edd2c0fd93a9cc270a5ec6d60", + "result_sha256": "480a3b50acaf9b9c1902e2669cf3aa7723a6703a00778da269b5b9757990ceef", + "trial_id": "cells/tp1_mns64/high-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.008196721311475419, + "admitted_input_tokens_mean_over_limit": 0.07372581845238091, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.09285714285714286, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.09285714285714285, + "tpot_max_over_slo": 0.1702932155907418, + "tpot_mean_over_slo": 0.1739500633542023, + "ttft_max_over_slo_max": 0.02082575849878293, + "ttft_mean_over_slo_max": 0.029532855802237655 + }, + "delta_state": { + "batch_size.mean": 1.3717277486910993, + "batch_tokens.mean": 56.51418492510525, + "decode_batch_size.mean": 1.3587752164958689, + "graph_full_share": -0.010646964922293112, + "graph_none_share": 0.011895900019023516, + "graph_padding_fraction": 0.0086692890425959, + "kv_usage_end_minus_start": -0.017684084324108285, + "kv_usage_max": 0.05015486062543706, + "kv_usage_mean": 0.03571575463439401, + "preemptions": 0.0, + "prefill_token_fraction": 0.011636868678620127, + "queue_running_mean": 2.2687814585000003, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -27.10000000000001 + }, + "group": { + "cell": "tp1_mns64", + "level": "low", + "mns": 64, + "source_replicate": 1, + "target_replicate": 2, + "tp": 1 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "44d729c1ace9b09fe2ea4c4e110bd0f95fee69f83edc78ee0f65256922dcd2c2", + "result_sha256": "d2c08e71c5d4e0dd99662bab8457bd1ad0a4aba6bda57794dbfce62e36720f62", + "trial_id": "cells/tp1_mns64/low-rep1/result.json" + }, + "target": { + "requests_sha256": "97de3cced64fd88a5c2dcd087ca6c29d8b09a0a43d7bf0e4e844ef246462465b", + "result_sha256": "f194db2f7abb995171cd24626e4e515c000481f81283c929b9a5b63f110f1506", + "trial_id": "cells/tp1_mns64/low-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.02459016393442623, + "admitted_input_tokens_mean_over_limit": -0.09311379825367644, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.24411764705882344, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.24411764705882355, + "tpot_max_over_slo": -0.3084416280309926, + "tpot_mean_over_slo": -0.21178917402407615, + "ttft_max_over_slo_max": -0.07642558032724386, + "ttft_mean_over_slo_max": -0.031511784150528625 + }, + "delta_state": { + "batch_size.mean": -1.5740567152122056, + "batch_tokens.mean": -51.36471646204379, + "decode_batch_size.mean": -1.5621527698392743, + "graph_full_share": 0.008413543976771298, + "graph_none_share": -0.010745558883376392, + "graph_padding_fraction": -0.007424342975302652, + "kv_usage_end_minus_start": 0.022080127884903655, + "kv_usage_max": -0.0773803576780897, + "kv_usage_mean": -0.04564071465536516, + "preemptions": 0.0, + "prefill_token_fraction": -0.004854783915929306, + "queue_running_mean": -3.3275810848000003, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 11.400000000000006 + }, + "group": { + "cell": "tp1_mns64", + "level": "low", + "mns": 64, + "source_replicate": 2, + "target_replicate": 3, + "tp": 1 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "97de3cced64fd88a5c2dcd087ca6c29d8b09a0a43d7bf0e4e844ef246462465b", + "result_sha256": "f194db2f7abb995171cd24626e4e515c000481f81283c929b9a5b63f110f1506", + "trial_id": "cells/tp1_mns64/low-rep2/result.json" + }, + "target": { + "requests_sha256": "30f4623100c58580cfa4efe9a1998c63581c8e44511c556ef39cf5bd777883bf", + "result_sha256": "0be457aa2fc9ce95bdb92a6eb25544f5f61a9739932ee5afa40535d26456e348", + "trial_id": "cells/tp1_mns64/low-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.027932960893854747, + "admitted_input_tokens_mean_over_limit": 0.06722636258335918, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.18238213399503722, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.18238213399503722, + "tpot_max_over_slo": 0.10448095039464528, + "tpot_mean_over_slo": 0.1638344212807229, + "ttft_max_over_slo_max": 0.1201083790025829, + "ttft_mean_over_slo_max": 0.10836152705554111 + }, + "delta_state": { + "batch_size.mean": 3.5304685668352485, + "batch_tokens.mean": 120.01419432628865, + "decode_batch_size.mean": 3.503590447801186, + "graph_full_share": -0.028247982047761022, + "graph_none_share": 0.028247982047760994, + "graph_padding_fraction": -0.005012808300211052, + "kv_usage_end_minus_start": 0.027314791780685632, + "kv_usage_max": 0.0341310512960844, + "kv_usage_mean": 0.042088077242922566, + "preemptions": 0.0, + "prefill_token_fraction": 0.006312963723005605, + "queue_running_mean": 2.7321242744000003, + "queue_waiting_mean": 2.2523028899, + "scheduler_steps_per_s": -32.1 + }, + "group": { + "cell": "tp1_mns8", + "level": "high", + "mns": 8, + "source_replicate": 1, + "target_replicate": 2, + "tp": 1 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "5f5bfa4a16846190f488d3369eed89ae895804145cea59197e6b544b703223fc", + "result_sha256": "f65e3dc91d17f5caab1f4aecf97d2c464b99f107231af2cfd96f5484880136ee", + "trial_id": "cells/tp1_mns8/high-rep1/result.json" + }, + "target": { + "requests_sha256": "e802d09057de349e5164c18314fcad09dc88c9673834fd6e5b100a57dd12fbf4", + "result_sha256": "4835a985dd6bb8dc50b53737baa980ac95a49e22f0b8a7326e1fb416c9b4e798", + "trial_id": "cells/tp1_mns8/high-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.03910614525139666, + "admitted_input_tokens_mean_over_limit": -0.15364353631132394, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.28494623655913986, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.28494623655913975, + "tpot_max_over_slo": -0.12018486661249839, + "tpot_mean_over_slo": -0.17840573280620411, + "ttft_max_over_slo_max": -0.19140850950498134, + "ttft_mean_over_slo_max": -0.12097088902259666 + }, + "delta_state": { + "batch_size.mean": -4.121161249471228, + "batch_tokens.mean": -153.761987490389, + "decode_batch_size.mean": -4.086151897858574, + "graph_full_share": 0.035009351612654394, + "graph_none_share": -0.03500935161265434, + "graph_padding_fraction": 0.0059864236218592284, + "kv_usage_end_minus_start": -0.07269018359122359, + "kv_usage_max": -0.041146325687845264, + "kv_usage_mean": -0.050474056816329174, + "preemptions": 0.0, + "prefill_token_fraction": -0.020411045123935767, + "queue_running_mean": -3.1130893942, + "queue_waiting_mean": -2.2981420619, + "scheduler_steps_per_s": 51.00000000000001 + }, + "group": { + "cell": "tp1_mns8", + "level": "high", + "mns": 8, + "source_replicate": 2, + "target_replicate": 3, + "tp": 1 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "e802d09057de349e5164c18314fcad09dc88c9673834fd6e5b100a57dd12fbf4", + "result_sha256": "4835a985dd6bb8dc50b53737baa980ac95a49e22f0b8a7326e1fb416c9b4e798", + "trial_id": "cells/tp1_mns8/high-rep2/result.json" + }, + "target": { + "requests_sha256": "cf1226185845cd8aa4e7ead0cb5c95c77eda62b84851a2ab579f984bbb88e8ac", + "result_sha256": "95d5103f2678eeb99c258870324566932e2f2853c3b6e0ee94bc6c538f6ed865", + "trial_id": "cells/tp1_mns8/high-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.008196721311475419, + "admitted_input_tokens_mean_over_limit": 0.07372581845238091, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.09285714285714286, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.09285714285714285, + "tpot_max_over_slo": 0.07788703385483853, + "tpot_mean_over_slo": 0.11354477774155977, + "ttft_max_over_slo_max": 0.05746760650072247, + "ttft_mean_over_slo_max": 0.039438710274106954 + }, + "delta_state": { + "batch_size.mean": 1.019865199718518, + "batch_tokens.mean": 47.38582145042871, + "decode_batch_size.mean": 1.0098558502165935, + "graph_full_share": -0.00959071508738496, + "graph_none_share": 0.009590715087384953, + "graph_padding_fraction": 0.00015425258823864325, + "kv_usage_end_minus_start": -0.017612816558037703, + "kv_usage_max": 0.032091148813373827, + "kv_usage_mean": 0.02651453604071853, + "preemptions": 0.0, + "prefill_token_fraction": 0.01154755727765977, + "queue_running_mean": 1.4312732661999998, + "queue_waiting_mean": 0.2325565183, + "scheduler_steps_per_s": -20.999999999999993 + }, + "group": { + "cell": "tp1_mns8", + "level": "low", + "mns": 8, + "source_replicate": 1, + "target_replicate": 2, + "tp": 1 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "0edc6095b16e5361a88b2765cc5d0f267ed16ff1d883e02fc3c58f57e59c1018", + "result_sha256": "8bc433109d148c21b7d158df95ec5d1fa4fa619243114563068e1e98c4241a29", + "trial_id": "cells/tp1_mns8/low-rep1/result.json" + }, + "target": { + "requests_sha256": "ca58a878401ac44cca0a9a595eab41659dafec2e923f7f2474e4f82ea30b2f3e", + "result_sha256": "42a56af136d1d3bdfc16d9cea092c2db0eaa4db2c85dceb21958c5e418122900", + "trial_id": "cells/tp1_mns8/low-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.02459016393442623, + "admitted_input_tokens_mean_over_limit": -0.09311379825367644, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.24411764705882344, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.24411764705882355, + "tpot_max_over_slo": -0.21798514897612425, + "tpot_mean_over_slo": -0.15828861005604167, + "ttft_max_over_slo_max": -0.11402056416651855, + "ttft_mean_over_slo_max": -0.04265748804665922 + }, + "delta_state": { + "batch_size.mean": -1.2912540169878262, + "batch_tokens.mean": -43.844839738612066, + "decode_batch_size.mean": -1.2817016498022866, + "graph_full_share": 0.007915705974410825, + "graph_none_share": -0.007915705974410745, + "graph_padding_fraction": 0.0010647023441478472, + "kv_usage_end_minus_start": 0.021991143838001936, + "kv_usage_max": -0.05920692571769737, + "kv_usage_mean": -0.036399986965406333, + "preemptions": 0.0, + "prefill_token_fraction": -0.004923691387543783, + "queue_running_mean": -2.5808782296, + "queue_waiting_mean": -0.2325565183, + "scheduler_steps_per_s": 7.399999999999999 + }, + "group": { + "cell": "tp1_mns8", + "level": "low", + "mns": 8, + "source_replicate": 2, + "target_replicate": 3, + "tp": 1 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "ca58a878401ac44cca0a9a595eab41659dafec2e923f7f2474e4f82ea30b2f3e", + "result_sha256": "42a56af136d1d3bdfc16d9cea092c2db0eaa4db2c85dceb21958c5e418122900", + "trial_id": "cells/tp1_mns8/low-rep2/result.json" + }, + "target": { + "requests_sha256": "8a72240f84bbeff9eabd63e7d1f7a4c19910660fc5e4335013a1deaf89408127", + "result_sha256": "0214f32cabbcdc1bad6c331c3945c505cad136fcc9a8336aeb6314b27474e279", + "trial_id": "cells/tp1_mns8/low-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.014492753623188387, + "admitted_input_tokens_mean_over_limit": 0.04833232120973391, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.008403361344537785, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.00840336134453784, + "tpot_max_over_slo": 0.09841551118728337, + "tpot_mean_over_slo": 0.06345367037672023, + "ttft_max_over_slo_max": 0.010166719497647136, + "ttft_mean_over_slo_max": 0.006259302157370798 + }, + "delta_state": { + "batch_size.mean": 2.0365641486137616, + "batch_tokens.mean": 91.46332360534291, + "decode_batch_size.mean": 2.012753956673029, + "graph_full_share": -0.01772084409024577, + "graph_none_share": 0.021565271074194634, + "graph_padding_fraction": -0.002786564210742205, + "kv_usage_end_minus_start": 0.007622555602450509, + "kv_usage_max": 0.020987606952384663, + "kv_usage_mean": 0.007630049119681775, + "preemptions": 0.0, + "prefill_token_fraction": 0.005242973565105835, + "queue_running_mean": 0.8145702931999992, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -13.199999999999996 + }, + "group": { + "cell": "tp2_mns64", + "level": "high", + "mns": 64, + "source_replicate": 1, + "target_replicate": 2, + "tp": 2 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "19537ec05f1e40543d850964db12234561954c209d55cd8cd0a1da716ab105fc", + "result_sha256": "49523b3269db649aa70072058666da3309ecabb93242cd504b6413f5de0f5d54", + "trial_id": "cells/tp2_mns64/high-rep1/result.json" + }, + "target": { + "requests_sha256": "d95e0dcc3d3f25af11b4d3da0ca133f647fd602753b804337a7131cc56e03059", + "result_sha256": "f37a42d9ea734adcb24892f86cbb19e2c51cee0baf8a6a63c5d51c4820bd28f2", + "trial_id": "cells/tp2_mns64/high-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.005797101449275366, + "admitted_input_tokens_mean_over_limit": -0.08122977688274602, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.10543840177580466, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.10543840177580466, + "tpot_max_over_slo": -0.08790813528521968, + "tpot_mean_over_slo": -0.0976607763237542, + "ttft_max_over_slo_max": -0.031135886325500906, + "ttft_mean_over_slo_max": -0.007596982260572371 + }, + "delta_state": { + "batch_size.mean": -2.9554801420404466, + "batch_tokens.mean": -131.8397196967789, + "decode_batch_size.mean": -2.9177065756667595, + "graph_full_share": 0.026547586067947715, + "graph_none_share": -0.024236910368448883, + "graph_padding_fraction": 0.005457793019402147, + "kv_usage_end_minus_start": -0.01758559388149228, + "kv_usage_max": -0.016281062553556214, + "kv_usage_mean": -0.00956401468061617, + "preemptions": 0.0, + "prefill_token_fraction": -0.00900387036264405, + "queue_running_mean": -2.3104728836999993, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 20.099999999999994 + }, + "group": { + "cell": "tp2_mns64", + "level": "high", + "mns": 64, + "source_replicate": 2, + "target_replicate": 3, + "tp": 2 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "d95e0dcc3d3f25af11b4d3da0ca133f647fd602753b804337a7131cc56e03059", + "result_sha256": "f37a42d9ea734adcb24892f86cbb19e2c51cee0baf8a6a63c5d51c4820bd28f2", + "trial_id": "cells/tp2_mns64/high-rep2/result.json" + }, + "target": { + "requests_sha256": "ccba3d0a18a7db8501b040855bda473ee55c519619d4b9fa2a105bda06300829", + "result_sha256": "55447419925f30397d19863e149890241c58863d0adaafd511a67757905cb58e", + "trial_id": "cells/tp2_mns64/high-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.004255319148936149, + "admitted_input_tokens_mean_over_limit": 0.027061719804318485, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.002024291497975783, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0020242914979757, + "tpot_max_over_slo": 0.09163424472588544, + "tpot_mean_over_slo": 0.0001830762350235604, + "ttft_max_over_slo_max": 0.0006185861614843163, + "ttft_mean_over_slo_max": 0.0017585336856889636 + }, + "delta_state": { + "batch_size.mean": -0.4199832128448815, + "batch_tokens.mean": 6.494131535483177, + "decode_batch_size.mean": -0.4188308558741509, + "graph_full_share": 0.0010917066038501622, + "graph_none_share": 0.00011588552243251699, + "graph_padding_fraction": -0.0006875875614091621, + "kv_usage_end_minus_start": 0.009093350727084282, + "kv_usage_max": 0.011843098134008634, + "kv_usage_mean": 0.0004732259215823806, + "preemptions": 0.0, + "prefill_token_fraction": 0.005759524366025781, + "queue_running_mean": -0.24529797560000066, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 2.799999999999997 + }, + "group": { + "cell": "tp2_mns64", + "level": "low", + "mns": 64, + "source_replicate": 1, + "target_replicate": 2, + "tp": 2 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "8685f475baf247f7ce52a6a2df1183cd77a6f580cd57a3a87c407b4e96c0895f", + "result_sha256": "7a8f4e462d496bd56b73b0a686653cbe499a238df62a8c9a50db687d051f1c68", + "trial_id": "cells/tp2_mns64/low-rep1/result.json" + }, + "target": { + "requests_sha256": "016a76ab3dc015700f9ecb9f60a20fcc3b456c45270472fdd1d04e84ebd7021a", + "result_sha256": "368869326f44a0032985eff9d3abfa2306b93c25370caa05cee255a8c0208c0e", + "trial_id": "cells/tp2_mns64/low-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.008510638297872353, + "admitted_input_tokens_mean_over_limit": -0.08059763768960165, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.2543859649122807, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.2543859649122807, + "tpot_max_over_slo": -0.1787865552757903, + "tpot_mean_over_slo": -0.07545848129810456, + "ttft_max_over_slo_max": -0.024053132161498063, + "ttft_mean_over_slo_max": -0.008335181439306037 + }, + "delta_state": { + "batch_size.mean": -1.151692859112214, + "batch_tokens.mean": -39.27803694513372, + "decode_batch_size.mean": -1.1449765646539838, + "graph_full_share": 0.006457127102288407, + "graph_none_share": -0.008497380755445271, + "graph_padding_fraction": 6.114154911974513e-05, + "kv_usage_end_minus_start": 0.012533732366445394, + "kv_usage_max": -0.022906035375820144, + "kv_usage_mean": -0.00690787067830519, + "preemptions": 0.0, + "prefill_token_fraction": -0.0041771367400077475, + "queue_running_mean": -1.9580360157999994, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 14.099999999999994 + }, + "group": { + "cell": "tp2_mns64", + "level": "low", + "mns": 64, + "source_replicate": 2, + "target_replicate": 3, + "tp": 2 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "016a76ab3dc015700f9ecb9f60a20fcc3b456c45270472fdd1d04e84ebd7021a", + "result_sha256": "368869326f44a0032985eff9d3abfa2306b93c25370caa05cee255a8c0208c0e", + "trial_id": "cells/tp2_mns64/low-rep2/result.json" + }, + "target": { + "requests_sha256": "5455d1cdeabd6e2937130ed8fc8d86c1aaa6f04cbf26c3cb69ad064be5f67d73", + "result_sha256": "2de5c4524f57b487b94b4fdba35defc2a566236fc3e31f7260b7a9f85bb7007c", + "trial_id": "cells/tp2_mns64/low-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.01785714285714285, + "admitted_input_tokens_mean_over_limit": 0.0482287348533163, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.007421150278293154, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.007421150278293098, + "tpot_max_over_slo": 0.045055894329630586, + "tpot_mean_over_slo": 0.03517161669315627, + "ttft_max_over_slo_max": 0.04909342716564424, + "ttft_mean_over_slo_max": 0.07044418369208602 + }, + "delta_state": { + "batch_size.mean": 0.9503756445719365, + "batch_tokens.mean": 50.369704265273356, + "decode_batch_size.mean": 0.9415787316873265, + "graph_full_share": -0.009959703582283153, + "graph_none_share": 0.009959703582283236, + "graph_padding_fraction": -0.001694114329886807, + "kv_usage_end_minus_start": 0.0035348314893507427, + "kv_usage_max": 0.009660171254290972, + "kv_usage_mean": 0.004883298444633271, + "preemptions": 0.0, + "prefill_token_fraction": 0.007370470138318375, + "queue_running_mean": 0.27710616229999996, + "queue_waiting_mean": 1.1825361130999998, + "scheduler_steps_per_s": -17.900000000000006 + }, + "group": { + "cell": "tp2_mns8", + "level": "high", + "mns": 8, + "source_replicate": 1, + "target_replicate": 2, + "tp": 2 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "5d41a316f329e2efcc0f728cdd078377eb72fb9258738636db704bcfcca290aa", + "result_sha256": "466b3991190a232c984ba1b54fb6793cac1893dce196ee96f7238d70d49fcb74", + "trial_id": "cells/tp2_mns8/high-rep1/result.json" + }, + "target": { + "requests_sha256": "1b7d2cdcd639c00c443ae1b98287a7cc57323eb80c7f0e1bfc72f1ab5bf48760", + "result_sha256": "e9156d6871cc8cb3a20dce629cc2443df7473f3836b46eb0944d491ea5c4d44c", + "trial_id": "cells/tp2_mns8/high-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.005952380952380931, + "admitted_input_tokens_mean_over_limit": -0.08086076227365946, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.04961984793917562, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.049619847939175676, + "tpot_max_over_slo": -0.03665214141383477, + "tpot_mean_over_slo": -0.03040908016559296, + "ttft_max_over_slo_max": -0.06188676766275117, + "ttft_mean_over_slo_max": -0.061940220978646765 + }, + "delta_state": { + "batch_size.mean": -0.9308857183911066, + "batch_tokens.mean": -38.259866334430285, + "decode_batch_size.mean": -0.9247474057174099, + "graph_full_share": 0.009718264940999921, + "graph_none_share": -0.009718264941000004, + "graph_padding_fraction": 0.0021664497846041342, + "kv_usage_end_minus_start": 0.0029988642599186877, + "kv_usage_max": -0.006508173500248904, + "kv_usage_mean": -0.0037595414789287837, + "preemptions": 0.0, + "prefill_token_fraction": -0.0037648024150247483, + "queue_running_mean": -0.21647643550000062, + "queue_waiting_mean": -1.8036835434999998, + "scheduler_steps_per_s": 15.700000000000003 + }, + "group": { + "cell": "tp2_mns8", + "level": "high", + "mns": 8, + "source_replicate": 2, + "target_replicate": 3, + "tp": 2 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "1b7d2cdcd639c00c443ae1b98287a7cc57323eb80c7f0e1bfc72f1ab5bf48760", + "result_sha256": "e9156d6871cc8cb3a20dce629cc2443df7473f3836b46eb0944d491ea5c4d44c", + "trial_id": "cells/tp2_mns8/high-rep2/result.json" + }, + "target": { + "requests_sha256": "5a68fda7c29477e5b3e66d70ad2b05b77be6c200cc1beea89d663a5729f31a2b", + "result_sha256": "f2116797b35057949acbf1f57898befac9ff18b94c808462f0969bdcf2446dd6", + "trial_id": "cells/tp2_mns8/high-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.008733624454148492, + "admitted_input_tokens_mean_over_limit": 0.03586033261326227, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.07276507276507282, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.07276507276507277, + "tpot_max_over_slo": -0.00577808629105403, + "tpot_mean_over_slo": -0.014899153875042898, + "ttft_max_over_slo_max": 0.014347533161829545, + "ttft_mean_over_slo_max": 0.00013980660489499447 + }, + "delta_state": { + "batch_size.mean": -0.6370388590554663, + "batch_tokens.mean": -0.24447251544799542, + "decode_batch_size.mean": -0.6327331911137266, + "graph_full_share": 0.004305667941739033, + "graph_none_share": -0.004305667941739005, + "graph_padding_fraction": 0.0015483374551458753, + "kv_usage_end_minus_start": -0.0016334239373173576, + "kv_usage_max": 0.0027181195206921194, + "kv_usage_mean": -0.0003114508043975048, + "preemptions": 0.0, + "prefill_token_fraction": 0.005116159306570833, + "queue_running_mean": -0.5131508089000008, + "queue_waiting_mean": -0.13343640950000002, + "scheduler_steps_per_s": 7.5 + }, + "group": { + "cell": "tp2_mns8", + "level": "low", + "mns": 8, + "source_replicate": 1, + "target_replicate": 2, + "tp": 2 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "455a1c21388700f0e3234c49d15ba3b0426b0d3b551c096b297793afa0d4602f", + "result_sha256": "13de25a73cc8c752245f169cfca1d64220f697c15b835a646ba969a1005047fb", + "trial_id": "cells/tp2_mns8/low-rep1/result.json" + }, + "target": { + "requests_sha256": "cae4b764ebb8714a6f5d2009bb79497420b388986a992c51410f1681810c14c6", + "result_sha256": "92f92a576e990946c4767a2107d2ce0e6ba1b066337911399360c199fae3f5e4", + "trial_id": "cells/tp2_mns8/low-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.008733624454148464, + "admitted_input_tokens_mean_over_limit": -0.08428101999879345, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.23320463320463325, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.2332046332046332, + "tpot_max_over_slo": -0.02744940897875181, + "tpot_mean_over_slo": -0.059641868091682704, + "ttft_max_over_slo_max": -0.10803754650017558, + "ttft_mean_over_slo_max": -0.032639687448619975 + }, + "delta_state": { + "batch_size.mean": -1.203120302435217, + "batch_tokens.mean": -43.54127643086966, + "decode_batch_size.mean": -1.1934092223734076, + "graph_full_share": 0.010598391508127913, + "graph_none_share": -0.0105983915081279, + "graph_padding_fraction": 0.004406513335831962, + "kv_usage_end_minus_start": 0.00967293237880118, + "kv_usage_max": -0.008881742659163083, + "kv_usage_mean": -0.005494017047481575, + "preemptions": 0.0, + "prefill_token_fraction": -0.0054281680100684015, + "queue_running_mean": -1.7641761531999998, + "queue_waiting_mean": -0.32204582449999997, + "scheduler_steps_per_s": 15.600000000000009 + }, + "group": { + "cell": "tp2_mns8", + "level": "low", + "mns": 8, + "source_replicate": 2, + "target_replicate": 3, + "tp": 2 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "cae4b764ebb8714a6f5d2009bb79497420b388986a992c51410f1681810c14c6", + "result_sha256": "92f92a576e990946c4767a2107d2ce0e6ba1b066337911399360c199fae3f5e4", + "trial_id": "cells/tp2_mns8/low-rep2/result.json" + }, + "target": { + "requests_sha256": "cc8c01e4560411cd5b84cdc9bebdda9a86b88615ee1c39a996affe669d26c167", + "result_sha256": "c9a7d026bd89cc027c45b8f1899d56e21d53d4234e95db365a9492f670a78d36", + "trial_id": "cells/tp2_mns8/low-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.005333333333333329, + "admitted_input_tokens_mean_over_limit": 0.00313201278155667, + "completed_fail_fraction_of_total": 0.004, + "completed_over_admitted": 0.00267618198037467, + "completed_pass_rate": -0.048128342245989275, + "outstanding_over_admitted": -0.00267618198037467, + "tpot_max_over_slo": 0.01678296282344799, + "tpot_mean_over_slo": 0.03333551369285431, + "ttft_max_over_slo_max": 0.04254181500679499, + "ttft_mean_over_slo_max": 0.027600657575446386 + }, + "delta_state": { + "batch_size.mean": 2.617504152501592, + "batch_tokens.mean": 83.36000248373932, + "decode_batch_size.mean": 2.6009989288874404, + "graph_full_share": -0.030922553904903838, + "graph_none_share": 0.027604433474596013, + "graph_padding_fraction": -8.22833741022888e-05, + "kv_usage_end_minus_start": -0.001912904196656684, + "kv_usage_max": 0.0016425470701958433, + "kv_usage_mean": 0.002711903470305097, + "preemptions": 0.0, + "prefill_token_fraction": 0.0031462566114718316, + "queue_running_mean": 1.2096441719000008, + "queue_waiting_mean": 2.3781169752999993, + "scheduler_steps_per_s": -13.399999999999991 + }, + "group": { + "cell": "tp4_mns16", + "level": "high", + "mns": 16, + "source_replicate": 1, + "target_replicate": 2, + "tp": 4 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "8695422a0f1e0e1c097a08a7f4c082f1ba2e3bb26b58caadae0afc7438aeca90", + "result_sha256": "c54a2bb5c1b78bdef3fa09c39f5974215370d6fc7c2b6d403a3d376d53a1d9eb", + "trial_id": "cells/tp4_mns16/high-rep1/result.json" + }, + "target": { + "requests_sha256": "aa0482b10414d911af227e2df172c19ecca0add2e0024dc88f85ffbd063854bb", + "result_sha256": "91e2491de4ca0868b667e7d126b755f03bdd78378681b8070a2bbc0c38fb9055", + "trial_id": "cells/tp4_mns16/high-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.006666666666666654, + "admitted_input_tokens_mean_over_limit": -0.045669445146517385, + "completed_fail_fraction_of_total": -0.012, + "completed_over_admitted": 0.09077740222114916, + "completed_pass_rate": 0.13636363636363635, + "outstanding_over_admitted": -0.09077740222114916, + "tpot_max_over_slo": 0.006141297796037171, + "tpot_mean_over_slo": -0.021391237509928385, + "ttft_max_over_slo_max": -0.2410895283295152, + "ttft_mean_over_slo_max": -0.12306385203762282 + }, + "delta_state": { + "batch_size.mean": 0.023181238824859562, + "batch_tokens.mean": 2.5905518549044473, + "decode_batch_size.mean": 0.011653673102646422, + "graph_full_share": 0.0012768372268986417, + "graph_none_share": 0.001837564686330409, + "graph_padding_fraction": 0.0006975685594821312, + "kv_usage_end_minus_start": 0.001387493177308352, + "kv_usage_max": 0.0022138677902641124, + "kv_usage_mean": -0.0011780510256150529, + "preemptions": 0.0, + "prefill_token_fraction": 0.0002680116189619497, + "queue_running_mean": 0.1377117206999987, + "queue_waiting_mean": -11.247484751, + "scheduler_steps_per_s": 2.5 + }, + "group": { + "cell": "tp4_mns16", + "level": "high", + "mns": 16, + "source_replicate": 2, + "target_replicate": 3, + "tp": 4 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "aa0482b10414d911af227e2df172c19ecca0add2e0024dc88f85ffbd063854bb", + "result_sha256": "91e2491de4ca0868b667e7d126b755f03bdd78378681b8070a2bbc0c38fb9055", + "trial_id": "cells/tp4_mns16/high-rep2/result.json" + }, + "target": { + "requests_sha256": "3a23ffbe0e724648226152ef5433cc5825ad640c37cf47997c2f091210eceeec", + "result_sha256": "136dde0451f806841ff3996b081fe0bffac0a76f60c3d78b7516f2689e106008", + "trial_id": "cells/tp4_mns16/high-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0215686274509804, + "admitted_input_tokens_mean_over_limit": -0.007677676072761208, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.12342135476463834, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.12342135476463834, + "tpot_max_over_slo": 0.042659409131532156, + "tpot_mean_over_slo": 0.06205778281240598, + "ttft_max_over_slo_max": 0.09863472549477593, + "ttft_mean_over_slo_max": 0.04373689836276399 + }, + "delta_state": { + "batch_size.mean": 3.8253490040280305, + "batch_tokens.mean": 77.16682826085241, + "decode_batch_size.mean": 3.80033895365794, + "graph_full_share": -0.03715089743892919, + "graph_none_share": 0.034228013337432306, + "graph_padding_fraction": -0.0048637698153220395, + "kv_usage_end_minus_start": -0.007677122175915652, + "kv_usage_max": 0.005958058937853616, + "kv_usage_mean": 0.004083928262059737, + "preemptions": 0.0, + "prefill_token_fraction": -0.0015885934328132567, + "queue_running_mean": 4.248843562199999, + "queue_waiting_mean": 1.9316373561999998, + "scheduler_steps_per_s": -12.799999999999997 + }, + "group": { + "cell": "tp4_mns16", + "level": "low", + "mns": 16, + "source_replicate": 1, + "target_replicate": 2, + "tp": 4 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "d7eaaefd35cc906f759dcaa7ac3c8543249a1a02f28572bec953e41e4379a29f", + "result_sha256": "11d9ec9d2a3358aed988f7b4b1014af087d61c4b5a72eb9076ae19bb5cc481cd", + "trial_id": "cells/tp4_mns16/low-rep1/result.json" + }, + "target": { + "requests_sha256": "7fdb90f679bba1747fbacc92fac2045c820e09965c755b20f6dafd4fd9e26a7a", + "result_sha256": "800e2aa6e9f9efef4be12ee778be6817364914d63ae0624775d02729089768d6", + "trial_id": "cells/tp4_mns16/low-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.023529411764705882, + "admitted_input_tokens_mean_over_limit": 0.013229832504734862, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.1573426573426573, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.15734265734265732, + "tpot_max_over_slo": -0.05231123732945586, + "tpot_mean_over_slo": -0.08222875707353122, + "ttft_max_over_slo_max": -0.11858330133448666, + "ttft_mean_over_slo_max": -0.04432077170326809 + }, + "delta_state": { + "batch_size.mean": -6.136759360232643, + "batch_tokens.mean": -119.22330840733238, + "decode_batch_size.mean": -6.093614010489692, + "graph_full_share": 0.04765150334943136, + "graph_none_share": -0.049295061536064805, + "graph_padding_fraction": 0.001850754011979509, + "kv_usage_end_minus_start": 0.00735065319301953, + "kv_usage_max": -0.003932931028326259, + "kv_usage_mean": -0.00419858552745247, + "preemptions": 0.0, + "prefill_token_fraction": 0.0044424417100876745, + "queue_running_mean": -5.4834562212, + "queue_waiting_mean": -1.804526599, + "scheduler_steps_per_s": 31.299999999999997 + }, + "group": { + "cell": "tp4_mns16", + "level": "low", + "mns": 16, + "source_replicate": 2, + "target_replicate": 3, + "tp": 4 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "7fdb90f679bba1747fbacc92fac2045c820e09965c755b20f6dafd4fd9e26a7a", + "result_sha256": "800e2aa6e9f9efef4be12ee778be6817364914d63ae0624775d02729089768d6", + "trial_id": "cells/tp4_mns16/low-rep2/result.json" + }, + "target": { + "requests_sha256": "b592fe254476957265fc9139aefdea77be4b5540f3c1b8a429c10ea6ccab9a6a", + "result_sha256": "cc1b1f7f5280c2892d0c6deac9df9f34ce126b0284e6ad893ffbf0ddbf27969f", + "trial_id": "cells/tp4_mns16/low-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.005333333333333329, + "admitted_input_tokens_mean_over_limit": 0.00313201278155667, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.005798394290811748, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.005798394290811748, + "tpot_max_over_slo": 0.1973487464546072, + "tpot_mean_over_slo": 0.10397174357977113, + "ttft_max_over_slo_max": -0.001787370662592977, + "ttft_mean_over_slo_max": 0.007868458508126674 + }, + "delta_state": { + "batch_size.mean": 11.95331828442438, + "batch_tokens.mean": 306.10848411182496, + "decode_batch_size.mean": 11.860920298662965, + "graph_full_share": -0.07233894773398164, + "graph_none_share": 0.06742142733113388, + "graph_padding_fraction": -0.0007349090714318936, + "kv_usage_end_minus_start": -0.0056930844866003305, + "kv_usage_max": -0.005488664935914489, + "kv_usage_mean": 0.006218554442237678, + "preemptions": 0.0, + "prefill_token_fraction": -0.002281107503328106, + "queue_running_mean": 4.266314980200001, + "queue_waiting_mean": -0.038871071699999996, + "scheduler_steps_per_s": -11.799999999999997 + }, + "group": { + "cell": "tp4_mns64", + "level": "high", + "mns": 64, + "source_replicate": 1, + "target_replicate": 2, + "tp": 4 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "b4aeaf5b3175bd85fdac08a2ef657fb2278c6c4b36ea32ea417275ab6d55812d", + "result_sha256": "111a1042b614d05772888665a7cfbc86925b2fbeb52ffe19edb62a5bcfe2f513", + "trial_id": "cells/tp4_mns64/high-rep1/result.json" + }, + "target": { + "requests_sha256": "5d82b6885f5c654ae1ae126696df4bafe04f5be1c1109db24c7856b0456d9443", + "result_sha256": "6ed574c18723e265ab1327c5e43d3c76d37b3fc5711d06c77a51718e4ba16601", + "trial_id": "cells/tp4_mns64/high-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.006666666666666654, + "admitted_input_tokens_mean_over_limit": -0.045669445146517385, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.1202317720907774, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.1202317720907774, + "tpot_max_over_slo": -0.2208674842524746, + "tpot_mean_over_slo": -0.3211407366971467, + "ttft_max_over_slo_max": -0.007970502677684024, + "ttft_mean_over_slo_max": -0.009044386120380173 + }, + "delta_state": { + "batch_size.mean": -17.26249134948097, + "batch_tokens.mean": -537.2182539260048, + "decode_batch_size.mean": -17.083955283470857, + "graph_full_share": 0.10428533404311957, + "graph_none_share": -0.09966462603140805, + "graph_padding_fraction": 0.004369015708508233, + "kv_usage_end_minus_start": -0.020942782967763085, + "kv_usage_max": -0.019762260062552328, + "kv_usage_mean": -0.018462380855259714, + "preemptions": 0.0, + "prefill_token_fraction": -0.0019184118517130422, + "queue_running_mean": -18.914856451900004, + "queue_waiting_mean": -0.0374047109, + "scheduler_steps_per_s": 25.299999999999997 + }, + "group": { + "cell": "tp4_mns64", + "level": "high", + "mns": 64, + "source_replicate": 2, + "target_replicate": 3, + "tp": 4 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "5d82b6885f5c654ae1ae126696df4bafe04f5be1c1109db24c7856b0456d9443", + "result_sha256": "6ed574c18723e265ab1327c5e43d3c76d37b3fc5711d06c77a51718e4ba16601", + "trial_id": "cells/tp4_mns64/high-rep2/result.json" + }, + "target": { + "requests_sha256": "5c5861d716181f06dc3f951ae0d6ee479b1b61fc21ebf24541c5b45053fcc167", + "result_sha256": "35b8f0465f6b0c197f71b5ecaf4e771e6613f86998ae44f8701ab9e4e1e440b4", + "trial_id": "cells/tp4_mns64/high-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0215686274509804, + "admitted_input_tokens_mean_over_limit": -0.007677676072761208, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.17470340604668966, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.17470340604668963, + "tpot_max_over_slo": 0.048643136378671015, + "tpot_mean_over_slo": 0.059397215574861106, + "ttft_max_over_slo_max": 0.01085578350466676, + "ttft_mean_over_slo_max": 0.0005479300281428442 + }, + "delta_state": { + "batch_size.mean": 3.3204058305140567, + "batch_tokens.mean": 67.87133685773168, + "decode_batch_size.mean": 3.294527414991851, + "graph_full_share": -0.02348818567751032, + "graph_none_share": 0.022293070755162486, + "graph_padding_fraction": -0.0005360553529459057, + "kv_usage_end_minus_start": -0.010353850242237006, + "kv_usage_max": 0.004231484699196653, + "kv_usage_mean": 0.0043582810100513195, + "preemptions": 0.0, + "prefill_token_fraction": -0.001346586578878206, + "queue_running_mean": 4.306671635299999, + "queue_waiting_mean": -0.000525882, + "scheduler_steps_per_s": -14.100000000000009 + }, + "group": { + "cell": "tp4_mns64", + "level": "low", + "mns": 64, + "source_replicate": 1, + "target_replicate": 2, + "tp": 4 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "931d372ded1e3aaa9c67f06653fac8f79b4ee4f369028c1148163580386681ce", + "result_sha256": "325ce57f33133670a6761b5dffb831ed3b2f464755ff743b56976b4a974c1215", + "trial_id": "cells/tp4_mns64/low-rep1/result.json" + }, + "target": { + "requests_sha256": "72c8f70a240f79edcaa4eb0a2fe9bade8ac07093339c5b2d09c49ddd1cf27d16", + "result_sha256": "ff8b2283d02d5967893ca6225b561ae11011da6149d1f6cff7b4275c00170e1d", + "trial_id": "cells/tp4_mns64/low-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.023529411764705882, + "admitted_input_tokens_mean_over_limit": 0.013229832504734862, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.2086247086247086, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.2086247086247086, + "tpot_max_over_slo": -0.049497940473067825, + "tpot_mean_over_slo": -0.07097474071479923, + "ttft_max_over_slo_max": 0.0113132930030891, + "ttft_mean_over_slo_max": 0.0027290206974555573 + }, + "delta_state": { + "batch_size.mean": -4.516372320824056, + "batch_tokens.mean": -75.80960433136954, + "decode_batch_size.mean": -4.485458989758809, + "graph_full_share": 0.024653639771889058, + "graph_none_share": -0.023884511783377624, + "graph_padding_fraction": -0.0020895387371847093, + "kv_usage_end_minus_start": 0.011432163372104731, + "kv_usage_max": -0.003015188372616051, + "kv_usage_mean": -0.004106181760760127, + "preemptions": 0.0, + "prefill_token_fraction": 0.006171117400546322, + "queue_running_mean": -5.0125981476999995, + "queue_waiting_mean": 0.000454857, + "scheduler_steps_per_s": 21.80000000000001 + }, + "group": { + "cell": "tp4_mns64", + "level": "low", + "mns": 64, + "source_replicate": 2, + "target_replicate": 3, + "tp": 4 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "72c8f70a240f79edcaa4eb0a2fe9bade8ac07093339c5b2d09c49ddd1cf27d16", + "result_sha256": "ff8b2283d02d5967893ca6225b561ae11011da6149d1f6cff7b4275c00170e1d", + "trial_id": "cells/tp4_mns64/low-rep2/result.json" + }, + "target": { + "requests_sha256": "8b2c779d2097c5e4038e94e1ac9b986e29f55d9acfd67eeb366c4e0e9270f6e0", + "result_sha256": "f026b69356701894fab184e0f2f7504a30b537982e823373899c9ab123a33891", + "trial_id": "cells/tp4_mns64/low-rep3/result.json" + } + } + ], + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.6185880702088911, + "action_delta": { + "distinct_n": 12, + "max": 19.42960244648318, + "min": -1.4718645603306726, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 8, + "zero": 0 + }, + "effect_to_repeat_median": 0.2658268122367889, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 2.327034150557677, + "repeat_abs_p95": 11.456264844377335, + "repeat_delta": { + "distinct_n": 24, + "max": 11.95331828442438, + "min": -17.26249134948097, + "n": 24 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 17.059106336421053, + "action_delta": { + "distinct_n": 12, + "max": 656.7778499176663, + "min": -31.86018426395526, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.2374581652269279, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 71.84047059455061, + "repeat_abs_p95": 327.5325978555095, + "repeat_delta": { + "distinct_n": 24, + "max": 306.10848411182496, + "min": -537.2182539260048, + "n": 24 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.6115154242137617, + "action_delta": { + "distinct_n": 12, + "max": 19.184206069160197, + "min": -1.4608669627184998, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 8, + "zero": 0 + }, + "effect_to_repeat_median": 0.2650837352505827, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 2.3068764427802346, + "repeat_abs_p95": 11.363988993022012, + "repeat_delta": { + "distinct_n": 24, + "max": 11.860920298662965, + "min": -17.083955283470857, + "n": 24 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.005588197661352101, + "action_delta": { + "distinct_n": 12, + "max": 0.02356281394840931, + "min": -0.13147494707127738, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 7, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 0.27121228977463446, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.020604514883878045, + "repeat_abs_p95": 0.08279928773704351, + "repeat_delta": { + "distinct_n": 24, + "max": 0.10428533404311957, + "min": -0.07611315423133669, + "n": 24 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.011395045691744581, + "action_delta": { + "distinct_n": 12, + "max": 0.11760056457304163, + "min": -0.027804605789997328, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 8, + "positive": 4, + "zero": 0 + }, + "effect_to_repeat_median": 0.5196295717736035, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.02192917091467856, + "repeat_abs_p95": 0.08301950470733124, + "repeat_delta": { + "distinct_n": 24, + "max": 0.08050759417399073, + "min": -0.09966462603140805, + "n": 24 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.0024624382794725458, + "action_delta": { + "distinct_n": 12, + "max": 0.008784696835623326, + "min": -0.0007445684074620391, + "n": 12 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 2, + "positive": 10, + "zero": 0 + }, + "effect_to_repeat_median": 1.15716396642797, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0021279942608944218, + "repeat_abs_p95": 0.007335491197810787, + "repeat_delta": { + "distinct_n": 24, + "max": 0.0086692890425959, + "min": -0.007424342975302652, + "n": 24 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0009122627395202754, + "action_delta": { + "distinct_n": 12, + "max": 0.05537535994708087, + "min": -0.002621229743826836, + "n": 12 + }, + "action_signs": { + "consistency": 0.9166666666666666, + "negative": 1, + "positive": 11, + "zero": 0 + }, + "effect_to_repeat_median": 0.08374755984910728, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.010893006807170869, + "repeat_abs_p95": 0.08088054822007018, + "repeat_delta": { + "distinct_n": 24, + "max": 0.08232590668398432, + "min": -0.12798481366769898, + "n": 24 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0070274347692272365, + "action_delta": { + "distinct_n": 12, + "max": 0.08922043547542868, + "min": 0.0002683613903166071, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 12, + "zero": 0 + }, + "effect_to_repeat_median": 0.49974360815926006, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.014062080343782424, + "repeat_abs_p95": 0.10858976920771296, + "repeat_delta": { + "distinct_n": 24, + "max": 0.11409731241882304, + "min": -0.12688580277750028, + "n": 24 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0019193628331501261, + "action_delta": { + "distinct_n": 12, + "max": 0.04639671324585863, + "min": -0.0003621004636635179, + "n": 12 + }, + "action_signs": { + "consistency": 0.75, + "negative": 3, + "positive": 9, + "zero": 0 + }, + "effect_to_repeat_median": 0.29244258288516367, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.006563212560271434, + "repeat_abs_p95": 0.08042579042638397, + "repeat_delta": { + "distinct_n": 24, + "max": 0.08571139047521727, + "min": -0.09580544440945148, + "n": 24 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 12 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 12 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 24 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.00026550413778614335, + "action_delta": { + "distinct_n": 12, + "max": 0.013365962658000052, + "min": -0.0003954091503666213, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 8, + "zero": 0 + }, + "effect_to_repeat_median": 0.05125991549197503, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.005179566435838334, + "repeat_abs_p95": 0.0190949186571384, + "repeat_delta": { + "distinct_n": 24, + "max": 0.011636868678620127, + "min": -0.024511024899494305, + "n": 24 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.43384798364999977, + "action_delta": { + "distinct_n": 12, + "max": 24.0049637595, + "min": -0.46276202019999957, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 8, + "zero": 0 + }, + "effect_to_repeat_median": 0.1773939239284337, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 2.4456755566499995, + "repeat_abs_p95": 7.584720893309999, + "repeat_delta": { + "distinct_n": 24, + "max": 7.2005321570000005, + "min": -18.914856451900004, + "n": 24 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.22497528095, + "action_delta": { + "distinct_n": 11, + "max": 0.0, + "min": -15.5760687653, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 10, + "positive": 0, + "zero": 2 + }, + "effect_to_repeat_median": 2.611323424650003, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0861537406, + "repeat_abs_p95": 2.3661207382899994, + "repeat_delta": { + "distinct_n": 17, + "max": 2.3781169752999993, + "min": -11.247484751, + "n": 24 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 6.950000000000006, + "action_delta": { + "distinct_n": 11, + "max": 11.900000000000006, + "min": -34.5, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.44408945686900986, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 15.650000000000006, + "repeat_abs_p95": 50.385, + "repeat_delta": { + "distinct_n": 24, + "max": 65.0, + "min": -46.900000000000006, + "n": 24 + } + } + }, + "sanity": { + "action_pairs": 12, + "invariants": { + "efficacy_label_balance": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "matched_action_request_hashes": true, + "probabilities_bounded": true + }, + "red_flags": [], + "repeat_pairs": 24, + "trials": 36 + } + }, + "5": { + "actions": [ + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": -0.0008452736944928041, + "tpot_mean_over_slo": -0.0021708008398531042, + "ttft_max_over_slo_max": -0.00014104516594670713, + "ttft_mean_over_slo_max": -0.00020825385150534392 + }, + "delta_state": { + "batch_size.mean": -0.06880455035794819, + "batch_tokens.mean": -1.0156712758654507, + "decode_batch_size.mean": -0.067941551436697, + "graph_full_share": 0.0008629989212514122, + "graph_none_share": -0.006688241639697952, + "graph_padding_fraction": 0.00030316259031496623, + "kv_usage_end_minus_start": 2.700440393244019e-05, + "kv_usage_max": 0.00012663114366773254, + "kv_usage_mean": -6.934750080297807e-05, + "preemptions": 0.0, + "prefill_token_fraction": -0.000948468871704522, + "queue_running_mean": -0.028601565200000234, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 4.0 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "false->false", + "group": { + "level": "high", + "offered_rate_per_gpu": 2.9833333333333334, + "replicate": 1, + "request_hash": "3f0ab9be34dbee884f5dbe2135b983d31a47578a43f606a5f390b0528dbedbe3", + "tp": 1 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp1_mns8", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.20670391061452514, + "mns": 8, + "requests_sha256": "5f5bfa4a16846190f488d3369eed89ae895804145cea59197e6b544b703223fc", + "result_sha256": "f65e3dc91d17f5caab1f4aecf97d2c464b99f107231af2cfd96f5484880136ee", + "trial_id": "cells/tp1_mns8/high-rep1/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.24581005586592178, + "mns": 64, + "requests_sha256": "e970dcac20144cb1e13a35997377f6695a9c57d788a88bad7f18cf6c64e35cec", + "result_sha256": "3233686b60b920dcd6cc2f45193a81f3af2d46f781ef42c4d117630612084feb", + "trial_id": "cells/tp1_mns64/high-rep1/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.375, + "completed_pass_rate": -1.0, + "outstanding_over_admitted": 0.375, + "tpot_max_over_slo": -0.562845061415821, + "tpot_mean_over_slo": -0.49685783692809143, + "ttft_max_over_slo_max": -0.1160576971694051, + "ttft_mean_over_slo_max": -0.06709235994559196 + }, + "delta_state": { + "batch_size.mean": 1.9650793650793652, + "batch_tokens.mean": 127.72380952380956, + "decode_batch_size.mean": 1.9158730158730162, + "graph_full_share": -0.041269841269841345, + "graph_none_share": 0.033333333333333326, + "graph_padding_fraction": 0.0032253509426556993, + "kv_usage_end_minus_start": 0.08606331121732713, + "kv_usage_max": 0.04171319199456147, + "kv_usage_mean": 0.01377304273482051, + "preemptions": 0.0, + "prefill_token_fraction": 0.0018893720284961502, + "queue_running_mean": 1.9109891312, + "queue_waiting_mean": -1.558876379, + "scheduler_steps_per_s": -10.8 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "false->false", + "group": { + "level": "high", + "offered_rate_per_gpu": 2.9833333333333334, + "replicate": 2, + "request_hash": "b19c91876a8417fe06cb2775f65a1111b202f88ce10ec33576af3d84c059f02a", + "tp": 1 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp1_mns8", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.7988826815642458, + "mns": 8, + "requests_sha256": "e802d09057de349e5164c18314fcad09dc88c9673834fd6e5b100a57dd12fbf4", + "result_sha256": "4835a985dd6bb8dc50b53737baa980ac95a49e22f0b8a7326e1fb416c9b4e798", + "trial_id": "cells/tp1_mns8/high-rep2/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.88268156424581, + "mns": 64, + "requests_sha256": "ec06878e2aefdc4853d972197cedc29ab94df7147abb2701e0d88b5dabf29728", + "result_sha256": "8215477a014f061c55e94d879a25bbf1676ee2ef1c5e81efadd23d20d337e07a", + "trial_id": "cells/tp1_mns64/high-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": -0.003354239532372044, + "tpot_mean_over_slo": -0.002150681978709712, + "ttft_max_over_slo_max": -0.0004135148386315776, + "ttft_mean_over_slo_max": -0.00028004554942959864 + }, + "delta_state": { + "batch_size.mean": -0.03143779072203845, + "batch_tokens.mean": -11.489561245442736, + "decode_batch_size.mean": -0.02922096970204935, + "graph_full_share": 0.0022168210199891014, + "graph_none_share": -0.00629845367305033, + "graph_padding_fraction": 0.001725218661472923, + "kv_usage_end_minus_start": -0.01687018923627459, + "kv_usage_max": -0.01682167078536223, + "kv_usage_mean": -0.00010651327558790225, + "preemptions": 0.0, + "prefill_token_fraction": -0.009907841139181417, + "queue_running_mean": -0.027900191600000124, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.5999999999999943 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "false->false", + "group": { + "level": "high", + "offered_rate_per_gpu": 2.9833333333333334, + "replicate": 3, + "request_hash": "a85849f1f057a67c82523cff57a52af22d2eb9514a953e076990aa6c25c1ef7e", + "tp": 1 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp1_mns8", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.6424581005586593, + "mns": 8, + "requests_sha256": "cf1226185845cd8aa4e7ead0cb5c95c77eda62b84851a2ab579f984bbb88e8ac", + "result_sha256": "95d5103f2678eeb99c258870324566932e2f2853c3b6e0ee94bc6c538f6ed865", + "trial_id": "cells/tp1_mns8/high-rep3/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.9273743016759777, + "mns": 64, + "requests_sha256": "18d72d1fe16136359e28a1909e395ffd553dbe3edd2c0fd93a9cc270a5ec6d60", + "result_sha256": "480a3b50acaf9b9c1902e2669cf3aa7723a6703a00778da269b5b9757990ceef", + "trial_id": "cells/tp1_mns64/high-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": -0.019319413383821316, + "tpot_mean_over_slo": -0.01717012501436968, + "ttft_max_over_slo_max": -0.0006420235052549544, + "ttft_mean_over_slo_max": -0.0016948439976355693 + }, + "delta_state": { + "batch_size.mean": -0.06047484495981781, + "batch_tokens.mean": -1.7159186818832382, + "decode_batch_size.mean": -0.059649187185791375, + "graph_full_share": 0.0008256577740266557, + "graph_none_share": -0.007128178782430002, + "graph_padding_fraction": 0.0006142370124512038, + "kv_usage_end_minus_start": 9.20038674978585e-05, + "kv_usage_max": 0.00016629145416469093, + "kv_usage_mean": -5.445550568086016e-05, + "preemptions": 0.0, + "prefill_token_fraction": -0.00036424705933824075, + "queue_running_mean": -0.16474551560000017, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 3.6000000000000085 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 2.033333333333333, + "replicate": 1, + "request_hash": "c8ea9278f180ebb23cee595e167627cbed711032e70c9ca5ddc7205320cd75c0", + "tp": 1 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 8, + "requests_sha256": "0edc6095b16e5361a88b2765cc5d0f267ed16ff1d883e02fc3c58f57e59c1018", + "result_sha256": "8bc433109d148c21b7d158df95ec5d1fa4fa619243114563068e1e98c4241a29", + "trial_id": "cells/tp1_mns8/low-rep1/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "44d729c1ace9b09fe2ea4c4e110bd0f95fee69f83edc78ee0f65256922dcd2c2", + "result_sha256": "d2c08e71c5d4e0dd99662bab8457bd1ad0a4aba6bda57794dbfce62e36720f62", + "trial_id": "cells/tp1_mns64/low-rep1/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.15384615384615385, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.15384615384615385, + "tpot_max_over_slo": 0.040732697633234394, + "tpot_mean_over_slo": 0.014527851366132327, + "ttft_max_over_slo_max": -0.03623925517119157, + "ttft_mean_over_slo_max": -0.015724865515949205 + }, + "delta_state": { + "batch_size.mean": 0.05866351271436976, + "batch_tokens.mean": 16.67403903015969, + "decode_batch_size.mean": 0.053696037847427824, + "graph_full_share": 0.0010053222945002327, + "graph_none_share": -0.012241277350680077, + "graph_padding_fraction": 0.0038721626639551332, + "kv_usage_end_minus_start": 0.02762750849475526, + "kv_usage_max": 0.01654350181309283, + "kv_usage_mean": 0.004906340576442361, + "preemptions": 0.0, + "prefill_token_fraction": 0.0010152672534560203, + "queue_running_mean": 0.6418094667999998, + "queue_waiting_mean": -0.3949698864, + "scheduler_steps_per_s": -2.3999999999999986 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 2.033333333333333, + "replicate": 2, + "request_hash": "1f27fe31ca32fd67f2f70a54eee83146329a7838ff2a33be48ce4372640a858c", + "tp": 1 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 8, + "requests_sha256": "ca58a878401ac44cca0a9a595eab41659dafec2e923f7f2474e4f82ea30b2f3e", + "result_sha256": "42a56af136d1d3bdfc16d9cea092c2db0eaa4db2c85dceb21958c5e418122900", + "trial_id": "cells/tp1_mns8/low-rep2/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "97de3cced64fd88a5c2dcd087ca6c29d8b09a0a43d7bf0e4e844ef246462465b", + "result_sha256": "f194db2f7abb995171cd24626e4e515c000481f81283c929b9a5b63f110f1506", + "trial_id": "cells/tp1_mns64/low-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": -0.0007078415662871984, + "tpot_mean_over_slo": -0.000578866876769335, + "ttft_max_over_slo_max": -5.364099827906421e-05, + "ttft_mean_over_slo_max": 4.2679616146623633e-05 + }, + "delta_state": { + "batch_size.mean": -0.001802258455126493, + "batch_tokens.mean": -0.2360817774774091, + "decode_batch_size.mean": -0.00173185773422313, + "graph_full_share": 7.040072090336302e-05, + "graph_none_share": -7.040072090338037e-05, + "graph_padding_fraction": 0.0, + "kv_usage_end_minus_start": 0.00011716903913294718, + "kv_usage_max": 0.00013871042605262396, + "kv_usage_mean": 8.130641993444807e-06, + "preemptions": 0.0, + "prefill_token_fraction": -3.2355679021600636e-05, + "queue_running_mean": -0.0025477285999999655, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.19999999999999574 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 2.033333333333333, + "replicate": 3, + "request_hash": "a46c62ebd8b7df5534831feb70aed45d23b1a5e7f8bf3d4cd82a79f182fbc69e", + "tp": 1 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 8, + "requests_sha256": "8a72240f84bbeff9eabd63e7d1f7a4c19910660fc5e4335013a1deaf89408127", + "result_sha256": "0214f32cabbcdc1bad6c331c3945c505cad136fcc9a8336aeb6314b27474e279", + "trial_id": "cells/tp1_mns8/low-rep3/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "30f4623100c58580cfa4efe9a1998c63581c8e44511c556ef39cf5bd777883bf", + "result_sha256": "0be457aa2fc9ce95bdb92a6eb25544f5f61a9739932ee5afa40535d26456e348", + "trial_id": "cells/tp1_mns64/low-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.27586206896551724, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.27586206896551724, + "tpot_max_over_slo": 0.207529695590914, + "tpot_mean_over_slo": 0.05943365186246716, + "ttft_max_over_slo_max": -0.10603234449323887, + "ttft_mean_over_slo_max": -0.017237129418515705 + }, + "delta_state": { + "batch_size.mean": 4.237397287896355, + "batch_tokens.mean": 201.35936633563546, + "decode_batch_size.mean": 4.139436406714726, + "graph_full_share": -0.05746836719261983, + "graph_none_share": 0.044500176437969455, + "graph_padding_fraction": 0.0022441507829442276, + "kv_usage_end_minus_start": 0.02343349054119659, + "kv_usage_max": 0.0207004044749316, + "kv_usage_mean": 0.007304490614720681, + "preemptions": 0.0, + "prefill_token_fraction": 0.016050535408914768, + "queue_running_mean": 9.929849956000002, + "queue_waiting_mean": -5.506802823999999, + "scheduler_steps_per_s": -29.19999999999999 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "high", + "offered_rate_per_gpu": 3.125, + "replicate": 1, + "request_hash": "5240182174ec5e25247be01c017cbefe2f1e59426bb24a57746dc40be4a82208", + "tp": 4 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.10666666666666667, + "mns": 16, + "requests_sha256": "8695422a0f1e0e1c097a08a7f4c082f1ba2e3bb26b58caadae0afc7438aeca90", + "result_sha256": "c54a2bb5c1b78bdef3fa09c39f5974215370d6fc7c2b6d403a3d376d53a1d9eb", + "trial_id": "cells/tp4_mns16/high-rep1/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "b4aeaf5b3175bd85fdac08a2ef657fb2278c6c4b36ea32ea417275ab6d55812d", + "result_sha256": "111a1042b614d05772888665a7cfbc86925b2fbeb52ffe19edb62a5bcfe2f513", + "trial_id": "cells/tp4_mns64/high-rep1/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.2631578947368421, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.26315789473684204, + "tpot_max_over_slo": 0.2331512886664177, + "tpot_mean_over_slo": 0.22120977902069205, + "ttft_max_over_slo_max": -0.17202288067589203, + "ttft_mean_over_slo_max": -0.05936201432352671 + }, + "delta_state": { + "batch_size.mean": 10.325762195121952, + "batch_tokens.mean": 461.67439024390245, + "decode_batch_size.mean": 10.164329268292683, + "graph_full_share": -0.08612804878048774, + "graph_none_share": 0.07111280487804877, + "graph_padding_fraction": 0.0007778735353099922, + "kv_usage_end_minus_start": 0.03093665701006798, + "kv_usage_max": 0.026463529328836888, + "kv_usage_mean": 0.010597923626744828, + "preemptions": 0.0, + "prefill_token_fraction": 0.009329314654311416, + "queue_running_mean": 12.835752482200002, + "queue_waiting_mean": -6.7273728942, + "scheduler_steps_per_s": -23.0 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "high", + "offered_rate_per_gpu": 3.125, + "replicate": 2, + "request_hash": "51be592ba9a59c8d66eb1bcc70c3d2b011c87c66128aff36574973b40389ad96", + "tp": 4 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.092, + "mns": 16, + "requests_sha256": "aa0482b10414d911af227e2df172c19ecca0add2e0024dc88f85ffbd063854bb", + "result_sha256": "91e2491de4ca0868b667e7d126b755f03bdd78378681b8070a2bbc0c38fb9055", + "trial_id": "cells/tp4_mns16/high-rep2/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "5d82b6885f5c654ae1ae126696df4bafe04f5be1c1109db24c7856b0456d9443", + "result_sha256": "6ed574c18723e265ab1327c5e43d3c76d37b3fc5711d06c77a51718e4ba16601", + "trial_id": "cells/tp4_mns64/high-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.07843137254901955, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0784313725490196, + "tpot_max_over_slo": 0.05021820802747551, + "tpot_mean_over_slo": -0.012818677862446504, + "ttft_max_over_slo_max": -0.08022667251255673, + "ttft_mean_over_slo_max": -0.02098792773025101 + }, + "delta_state": { + "batch_size.mean": 0.32844164919636576, + "batch_tokens.mean": 20.103174603174637, + "decode_batch_size.mean": 0.3230008984725963, + "graph_full_share": 0.004542278127183774, + "graph_none_share": -0.017769791354697015, + "graph_padding_fraction": 0.006616409980351494, + "kv_usage_end_minus_start": 0.007401906100479305, + "kv_usage_max": 0.004352176770658023, + "kv_usage_mean": 0.0009581054950093242, + "preemptions": 0.0, + "prefill_token_fraction": 0.0019433062474734086, + "queue_running_mean": 0.822417270599999, + "queue_waiting_mean": -1.9255238218, + "scheduler_steps_per_s": 1.3999999999999915 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "high", + "offered_rate_per_gpu": 3.125, + "replicate": 3, + "request_hash": "604e338cf576d187157293922dada2bceb311567268ddae74eb7a089debf0242", + "tp": 4 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.14666666666666667, + "mns": 16, + "requests_sha256": "3a23ffbe0e724648226152ef5433cc5825ad640c37cf47997c2f091210eceeec", + "result_sha256": "136dde0451f806841ff3996b081fe0bffac0a76f60c3d78b7516f2689e106008", + "trial_id": "cells/tp4_mns16/high-rep3/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "5c5861d716181f06dc3f951ae0d6ee479b1b61fc21ebf24541c5b45053fcc167", + "result_sha256": "35b8f0465f6b0c197f71b5ecaf4e771e6613f86998ae44f8701ab9e4e1e440b4", + "trial_id": "cells/tp4_mns64/high-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.03209297007177109, + "tpot_mean_over_slo": 0.00165350785497842, + "ttft_max_over_slo_max": -0.04185720401195188, + "ttft_mean_over_slo_max": -0.008807509172654063 + }, + "delta_state": { + "batch_size.mean": -1.1226839723743751, + "batch_tokens.mean": -34.76613479399856, + "decode_batch_size.mean": -1.1063943796141942, + "graph_full_share": 0.012598237675637058, + "graph_none_share": -0.01938556799237913, + "graph_padding_fraction": -9.478587342680138e-05, + "kv_usage_end_minus_start": 3.947117882996398e-05, + "kv_usage_max": 0.004831301652910902, + "kv_usage_mean": 1.4883144002681564e-05, + "preemptions": 0.0, + "prefill_token_fraction": -0.00096100288388834, + "queue_running_mean": -0.07484160059999923, + "queue_waiting_mean": -0.1708327272, + "scheduler_steps_per_s": 12.400000000000006 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 2.125, + "replicate": 1, + "request_hash": "6fb170f366b68a4e4e0fc976d370cb1f5b7b915d4dc22ebe5db3c6b9615afe68", + "tp": 4 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.6196078431372549, + "mns": 16, + "requests_sha256": "d7eaaefd35cc906f759dcaa7ac3c8543249a1a02f28572bec953e41e4379a29f", + "result_sha256": "11d9ec9d2a3358aed988f7b4b1014af087d61c4b5a72eb9076ae19bb5cc481cd", + "trial_id": "cells/tp4_mns16/low-rep1/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "931d372ded1e3aaa9c67f06653fac8f79b4ee4f369028c1148163580386681ce", + "result_sha256": "325ce57f33133670a6761b5dffb831ed3b2f464755ff743b56976b4a974c1215", + "trial_id": "cells/tp4_mns64/low-rep1/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.10869565217391303, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.10869565217391308, + "tpot_max_over_slo": 0.031286743469536304, + "tpot_mean_over_slo": -0.024426390991776098, + "ttft_max_over_slo_max": -0.014295891169846676, + "ttft_mean_over_slo_max": -0.003572963410477855 + }, + "delta_state": { + "batch_size.mean": 0.7171517639433311, + "batch_tokens.mean": 19.4169881786475, + "decode_batch_size.mean": 0.701919812339888, + "graph_full_share": 0.004382851322571657, + "graph_none_share": -0.007268742862434022, + "graph_padding_fraction": 0.002283307241934776, + "kv_usage_end_minus_start": 0.008891646331802172, + "kv_usage_max": 0.0030741209471074615, + "kv_usage_mean": 0.0003701503920676315, + "preemptions": 0.0, + "prefill_token_fraction": 0.00017846155000900144, + "queue_running_mean": 0.8948316157999994, + "queue_waiting_mean": -1.1446055668, + "scheduler_steps_per_s": -0.8000000000000114 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 2.125, + "replicate": 2, + "request_hash": "eb4c0445556d23d73d27d728b45e63fec44e17e84a4f2f341787108b7f7a4901", + "tp": 4 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.6803921568627451, + "mns": 16, + "requests_sha256": "7fdb90f679bba1747fbacc92fac2045c820e09965c755b20f6dafd4fd9e26a7a", + "result_sha256": "800e2aa6e9f9efef4be12ee778be6817364914d63ae0624775d02729089768d6", + "trial_id": "cells/tp4_mns16/low-rep2/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "72c8f70a240f79edcaa4eb0a2fe9bade8ac07093339c5b2d09c49ddd1cf27d16", + "result_sha256": "ff8b2283d02d5967893ca6225b561ae11011da6149d1f6cff7b4275c00170e1d", + "trial_id": "cells/tp4_mns64/low-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.06666666666666676, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.06666666666666665, + "tpot_max_over_slo": -0.0004955744884145297, + "tpot_mean_over_slo": -0.0060310517702504285, + "ttft_max_over_slo_max": 0.0002604483355147097, + "ttft_mean_over_slo_max": -0.003228990856401895 + }, + "delta_state": { + "batch_size.mean": -0.09294265910522981, + "batch_tokens.mean": -6.586326402016368, + "decode_batch_size.mean": -0.0904221802142402, + "graph_full_share": 0.0001575299306868505, + "graph_none_share": -0.0036231884057971037, + "graph_padding_fraction": 0.0004465429838699502, + "kv_usage_end_minus_start": -0.002635620484154888, + "kv_usage_max": 0.0003123116904728551, + "kv_usage_mean": -0.00022787683290566817, + "preemptions": 0.0, + "prefill_token_fraction": -0.0009035229748303175, + "queue_running_mean": -0.21642387019999987, + "queue_waiting_mean": -0.0184107002, + "scheduler_steps_per_s": 4.6000000000000085 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 2.125, + "replicate": 3, + "request_hash": "a660c44d1f8b9c0ada5a19285fa579871bcfcae7c6fa71649546efda9aa3baef", + "tp": 4 + }, + "kind": "matched_mns_increase", + "source": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.6294117647058823, + "mns": 16, + "requests_sha256": "b592fe254476957265fc9139aefdea77be4b5540f3c1b8a429c10ea6ccab9a6a", + "result_sha256": "cc1b1f7f5280c2892d0c6deac9df9f34ce126b0284e6ad893ffbf0ddbf27969f", + "trial_id": "cells/tp4_mns16/low-rep3/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "8b2c779d2097c5e4038e94e1ac9b986e29f55d9acfd67eeb366c4e0e9270f6e0", + "result_sha256": "f026b69356701894fab184e0f2f7504a30b537982e823373899c9ab123a33891", + "trial_id": "cells/tp4_mns64/low-rep3/result.json" + } + } + ], + "efficacy": { + "feasibility_transitions": { + "false->false": 3, + "false->true": 6, + "true->true": 3 + }, + "minimum_balanced_accuracy": 0.75, + "minimum_delta_over_best_outcome": 0.15, + "outcome_delta": { + "best_accuracy": 0.8333333333333334, + "best_balanced_accuracy": 0.8333333333333334, + "best_feature": "tpot_max_over_slo", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_over_admitted": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.03333333333333338, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.039215686274509776, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.05434782608695651, + "train_balanced_accuracy": 0.625 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.5, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.5, + "train_balanced_accuracy": 0.625 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.033333333333333326, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0392156862745098, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.05434782608695654, + "train_balanced_accuracy": 0.625 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333334, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.000601708027350864, + "train_balanced_accuracy": 0.875 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 1 + ], + "threshold": -0.000601708027350864, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.01522073488752175, + "train_balanced_accuracy": 0.875 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.41666666666666663, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 0 + ], + "threshold": -0.00409086687448007, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 0 + ], + "threshold": 0.0005373204891045424, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.00025864649243734217, + "train_balanced_accuracy": 0.75 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.00010340366861782274, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 0 + ], + "threshold": -0.021249613758603417, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.0074689573375508155, + "train_balanced_accuracy": 0.75 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.0017545182029157468, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.002461917427018732, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.002633903704056712, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_delta": { + "best_accuracy": 0.75, + "best_balanced_accuracy": 0.75, + "best_feature": "queue_waiting_mean", + "features": { + "decode_batch_size.mean": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.18834846816001205, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.0791818658254686, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.3778079250936579, + "train_balanced_accuracy": 0.75 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 0.25, + "balanced_accuracy": 0.25, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 0 + ], + "threshold": 0.0002232714919349751, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 0, + 1 + ], + "threshold": 0.0019846847222085755, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 0, + 1 + ], + "threshold": 0.0027543290922952377, + "train_balanced_accuracy": 0.75 + } + ] + }, + "kv_usage_mean": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.0001671950542467852, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 1.1506892998063185e-05, + "train_balanced_accuracy": 0.875 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 0 + ], + "threshold": -1.97861808390893e-05, + "train_balanced_accuracy": 0.75 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.0019163391379847794, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 0 + ], + "threshold": 0.000955475284225904, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.005609343341403783, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_running_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.7321133686999994, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.40993477099999953, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.7683205412999996, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.0092053501, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.0092053501, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.0854163636, + "train_balanced_accuracy": 0.75 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.41666666666666663, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 0, + 1 + ], + "threshold": 0.9999999999999929, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 4.300000000000004, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -16.9, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_qualifying_features": [] + }, + "horizon_s": 5.0, + "qualifying_response_features": [], + "repeats": [ + { + "delta_outcome": { + "admitted_fraction": 0.027932960893854754, + "admitted_input_tokens_mean_over_limit": 0.25888824462890625, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.8181818181818182, + "completed_pass_rate": -1.0, + "outstanding_over_admitted": 0.8181818181818181, + "tpot_max_over_slo": -0.22724595748614082, + "tpot_mean_over_slo": -0.16274999088627282, + "ttft_max_over_slo_max": -0.034610742829196774, + "ttft_mean_over_slo_max": -0.0228428806456375 + }, + "delta_state": { + "batch_size.mean": 5.528833410386808, + "batch_tokens.mean": 384.9526121128063, + "decode_batch_size.mean": 5.423208506703652, + "graph_full_share": -0.09768839574664823, + "graph_none_share": 0.09557713052858684, + "graph_padding_fraction": -0.01691520035196146, + "kv_usage_end_minus_start": 0.16350284743730636, + "kv_usage_max": 0.1339294634828655, + "kv_usage_mean": 0.07938311082464784, + "preemptions": 0.0, + "prefill_token_fraction": 0.07360971724683552, + "queue_running_mean": 5.2693423148, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -77.8 + }, + "group": { + "cell": "tp1_mns64", + "level": "high", + "mns": 64, + "source_replicate": 1, + "target_replicate": 2, + "tp": 1 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "e970dcac20144cb1e13a35997377f6695a9c57d788a88bad7f18cf6c64e35cec", + "result_sha256": "3233686b60b920dcd6cc2f45193a81f3af2d46f781ef42c4d117630612084feb", + "trial_id": "cells/tp1_mns64/high-rep1/result.json" + }, + "target": { + "requests_sha256": "ec06878e2aefdc4853d972197cedc29ab94df7147abb2701e0d88b5dabf29728", + "result_sha256": "8215477a014f061c55e94d879a25bbf1676ee2ef1c5e81efadd23d20d337e07a", + "trial_id": "cells/tp1_mns64/high-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.011173184357541902, + "admitted_input_tokens_mean_over_limit": -0.18295179094587052, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.5, + "completed_pass_rate": 1.0, + "outstanding_over_admitted": -0.5, + "tpot_max_over_slo": 0.16581994141305348, + "tpot_mean_over_slo": 0.14577003201512728, + "ttft_max_over_slo_max": 0.04351074216538109, + "ttft_mean_over_slo_max": 0.02181711295270361 + }, + "delta_state": { + "batch_size.mean": -5.595011337868481, + "batch_tokens.mean": -365.5517006802721, + "decode_batch_size.mean": -5.494557823129252, + "graph_full_share": 0.09251700680272112, + "graph_none_share": -0.08866213151927438, + "graph_padding_fraction": 0.004841272439424705, + "kv_usage_end_minus_start": -0.1340793286042561, + "kv_usage_max": -0.12953341992207013, + "kv_usage_mean": -0.0768057629061545, + "preemptions": 0.0, + "prefill_token_fraction": -0.033037405519596885, + "queue_running_mean": -4.789778502000001, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 72.8 + }, + "group": { + "cell": "tp1_mns64", + "level": "high", + "mns": 64, + "source_replicate": 2, + "target_replicate": 3, + "tp": 1 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "ec06878e2aefdc4853d972197cedc29ab94df7147abb2701e0d88b5dabf29728", + "result_sha256": "8215477a014f061c55e94d879a25bbf1676ee2ef1c5e81efadd23d20d337e07a", + "trial_id": "cells/tp1_mns64/high-rep2/result.json" + }, + "target": { + "requests_sha256": "18d72d1fe16136359e28a1909e395ffd553dbe3edd2c0fd93a9cc270a5ec6d60", + "result_sha256": "480a3b50acaf9b9c1902e2669cf3aa7723a6703a00778da269b5b9757990ceef", + "trial_id": "cells/tp1_mns64/high-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.02459016393442623, + "admitted_input_tokens_mean_over_limit": 0.20505746694711535, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.4692307692307692, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.4692307692307693, + "tpot_max_over_slo": 0.38761381102051196, + "tpot_mean_over_slo": 0.33823327695869493, + "ttft_max_over_slo_max": -0.043946471996605396, + "ttft_mean_over_slo_max": -0.0057849611233298966 + }, + "delta_state": { + "batch_size.mean": 3.1123123406666036, + "batch_tokens.mean": 225.4005759607214, + "decode_batch_size.mean": 3.0546690586346896, + "graph_full_share": -0.04640732697573413, + "graph_none_share": 0.041473892927957695, + "graph_padding_fraction": 0.005316401000291102, + "kv_usage_end_minus_start": 0.11649515436107505, + "kv_usage_max": 0.10080927165551001, + "kv_usage_mean": 0.05787529741227895, + "preemptions": 0.0, + "prefill_token_fraction": 0.025873876159429043, + "queue_running_mean": 4.3325417076, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -59.6 + }, + "group": { + "cell": "tp1_mns64", + "level": "low", + "mns": 64, + "source_replicate": 1, + "target_replicate": 2, + "tp": 1 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "44d729c1ace9b09fe2ea4c4e110bd0f95fee69f83edc78ee0f65256922dcd2c2", + "result_sha256": "d2c08e71c5d4e0dd99662bab8457bd1ad0a4aba6bda57794dbfce62e36720f62", + "trial_id": "cells/tp1_mns64/low-rep1/result.json" + }, + "target": { + "requests_sha256": "97de3cced64fd88a5c2dcd087ca6c29d8b09a0a43d7bf0e4e844ef246462465b", + "result_sha256": "f194db2f7abb995171cd24626e4e515c000481f81283c929b9a5b63f110f1506", + "trial_id": "cells/tp1_mns64/low-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.06557377049180327, + "admitted_input_tokens_mean_over_limit": 0.058956204927884615, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.3692307692307692, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.36923076923076925, + "tpot_max_over_slo": -0.41323108928586083, + "tpot_mean_over_slo": -0.3718732634627717, + "ttft_max_over_slo_max": 0.05646172333702755, + "ttft_mean_over_slo_max": 0.02716241966764857 + }, + "delta_state": { + "batch_size.mean": -3.6161048689138573, + "batch_tokens.mean": -189.70786516853934, + "decode_batch_size.mean": -3.556179775280899, + "graph_full_share": 0.048689138576779034, + "graph_none_share": -0.037453183520599245, + "graph_padding_fraction": -0.008781536045622663, + "kv_usage_end_minus_start": -0.11025077430312713, + "kv_usage_max": -0.10765311219902085, + "kv_usage_mean": -0.06249167924800681, + "preemptions": 0.0, + "prefill_token_fraction": 0.00031032253091034523, + "queue_running_mean": -5.3659050614, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 17.799999999999997 + }, + "group": { + "cell": "tp1_mns64", + "level": "low", + "mns": 64, + "source_replicate": 2, + "target_replicate": 3, + "tp": 1 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "97de3cced64fd88a5c2dcd087ca6c29d8b09a0a43d7bf0e4e844ef246462465b", + "result_sha256": "f194db2f7abb995171cd24626e4e515c000481f81283c929b9a5b63f110f1506", + "trial_id": "cells/tp1_mns64/low-rep2/result.json" + }, + "target": { + "requests_sha256": "30f4623100c58580cfa4efe9a1998c63581c8e44511c556ef39cf5bd777883bf", + "result_sha256": "0be457aa2fc9ce95bdb92a6eb25544f5f61a9739932ee5afa40535d26456e348", + "trial_id": "cells/tp1_mns64/low-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.027932960893854754, + "admitted_input_tokens_mean_over_limit": 0.25888824462890625, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.44318181818181823, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.4431818181818182, + "tpot_max_over_slo": 0.3347538302351874, + "tpot_mean_over_slo": 0.33193704520196554, + "ttft_max_over_slo_max": 0.08130590917426161, + "ttft_mean_over_slo_max": 0.044041225448449114 + }, + "delta_state": { + "batch_size.mean": 3.494949494949495, + "batch_tokens.mean": 256.21313131313127, + "decode_batch_size.mean": 3.4393939393939394, + "graph_full_share": -0.05555555555555547, + "graph_none_share": 0.05555555555555555, + "graph_padding_fraction": -0.01983738870430219, + "kv_usage_end_minus_start": 0.07746654062391167, + "kv_usage_max": 0.09234290263197176, + "kv_usage_mean": 0.06554072058902435, + "preemptions": 0.0, + "prefill_token_fraction": 0.07077187634663484, + "queue_running_mean": 3.3297516184, + "queue_waiting_mean": 1.558876379, + "scheduler_steps_per_s": -63.0 + }, + "group": { + "cell": "tp1_mns8", + "level": "high", + "mns": 8, + "source_replicate": 1, + "target_replicate": 2, + "tp": 1 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "5f5bfa4a16846190f488d3369eed89ae895804145cea59197e6b544b703223fc", + "result_sha256": "f65e3dc91d17f5caab1f4aecf97d2c464b99f107231af2cfd96f5484880136ee", + "trial_id": "cells/tp1_mns8/high-rep1/result.json" + }, + "target": { + "requests_sha256": "e802d09057de349e5164c18314fcad09dc88c9673834fd6e5b100a57dd12fbf4", + "result_sha256": "4835a985dd6bb8dc50b53737baa980ac95a49e22f0b8a7326e1fb416c9b4e798", + "trial_id": "cells/tp1_mns8/high-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.011173184357541902, + "admitted_input_tokens_mean_over_limit": -0.18295179094587052, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.125, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.125, + "tpot_max_over_slo": -0.3936708804703955, + "tpot_mean_over_slo": -0.34893712293425444, + "ttft_max_over_slo_max": -0.07213344016539244, + "ttft_mean_over_slo_max": -0.04499520144345875 + }, + "delta_state": { + "batch_size.mean": -3.598494182067077, + "batch_tokens.mean": -226.33832991101983, + "decode_batch_size.mean": -3.5494638375541867, + "graph_full_share": 0.04903034451289068, + "graph_none_share": -0.04903034451289072, + "graph_padding_fraction": 0.006341404720607481, + "kv_usage_end_minus_start": -0.031145828150654364, + "kv_usage_max": -0.07099855714214642, + "kv_usage_mean": -0.06292620689574609, + "preemptions": 0.0, + "prefill_token_fraction": -0.021240192351919318, + "queue_running_mean": -2.8508891792, + "queue_waiting_mean": -1.558876379, + "scheduler_steps_per_s": 61.400000000000006 + }, + "group": { + "cell": "tp1_mns8", + "level": "high", + "mns": 8, + "source_replicate": 2, + "target_replicate": 3, + "tp": 1 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "e802d09057de349e5164c18314fcad09dc88c9673834fd6e5b100a57dd12fbf4", + "result_sha256": "4835a985dd6bb8dc50b53737baa980ac95a49e22f0b8a7326e1fb416c9b4e798", + "trial_id": "cells/tp1_mns8/high-rep2/result.json" + }, + "target": { + "requests_sha256": "cf1226185845cd8aa4e7ead0cb5c95c77eda62b84851a2ab579f984bbb88e8ac", + "result_sha256": "95d5103f2678eeb99c258870324566932e2f2853c3b6e0ee94bc6c538f6ed865", + "trial_id": "cells/tp1_mns8/high-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.02459016393442623, + "admitted_input_tokens_mean_over_limit": 0.20505746694711535, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.3153846153846153, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.31538461538461543, + "tpot_max_over_slo": 0.3275617000034563, + "tpot_mean_over_slo": 0.3065353005781929, + "ttft_max_over_slo_max": -0.008349240330668778, + "ttft_mean_over_slo_max": 0.008245060394983739 + }, + "delta_state": { + "batch_size.mean": 2.993173982992416, + "batch_tokens.mean": 207.01061824867847, + "decode_batch_size.mean": 2.9413238336014706, + "graph_full_share": -0.04658699149620771, + "graph_none_share": 0.04658699149620778, + "graph_padding_fraction": 0.0020584753487871725, + "kv_usage_end_minus_start": 0.08895964973381765, + "kv_usage_max": 0.08443206129658187, + "kv_usage_mean": 0.05291450133015573, + "preemptions": 0.0, + "prefill_token_fraction": 0.024494361846634782, + "queue_running_mean": 3.5259867252, + "queue_waiting_mean": 0.3949698864, + "scheduler_steps_per_s": -53.599999999999994 + }, + "group": { + "cell": "tp1_mns8", + "level": "low", + "mns": 8, + "source_replicate": 1, + "target_replicate": 2, + "tp": 1 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "0edc6095b16e5361a88b2765cc5d0f267ed16ff1d883e02fc3c58f57e59c1018", + "result_sha256": "8bc433109d148c21b7d158df95ec5d1fa4fa619243114563068e1e98c4241a29", + "trial_id": "cells/tp1_mns8/low-rep1/result.json" + }, + "target": { + "requests_sha256": "ca58a878401ac44cca0a9a595eab41659dafec2e923f7f2474e4f82ea30b2f3e", + "result_sha256": "42a56af136d1d3bdfc16d9cea092c2db0eaa4db2c85dceb21958c5e418122900", + "trial_id": "cells/tp1_mns8/low-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.06557377049180327, + "admitted_input_tokens_mean_over_limit": 0.058956204927884615, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.21538461538461534, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.2153846153846154, + "tpot_max_over_slo": -0.3717905500863392, + "tpot_mean_over_slo": -0.35676654521987006, + "ttft_max_over_slo_max": 0.02027610916411504, + "ttft_mean_over_slo_max": 0.01139487453555274 + }, + "delta_state": { + "batch_size.mean": -3.5556390977443613, + "batch_tokens.mean": -172.79774436090224, + "decode_batch_size.mean": -3.500751879699248, + "graph_full_share": 0.049624060150375904, + "graph_none_share": -0.049624060150375945, + "graph_padding_fraction": -0.00490937338166753, + "kv_usage_end_minus_start": -0.08274043484750482, + "kv_usage_max": -0.09124832081198064, + "kv_usage_mean": -0.057593469313557896, + "preemptions": 0.0, + "prefill_token_fraction": 0.0013579454633879662, + "queue_running_mean": -4.721547866, + "queue_waiting_mean": -0.3949698864, + "scheduler_steps_per_s": 15.200000000000003 + }, + "group": { + "cell": "tp1_mns8", + "level": "low", + "mns": 8, + "source_replicate": 2, + "target_replicate": 3, + "tp": 1 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "ca58a878401ac44cca0a9a595eab41659dafec2e923f7f2474e4f82ea30b2f3e", + "result_sha256": "42a56af136d1d3bdfc16d9cea092c2db0eaa4db2c85dceb21958c5e418122900", + "trial_id": "cells/tp1_mns8/low-rep2/result.json" + }, + "target": { + "requests_sha256": "8a72240f84bbeff9eabd63e7d1f7a4c19910660fc5e4335013a1deaf89408127", + "result_sha256": "0214f32cabbcdc1bad6c331c3945c505cad136fcc9a8336aeb6314b27474e279", + "trial_id": "cells/tp1_mns8/low-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.005797101449275366, + "admitted_input_tokens_mean_over_limit": 0.08894061053240737, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.4148148148148148, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.41481481481481475, + "tpot_max_over_slo": 0.18239212881473055, + "tpot_mean_over_slo": 0.22534509859074725, + "ttft_max_over_slo_max": 0.009043970329609387, + "ttft_mean_over_slo_max": 0.0034825192906686835 + }, + "delta_state": { + "batch_size.mean": 4.136376094514087, + "batch_tokens.mean": 294.2141424923907, + "decode_batch_size.mean": 4.035904389596526, + "graph_full_share": -0.08650808533964194, + "graph_none_share": 0.08037736393404786, + "graph_padding_fraction": -0.007242852466043042, + "kv_usage_end_minus_start": 0.03774188185038829, + "kv_usage_max": 0.02683241888245158, + "kv_usage_mean": 0.01589560565810534, + "preemptions": 0.0, + "prefill_token_fraction": 0.01670564225806337, + "queue_running_mean": 3.658735814, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -40.39999999999999 + }, + "group": { + "cell": "tp2_mns64", + "level": "high", + "mns": 64, + "source_replicate": 1, + "target_replicate": 2, + "tp": 2 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "19537ec05f1e40543d850964db12234561954c209d55cd8cd0a1da716ab105fc", + "result_sha256": "49523b3269db649aa70072058666da3309ecabb93242cd504b6413f5de0f5d54", + "trial_id": "cells/tp2_mns64/high-rep1/result.json" + }, + "target": { + "requests_sha256": "d95e0dcc3d3f25af11b4d3da0ca133f647fd602753b804337a7131cc56e03059", + "result_sha256": "f37a42d9ea734adcb24892f86cbb19e2c51cee0baf8a6a63c5d51c4820bd28f2", + "trial_id": "cells/tp2_mns64/high-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": -0.07515914351851849, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.3333333333333333, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.3333333333333333, + "tpot_max_over_slo": -0.2056584360561972, + "tpot_mean_over_slo": -0.22610848720556842, + "ttft_max_over_slo_max": 0.0024637060026483013, + "ttft_mean_over_slo_max": -0.0009699875146560837 + }, + "delta_state": { + "batch_size.mean": -3.8698393491171377, + "batch_tokens.mean": -263.8179979938542, + "decode_batch_size.mean": -3.7790214466540357, + "graph_full_share": 0.0771251612081455, + "graph_none_share": -0.06919610871399684, + "graph_padding_fraction": 0.008047836044516822, + "kv_usage_end_minus_start": -0.028942690148230588, + "kv_usage_max": -0.029901904359948328, + "kv_usage_mean": -0.01598686261764442, + "preemptions": 0.0, + "prefill_token_fraction": -0.011907711608659222, + "queue_running_mean": -3.4762310214000003, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 33.2 + }, + "group": { + "cell": "tp2_mns64", + "level": "high", + "mns": 64, + "source_replicate": 2, + "target_replicate": 3, + "tp": 2 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "d95e0dcc3d3f25af11b4d3da0ca133f647fd602753b804337a7131cc56e03059", + "result_sha256": "f37a42d9ea734adcb24892f86cbb19e2c51cee0baf8a6a63c5d51c4820bd28f2", + "trial_id": "cells/tp2_mns64/high-rep2/result.json" + }, + "target": { + "requests_sha256": "ccba3d0a18a7db8501b040855bda473ee55c519619d4b9fa2a105bda06300829", + "result_sha256": "55447419925f30397d19863e149890241c58863d0adaafd511a67757905cb58e", + "trial_id": "cells/tp2_mns64/high-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.017021276595744678, + "admitted_input_tokens_mean_over_limit": 0.045296223958333326, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.33333333333333337, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.33333333333333337, + "tpot_max_over_slo": 0.22320900677050662, + "tpot_mean_over_slo": 0.15232562887487847, + "ttft_max_over_slo_max": 0.0061178085064360205, + "ttft_mean_over_slo_max": 0.018187332690771045 + }, + "delta_state": { + "batch_size.mean": 4.4320356115196375, + "batch_tokens.mean": 99.22318191879197, + "decode_batch_size.mean": 4.402188829591008, + "graph_full_share": -0.026162662010280058, + "graph_none_share": 0.02238508644647205, + "graph_padding_fraction": 0.010726240511491474, + "kv_usage_end_minus_start": -0.01923544232564689, + "kv_usage_max": 0.02004118226348972, + "kv_usage_mean": 0.014893345374172863, + "preemptions": 0.0, + "prefill_token_fraction": -0.00708953487763786, + "queue_running_mean": 4.799239215399999, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -38.400000000000006 + }, + "group": { + "cell": "tp2_mns64", + "level": "low", + "mns": 64, + "source_replicate": 1, + "target_replicate": 2, + "tp": 2 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "8685f475baf247f7ce52a6a2df1183cd77a6f580cd57a3a87c407b4e96c0895f", + "result_sha256": "7a8f4e462d496bd56b73b0a686653cbe499a238df62a8c9a50db687d051f1c68", + "trial_id": "cells/tp2_mns64/low-rep1/result.json" + }, + "target": { + "requests_sha256": "016a76ab3dc015700f9ecb9f60a20fcc3b456c45270472fdd1d04e84ebd7021a", + "result_sha256": "368869326f44a0032985eff9d3abfa2306b93c25370caa05cee255a8c0208c0e", + "trial_id": "cells/tp2_mns64/low-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.042553191489361694, + "admitted_input_tokens_mean_over_limit": -0.026056198846726164, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.11904761904761907, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.11904761904761904, + "tpot_max_over_slo": -0.2355341601559496, + "tpot_mean_over_slo": -0.1643432670810964, + "ttft_max_over_slo_max": -0.026516160830700145, + "ttft_mean_over_slo_max": -0.01345709933763525 + }, + "delta_state": { + "batch_size.mean": -5.32536868824603, + "batch_tokens.mean": -133.89198850405924, + "decode_batch_size.mean": -5.279523379974407, + "graph_full_share": 0.04036585621682853, + "graph_none_share": -0.034001132811680546, + "graph_padding_fraction": -0.008571843774165616, + "kv_usage_end_minus_start": 0.0019567969919042083, + "kv_usage_max": -0.028865953011293133, + "kv_usage_mean": -0.018137793999508895, + "preemptions": 0.0, + "prefill_token_fraction": 0.0041833878419771375, + "queue_running_mean": -6.512575867199999, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 57.599999999999994 + }, + "group": { + "cell": "tp2_mns64", + "level": "low", + "mns": 64, + "source_replicate": 2, + "target_replicate": 3, + "tp": 2 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "016a76ab3dc015700f9ecb9f60a20fcc3b456c45270472fdd1d04e84ebd7021a", + "result_sha256": "368869326f44a0032985eff9d3abfa2306b93c25370caa05cee255a8c0208c0e", + "trial_id": "cells/tp2_mns64/low-rep2/result.json" + }, + "target": { + "requests_sha256": "5455d1cdeabd6e2937130ed8fc8d86c1aaa6f04cbf26c3cb69ad064be5f67d73", + "result_sha256": "2de5c4524f57b487b94b4fdba35defc2a566236fc3e31f7260b7a9f85bb7007c", + "trial_id": "cells/tp2_mns64/low-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.005952380952380959, + "admitted_input_tokens_mean_over_limit": 0.11506300706129807, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.16666666666666663, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.16666666666666669, + "tpot_max_over_slo": 0.0634281492909402, + "tpot_mean_over_slo": 0.06688242088001703, + "ttft_max_over_slo_max": 0.13068378849614723, + "ttft_mean_over_slo_max": 0.05409936663613152 + }, + "delta_state": { + "batch_size.mean": 1.552585603284773, + "batch_tokens.mean": 89.12552429250253, + "decode_batch_size.mean": 1.5358014452458475, + "graph_full_share": -0.018898322943787682, + "graph_none_share": 0.01889832294378769, + "graph_padding_fraction": -0.003394579943621764, + "kv_usage_end_minus_start": 0.000535967229432166, + "kv_usage_max": 0.01052792772099076, + "kv_usage_mean": 0.006505251073294799, + "preemptions": 0.0, + "prefill_token_fraction": 0.010329730802724923, + "queue_running_mean": 0.5542123245999999, + "queue_waiting_mean": 2.0910913376, + "scheduler_steps_per_s": -32.39999999999999 + }, + "group": { + "cell": "tp2_mns8", + "level": "high", + "mns": 8, + "source_replicate": 1, + "target_replicate": 2, + "tp": 2 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "5d41a316f329e2efcc0f728cdd078377eb72fb9258738636db704bcfcca290aa", + "result_sha256": "466b3991190a232c984ba1b54fb6793cac1893dce196ee96f7238d70d49fcb74", + "trial_id": "cells/tp2_mns8/high-rep1/result.json" + }, + "target": { + "requests_sha256": "1b7d2cdcd639c00c443ae1b98287a7cc57323eb80c7f0e1bfc72f1ab5bf48760", + "result_sha256": "e9156d6871cc8cb3a20dce629cc2443df7473f3836b46eb0944d491ea5c4d44c", + "trial_id": "cells/tp2_mns8/high-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.0029761904761904795, + "admitted_input_tokens_mean_over_limit": -0.09617018479567307, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.14, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.14, + "tpot_max_over_slo": -0.07476021338613548, + "tpot_mean_over_slo": -0.06040260059613095, + "ttft_max_over_slo_max": -0.0990663813330078, + "ttft_mean_over_slo_max": -0.03931008928012512 + }, + "delta_state": { + "batch_size.mean": -1.5731600373589236, + "batch_tokens.mean": -75.68624207727024, + "decode_batch_size.mean": -1.5588509992086177, + "graph_full_share": 0.01874362795074891, + "graph_none_share": -0.018743627950748964, + "graph_padding_fraction": 0.004413545938114254, + "kv_usage_end_minus_start": 0.003471025866799371, + "kv_usage_max": -0.012199635031839073, + "kv_usage_mean": -0.007236177184589677, + "preemptions": 0.0, + "prefill_token_fraction": -0.006096255385559024, + "queue_running_mean": -0.43295287099999946, + "queue_waiting_mean": -1.9712983662, + "scheduler_steps_per_s": 28.0 + }, + "group": { + "cell": "tp2_mns8", + "level": "high", + "mns": 8, + "source_replicate": 2, + "target_replicate": 3, + "tp": 2 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "1b7d2cdcd639c00c443ae1b98287a7cc57323eb80c7f0e1bfc72f1ab5bf48760", + "result_sha256": "e9156d6871cc8cb3a20dce629cc2443df7473f3836b46eb0944d491ea5c4d44c", + "trial_id": "cells/tp2_mns8/high-rep2/result.json" + }, + "target": { + "requests_sha256": "5a68fda7c29477e5b3e66d70ad2b05b77be6c200cc1beea89d663a5729f31a2b", + "result_sha256": "f2116797b35057949acbf1f57898befac9ff18b94c808462f0969bdcf2446dd6", + "trial_id": "cells/tp2_mns8/high-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.017467248908296942, + "admitted_input_tokens_mean_over_limit": 0.045296223958333326, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.16666666666666663, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.16666666666666669, + "tpot_max_over_slo": 0.06375630676753702, + "tpot_mean_over_slo": 0.0900300343637355, + "ttft_max_over_slo_max": 0.051512272330001, + "ttft_mean_over_slo_max": 0.0365448741568495 + }, + "delta_state": { + "batch_size.mean": 3.2164733311475753, + "batch_tokens.mean": 83.75307177093255, + "decode_batch_size.mean": 3.188651751468882, + "graph_full_share": -0.027821579678693587, + "graph_none_share": 0.027821579678693545, + "graph_padding_fraction": -0.0015661122787500655, + "kv_usage_end_minus_start": -0.011063894950423037, + "kv_usage_max": 0.003075431006980267, + "kv_usage_mean": 0.009516064845812437, + "preemptions": 0.0, + "prefill_token_fraction": -0.003183579698054051, + "queue_running_mean": 2.5618635706, + "queue_waiting_mean": 1.0030269042, + "scheduler_steps_per_s": -25.60000000000001 + }, + "group": { + "cell": "tp2_mns8", + "level": "low", + "mns": 8, + "source_replicate": 1, + "target_replicate": 2, + "tp": 2 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "455a1c21388700f0e3234c49d15ba3b0426b0d3b551c096b297793afa0d4602f", + "result_sha256": "13de25a73cc8c752245f169cfca1d64220f697c15b835a646ba969a1005047fb", + "trial_id": "cells/tp2_mns8/low-rep1/result.json" + }, + "target": { + "requests_sha256": "cae4b764ebb8714a6f5d2009bb79497420b388986a992c51410f1681810c14c6", + "result_sha256": "92f92a576e990946c4767a2107d2ce0e6ba1b066337911399360c199fae3f5e4", + "trial_id": "cells/tp2_mns8/low-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.04803493449781659, + "admitted_input_tokens_mean_over_limit": -0.009297688802083315, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.10256410256410264, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.10256410256410253, + "tpot_max_over_slo": -0.07959600126436789, + "tpot_mean_over_slo": -0.121404858913335, + "ttft_max_over_slo_max": -0.07178450832725503, + "ttft_mean_over_slo_max": -0.03313614673934353 + }, + "delta_state": { + "batch_size.mean": -4.304456872572413, + "batch_tokens.mean": -123.40500688035755, + "decode_batch_size.mean": -4.261865986626437, + "graph_full_share": 0.04259088594597549, + "graph_none_share": -0.04259088594597552, + "graph_padding_fraction": 0.005238254196308781, + "kv_usage_end_minus_start": -0.0053979556678535046, + "kv_usage_max": -0.011102178323953882, + "kv_usage_mean": -0.01280541269135944, + "preemptions": 0.0, + "prefill_token_fraction": 0.000758348233463435, + "queue_running_mean": -4.5151126308, + "queue_waiting_mean": -1.010089547, + "scheduler_steps_per_s": 49.60000000000001 + }, + "group": { + "cell": "tp2_mns8", + "level": "low", + "mns": 8, + "source_replicate": 2, + "target_replicate": 3, + "tp": 2 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "cae4b764ebb8714a6f5d2009bb79497420b388986a992c51410f1681810c14c6", + "result_sha256": "92f92a576e990946c4767a2107d2ce0e6ba1b066337911399360c199fae3f5e4", + "trial_id": "cells/tp2_mns8/low-rep2/result.json" + }, + "target": { + "requests_sha256": "cc8c01e4560411cd5b84cdc9bebdda9a86b88615ee1c39a996affe669d26c167", + "result_sha256": "c9a7d026bd89cc027c45b8f1899d56e21d53d4234e95db365a9492f670a78d36", + "trial_id": "cells/tp2_mns8/low-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.0013333333333333391, + "admitted_input_tokens_mean_over_limit": 0.0868109341325431, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.025105868118572294, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.025105868118572294, + "tpot_max_over_slo": 0.029982028815044692, + "tpot_mean_over_slo": 0.05498080777948092, + "ttft_max_over_slo_max": 0.05707128983340226, + "ttft_mean_over_slo_max": 0.04147020173474 + }, + "delta_state": { + "batch_size.mean": 3.8799424686192463, + "batch_tokens.mean": 164.1232217573222, + "decode_batch_size.mean": 3.8344665271966534, + "graph_full_share": -0.04971234309623429, + "graph_none_share": 0.042429393305439336, + "graph_padding_fraction": -0.000239797608965738, + "kv_usage_end_minus_start": 0.003295296296107475, + "kv_usage_max": 0.004723598096277648, + "kv_usage_mean": 0.003950784826693938, + "preemptions": 0.0, + "prefill_token_fraction": 0.013157938495005683, + "queue_running_mean": 2.4192883438, + "queue_waiting_mean": 1.2621521842000005, + "scheduler_steps_per_s": -31.599999999999994 + }, + "group": { + "cell": "tp4_mns16", + "level": "high", + "mns": 16, + "source_replicate": 1, + "target_replicate": 2, + "tp": 4 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "8695422a0f1e0e1c097a08a7f4c082f1ba2e3bb26b58caadae0afc7438aeca90", + "result_sha256": "c54a2bb5c1b78bdef3fa09c39f5974215370d6fc7c2b6d403a3d376d53a1d9eb", + "trial_id": "cells/tp4_mns16/high-rep1/result.json" + }, + "target": { + "requests_sha256": "aa0482b10414d911af227e2df172c19ecca0add2e0024dc88f85ffbd063854bb", + "result_sha256": "91e2491de4ca0868b667e7d126b755f03bdd78378681b8070a2bbc0c38fb9055", + "trial_id": "cells/tp4_mns16/high-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.007999999999999993, + "admitted_input_tokens_mean_over_limit": -0.08157169117647056, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.1713106295149639, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.1713106295149639, + "tpot_max_over_slo": -0.04532225479511176, + "tpot_mean_over_slo": -0.04295654146532207, + "ttft_max_over_slo_max": -0.031098171496220545, + "ttft_mean_over_slo_max": -0.018361275960999665 + }, + "delta_state": { + "batch_size.mean": 0.28468665768194157, + "batch_tokens.mean": -55.978571428571456, + "decode_batch_size.mean": 0.2811657681940698, + "graph_full_share": 0.016324123989218342, + "graph_none_share": -0.006949123989218334, + "graph_padding_fraction": 0.0015473261462516127, + "kv_usage_end_minus_start": -0.0032850941403919087, + "kv_usage_max": -0.00460117222769163, + "kv_usage_mean": -0.0019998572533929705, + "preemptions": 0.0, + "prefill_token_fraction": -0.008209306549717632, + "queue_running_mean": 0.27542344140000097, + "queue_waiting_mean": -4.8449581018000005, + "scheduler_steps_per_s": 10.200000000000003 + }, + "group": { + "cell": "tp4_mns16", + "level": "high", + "mns": 16, + "source_replicate": 2, + "target_replicate": 3, + "tp": 4 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "aa0482b10414d911af227e2df172c19ecca0add2e0024dc88f85ffbd063854bb", + "result_sha256": "91e2491de4ca0868b667e7d126b755f03bdd78378681b8070a2bbc0c38fb9055", + "trial_id": "cells/tp4_mns16/high-rep2/result.json" + }, + "target": { + "requests_sha256": "3a23ffbe0e724648226152ef5433cc5825ad640c37cf47997c2f091210eceeec", + "result_sha256": "136dde0451f806841ff3996b081fe0bffac0a76f60c3d78b7516f2689e106008", + "trial_id": "cells/tp4_mns16/high-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0196078431372549, + "admitted_input_tokens_mean_over_limit": -0.009788771182442646, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.20048309178743962, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.20048309178743962, + "tpot_max_over_slo": 0.042659409131532156, + "tpot_mean_over_slo": 0.04424057361688544, + "ttft_max_over_slo_max": -0.048664552169308685, + "ttft_mean_over_slo_max": -0.015179490139230319 + }, + "delta_state": { + "batch_size.mean": 2.2298197150334396, + "batch_tokens.mean": 48.29412619947661, + "decode_batch_size.mean": 2.216560046525153, + "graph_full_share": -0.03405059610351846, + "graph_none_share": 0.02576330328583891, + "graph_padding_fraction": -0.0005586075805253821, + "kv_usage_end_minus_start": 0.005978463249284638, + "kv_usage_max": 0.005988665405000093, + "kv_usage_mean": 0.0033715964225947276, + "preemptions": 0.0, + "prefill_token_fraction": -0.0013328880320482295, + "queue_running_mean": 2.578025910000001, + "queue_waiting_mean": 0.9737728395999999, + "scheduler_steps_per_s": -3.5999999999999943 + }, + "group": { + "cell": "tp4_mns16", + "level": "low", + "mns": 16, + "source_replicate": 1, + "target_replicate": 2, + "tp": 4 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "d7eaaefd35cc906f759dcaa7ac3c8543249a1a02f28572bec953e41e4379a29f", + "result_sha256": "11d9ec9d2a3358aed988f7b4b1014af087d61c4b5a72eb9076ae19bb5cc481cd", + "trial_id": "cells/tp4_mns16/low-rep1/result.json" + }, + "target": { + "requests_sha256": "7fdb90f679bba1747fbacc92fac2045c820e09965c755b20f6dafd4fd9e26a7a", + "result_sha256": "800e2aa6e9f9efef4be12ee778be6817364914d63ae0624775d02729089768d6", + "trial_id": "cells/tp4_mns16/low-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.03137254901960784, + "admitted_input_tokens_mean_over_limit": 0.04353239639945655, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.17826086956521736, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.17826086956521742, + "tpot_max_over_slo": -0.06113145905583517, + "tpot_mean_over_slo": -0.07880983479690457, + "ttft_max_over_slo_max": 0.028715976329597957, + "ttft_mean_over_slo_max": 0.012690293574927458 + }, + "delta_state": { + "batch_size.mean": -5.802285141359179, + "batch_tokens.mean": -131.81382573186144, + "decode_batch_size.mean": -5.749516966234634, + "graph_full_share": 0.0604758274237851, + "graph_none_share": -0.055969252942589476, + "graph_padding_fraction": 0.00012825203851504416, + "kv_usage_end_minus_start": -0.0044583420476747015, + "kv_usage_max": -0.003932931028326259, + "kv_usage_mean": -0.00443657185283492, + "preemptions": 0.0, + "prefill_token_fraction": 0.003659185530759501, + "queue_running_mean": -4.786444050200001, + "queue_waiting_mean": -1.1252851525999998, + "scheduler_steps_per_s": 33.39999999999999 + }, + "group": { + "cell": "tp4_mns16", + "level": "low", + "mns": 16, + "source_replicate": 2, + "target_replicate": 3, + "tp": 4 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "7fdb90f679bba1747fbacc92fac2045c820e09965c755b20f6dafd4fd9e26a7a", + "result_sha256": "800e2aa6e9f9efef4be12ee778be6817364914d63ae0624775d02729089768d6", + "trial_id": "cells/tp4_mns16/low-rep2/result.json" + }, + "target": { + "requests_sha256": "b592fe254476957265fc9139aefdea77be4b5540f3c1b8a429c10ea6ccab9a6a", + "result_sha256": "cc1b1f7f5280c2892d0c6deac9df9f34ce126b0284e6ad893ffbf0ddbf27969f", + "trial_id": "cells/tp4_mns16/low-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.0013333333333333391, + "admitted_input_tokens_mean_over_limit": 0.0868109341325431, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.03781004234724741, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.0378100423472475, + "tpot_max_over_slo": 0.055603621890548405, + "tpot_mean_over_slo": 0.21675693493770581, + "ttft_max_over_slo_max": -0.008919246349250898, + "ttft_mean_over_slo_max": -0.0006546831702710018 + }, + "delta_state": { + "batch_size.mean": 9.968307375844843, + "batch_tokens.mean": 424.4382456655892, + "decode_batch_size.mean": 9.85935938877461, + "graph_full_share": -0.0783720246841022, + "graph_none_share": 0.06904202174551866, + "graph_padding_fraction": -0.0017060748565999734, + "kv_usage_end_minus_start": 0.010798462764978867, + "kv_usage_max": 0.010486722950182936, + "kv_usage_mean": 0.007244217838718085, + "preemptions": 0.0, + "prefill_token_fraction": 0.006436717740402331, + "queue_running_mean": 5.32519087, + "queue_waiting_mean": 0.041582113999999996, + "scheduler_steps_per_s": -25.400000000000006 + }, + "group": { + "cell": "tp4_mns64", + "level": "high", + "mns": 64, + "source_replicate": 1, + "target_replicate": 2, + "tp": 4 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "b4aeaf5b3175bd85fdac08a2ef657fb2278c6c4b36ea32ea417275ab6d55812d", + "result_sha256": "111a1042b614d05772888665a7cfbc86925b2fbeb52ffe19edb62a5bcfe2f513", + "trial_id": "cells/tp4_mns64/high-rep1/result.json" + }, + "target": { + "requests_sha256": "5d82b6885f5c654ae1ae126696df4bafe04f5be1c1109db24c7856b0456d9443", + "result_sha256": "6ed574c18723e265ab1327c5e43d3c76d37b3fc5711d06c77a51718e4ba16601", + "trial_id": "cells/tp4_mns64/high-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.007999999999999993, + "admitted_input_tokens_mean_over_limit": -0.08157169117647056, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.35603715170278644, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.3560371517027863, + "tpot_max_over_slo": -0.22825533543405396, + "tpot_mean_over_slo": -0.27698499834846063, + "ttft_max_over_slo_max": 0.06069803666711475, + "ttft_mean_over_slo_max": 0.020012810632276036 + }, + "delta_state": { + "batch_size.mean": -9.712633888243644, + "batch_tokens.mean": -497.54978706929927, + "decode_batch_size.mean": -9.560162601626017, + "graph_full_share": 0.10699445089688986, + "graph_none_share": -0.09583172022196412, + "graph_padding_fraction": 0.007385862591293114, + "kv_usage_end_minus_start": -0.026819845049980584, + "kv_usage_max": -0.026712524785870495, + "kv_usage_mean": -0.011639675385128475, + "preemptions": 0.0, + "prefill_token_fraction": -0.01559531495655564, + "queue_running_mean": -11.737911770200002, + "queue_waiting_mean": -0.0431090294, + "scheduler_steps_per_s": 34.599999999999994 + }, + "group": { + "cell": "tp4_mns64", + "level": "high", + "mns": 64, + "source_replicate": 2, + "target_replicate": 3, + "tp": 4 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "5d82b6885f5c654ae1ae126696df4bafe04f5be1c1109db24c7856b0456d9443", + "result_sha256": "6ed574c18723e265ab1327c5e43d3c76d37b3fc5711d06c77a51718e4ba16601", + "trial_id": "cells/tp4_mns64/high-rep2/result.json" + }, + "target": { + "requests_sha256": "5c5861d716181f06dc3f951ae0d6ee479b1b61fc21ebf24541c5b45053fcc167", + "result_sha256": "35b8f0465f6b0c197f71b5ecaf4e771e6613f86998ae44f8701ab9e4e1e440b4", + "trial_id": "cells/tp4_mns64/high-rep3/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0196078431372549, + "admitted_input_tokens_mean_over_limit": -0.009788771182442646, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": -0.30917874396135264, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.3091787439613527, + "tpot_max_over_slo": 0.04185318252929737, + "tpot_mean_over_slo": 0.01816067477013092, + "ttft_max_over_slo_max": -0.021103239327203482, + "ttft_mean_over_slo_max": -0.00994494437705411 + }, + "delta_state": { + "batch_size.mean": 4.069655451351146, + "batch_tokens.mean": 102.47724917212267, + "decode_batch_size.mean": 4.024874238479235, + "graph_full_share": -0.04226598245658386, + "graph_none_share": 0.03788012841578402, + "graph_padding_fraction": 0.0018194855348361953, + "kv_usage_end_minus_start": 0.014830638402256846, + "kv_usage_max": 0.004231484699196653, + "kv_usage_mean": 0.0037268636706596775, + "preemptions": 0.0, + "prefill_token_fraction": -0.00019342359815088805, + "queue_running_mean": 3.5476991263999995, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -16.80000000000001 + }, + "group": { + "cell": "tp4_mns64", + "level": "low", + "mns": 64, + "source_replicate": 1, + "target_replicate": 2, + "tp": 4 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "931d372ded1e3aaa9c67f06653fac8f79b4ee4f369028c1148163580386681ce", + "result_sha256": "325ce57f33133670a6761b5dffb831ed3b2f464755ff743b56976b4a974c1215", + "trial_id": "cells/tp4_mns64/low-rep1/result.json" + }, + "target": { + "requests_sha256": "72c8f70a240f79edcaa4eb0a2fe9bade8ac07093339c5b2d09c49ddd1cf27d16", + "result_sha256": "ff8b2283d02d5967893ca6225b561ae11011da6149d1f6cff7b4275c00170e1d", + "trial_id": "cells/tp4_mns64/low-rep2/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": -0.03137254901960784, + "admitted_input_tokens_mean_over_limit": 0.04353239639945655, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.35362318840579715, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.35362318840579715, + "tpot_max_over_slo": -0.092913777013786, + "tpot_mean_over_slo": -0.060414495575378896, + "ttft_max_over_slo_max": 0.04327231583495934, + "ttft_mean_over_slo_max": 0.013034266129003418 + }, + "delta_state": { + "batch_size.mean": -6.61237956440774, + "batch_tokens.mean": -157.8171403125253, + "decode_batch_size.mean": -6.541858958788763, + "graph_full_share": 0.056250506031900294, + "graph_none_share": -0.05232369848595256, + "graph_padding_fraction": -0.0017085122195497817, + "kv_usage_end_minus_start": -0.01598560886363176, + "kv_usage_max": -0.006694740284960865, + "kv_usage_mean": -0.00503459907780822, + "preemptions": 0.0, + "prefill_token_fraction": 0.002577201005920182, + "queue_running_mean": -5.8976995362, + "queue_waiting_mean": 0.000909714, + "scheduler_steps_per_s": 38.80000000000001 + }, + "group": { + "cell": "tp4_mns64", + "level": "low", + "mns": 64, + "source_replicate": 2, + "target_replicate": 3, + "tp": 4 + }, + "kind": "same_config_workload_repeat", + "source": { + "requests_sha256": "72c8f70a240f79edcaa4eb0a2fe9bade8ac07093339c5b2d09c49ddd1cf27d16", + "result_sha256": "ff8b2283d02d5967893ca6225b561ae11011da6149d1f6cff7b4275c00170e1d", + "trial_id": "cells/tp4_mns64/low-rep2/result.json" + }, + "target": { + "requests_sha256": "8b2c779d2097c5e4038e94e1ac9b986e29f55d9acfd67eeb366c4e0e9270f6e0", + "result_sha256": "f026b69356701894fab184e0f2f7504a30b537982e823373899c9ab123a33891", + "trial_id": "cells/tp4_mns64/low-rep3/result.json" + } + } + ], + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.21069215415079778, + "action_delta": { + "distinct_n": 12, + "max": 10.325762195121952, + "min": -1.1226839723743751, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.054373699571412804, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 3.874890908868192, + "repeat_abs_p95": 9.247595739668252, + "repeat_delta": { + "distinct_n": 24, + "max": 9.968307375844843, + "min": -9.712633888243644, + "n": 24 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 18.045513604403595, + "action_delta": { + "distinct_n": 12, + "max": 461.67439024390245, + "min": -34.76613479399856, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.10712015825142496, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 168.46048305911222, + "repeat_abs_p95": 418.5154006326717, + "repeat_delta": { + "distinct_n": 24, + "max": 424.4382456655892, + "min": -497.54978706929927, + "n": 24 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.20671153934341824, + "action_delta": { + "distinct_n": 12, + "max": 10.164329268292683, + "min": -1.1063943796141942, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.05430140299778245, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 3.8067439869253445, + "repeat_abs_p95": 9.107417055200422, + "repeat_delta": { + "distinct_n": 24, + "max": 9.85935938877461, + "min": -9.560162601626017, + "n": 24 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.003299836171280379, + "action_delta": { + "distinct_n": 12, + "max": 0.012598237675637058, + "min": -0.08612804878048774, + "n": 12 + }, + "action_signs": { + "consistency": 0.75, + "negative": 3, + "positive": 9, + "zero": 0 + }, + "effect_to_repeat_median": 0.06753691417406232, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.048859741544834856, + "repeat_abs_p95": 0.09691268740505915, + "repeat_delta": { + "distinct_n": 24, + "max": 0.10699445089688986, + "min": -0.09768839574664823, + "n": 24 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.00975501010655705, + "action_delta": { + "distinct_n": 12, + "max": 0.07111280487804877, + "min": -0.01938556799237913, + "n": 12 + }, + "action_signs": { + "consistency": 0.75, + "negative": 9, + "positive": 3, + "zero": 0 + }, + "effect_to_repeat_median": 0.21877645860951367, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.04458893872109165, + "repeat_abs_p95": 0.09453988067718995, + "repeat_delta": { + "distinct_n": 24, + "max": 0.09557713052858684, + "min": -0.09583172022196412, + "n": 24 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0012515460983914576, + "action_delta": { + "distinct_n": 12, + "max": 0.006616409980351494, + "min": -9.478587342680138e-05, + "n": 12 + }, + "action_signs": { + "consistency": 0.9090909090909091, + "negative": 1, + "positive": 10, + "zero": 1 + }, + "effect_to_repeat_median": 0.2567104007991265, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.004875322910546117, + "repeat_abs_p95": 0.015986856375890948, + "repeat_delta": { + "distinct_n": 24, + "max": 0.010726240511491474, + "min": -0.01983738870430219, + "n": 24 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.008146776216140739, + "action_delta": { + "distinct_n": 12, + "max": 0.08606331121732713, + "min": -0.01687018923627459, + "n": 12 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 2, + "positive": 10, + "zero": 0 + }, + "effect_to_repeat_median": 0.4626083515997178, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.017610525594639326, + "repeat_abs_p95": 0.1314417024677789, + "repeat_delta": { + "distinct_n": 24, + "max": 0.16350284743730636, + "min": -0.1340793286042561, + "n": 24 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.004591739211784462, + "action_delta": { + "distinct_n": 12, + "max": 0.04171319199456147, + "min": -0.01682167078536223, + "n": 12 + }, + "action_signs": { + "consistency": 0.9166666666666666, + "negative": 1, + "positive": 11, + "zero": 0 + }, + "effect_to_repeat_median": 0.19642246579235886, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.023376853524680108, + "repeat_abs_p95": 0.1262513737636127, + "repeat_delta": { + "distinct_n": 24, + "max": 0.1339294634828655, + "min": -0.12953341992207013, + "n": 24 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0002990136124866498, + "action_delta": { + "distinct_n": 12, + "max": 0.01377304273482051, + "min": -0.00022787683290566817, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 8, + "zero": 0 + }, + "effect_to_repeat_median": 0.021590398513840623, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.013849379032766153, + "repeat_abs_p95": 0.07511600655858494, + "repeat_delta": { + "distinct_n": 24, + "max": 0.07938311082464784, + "min": -0.0768057629061545, + "n": 24 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 12 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 12 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 24 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0009881350686721802, + "action_delta": { + "distinct_n": 12, + "max": 0.016050535408914768, + "min": -0.009907841139181417, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.12917776465155323, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.007649420713677746, + "repeat_abs_p95": 0.06511170572257907, + "repeat_delta": { + "distinct_n": 24, + "max": 0.07360971724683552, + "min": -0.033037405519596885, + "n": 24 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.42911666849999985, + "action_delta": { + "distinct_n": 12, + "max": 12.835752482200002, + "min": -0.21642387019999987, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.10739626232229334, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 3.9956387608, + "repeat_abs_p95": 6.420344417549997, + "repeat_delta": { + "distinct_n": 24, + "max": 5.32519087, + "min": -11.737911770200002, + "n": 24 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.28290130680000003, + "action_delta": { + "distinct_n": 9, + "max": 0.0, + "min": -6.7273728942, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 8, + "positive": 0, + "zero": 4 + }, + "effect_to_repeat_median": 1.2915540857901293, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.2190394579, + "repeat_abs_p95": 2.0731223918899997, + "repeat_delta": { + "distinct_n": 16, + "max": 2.0910913376, + "min": -4.8449581018000005, + "n": 24 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 3.8000000000000043, + "action_delta": { + "distinct_n": 12, + "max": 12.400000000000006, + "min": -29.19999999999999, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.11176470588235309, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 33.99999999999999, + "repeat_abs_p95": 71.32999999999997, + "repeat_delta": { + "distinct_n": 24, + "max": 72.8, + "min": -77.8, + "n": 24 + } + } + }, + "sanity": { + "action_pairs": 12, + "invariants": { + "efficacy_label_balance": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "matched_action_request_hashes": true, + "probabilities_bounded": true + }, + "red_flags": [], + "repeat_pairs": 24, + "trials": 36 + } + } + }, + "provenance": { + "analysis_script": "/home/gahow/phd/aituner/runs/intervention-response-v0/analyze_p1.py", + "analysis_script_sha256": "902adcba58bcb73f97c455fdea3710f6cb28fa8d8f4ea5bc4915143ed106b17a", + "manifest": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/pilot-manifest.json", + "manifest_sha256": "86b249a087ab1bcd51cd8de3d1da6bf34afd41af977eb58c93229b941dbf5c8a", + "manifest_validation": { + "expected_trials": 36, + "matched_trials": 36, + "schema": "fidelity-prefix-pilot-manifest-v1" + }, + "phase6_v0_script_sha256": "f1b40c350f22e797b08b129df7b55386b407bd5ae7f466d36987e94d768ded2f", + "run_root": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b", + "streams": [ + { + "bytes": 24501686, + "path": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/cells/tp1_mns64/opprof/opprof-v1-dp0-pid234511-1784005631788937266.jsonl", + "sha256": "a813076afe677e5c13010b2ec44f46998794ca44cdf30ee81be836e3623f0068" + }, + { + "bytes": 25524027, + "path": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/cells/tp1_mns8/opprof/opprof-v1-dp0-pid227734-1784005177936445360.jsonl", + "sha256": "1708759d2e77c42bff1940d43214fb2d927c3eebeda62d53a3fb7d04817e9b22" + }, + { + "bytes": 29431988, + "path": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/cells/tp2_mns64/opprof/opprof-v1-dp0-pid247791-1784006495467408663.jsonl", + "sha256": "65355de883f2eb5d31be161d82b26dddc53c41a0a35a0fdb807777bc5b21c5ec" + }, + { + "bytes": 26036993, + "path": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/cells/tp2_mns8/opprof/opprof-v1-dp0-pid241594-1784006096655191299.jsonl", + "sha256": "485e203d37d45fdd3c4de768b95da2050478f111f4d739b129d4f4ebd4737b70" + }, + { + "bytes": 17449143, + "path": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/cells/tp4_mns16/opprof/opprof-v1-dp0-pid256090-1784007028237607411.jsonl", + "sha256": "59e98014d01bfc68d9f97112ecd782cb03571ad437382ed67416c7b85ae48c56" + }, + { + "bytes": 22674678, + "path": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/cells/tp4_mns64/opprof/opprof-v1-dp0-pid262293-1784007376383569515.jsonl", + "sha256": "03e4f3cca3023a78e859b663703b9468dedf89e58320f0292e2973070e7616c8" + } + ] + }, + "sanity": { + "red_flags": [], + "stream_bytes": { + "distinct_n": 6, + "max": 29431988.0, + "min": 17449143.0, + "n": 6 + }, + "stream_count": 6 + }, + "schema": "intervention-response-p1-confirmation-v1", + "stable_incremental_efficacy_features": [], + "stable_response_features": [], + "status": "COMPLETE" +} diff --git a/runs/intervention-response-v0/phase6-audit.json b/runs/intervention-response-v0/phase6-audit.json new file mode 100644 index 0000000..6231482 --- /dev/null +++ b/runs/intervention-response-v0/phase6-audit.json @@ -0,0 +1,5121 @@ +{ + "claim_boundary": "Development-only identifiability gate. Passing opens a controlled real-GPU pilot; it does not establish tuning benefit or causality.", + "decision": "STOP_NO_IDENTIFIABLE_RESPONSE", + "frozen_gate": { + "expected_action_pairs": 17, + "gate_features": [ + "scheduler_steps_per_s", + "decode_batch_size.mean", + "prefill_token_fraction", + "queue_waiting_mean", + "queue_running_mean", + "kv_usage_mean", + "graph_padding_fraction" + ], + "horizons_s": [ + 5.0, + 10.0 + ], + "minimum_action_above_repeat_p95_fraction": 0.5, + "minimum_effect_to_repeat_median": 2.0, + "minimum_repeat_pairs": 20, + "minimum_sign_consistency": 0.75, + "minimum_stable_features": 2 + }, + "horizons": { + "10": { + "actions": [ + { + "delta_state": { + "batch_size.mean": 1.5764389277324504, + "batch_tokens.mean": 81.00995765564915, + "decode_batch_size.mean": 1.5631264281332364, + "graph_full_share": -0.011964790704335537, + "graph_none_share": 0.020610006604949223, + "graph_padding_fraction": -0.01937281246147585, + "kv_usage_end_minus_start": -0.02063938485088157, + "kv_usage_max": 0.05576515049492037, + "kv_usage_mean": 0.02372409715700479, + "preemptions": 0.0, + "prefill_token_fraction": 0.09894954087875252, + "queue_running_mean": 1.8681007873, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -51.89999999999999 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.24609375, + "request_hash": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "mns_increase", + "source": { + "cell": "tp1_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp1_mns16/anchor-0.24609375/result.json" + }, + "target": { + "cell": "tp1_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp1_mns32/anchor-0.24609375/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.003449406279594669, + "batch_tokens.mean": 0.2550229474757799, + "decode_batch_size.mean": 0.003361987324251192, + "graph_full_share": -8.37764988708134e-05, + "graph_none_share": -0.0012785022218984483, + "graph_padding_fraction": 0.00024814146059230855, + "kv_usage_end_minus_start": 4.607711554860394e-05, + "kv_usage_max": 0.00030157213266412963, + "kv_usage_mean": 0.0007577042092319561, + "preemptions": 0.0, + "prefill_token_fraction": 6.688650356756476e-05, + "queue_running_mean": 0.05724569279999958, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -0.20000000000000284 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.24609375, + "request_hash": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "mns_increase", + "source": { + "cell": "tp1_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp1_mns32/anchor-0.24609375/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp1_mns64/anchor-0.24609375/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.012661422160780589, + "batch_tokens.mean": -0.39155502891061644, + "decode_batch_size.mean": -0.012529760925139577, + "graph_full_share": 0.0001261753508223773, + "graph_none_share": -0.0014757030161394735, + "graph_padding_fraction": 0.0004267789129811959, + "kv_usage_end_minus_start": 7.934839022727402e-06, + "kv_usage_max": 0.03040170172083234, + "kv_usage_mean": 0.0025619580859006993, + "preemptions": 0.0, + "prefill_token_fraction": -2.6753484588915022e-05, + "queue_running_mean": 0.22427197499999973, + "queue_waiting_mean": -0.0255129477, + "scheduler_steps_per_s": 0.29999999999999716 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.04195804195804198, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.25, + "request_hash": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "mns_increase", + "source": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 0.958041958041958, + "mns": 8, + "trial_id": "tp1_mns8/anchor-0.25/result.json" + }, + "target": { + "cell": "tp1_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp1_mns16/anchor-0.25/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.012870461691032453, + "batch_tokens.mean": -0.2637663627959057, + "decode_batch_size.mean": -0.01278327820794356, + "graph_full_share": 8.355083796041107e-05, + "graph_none_share": -0.002771708233208551, + "graph_padding_fraction": -4.1151267345866487e-05, + "kv_usage_end_minus_start": 1.7658454245395028e-05, + "kv_usage_max": 0.00011557359098801623, + "kv_usage_mean": -0.00031444354435781335, + "preemptions": 0.0, + "prefill_token_fraction": 2.6753484588915022e-05, + "queue_running_mean": -0.05587743330000006, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.20000000000000284 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.25, + "request_hash": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "mns_increase", + "source": { + "cell": "tp1_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp1_mns16/anchor-0.25/result.json" + }, + "target": { + "cell": "tp1_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp1_mns32/anchor-0.25/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.021751047667267098, + "batch_tokens.mean": 0.5249008791116694, + "decode_batch_size.mean": 0.021576208801315833, + "graph_full_share": -0.00016755391320344426, + "graph_none_share": -0.0012074809179768962, + "graph_padding_fraction": 9.628532857289349e-05, + "kv_usage_end_minus_start": 4.607711554860394e-05, + "kv_usage_max": 0.00030157213266412963, + "kv_usage_mean": 0.00019975354503918435, + "preemptions": 0.0, + "prefill_token_fraction": -1.337692842384719e-05, + "queue_running_mean": 0.0040136895999998146, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -0.3999999999999915 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.25, + "request_hash": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "mns_increase", + "source": { + "cell": "tp1_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp1_mns32/anchor-0.25/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp1_mns64/anchor-0.25/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 6.1794369551023545, + "batch_tokens.mean": 175.96941851979875, + "decode_batch_size.mean": 6.118283596420478, + "graph_full_share": -0.05096593309521069, + "graph_none_share": 0.047163651726389366, + "graph_padding_fraction": 0.0032959117320311947, + "kv_usage_end_minus_start": 0.07533673225448412, + "kv_usage_max": 0.06162226941794913, + "kv_usage_mean": 0.056370734075687004, + "preemptions": 0.0, + "prefill_token_fraction": -0.0018186060572318308, + "queue_running_mean": 5.9180508831, + "queue_waiting_mean": -2.7288210595, + "scheduler_steps_per_s": -13.3 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.06521739130434782, + "feasibility_transition": "false->false" + }, + "group": { + "anchor": 0.5, + "request_hash": "3160017dbd8249f711eb115f1e72ddbb37eab3a3333a48762cc3104e94bd2f15", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "mns_increase", + "source": { + "cell": "tp1_mns8", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.09057971014492754, + "mns": 8, + "trial_id": "tp1_mns8/anchor-0.5/result.json" + }, + "target": { + "cell": "tp1_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.15579710144927536, + "mns": 16, + "trial_id": "tp1_mns16/anchor-0.5/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 4.764302767707134, + "batch_tokens.mean": 277.81382085064996, + "decode_batch_size.mean": 4.6529755062339735, + "graph_full_share": -0.09344327526748608, + "graph_none_share": 0.0681757891944469, + "graph_padding_fraction": 0.000877004096309171, + "kv_usage_end_minus_start": 0.13999257948102506, + "kv_usage_max": 0.13096340508568438, + "kv_usage_mean": 0.05558263246404621, + "preemptions": 0.0, + "prefill_token_fraction": 0.006009982585939322, + "queue_running_mean": 6.513921786700001, + "queue_waiting_mean": -2.611976143, + "scheduler_steps_per_s": -9.100000000000001 + }, + "descriptive_full_outcome": { + "delta_pass_rate": -0.06521739130434782, + "feasibility_transition": "false->false" + }, + "group": { + "anchor": 0.5, + "request_hash": "3160017dbd8249f711eb115f1e72ddbb37eab3a3333a48762cc3104e94bd2f15", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "mns_increase", + "source": { + "cell": "tp1_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.15579710144927536, + "mns": 16, + "trial_id": "tp1_mns16/anchor-0.5/result.json" + }, + "target": { + "cell": "tp1_mns32", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.09057971014492754, + "mns": 32, + "trial_id": "tp1_mns32/anchor-0.5/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.8274828140997492, + "batch_tokens.mean": 96.29113646336111, + "decode_batch_size.mean": -0.8609770367120078, + "graph_full_share": -0.017588123445955728, + "graph_none_share": 0.015211350007313162, + "graph_padding_fraction": 7.496352550239935e-05, + "kv_usage_end_minus_start": 0.03212970309780483, + "kv_usage_max": 0.01812081632758722, + "kv_usage_mean": 0.003998827610290162, + "preemptions": 0.0, + "prefill_token_fraction": 0.004183520933583096, + "queue_running_mean": 0.37492188330000076, + "queue_waiting_mean": -0.1123975519, + "scheduler_steps_per_s": -1.299999999999999 + }, + "descriptive_full_outcome": { + "delta_pass_rate": -0.02898550724637681, + "feasibility_transition": "false->false" + }, + "group": { + "anchor": 0.5, + "request_hash": "3160017dbd8249f711eb115f1e72ddbb37eab3a3333a48762cc3104e94bd2f15", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "mns_increase", + "source": { + "cell": "tp1_mns32", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.09057971014492754, + "mns": 32, + "trial_id": "tp1_mns32/anchor-0.5/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.06159420289855073, + "mns": 64, + "trial_id": "tp1_mns64/anchor-0.5/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.07209087471218689, + "batch_tokens.mean": 0.5870128720027967, + "decode_batch_size.mean": 0.071372838111468, + "graph_full_share": -0.0007180366007190075, + "graph_none_share": 0.00017950915017974842, + "graph_padding_fraction": -0.00012257265767716408, + "kv_usage_end_minus_start": 0.0005957119907479269, + "kv_usage_max": 0.003445359565031225, + "kv_usage_mean": 0.0015147564690633879, + "preemptions": 0.0, + "prefill_token_fraction": 0.0006457635751305757, + "queue_running_mean": 0.2144143154, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -5.000000000000014 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.2421875, + "request_hash": "adb62bea7f7a12c1e33fa1572ec1d0e274100013ad90e14c6ef5c549b0e0d017", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "mns_increase", + "source": { + "cell": "tp1_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp1_mns32/anchor-0.2421875/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp1_mns64/anchor-0.2421875/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.36245150019298356, + "batch_tokens.mean": -0.3978304588945094, + "decode_batch_size.mean": -0.35854104454872315, + "graph_full_share": 0.003837325045198847, + "graph_none_share": -0.010532837670384139, + "graph_padding_fraction": -0.04819081852048884, + "kv_usage_end_minus_start": -0.006582719323285691, + "kv_usage_max": -0.0005696857286345836, + "kv_usage_mean": -0.0009069497569939534, + "preemptions": 0.0, + "prefill_token_fraction": -0.0024232654720200003, + "queue_running_mean": -0.4996837289, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 21.599999999999994 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.4921875, + "request_hash": "cd227aee3be472a35e8c35af60a44a1411c4dc689cbd98746b3fb8a8d329b44c", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "mns_increase", + "source": { + "cell": "tp2_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 8, + "trial_id": "tp2_mns8/anchor-0.4921875/result.json" + }, + "target": { + "cell": "tp2_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp2_mns16/anchor-0.4921875/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.11406418543540475, + "batch_tokens.mean": 9.343502830968255, + "decode_batch_size.mean": -0.11416962019156696, + "graph_full_share": -0.0001054347561618707, + "graph_none_share": -0.006673028798106863, + "graph_padding_fraction": 0.00039894330767695045, + "kv_usage_end_minus_start": 0.013684335607858866, + "kv_usage_max": 0.015105694785835122, + "kv_usage_mean": 0.003178130879397629, + "preemptions": 0.0, + "prefill_token_fraction": 0.010932399371689994, + "queue_running_mean": 1.5075751790999998, + "queue_waiting_mean": -1.0976031745, + "scheduler_steps_per_s": 7.0 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.728937728937729, + "feasibility_transition": "false->true" + }, + "group": { + "anchor": 0.49609375, + "request_hash": "a78992ce59bb3b857a7ef8844e0a690ea9dd529a659cf5edd599b63fef8b6a80", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "mns_increase", + "source": { + "cell": "tp2_mns8", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.27106227106227104, + "mns": 8, + "trial_id": "tp2_mns8/anchor-0.49609375/result.json" + }, + "target": { + "cell": "tp2_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp2_mns16/anchor-0.49609375/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 5.413566807825205, + "batch_tokens.mean": 200.71125302018655, + "decode_batch_size.mean": 5.3571681691824065, + "graph_full_share": -0.05071523213218676, + "graph_none_share": 0.05388272583201268, + "graph_padding_fraction": -0.227748799260124, + "kv_usage_end_minus_start": 0.0335338856501185, + "kv_usage_max": 0.060946820037647065, + "kv_usage_mean": 0.028804550592809862, + "preemptions": 0.0, + "prefill_token_fraction": 0.8741864436917179, + "queue_running_mean": 10.8362644165, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -89.4 + }, + "descriptive_full_outcome": { + "delta_pass_rate": -0.7698209718670077, + "feasibility_transition": "true->false" + }, + "group": { + "anchor": 0.75, + "request_hash": "4f4c792e90f5c1bf33978fea26debcadfce5f8b872a31be86a88a1bf6b09a21c", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "mns_increase", + "source": { + "cell": "tp2_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp2_mns32/anchor-0.75/result.json" + }, + "target": { + "cell": "tp2_mns64", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.23017902813299232, + "mns": 64, + "trial_id": "tp2_mns64/anchor-0.75/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 2.075957134599138, + "batch_tokens.mean": 46.93942520046838, + "decode_batch_size.mean": 2.0480560866794146, + "graph_full_share": -0.0074377102879925605, + "graph_none_share": -0.0018112558732417336, + "graph_padding_fraction": 0.0006496717251760695, + "kv_usage_end_minus_start": 0.018695600546584812, + "kv_usage_max": 0.017004326138624792, + "kv_usage_mean": 0.00908001267992682, + "preemptions": 0.0, + "prefill_token_fraction": 0.0002714485379902154, + "queue_running_mean": 8.942033995400001, + "queue_waiting_mean": -3.5751631934000008, + "scheduler_steps_per_s": -12.200000000000003 + }, + "descriptive_full_outcome": { + "delta_pass_rate": -0.032423208191126277, + "feasibility_transition": "false->false" + }, + "group": { + "anchor": 0.033717411016, + "request_hash": "472a874ecc0e844b5b36a715bbc309b63e73074d39327cd734869f76721d71db", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "mns_increase", + "source": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.06313993174061433, + "mns": 16, + "trial_id": "tp4_mns16/anchor-0.033717411016/result.json" + }, + "target": { + "cell": "tp4_mns32", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.030716723549488054, + "mns": 32, + "trial_id": "tp4_mns32/anchor-0.033717411016/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.35039583066624225, + "batch_tokens.mean": 122.17784504581891, + "decode_batch_size.mean": -0.430057526564223, + "graph_full_share": -0.016886297279774087, + "graph_none_share": 0.014833841973330465, + "graph_padding_fraction": -0.0008982073370737251, + "kv_usage_end_minus_start": 0.023420994136330364, + "kv_usage_max": 0.017803055174036908, + "kv_usage_mean": 0.0044696216831996635, + "preemptions": 0.0, + "prefill_token_fraction": 0.015941265487523948, + "queue_running_mean": 5.9603496991, + "queue_waiting_mean": -5.3947101023999995, + "scheduler_steps_per_s": -6.0 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.08361774744027303, + "feasibility_transition": "false->false" + }, + "group": { + "anchor": 0.033717411016, + "request_hash": "472a874ecc0e844b5b36a715bbc309b63e73074d39327cd734869f76721d71db", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "mns_increase", + "source": { + "cell": "tp4_mns32", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.030716723549488054, + "mns": 32, + "trial_id": "tp4_mns32/anchor-0.033717411016/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.11433447098976109, + "mns": 64, + "trial_id": "tp4_mns64/anchor-0.033717411016/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -1.3459024035954341, + "batch_tokens.mean": -76.90460019071148, + "decode_batch_size.mean": -1.335767993913132, + "graph_full_share": 0.014601482572194069, + "graph_none_share": -0.03533131042635292, + "graph_padding_fraction": 0.215661617603708, + "kv_usage_end_minus_start": -0.0019582245100192575, + "kv_usage_max": 0.004037693486432903, + "kv_usage_mean": -0.0013060500299530847, + "preemptions": 0.0, + "prefill_token_fraction": -0.7248107341009736, + "queue_running_mean": -1.1173600969999997, + "queue_waiting_mean": -5.5313452697, + "scheduler_steps_per_s": 55.0 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.8216666666666667, + "feasibility_transition": "false->true" + }, + "group": { + "anchor": 0.034252608017, + "request_hash": "21c1f3b7d972b19c952a894863a70d1e85bd0e24a7e7b6b0fcc088360d0e01bd", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "mns_increase", + "source": { + "cell": "tp4_mns8", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.17833333333333334, + "mns": 8, + "trial_id": "tp4_mns8/anchor-0.034252608017/result.json" + }, + "target": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp4_mns16/anchor-0.034252608017/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.001077049918674966, + "batch_tokens.mean": -0.009036864794174093, + "decode_batch_size.mean": 0.0014499324412282633, + "graph_full_share": -0.0018530405936193484, + "graph_none_share": -0.0005575282018593251, + "graph_padding_fraction": -0.005954146779449193, + "kv_usage_end_minus_start": 2.5931683662605565e-05, + "kv_usage_max": 1.368725807104898e-05, + "kv_usage_mean": -2.4961532697604337e-05, + "preemptions": 0.0, + "prefill_token_fraction": -0.0012108546438647505, + "queue_running_mean": -0.027323256100000748, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 1.4000000000000057 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.034252608017, + "request_hash": "21c1f3b7d972b19c952a894863a70d1e85bd0e24a7e7b6b0fcc088360d0e01bd", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "mns_increase", + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp4_mns16/anchor-0.034252608017/result.json" + }, + "target": { + "cell": "tp4_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp4_mns32/anchor-0.034252608017/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.012380697130775609, + "batch_tokens.mean": 0.015975045041020763, + "decode_batch_size.mean": 0.012248179747174959, + "graph_full_share": 0.0004243534193951737, + "graph_none_share": 2.977918732597739e-06, + "graph_padding_fraction": 0.0021324803565644923, + "kv_usage_end_minus_start": -4.493314282427718e-06, + "kv_usage_max": 1.9874666237429395e-05, + "kv_usage_mean": 2.2389395203431008e-05, + "preemptions": 0.0, + "prefill_token_fraction": 9.913223620203193e-05, + "queue_running_mean": 0.015902196000000757, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -0.5 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.034252608017, + "request_hash": "21c1f3b7d972b19c952a894863a70d1e85bd0e24a7e7b6b0fcc088360d0e01bd", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "mns_increase", + "source": { + "cell": "tp4_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp4_mns32/anchor-0.034252608017/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp4_mns64/anchor-0.034252608017/result.json" + } + } + ], + "descriptive_full_outcome": { + "delta_pass_rate": { + "distinct_n": 10, + "max": 0.8216666666666667, + "min": -0.7698209718670077, + "n": 17 + }, + "feasibility_transitions": { + "false->false": 5, + "false->true": 2, + "true->false": 1, + "true->true": 9 + }, + "limitation": "Full outcomes may use different elapsed durations when a trial early-stopped; they are descriptive and are not a gate input.", + "negative": 4, + "positive": 5, + "zero": 8 + }, + "feature_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.058823529411764705, + "action_abs_median": 0.35039583066624225, + "action_delta": { + "distinct_n": 17, + "max": 6.1794369551023545, + "min": -1.3459024035954341, + "n": 17 + }, + "action_signs": { + "consistency": 0.5882352941176471, + "negative": 7, + "positive": 10, + "zero": 0 + }, + "effect_to_repeat_median": 6.562183065127602, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.05339622914945741, + "repeat_abs_p95": 5.572276889098532, + "repeat_delta": { + "distinct_n": 29, + "max": 0.648283558063266, + "min": -5.684231334296602, + "n": 29 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 9.343502830968255, + "action_delta": { + "distinct_n": 17, + "max": 277.81382085064996, + "min": -76.90460019071148, + "n": 17 + }, + "action_signs": { + "consistency": 0.7058823529411765, + "negative": 5, + "positive": 12, + "zero": 0 + }, + "effect_to_repeat_median": 8.491093823400865, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 1.1003885983709436, + "repeat_abs_p95": 315.3187671058407, + "repeat_delta": { + "distinct_n": 29, + "max": 49.961881401529, + "min": -364.45844383082266, + "n": 29 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.058823529411764705, + "action_abs_median": 0.35854104454872315, + "action_delta": { + "distinct_n": 17, + "max": 6.118283596420478, + "min": -1.335767993913132, + "n": 17 + }, + "action_signs": { + "consistency": 0.5882352941176471, + "negative": 7, + "positive": 10, + "zero": 0 + }, + "effect_to_repeat_median": 6.76984750225626, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.05296146544353153, + "repeat_abs_p95": 5.508199821866692, + "repeat_delta": { + "distinct_n": 29, + "max": 0.6427639871873927, + "min": -5.614987163710968, + "n": 29 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.058823529411764705, + "action_abs_median": 0.003837325045198847, + "action_delta": { + "distinct_n": 17, + "max": 0.014601482572194069, + "min": -0.09344327526748608, + "n": 17 + }, + "action_signs": { + "consistency": 0.7058823529411765, + "negative": 12, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 4.142422601449927, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0009263480369809951, + "repeat_abs_p95": 0.05226163287124676, + "repeat_delta": { + "distinct_n": 28, + "max": 0.055525557955363625, + "min": -0.006350826071218729, + "n": 29 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.058823529411764705, + "action_abs_median": 0.006673028798106863, + "action_delta": { + "distinct_n": 17, + "max": 0.0681757891944469, + "min": -0.03533131042635292, + "n": 17 + }, + "action_signs": { + "consistency": 0.5294117647058824, + "negative": 9, + "positive": 8, + "zero": 0 + }, + "effect_to_repeat_median": 6.753166603769785, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0009881332994778795, + "repeat_abs_p95": 0.0591340838533234, + "repeat_delta": { + "distinct_n": 22, + "max": 0.014663378024668453, + "min": -0.06263498920086392, + "n": 29 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.000877004096309171, + "action_delta": { + "distinct_n": 17, + "max": 0.215661617603708, + "min": -0.227748799260124, + "n": 17 + }, + "action_signs": { + "consistency": 0.5882352941176471, + "negative": 7, + "positive": 10, + "zero": 0 + }, + "effect_to_repeat_median": 0.05354985977835987, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.016377336933075942, + "repeat_abs_p95": 0.24277823872412585, + "repeat_delta": { + "distinct_n": 29, + "max": 0.2448027632667215, + "min": -0.016377336933075942, + "n": 29 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.11764705882352941, + "action_abs_median": 0.006582719323285691, + "action_delta": { + "distinct_n": 16, + "max": 0.13999257948102506, + "min": -0.02063938485088157, + "n": 17 + }, + "action_signs": { + "consistency": 0.7647058823529411, + "negative": 4, + "positive": 13, + "zero": 0 + }, + "effect_to_repeat_median": 322.61413699448417, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 2.0404311431021505e-05, + "repeat_abs_p95": 0.050972874042718504, + "repeat_delta": { + "distinct_n": 19, + "max": 0.004504676952132969, + "min": -0.07211360414777734, + "n": 29 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.23529411764705882, + "action_abs_median": 0.015105694785835122, + "action_delta": { + "distinct_n": 16, + "max": 0.13096340508568438, + "min": -0.0005696857286345836, + "n": 17 + }, + "action_signs": { + "consistency": 0.9411764705882353, + "negative": 1, + "positive": 16, + "zero": 0 + }, + "effect_to_repeat_median": 2958.616486447546, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 5.1056616682254585e-06, + "repeat_abs_p95": 0.051096537054163664, + "repeat_delta": { + "distinct_n": 16, + "max": 0.004527588437235774, + "min": -0.0609037076826664, + "n": 29 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.17647058823529413, + "action_abs_median": 0.0025619580859006993, + "action_delta": { + "distinct_n": 17, + "max": 0.056370734075687004, + "min": -0.0013060500299530847, + "n": 17 + }, + "action_signs": { + "consistency": 0.7647058823529411, + "negative": 4, + "positive": 13, + "zero": 0 + }, + "effect_to_repeat_median": 7.0681027141206565, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0003624675799889579, + "repeat_abs_p95": 0.026328182817430976, + "repeat_delta": { + "distinct_n": 29, + "max": 0.00522340215966962, + "min": -0.034746555150565066, + "n": 29 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 17 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 17 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 29 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0018186060572318308, + "action_delta": { + "distinct_n": 17, + "max": 0.8741864436917179, + "min": -0.7248107341009736, + "n": 17 + }, + "action_signs": { + "consistency": 0.6470588235294118, + "negative": 6, + "positive": 11, + "zero": 0 + }, + "effect_to_repeat_median": 0.01492870595090838, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.12181940371872435, + "repeat_abs_p95": 0.904349172685805, + "repeat_delta": { + "distinct_n": 29, + "max": 0.12181940371872435, + "min": -0.9109256423065466, + "n": 29 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.4996837289, + "action_delta": { + "distinct_n": 17, + "max": 10.8362644165, + "min": -1.1173600969999997, + "n": 17 + }, + "action_signs": { + "consistency": 0.7647058823529411, + "negative": 4, + "positive": 13, + "zero": 0 + }, + "effect_to_repeat_median": 6.675982397855274, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.07484796979999953, + "repeat_abs_p95": 19.390367244779988, + "repeat_delta": { + "distinct_n": 29, + "max": 0.11568243179999982, + "min": -22.007979622, + "n": 29 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 9, + "max": 0.0, + "min": -5.5313452697, + "n": 17 + }, + "action_signs": { + "consistency": 1.0, + "negative": 8, + "positive": 0, + "zero": 9 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 5.569089138, + "repeat_delta": { + "distinct_n": 10, + "max": 0.0, + "min": -9.1442523314, + "n": 29 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 6.0, + "action_delta": { + "distinct_n": 17, + "max": 55.0, + "min": -89.4, + "n": 17 + }, + "action_signs": { + "consistency": 0.6470588235294118, + "negative": 11, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 2.8571428571428648, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 2.0999999999999943, + "repeat_abs_p95": 136.72000000000003, + "repeat_delta": { + "distinct_n": 24, + "max": 143.3, + "min": -28.89999999999999, + "n": 29 + } + } + }, + "horizon_s": 10.0, + "qualifying_features": [], + "repeats": [ + { + "delta_state": { + "batch_size.mean": -0.032046759505486744, + "batch_tokens.mean": -0.6567641003932749, + "decode_batch_size.mean": -0.031829677307312565, + "graph_full_share": 0.00020803710658368058, + "graph_none_share": -0.00019899201499314415, + "graph_padding_fraction": -0.0002750356645350475, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": -4.977600796418624e-05, + "kv_usage_mean": -0.0014423226092831615, + "preemptions": 0.0, + "prefill_token_fraction": 6.688650356756476e-05, + "queue_running_mean": -0.1142690622, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.5 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.25, + "request_hash": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp1_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp1_mns16/anchor-0.25/result.json" + }, + "target": { + "cell": "tp1_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp1_mns16/confirm-1-anchor-0.25/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.026614995471553993, + "batch_tokens.mean": -0.6496552246722729, + "decode_batch_size.mean": -0.02639849605068978, + "graph_full_share": 0.00020747861166137138, + "graph_none_share": -0.00018041618405337592, + "graph_padding_fraction": -9.631257646229227e-05, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": 0.0, + "kv_usage_mean": -0.00015576460470852899, + "preemptions": 0.0, + "prefill_token_fraction": 1.3376556165067832e-05, + "queue_running_mean": -0.04869389649999967, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.5 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.24609375, + "request_hash": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp1_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp1_mns32/anchor-0.24609375/result.json" + }, + "target": { + "cell": "tp1_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp1_mns32/confirm-1-anchor-0.24609375/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.010808600034953653, + "batch_tokens.mean": -0.5085699460020408, + "decode_batch_size.mean": -0.010635633606656203, + "graph_full_share": 0.0001657594937848339, + "graph_none_share": -0.00014413869024775677, + "graph_padding_fraction": -1.428081475827233e-05, + "kv_usage_end_minus_start": 9.965122072741472e-05, + "kv_usage_max": 0.0, + "kv_usage_mean": -4.734747194812261e-05, + "preemptions": 0.0, + "prefill_token_fraction": -9.363068207879355e-05, + "queue_running_mean": -0.009244884900000194, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.4000000000000057 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.25, + "request_hash": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp1_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp1_mns32/anchor-0.25/result.json" + }, + "target": { + "cell": "tp1_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp1_mns32/confirm-1-anchor-0.25/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.016288869514675675, + "batch_tokens.mean": -0.5180834059866299, + "decode_batch_size.mean": -0.016114501598372577, + "graph_full_share": 0.00016710258645746823, + "graph_none_share": -0.00013804126707352496, + "graph_padding_fraction": -0.00023422744992916602, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": -4.995504046367838e-05, + "kv_usage_mean": -0.0007559762079877713, + "preemptions": 0.0, + "prefill_token_fraction": -4.013301897864974e-05, + "queue_running_mean": -0.07447232549999949, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.4000000000000057 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.24609375, + "request_hash": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp1_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp1_mns64/anchor-0.24609375/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp1_mns64/confirm-1-anchor-0.24609375/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.04857934121216845, + "batch_tokens.mean": -1.5419494800797793, + "decode_batch_size.mean": -0.04806041200816624, + "graph_full_share": 0.0004973071538354468, + "graph_none_share": -0.0004108189531684403, + "graph_padding_fraction": -0.0002209229137654341, + "kv_usage_end_minus_start": 9.991008092724574e-05, + "kv_usage_max": -4.995504046367838e-05, + "kv_usage_mean": -0.0009120779275651192, + "preemptions": 0.0, + "prefill_token_fraction": -0.00012037560603939124, + "queue_running_mean": -0.07484796979999953, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 1.1999999999999886 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.25, + "request_hash": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp1_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp1_mns64/anchor-0.25/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp1_mns64/confirm-1-anchor-0.25/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.648283558063266, + "batch_tokens.mean": 49.961881401529, + "decode_batch_size.mean": 0.6427639871873927, + "graph_full_share": -0.006350826071218729, + "graph_none_share": 0.014663378024668453, + "graph_padding_fraction": -0.016377336933075942, + "kv_usage_end_minus_start": 4.9753719090550597e-05, + "kv_usage_max": 0.004527588437235774, + "kv_usage_mean": 0.00522340215966962, + "preemptions": 0.0, + "prefill_token_fraction": 0.12181940371872435, + "queue_running_mean": 0.11568243179999982, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -28.89999999999999 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.2265625, + "request_hash": "a05848006335fc9f595432bab793a13d427244886e4f04e51ba74d495453ef90", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 0.9920634920634921, + "mns": 8, + "trial_id": "tp1_mns8/anchor-0.2265625/result.json" + }, + "target": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 0.9920634920634921, + "mns": 8, + "trial_id": "tp1_mns8/confirm-1-anchor-0.2265625/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.003282275711160132, + "batch_tokens.mean": 0.03829321663019414, + "decode_batch_size.mean": 0.003282275711159688, + "graph_full_share": 0.0, + "graph_none_share": 0.0, + "graph_padding_fraction": 6.231700658618396e-05, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": 0.0, + "kv_usage_mean": -4.5836795611722825e-05, + "preemptions": 0.0, + "prefill_token_fraction": -2.6373628770337554e-05, + "queue_running_mean": -0.0024092132999999905, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.0 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.23046875, + "request_hash": "0f4c973153935c8abcd421eb631900670e31af0dc8a33bdf7e4d3d85e7ed7711", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 0.9923076923076923, + "mns": 8, + "trial_id": "tp1_mns8/anchor-0.23046875/result.json" + }, + "target": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 0.9923076923076923, + "mns": 8, + "trial_id": "tp1_mns8/confirm-1-anchor-0.23046875/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.016918595496772326, + "batch_tokens.mean": -0.47955440088175294, + "decode_batch_size.mean": -0.0167453944260747, + "graph_full_share": 0.00017320107069751511, + "graph_none_share": -0.000173201070697529, + "graph_padding_fraction": 3.159090949033175e-05, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": 0.0, + "kv_usage_mean": -1.8095624633056306e-05, + "preemptions": 0.0, + "prefill_token_fraction": -4.685685224570246e-05, + "queue_running_mean": -0.005980747899999805, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.5999999999999943 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.234375, + "request_hash": "14d2a74fffe25e7d9b3e7490c392eae0dc31bb1f3fe6566c9c0a567543bdd6ac", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 0.9924242424242424, + "mns": 8, + "trial_id": "tp1_mns8/anchor-0.234375/result.json" + }, + "target": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 0.9924242424242424, + "mns": 8, + "trial_id": "tp1_mns8/confirm-1-anchor-0.234375/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.05339622914945741, + "batch_tokens.mean": -1.3045555990319286, + "decode_batch_size.mean": -0.05296146544353153, + "graph_full_share": 0.0004166485515122442, + "graph_none_share": -0.00041664855151225116, + "graph_padding_fraction": -5.497908988029605e-05, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": 0.0, + "kv_usage_mean": -0.0003624675799889579, + "preemptions": 0.0, + "prefill_token_fraction": 2.675497368564983e-05, + "queue_running_mean": -0.03772782840000044, + "queue_waiting_mean": -0.0204415657, + "scheduler_steps_per_s": 1.0 + }, + "descriptive_full_outcome": { + "delta_pass_rate": -0.006993006993006978, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.25, + "request_hash": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 0.958041958041958, + "mns": 8, + "trial_id": "tp1_mns8/anchor-0.25/result.json" + }, + "target": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 0.951048951048951, + "mns": 8, + "trial_id": "tp1_mns8/confirm-1-anchor-0.25/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -1.5223872504055418, + "batch_tokens.mean": -69.8055007051478, + "decode_batch_size.mean": -1.5056685448852059, + "graph_full_share": 0.017265452376541424, + "graph_none_share": -0.025711662075298437, + "graph_padding_fraction": 0.15174913486120387, + "kv_usage_end_minus_start": -0.009853977330746444, + "kv_usage_max": -0.01946543449402638, + "kv_usage_mean": -0.008432216411693306, + "preemptions": 0.0, + "prefill_token_fraction": -0.8614186464282653, + "queue_running_mean": -3.6705211939, + "queue_waiting_mean": -0.0093291794, + "scheduler_steps_per_s": 74.0 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.49609375, + "request_hash": "a78992ce59bb3b857a7ef8844e0a690ea9dd529a659cf5edd599b63fef8b6a80", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp2_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp2_mns16/anchor-0.49609375/result.json" + }, + "target": { + "cell": "tp2_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp2_mns16/confirm-1-anchor-0.49609375/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.007251432291120796, + "batch_tokens.mean": -0.00762511250828668, + "decode_batch_size.mean": -0.007211010152244679, + "graph_full_share": 3.9523869123314626e-05, + "graph_none_share": 0.0, + "graph_padding_fraction": -0.0023343164243360137, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": 0.0, + "kv_usage_mean": -1.4972595796737716e-05, + "preemptions": 0.0, + "prefill_token_fraction": 5.034578599764916e-05, + "queue_running_mean": -0.006232385800000184, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.30000000000001137 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.5, + "request_hash": "3160017dbd8249f711eb115f1e72ddbb37eab3a3333a48762cc3104e94bd2f15", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp2_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp2_mns16/anchor-0.5/result.json" + }, + "target": { + "cell": "tp2_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp2_mns16/confirm-1-anchor-0.5/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.001311475409836227, + "batch_tokens.mean": -0.001311475409836227, + "decode_batch_size.mean": -0.001311475409836227, + "graph_full_share": 0.0, + "graph_none_share": 0.0, + "graph_padding_fraction": -0.00024782551484880955, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": 0.0, + "kv_usage_mean": -2.119802975337863e-05, + "preemptions": 0.0, + "prefill_token_fraction": 1.920108243574914e-05, + "queue_running_mean": -0.0037054131000004986, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.0 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.75, + "request_hash": "4f4c792e90f5c1bf33978fea26debcadfce5f8b872a31be86a88a1bf6b09a21c", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp2_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp2_mns32/anchor-0.75/result.json" + }, + "target": { + "cell": "tp2_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp2_mns32/confirm-1-anchor-0.75/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -5.253278252178259, + "batch_tokens.mean": -203.60648072702782, + "decode_batch_size.mean": -5.198083023705538, + "graph_full_share": 0.03511966842665615, + "graph_none_share": -0.043478260869565216, + "graph_padding_fraction": 0.24157911714869673, + "kv_usage_end_minus_start": -0.07211360414777734, + "kv_usage_max": -0.05893470570957904, + "kv_usage_mean": -0.034746555150565066, + "preemptions": 0.0, + "prefill_token_fraction": -0.8735909388483541, + "queue_running_mean": -12.366947709400002, + "queue_waiting_mean": -1.6375549495, + "scheduler_steps_per_s": 86.29999999999998 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.7385786802030457, + "feasibility_transition": "false->false" + }, + "group": { + "anchor": 0.75390625, + "request_hash": "2677a6a12ff0e104d6246dd121d0667ff4fe3d25f8d68699547906420246ebdd", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp2_mns32", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.20558375634517767, + "mns": 32, + "trial_id": "tp2_mns32/anchor-0.75390625/result.json" + }, + "target": { + "cell": "tp2_mns32", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.9441624365482234, + "mns": 32, + "trial_id": "tp2_mns32/confirm-1-anchor-0.75390625/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.0022141886965014734, + "batch_tokens.mean": -0.0023373966282491665, + "decode_batch_size.mean": -0.002200860915423153, + "graph_full_share": 1.3031608165570141e-05, + "graph_none_share": 0.0, + "graph_padding_fraction": 0.0003699622628544341, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": 0.0, + "kv_usage_mean": -6.601132195046752e-06, + "preemptions": 0.0, + "prefill_token_fraction": 1.2576053067120396e-05, + "queue_running_mean": 0.0006418031000001712, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.10000000000002274 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.5, + "request_hash": "3160017dbd8249f711eb115f1e72ddbb37eab3a3333a48762cc3104e94bd2f15", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp2_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp2_mns64/anchor-0.5/result.json" + }, + "target": { + "cell": "tp2_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp2_mns64/confirm-1-anchor-0.5/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -5.415534020939959, + "batch_tokens.mean": -200.71322023330129, + "decode_batch_size.mean": -5.35913538229716, + "graph_full_share": 0.05071523213218676, + "graph_none_share": -0.05388272583201268, + "graph_padding_fraction": 0.22642208124358779, + "kv_usage_end_minus_start": -0.03352133931883006, + "kv_usage_max": -0.0609037076826664, + "kv_usage_mean": -0.02880160581001033, + "preemptions": 0.0, + "prefill_token_fraction": -0.8741576384479508, + "queue_running_mean": -10.836573950600002, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 89.4 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.6624040920716113, + "feasibility_transition": "false->false" + }, + "group": { + "anchor": 0.75, + "request_hash": "4f4c792e90f5c1bf33978fea26debcadfce5f8b872a31be86a88a1bf6b09a21c", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp2_mns64", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.23017902813299232, + "mns": 64, + "trial_id": "tp2_mns64/anchor-0.75/result.json" + }, + "target": { + "cell": "tp2_mns64", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.8925831202046036, + "mns": 64, + "trial_id": "tp2_mns64/confirm-1-anchor-0.75/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.04236326637336152, + "batch_tokens.mean": -0.0506303786819462, + "decode_batch_size.mean": -0.041385582574491764, + "graph_full_share": 0.0009697259074838982, + "graph_none_share": -0.000746904948671601, + "graph_padding_fraction": -0.0024530500387316967, + "kv_usage_end_minus_start": -0.005895639523754825, + "kv_usage_max": 0.0, + "kv_usage_mean": -0.0001257250352028564, + "preemptions": 0.0, + "prefill_token_fraction": -0.0014409663380382937, + "queue_running_mean": -0.0696698146000001, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 2.0999999999999943 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.4921875, + "request_hash": "cd227aee3be472a35e8c35af60a44a1411c4dc689cbd98746b3fb8a8d329b44c", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp2_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 8, + "trial_id": "tp2_mns8/anchor-0.4921875/result.json" + }, + "target": { + "cell": "tp2_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 8, + "trial_id": "tp2_mns8/confirm-1-anchor-0.4921875/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.9687249841095076, + "batch_tokens.mean": -59.73987529620144, + "decode_batch_size.mean": -0.9579959096941315, + "graph_full_share": 0.011406581190443799, + "graph_none_share": -0.019512062147118037, + "graph_padding_fraction": 0.20562784534661108, + "kv_usage_end_minus_start": 0.004504676952132969, + "kv_usage_max": -0.0037772928550463547, + "kv_usage_mean": -0.0036990145914360204, + "preemptions": 0.0, + "prefill_token_fraction": -0.8491799860125482, + "queue_running_mean": -1.3942294618000002, + "queue_waiting_mean": -1.1069323539, + "scheduler_steps_per_s": 45.69999999999999 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.6190476190476191, + "feasibility_transition": "false->false" + }, + "group": { + "anchor": 0.49609375, + "request_hash": "a78992ce59bb3b857a7ef8844e0a690ea9dd529a659cf5edd599b63fef8b6a80", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp2_mns8", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.27106227106227104, + "mns": 8, + "trial_id": "tp2_mns8/anchor-0.49609375/result.json" + }, + "target": { + "cell": "tp2_mns8", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.8901098901098901, + "mns": 8, + "trial_id": "tp2_mns8/confirm-1-anchor-0.49609375/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.02348440960742426, + "batch_tokens.mean": -0.024462377280626946, + "decode_batch_size.mean": -0.02333965019280182, + "graph_full_share": -0.0009263480369809951, + "graph_none_share": 0.0, + "graph_padding_fraction": 0.0002862441287893591, + "kv_usage_end_minus_start": -5.101077857672109e-06, + "kv_usage_max": -1.0202155715566263e-05, + "kv_usage_mean": -2.2000789048993996e-05, + "preemptions": 0.0, + "prefill_token_fraction": 5.985572526742006e-05, + "queue_running_mean": -0.02092175199999957, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.5999999999999943 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.2469565217391304, + "feasibility_transition": "false->true" + }, + "group": { + "anchor": 0.033182214016, + "request_hash": "e6b624a0212807df3733c54b4a2160fba24f19ca4776b14f34918d83428bba7f", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.7530434782608696, + "mns": 16, + "trial_id": "tp4_mns16/anchor-0.033182214016/result.json" + }, + "target": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp4_mns16/confirm-1-anchor-0.033182214016/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -3.19743814378252, + "batch_tokens.mean": -194.8952745035574, + "decode_batch_size.mean": -3.1597031963470315, + "graph_full_share": 0.02834766910905817, + "graph_none_share": -0.04961240310077519, + "graph_padding_fraction": 0.24357765310774523, + "kv_usage_end_minus_start": -0.008034197625958361, + "kv_usage_max": -0.004550161449114243, + "kv_usage_mean": -0.008735063035535659, + "preemptions": 0.0, + "prefill_token_fraction": -0.8943849310253909, + "queue_running_mean": -6.701412029300001, + "queue_waiting_mean": -9.1442523314, + "scheduler_steps_per_s": 110.69999999999999 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.34982935153583616, + "feasibility_transition": "false->false" + }, + "group": { + "anchor": 0.033717411016, + "request_hash": "472a874ecc0e844b5b36a715bbc309b63e73074d39327cd734869f76721d71db", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.06313993174061433, + "mns": 16, + "trial_id": "tp4_mns16/anchor-0.033717411016/result.json" + }, + "target": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.4129692832764505, + "mns": 16, + "trial_id": "tp4_mns16/confirm-1-anchor-0.033717411016/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.07980086487246574, + "batch_tokens.mean": -1.0880711819756304, + "decode_batch_size.mean": -0.07880182167344785, + "graph_full_share": -0.0018140521245323393, + "graph_none_share": -0.0016474464579901153, + "graph_padding_fraction": 0.02197042071969288, + "kv_usage_end_minus_start": 2.0404311431021505e-05, + "kv_usage_max": 0.0, + "kv_usage_mean": -8.46355932073951e-05, + "preemptions": 0.0, + "prefill_token_fraction": -0.13021608847689226, + "queue_running_mean": -0.09728112210000006, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 3.8000000000000114 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.034252608017, + "request_hash": "21c1f3b7d972b19c952a894863a70d1e85bd0e24a7e7b6b0fcc088360d0e01bd", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp4_mns16/anchor-0.034252608017/result.json" + }, + "target": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp4_mns16/confirm-1-anchor-0.034252608017/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.006431552476177771, + "batch_tokens.mean": -0.006758232721721491, + "decode_batch_size.mean": -0.006383197052567624, + "graph_full_share": 0.000573798951080895, + "graph_none_share": 0.0, + "graph_padding_fraction": 0.0006374012583605093, + "kv_usage_end_minus_start": -5.101077857672109e-06, + "kv_usage_max": -5.101077857783132e-06, + "kv_usage_mean": -1.0214925773703067e-05, + "preemptions": 0.0, + "prefill_token_fraction": 5.436516758668131e-06, + "queue_running_mean": -0.009119564499999733, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.19999999999998863 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.2469565217391304, + "feasibility_transition": "false->true" + }, + "group": { + "anchor": 0.033182214016, + "request_hash": "e6b624a0212807df3733c54b4a2160fba24f19ca4776b14f34918d83428bba7f", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.7530434782608696, + "mns": 16, + "trial_id": "tp4_mns16/anchor-0.033182214016/result.json" + }, + "target": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp4_mns16/confirm-2-anchor-0.033182214016/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -5.684231334296602, + "batch_tokens.mean": -242.26991192050903, + "decode_batch_size.mean": -5.614987163710968, + "graph_full_share": 0.03905791041999285, + "graph_none_share": -0.04780114722753346, + "graph_padding_fraction": 0.23825248667744758, + "kv_usage_end_minus_start": -0.028101561821904375, + "kv_usage_max": -0.02156120922490945, + "kv_usage_mean": -0.01815563294515909, + "preemptions": 0.0, + "prefill_token_fraction": -0.89493540395689, + "queue_running_mean": -16.051436879700002, + "queue_waiting_mean": -5.569089138, + "scheduler_steps_per_s": 136.8 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.9692832764505119, + "feasibility_transition": "false->true" + }, + "group": { + "anchor": 0.033717411016, + "request_hash": "472a874ecc0e844b5b36a715bbc309b63e73074d39327cd734869f76721d71db", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns32", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.030716723549488054, + "mns": 32, + "trial_id": "tp4_mns32/anchor-0.033717411016/result.json" + }, + "target": { + "cell": "tp4_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp4_mns32/confirm-1-anchor-0.033717411016/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.07573170158789821, + "batch_tokens.mean": -1.0733751057032785, + "decode_batch_size.mean": -0.07518292539697491, + "graph_full_share": 0.0010444071220520046, + "graph_none_share": -0.0010899182561307902, + "graph_padding_fraction": 0.027146526691836648, + "kv_usage_end_minus_start": -5.105661668114436e-06, + "kv_usage_max": -5.1056616682254585e-06, + "kv_usage_mean": -6.434458670689751e-05, + "preemptions": 0.0, + "prefill_token_fraction": -0.1289638873088742, + "queue_running_mean": -0.0711253370999998, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 2.0999999999999943 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.034252608017, + "request_hash": "21c1f3b7d972b19c952a894863a70d1e85bd0e24a7e7b6b0fcc088360d0e01bd", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp4_mns32/anchor-0.034252608017/result.json" + }, + "target": { + "cell": "tp4_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp4_mns32/confirm-1-anchor-0.034252608017/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -5.676772134537581, + "batch_tokens.mean": -242.26212742181514, + "decode_batch_size.mean": -5.607576114913048, + "graph_full_share": 0.039014238618063524, + "graph_none_share": -0.04780114722753346, + "graph_padding_fraction": 0.23544552622699638, + "kv_usage_end_minus_start": -0.02810666748357249, + "kv_usage_max": -0.021556103563241225, + "kv_usage_mean": -0.01814814993745567, + "preemptions": 0.0, + "prefill_token_fraction": -0.8949517370921197, + "queue_running_mean": -16.0413130873, + "queue_waiting_mean": -5.569089138, + "scheduler_steps_per_s": 136.60000000000002 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.9692832764505119, + "feasibility_transition": "false->true" + }, + "group": { + "anchor": 0.033717411016, + "request_hash": "472a874ecc0e844b5b36a715bbc309b63e73074d39327cd734869f76721d71db", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns32", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.030716723549488054, + "mns": 32, + "trial_id": "tp4_mns32/anchor-0.033717411016/result.json" + }, + "target": { + "cell": "tp4_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp4_mns32/confirm-2-anchor-0.033717411016/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -4.9280804627990085, + "batch_tokens.mean": -364.01800389606234, + "decode_batch_size.mean": -4.782726800328038, + "graph_full_share": 0.05329256669728677, + "graph_none_share": -0.06263498920086392, + "graph_padding_fraction": 0.2448027632667215, + "kv_usage_end_minus_start": -0.05014161119802052, + "kv_usage_max": -0.039318834800568414, + "kv_usage_mean": -0.022291899962727596, + "preemptions": 0.0, + "prefill_token_fraction": -0.9106141297482617, + "queue_running_mean": -21.6163208215, + "queue_waiting_mean": -0.1743790356, + "scheduler_steps_per_s": 129.10000000000002 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.8430034129692833, + "feasibility_transition": "false->true" + }, + "group": { + "anchor": 0.033717411016, + "request_hash": "472a874ecc0e844b5b36a715bbc309b63e73074d39327cd734869f76721d71db", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns64", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.11433447098976109, + "mns": 64, + "trial_id": "tp4_mns64/anchor-0.033717411016/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 0.9573378839590444, + "mns": 64, + "trial_id": "tp4_mns64/confirm-1-anchor-0.033717411016/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.0986378480703829, + "batch_tokens.mean": -1.1003885983709436, + "decode_batch_size.mean": -0.09787917001031765, + "graph_full_share": 0.00015255866453844913, + "graph_none_share": -0.001092896174863388, + "graph_padding_fraction": 0.026001990897698585, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": 0.0, + "kv_usage_mean": -8.10913166477481e-05, + "preemptions": 0.0, + "prefill_token_fraction": -0.1290526738911407, + "queue_running_mean": -0.09155397830000034, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 2.9000000000000057 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.034252608017, + "request_hash": "21c1f3b7d972b19c952a894863a70d1e85bd0e24a7e7b6b0fcc088360d0e01bd", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp4_mns64/anchor-0.034252608017/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp4_mns64/confirm-1-anchor-0.034252608017/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -5.343712123283302, + "batch_tokens.mean": -364.45844383082266, + "decode_batch_size.mean": -5.194686323828272, + "graph_full_share": 0.055525557955363625, + "graph_none_share": -0.06263498920086392, + "graph_padding_fraction": 0.2382752171480348, + "kv_usage_end_minus_start": -0.05152704927251717, + "kv_usage_max": -0.0393392840710407, + "kv_usage_mean": -0.022618048328561978, + "preemptions": 0.0, + "prefill_token_fraction": -0.9109256423065466, + "queue_running_mean": -22.007979622, + "queue_waiting_mean": -0.1743790356, + "scheduler_steps_per_s": 143.3 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.8856655290102389, + "feasibility_transition": "false->true" + }, + "group": { + "anchor": 0.033717411016, + "request_hash": "472a874ecc0e844b5b36a715bbc309b63e73074d39327cd734869f76721d71db", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns64", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.11433447098976109, + "mns": 64, + "trial_id": "tp4_mns64/anchor-0.033717411016/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp4_mns64/confirm-2-anchor-0.033717411016/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.5747954473735191, + "batch_tokens.mean": -30.476142999621462, + "decode_batch_size.mean": -0.5686361081847746, + "graph_full_share": 0.006538565006766528, + "graph_none_share": -0.017039834454924494, + "graph_padding_fraction": 0.1583386005746088, + "kv_usage_end_minus_start": -0.002356121069944095, + "kv_usage_max": 0.0, + "kv_usage_mean": -0.0006074327722416259, + "preemptions": 0.0, + "prefill_token_fraction": -0.8386345381065662, + "queue_running_mean": -0.9726716715000001, + "queue_waiting_mean": -0.428824454, + "scheduler_steps_per_s": 40.900000000000006 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.016591107009, + "request_hash": "2407501cda53bf9c761c448dc50ab4780b4a55958cc00ba1b19cd09de1cf73e1", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 8, + "trial_id": "tp4_mns8/anchor-0.016591107009/result.json" + }, + "target": { + "cell": "tp4_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 8, + "trial_id": "tp4_mns8/confirm-1-anchor-0.016591107009/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.03238588118373942, + "batch_tokens.mean": -2.9133132020797436, + "decode_batch_size.mean": -0.0321242294926507, + "graph_full_share": 0.00025027553060685825, + "graph_none_share": -0.0009881332994778795, + "graph_padding_fraction": 0.06450969515475685, + "kv_usage_end_minus_start": 5.099829155752467e-06, + "kv_usage_max": 0.0, + "kv_usage_mean": -4.0493927196364344e-05, + "preemptions": 0.0, + "prefill_token_fraction": -0.4656521349077925, + "queue_running_mean": -0.06308746549999977, + "queue_waiting_mean": -0.0015971439000000004, + "scheduler_steps_per_s": 2.5 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.017126304009, + "request_hash": "93cd99ebf2a760cc440dce0cb7e51273686dcfbfb07650d27e9e16d2ff1d2eed", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 8, + "trial_id": "tp4_mns8/anchor-0.017126304009/result.json" + }, + "target": { + "cell": "tp4_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 8, + "trial_id": "tp4_mns8/confirm-1-anchor-0.017126304009/result.json" + } + } + ], + "sanity": { + "action_pairs": 17, + "distinct_action_vectors": 17, + "invariants": { + "all_pair_hashes_match": true, + "all_values_finite": true, + "expected_action_pair_count": true, + "nonnegative_counters": true, + "ratios_bounded": true, + "state_vectors_not_all_identical": true, + "sufficient_repeat_pair_count": true + }, + "red_flags": [], + "repeat_pairs": 29, + "trials": 66 + } + }, + "5": { + "actions": [ + { + "delta_state": { + "batch_size.mean": 3.5280207937111703, + "batch_tokens.mean": 189.10013313046787, + "decode_batch_size.mean": 3.495993406872068, + "graph_full_share": -0.027843286420692226, + "graph_none_share": 0.04184100418410042, + "graph_padding_fraction": -0.10888597553063217, + "kv_usage_end_minus_start": 0.04832989960722345, + "kv_usage_max": 0.05576515049492037, + "kv_usage_mean": 0.04101485276318234, + "preemptions": 0.0, + "prefill_token_fraction": 0.8861120723274618, + "queue_running_mean": 3.4413866424000004, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -84.2 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.24609375, + "request_hash": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "mns_increase", + "source": { + "cell": "tp1_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp1_mns16/anchor-0.24609375/result.json" + }, + "target": { + "cell": "tp1_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp1_mns32/anchor-0.24609375/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.05698177434224494, + "batch_tokens.mean": 2.4183568541238287, + "decode_batch_size.mean": 0.056343521736047286, + "graph_full_share": -0.0005850648890149968, + "graph_none_share": 0.000531877171831785, + "graph_padding_fraction": 0.00021749194258890905, + "kv_usage_end_minus_start": 7.313298449929118e-05, + "kv_usage_max": 0.00030157213266412963, + "kv_usage_mean": 0.0016340770443056124, + "preemptions": 0.0, + "prefill_token_fraction": 8.473923694440888e-05, + "queue_running_mean": 0.1474536237999997, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -0.5999999999999943 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.24609375, + "request_hash": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "mns_increase", + "source": { + "cell": "tp1_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp1_mns32/anchor-0.24609375/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp1_mns64/anchor-0.24609375/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.1278554259770317, + "batch_tokens.mean": 3.2897534051455466, + "decode_batch_size.mean": 0.127000801210718, + "graph_full_share": -0.0007834060357874151, + "graph_none_share": -0.0034719131131487613, + "graph_padding_fraction": 0.0008694893399684026, + "kv_usage_end_minus_start": 2.1174429976178466e-05, + "kv_usage_max": 0.03040170172083234, + "kv_usage_mean": 0.005773282024914478, + "preemptions": 0.0, + "prefill_token_fraction": -0.0001483422608830054, + "queue_running_mean": 0.5028872580000003, + "queue_waiting_mean": -0.0510258954, + "scheduler_steps_per_s": -0.7999999999999972 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.04195804195804198, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.25, + "request_hash": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "mns_increase", + "source": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 0.958041958041958, + "mns": 8, + "trial_id": "tp1_mns8/anchor-0.25/result.json" + }, + "target": { + "cell": "tp1_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp1_mns16/anchor-0.25/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.06163060969068468, + "batch_tokens.mean": -2.443018058287123, + "decode_batch_size.mean": -0.06098694797067772, + "graph_full_share": 0.0005900232433398989, + "graph_none_share": -0.0005363847666726221, + "graph_padding_fraction": -0.00017399768727638983, + "kv_usage_end_minus_start": 4.7122279587474836e-05, + "kv_usage_max": 0.00011557359098801623, + "kv_usage_mean": -0.0004086366142161957, + "preemptions": 0.0, + "prefill_token_fraction": -6.356136875318441e-05, + "queue_running_mean": -0.0843407528000002, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.6000000000000014 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.25, + "request_hash": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "mns_increase", + "source": { + "cell": "tp1_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp1_mns16/anchor-0.25/result.json" + }, + "target": { + "cell": "tp1_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp1_mns32/anchor-0.25/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.024589582668510523, + "batch_tokens.mean": 0.8116866999964429, + "decode_batch_size.mean": 0.02437683934333279, + "graph_full_share": -0.00019501471474669962, + "graph_none_share": 0.00017728610431513858, + "graph_padding_fraction": 8.69181567518763e-05, + "kv_usage_end_minus_start": 0.00012295859486299854, + "kv_usage_max": 0.00030157213266412963, + "kv_usage_mean": 0.00032121484848791904, + "preemptions": 0.0, + "prefill_token_fraction": 0.0, + "queue_running_mean": 0.013309304399999888, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -0.20000000000000284 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.25, + "request_hash": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "mns_increase", + "source": { + "cell": "tp1_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp1_mns32/anchor-0.25/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp1_mns64/anchor-0.25/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 4.473159643294283, + "batch_tokens.mean": 189.17581745060323, + "decode_batch_size.mean": 4.408288162266131, + "graph_full_share": -0.046424200034971164, + "graph_none_share": 0.03890540304248993, + "graph_padding_fraction": 0.005662578660898298, + "kv_usage_end_minus_start": 0.08863131728902607, + "kv_usage_max": 0.08590208433411406, + "kv_usage_mean": 0.0354576817886873, + "preemptions": 0.0, + "prefill_token_fraction": 0.0007191134183018866, + "queue_running_mean": 3.8361017661999997, + "queue_waiting_mean": -1.0501423682, + "scheduler_steps_per_s": -7.799999999999997 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.06521739130434782, + "feasibility_transition": "false->false" + }, + "group": { + "anchor": 0.5, + "request_hash": "3160017dbd8249f711eb115f1e72ddbb37eab3a3333a48762cc3104e94bd2f15", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "mns_increase", + "source": { + "cell": "tp1_mns8", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.09057971014492754, + "mns": 8, + "trial_id": "tp1_mns8/anchor-0.5/result.json" + }, + "target": { + "cell": "tp1_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.15579710144927536, + "mns": 16, + "trial_id": "tp1_mns16/anchor-0.5/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -1.4603817235396175, + "batch_tokens.mean": 220.88027761711976, + "decode_batch_size.mean": -1.5222672064777338, + "graph_full_share": -0.06998264893001727, + "graph_none_share": 0.055523423944476596, + "graph_padding_fraction": 0.0002698844942520852, + "kv_usage_end_minus_start": 0.03428564980259485, + "kv_usage_max": 0.03430172098005413, + "kv_usage_mean": 0.004527112114386889, + "preemptions": 0.0, + "prefill_token_fraction": 0.009687903063064107, + "queue_running_mean": 0.49739668660000014, + "queue_waiting_mean": -0.4414553072, + "scheduler_steps_per_s": -8.400000000000002 + }, + "descriptive_full_outcome": { + "delta_pass_rate": -0.06521739130434782, + "feasibility_transition": "false->false" + }, + "group": { + "anchor": 0.5, + "request_hash": "3160017dbd8249f711eb115f1e72ddbb37eab3a3333a48762cc3104e94bd2f15", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "mns_increase", + "source": { + "cell": "tp1_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.15579710144927536, + "mns": 16, + "trial_id": "tp1_mns16/anchor-0.5/result.json" + }, + "target": { + "cell": "tp1_mns32", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.09057971014492754, + "mns": 32, + "trial_id": "tp1_mns32/anchor-0.5/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.0, + "batch_tokens.mean": 0.0, + "decode_batch_size.mean": 0.0, + "graph_full_share": 0.0, + "graph_none_share": 0.0, + "graph_padding_fraction": 0.0, + "kv_usage_end_minus_start": 0.0004933875406503141, + "kv_usage_max": 0.0005353228930036202, + "kv_usage_mean": 0.0001535070867022259, + "preemptions": 0.0, + "prefill_token_fraction": 0.0, + "queue_running_mean": -0.00963330859999978, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.0 + }, + "descriptive_full_outcome": { + "delta_pass_rate": -0.02898550724637681, + "feasibility_transition": "false->false" + }, + "group": { + "anchor": 0.5, + "request_hash": "3160017dbd8249f711eb115f1e72ddbb37eab3a3333a48762cc3104e94bd2f15", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "mns_increase", + "source": { + "cell": "tp1_mns32", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.09057971014492754, + "mns": 32, + "trial_id": "tp1_mns32/anchor-0.5/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.06159420289855073, + "mns": 64, + "trial_id": "tp1_mns64/anchor-0.5/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.06154364887467256, + "batch_tokens.mean": 0.0673666044072565, + "decode_batch_size.mean": 0.06098464514354429, + "graph_full_share": -0.0005590037311280494, + "graph_none_share": 0.0, + "graph_padding_fraction": 0.005655833585839015, + "kv_usage_end_minus_start": -2.459171897339907e-06, + "kv_usage_max": 0.003445359565031225, + "kv_usage_mean": 0.0015597623110467189, + "preemptions": 0.0, + "prefill_token_fraction": 0.00030636946592198233, + "queue_running_mean": 0.12847152480000013, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -4.200000000000017 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.2421875, + "request_hash": "adb62bea7f7a12c1e33fa1572ec1d0e274100013ad90e14c6ef5c549b0e0d017", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "mns_increase", + "source": { + "cell": "tp1_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp1_mns32/anchor-0.2421875/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp1_mns64/anchor-0.2421875/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.23325211580663785, + "batch_tokens.mean": -0.2519983270912536, + "decode_batch_size.mean": -0.23134894714830168, + "graph_full_share": 0.001903168658336507, + "graph_none_share": -0.006795016987542469, + "graph_padding_fraction": -0.004536339467287626, + "kv_usage_end_minus_start": 5.075521287700724e-07, + "kv_usage_max": -2.1953909969063545e-05, + "kv_usage_mean": -0.0005894447334146543, + "preemptions": 0.0, + "prefill_token_fraction": -6.083338940097993e-05, + "queue_running_mean": -0.27085744499999986, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 16.200000000000017 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.4921875, + "request_hash": "cd227aee3be472a35e8c35af60a44a1411c4dc689cbd98746b3fb8a8d329b44c", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "mns_increase", + "source": { + "cell": "tp2_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 8, + "trial_id": "tp2_mns8/anchor-0.4921875/result.json" + }, + "target": { + "cell": "tp2_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp2_mns16/anchor-0.4921875/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.6005778377124296, + "batch_tokens.mean": -4.162510512731277, + "decode_batch_size.mean": -0.5951764688823156, + "graph_full_share": 0.005401368830114217, + "graph_none_share": -0.01149694043268952, + "graph_padding_fraction": -0.004587093073780198, + "kv_usage_end_minus_start": 1.4354961987761428e-05, + "kv_usage_max": -0.0029570125473361575, + "kv_usage_mean": -0.0013514937493694472, + "preemptions": 0.0, + "prefill_token_fraction": -0.0010977628546262608, + "queue_running_mean": -0.5356197792000001, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 29.799999999999983 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.728937728937729, + "feasibility_transition": "false->true" + }, + "group": { + "anchor": 0.49609375, + "request_hash": "a78992ce59bb3b857a7ef8844e0a690ea9dd529a659cf5edd599b63fef8b6a80", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "mns_increase", + "source": { + "cell": "tp2_mns8", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.27106227106227104, + "mns": 8, + "trial_id": "tp2_mns8/anchor-0.49609375/result.json" + }, + "target": { + "cell": "tp2_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp2_mns16/anchor-0.49609375/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.35712385038833183, + "batch_tokens.mean": 111.90975236574697, + "decode_batch_size.mean": 0.32775366295797426, + "graph_full_share": -0.025012888955412538, + "graph_none_share": 0.026143790849673203, + "graph_padding_fraction": -0.2241759801940655, + "kv_usage_end_minus_start": 0.03485111013888609, + "kv_usage_max": 0.01867744710795105, + "kv_usage_mean": 0.006535245430020315, + "preemptions": 0.0, + "prefill_token_fraction": 0.8766852234122235, + "queue_running_mean": 2.085466555, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -65.39999999999999 + }, + "descriptive_full_outcome": { + "delta_pass_rate": -0.7698209718670077, + "feasibility_transition": "true->false" + }, + "group": { + "anchor": 0.75, + "request_hash": "4f4c792e90f5c1bf33978fea26debcadfce5f8b872a31be86a88a1bf6b09a21c", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "mns_increase", + "source": { + "cell": "tp2_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp2_mns32/anchor-0.75/result.json" + }, + "target": { + "cell": "tp2_mns64", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.23017902813299232, + "mns": 64, + "trial_id": "tp2_mns64/anchor-0.75/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.8613841701826859, + "batch_tokens.mean": 99.28363780953853, + "decode_batch_size.mean": -0.9000309262409161, + "graph_full_share": -0.013132607303010913, + "graph_none_share": 0.007356027303452698, + "graph_padding_fraction": -0.002483113089604908, + "kv_usage_end_minus_start": 0.013972653351394082, + "kv_usage_max": 0.013923980316157603, + "kv_usage_mean": 0.002289615839773029, + "preemptions": 0.0, + "prefill_token_fraction": 0.027119159852510122, + "queue_running_mean": 2.255338312400001, + "queue_waiting_mean": -1.3976082232, + "scheduler_steps_per_s": -8.0 + }, + "descriptive_full_outcome": { + "delta_pass_rate": -0.032423208191126277, + "feasibility_transition": "false->false" + }, + "group": { + "anchor": 0.033717411016, + "request_hash": "472a874ecc0e844b5b36a715bbc309b63e73074d39327cd734869f76721d71db", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "mns_increase", + "source": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.06313993174061433, + "mns": 16, + "trial_id": "tp4_mns16/anchor-0.033717411016/result.json" + }, + "target": { + "cell": "tp4_mns32", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.030716723549488054, + "mns": 32, + "trial_id": "tp4_mns32/anchor-0.033717411016/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.00950293278090264, + "batch_tokens.mean": -1.5782094981150578, + "decode_batch_size.mean": -0.008726077059269421, + "graph_full_share": 0.0005239259517987316, + "graph_none_share": -0.00023486335770291772, + "graph_padding_fraction": 8.79877867886172e-05, + "kv_usage_end_minus_start": 3.819849616093496e-05, + "kv_usage_max": 4.159303058193231e-05, + "kv_usage_mean": 6.884522888032929e-05, + "preemptions": 0.0, + "prefill_token_fraction": -0.00013165055575314888, + "queue_running_mean": 0.02412652539999982, + "queue_waiting_mean": -9.661300000000927e-05, + "scheduler_steps_per_s": 0.5999999999999943 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.08361774744027303, + "feasibility_transition": "false->false" + }, + "group": { + "anchor": 0.033717411016, + "request_hash": "472a874ecc0e844b5b36a715bbc309b63e73074d39327cd734869f76721d71db", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "mns_increase", + "source": { + "cell": "tp4_mns32", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.030716723549488054, + "mns": 32, + "trial_id": "tp4_mns32/anchor-0.033717411016/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.11433447098976109, + "mns": 64, + "trial_id": "tp4_mns64/anchor-0.033717411016/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.7391144064912778, + "batch_tokens.mean": -97.02946180778741, + "decode_batch_size.mean": -0.7271776858566685, + "graph_full_share": 0.015299949334160123, + "graph_none_share": -0.04103743978705223, + "graph_padding_fraction": 0.21027653696749515, + "kv_usage_end_minus_start": -0.006308261401860582, + "kv_usage_max": 0.004195788190260341, + "kv_usage_mean": -0.0005148594359863203, + "preemptions": 0.0, + "prefill_token_fraction": -0.7248910361224853, + "queue_running_mean": -0.7297395955999999, + "queue_waiting_mean": -3.4976107514, + "scheduler_steps_per_s": 59.80000000000001 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.8216666666666667, + "feasibility_transition": "false->true" + }, + "group": { + "anchor": 0.034252608017, + "request_hash": "21c1f3b7d972b19c952a894863a70d1e85bd0e24a7e7b6b0fcc088360d0e01bd", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "mns_increase", + "source": { + "cell": "tp4_mns8", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.17833333333333334, + "mns": 8, + "trial_id": "tp4_mns8/anchor-0.034252608017/result.json" + }, + "target": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp4_mns16/anchor-0.034252608017/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.0033632286995510796, + "batch_tokens.mean": -0.0033632286995510796, + "decode_batch_size.mean": -0.003363228699551968, + "graph_full_share": 0.0, + "graph_none_share": 0.0, + "graph_padding_fraction": -0.0030699089127894374, + "kv_usage_end_minus_start": 8.342535059435718e-07, + "kv_usage_max": 1.368725807104898e-05, + "kv_usage_mean": -6.535810642852545e-06, + "preemptions": 0.0, + "prefill_token_fraction": 9.850983893985865e-05, + "queue_running_mean": -0.009463884000000533, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.0 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.034252608017, + "request_hash": "21c1f3b7d972b19c952a894863a70d1e85bd0e24a7e7b6b0fcc088360d0e01bd", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "mns_increase", + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp4_mns16/anchor-0.034252608017/result.json" + }, + "target": { + "cell": "tp4_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp4_mns32/anchor-0.034252608017/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.01835841147659245, + "batch_tokens.mean": 0.02343919454014376, + "decode_batch_size.mean": 0.01819951878212578, + "graph_full_share": -0.0012724026088667406, + "graph_none_share": 3.7831593920716316e-06, + "graph_padding_fraction": 0.0006955531292226758, + "kv_usage_end_minus_start": 1.2113828717907893e-06, + "kv_usage_max": 1.9874666237429395e-05, + "kv_usage_mean": 2.5481750126793087e-05, + "preemptions": 0.0, + "prefill_token_fraction": 3.285684502737918e-05, + "queue_running_mean": 0.018367366400000584, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -0.5999999999999943 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.034252608017, + "request_hash": "21c1f3b7d972b19c952a894863a70d1e85bd0e24a7e7b6b0fcc088360d0e01bd", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "mns_increase", + "source": { + "cell": "tp4_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp4_mns32/anchor-0.034252608017/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp4_mns64/anchor-0.034252608017/result.json" + } + } + ], + "descriptive_full_outcome": { + "delta_pass_rate": { + "distinct_n": 10, + "max": 0.8216666666666667, + "min": -0.7698209718670077, + "n": 17 + }, + "feasibility_transitions": { + "false->false": 5, + "false->true": 2, + "true->false": 1, + "true->true": 9 + }, + "limitation": "Full outcomes may use different elapsed durations when a trial early-stopped; they are descriptive and are not a gate input.", + "negative": 4, + "positive": 5, + "zero": 8 + }, + "feature_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.35294117647058826, + "action_abs_median": 0.1278554259770317, + "action_delta": { + "distinct_n": 17, + "max": 4.473159643294283, + "min": -1.4603817235396175, + "n": 17 + }, + "action_signs": { + "consistency": 0.5, + "negative": 8, + "positive": 8, + "zero": 1 + }, + "effect_to_repeat_median": 1.9564724489442589, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.06534997517906493, + "repeat_abs_p95": 0.4689572472409891, + "repeat_delta": { + "distinct_n": 29, + "max": 0.9280844155844155, + "min": -0.373100622158411, + "n": 29 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.058823529411764705, + "action_abs_median": 2.443018058287123, + "action_delta": { + "distinct_n": 17, + "max": 220.88027761711976, + "min": -97.02946180778741, + "n": 17 + }, + "action_signs": { + "consistency": 0.625, + "negative": 6, + "positive": 10, + "zero": 1 + }, + "effect_to_repeat_median": 1.016963189345596, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 2.402267932489451, + "repeat_abs_p95": 212.7553678215874, + "repeat_delta": { + "distinct_n": 29, + "max": 100.79992784992785, + "min": -213.38441949985196, + "n": 29 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.35294117647058826, + "action_abs_median": 0.127000801210718, + "action_delta": { + "distinct_n": 17, + "max": 4.408288162266131, + "min": -1.5222672064777338, + "n": 17 + }, + "action_signs": { + "consistency": 0.5, + "negative": 8, + "positive": 8, + "zero": 1 + }, + "effect_to_repeat_median": 1.9625625182162536, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.06471172257286728, + "repeat_abs_p95": 0.5074569452309783, + "repeat_delta": { + "distinct_n": 29, + "max": 0.9185245310245311, + "min": -0.36846434553720986, + "n": 29 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.11764705882352941, + "action_abs_median": 0.0012724026088667406, + "action_delta": { + "distinct_n": 16, + "max": 0.015299949334160123, + "min": -0.06998264893001727, + "n": 17 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 10, + "positive": 5, + "zero": 2 + }, + "effect_to_repeat_median": 1.8902399214885985, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0006731434430105043, + "repeat_abs_p95": 0.029715971438244844, + "repeat_delta": { + "distinct_n": 28, + "max": 0.029898516783762652, + "min": -0.009559884559884546, + "n": 29 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.23529411764705882, + "action_abs_median": 0.0034719131131487613, + "action_delta": { + "distinct_n": 15, + "max": 0.055523423944476596, + "min": -0.04103743978705223, + "n": 17 + }, + "action_signs": { + "consistency": 0.5714285714285714, + "negative": 6, + "positive": 8, + "zero": 3 + }, + "effect_to_repeat_median": 3.2771966527196663, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0010594155557517443, + "repeat_abs_p95": 0.031925759090416374, + "repeat_delta": { + "distinct_n": 21, + "max": 0.02168109668109668, + "min": -0.03201970443349754, + "n": 29 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.002483113089604908, + "action_delta": { + "distinct_n": 17, + "max": 0.21027653696749515, + "min": -0.2241759801940655, + "n": 17 + }, + "action_signs": { + "consistency": 0.5625, + "negative": 7, + "positive": 9, + "zero": 1 + }, + "effect_to_repeat_median": 0.07674048944322485, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.032357274596769375, + "repeat_abs_p95": 0.24385677131445133, + "repeat_delta": { + "distinct_n": 29, + "max": 0.26242965569766435, + "min": -0.18173061839089436, + "n": 29 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.23529411764705882, + "action_abs_median": 7.313298449929118e-05, + "action_delta": { + "distinct_n": 17, + "max": 0.08863131728902607, + "min": -0.006308261401860582, + "n": 17 + }, + "action_signs": { + "consistency": 0.8823529411764706, + "negative": 2, + "positive": 15, + "zero": 0 + }, + "effect_to_repeat_median": 1.4692416585900228, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 4.977600796418624e-05, + "repeat_abs_p95": 0.03241204552355029, + "repeat_delta": { + "distinct_n": 14, + "max": 0.048360614955967995, + "min": -0.03483866016958903, + "n": 29 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.29411764705882354, + "action_abs_median": 0.0029570125473361575, + "action_delta": { + "distinct_n": 16, + "max": 0.08590208433411406, + "min": -0.0029570125473361575, + "n": 17 + }, + "action_signs": { + "consistency": 0.8823529411764706, + "negative": 2, + "positive": 15, + "zero": 0 + }, + "effect_to_repeat_median": null, + "effect_to_repeat_median_is_infinite": true, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.016674847306688556, + "repeat_delta": { + "distinct_n": 14, + "max": 0.004527588437235774, + "min": -0.018634334752970383, + "n": 29 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.11764705882352941, + "action_abs_median": 0.0013514937493694472, + "action_delta": { + "distinct_n": 17, + "max": 0.04101485276318234, + "min": -0.0013514937493694472, + "n": 17 + }, + "action_signs": { + "consistency": 0.7058823529411765, + "negative": 5, + "positive": 12, + "zero": 0 + }, + "effect_to_repeat_median": 2.827343009345717, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0004780084145793828, + "repeat_abs_p95": 0.006592699934625479, + "repeat_delta": { + "distinct_n": 29, + "max": 0.004456117360644808, + "min": -0.006626073152505554, + "n": 29 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 17 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 17 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 29 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0001483422608830054, + "action_delta": { + "distinct_n": 16, + "max": 0.8861120723274618, + "min": -0.7248910361224853, + "n": 17 + }, + "action_signs": { + "consistency": 0.6, + "negative": 6, + "positive": 9, + "zero": 2 + }, + "effect_to_repeat_median": 0.0009890369063830554, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.1499865777764538, + "repeat_abs_p95": 0.9129144795909149, + "repeat_delta": { + "distinct_n": 28, + "max": 0.8754134325195487, + "min": -0.9129695381069503, + "n": 29 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.27085744499999986, + "action_delta": { + "distinct_n": 17, + "max": 3.8361017661999997, + "min": -0.7297395955999999, + "n": 17 + }, + "action_signs": { + "consistency": 0.6470588235294118, + "negative": 6, + "positive": 11, + "zero": 0 + }, + "effect_to_repeat_median": 2.6248225517066213, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.10319076419999984, + "repeat_abs_p95": 6.572392109320001, + "repeat_delta": { + "distinct_n": 29, + "max": 0.07105187159999993, + "min": -6.5953128982, + "n": 29 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.23529411764705882, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 7, + "max": 0.0, + "min": -3.4976107514, + "n": 17 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 11 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.10560919299999999, + "repeat_delta": { + "distinct_n": 5, + "max": 0.0, + "min": -1.5032174162, + "n": 29 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 4.200000000000017, + "action_delta": { + "distinct_n": 15, + "max": 59.80000000000001, + "min": -84.2, + "n": 17 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 10, + "positive": 5, + "zero": 2 + }, + "effect_to_repeat_median": 2.333333333333365, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 1.799999999999983, + "repeat_abs_p95": 101.72, + "repeat_delta": { + "distinct_n": 24, + "max": 101.8, + "min": -31.799999999999997, + "n": 29 + } + } + }, + "horizon_s": 5.0, + "qualifying_features": [], + "repeats": [ + { + "delta_state": { + "batch_size.mean": -0.11342809216915306, + "batch_tokens.mean": -4.816915335040164, + "decode_batch_size.mean": -0.11215679350225116, + "graph_full_share": 0.0011653571113269257, + "graph_none_share": -0.0010594155557517443, + "graph_padding_fraction": -0.0002394644309091495, + "kv_usage_end_minus_start": 4.977600796418624e-05, + "kv_usage_max": -4.977600796418624e-05, + "kv_usage_mean": -0.002650866963514159, + "preemptions": 0.0, + "prefill_token_fraction": -0.00016947847485904166, + "queue_running_mean": -0.21751143979999998, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 1.2000000000000028 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.25, + "request_hash": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp1_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp1_mns16/anchor-0.25/result.json" + }, + "target": { + "cell": "tp1_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp1_mns16/confirm-1-anchor-0.25/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.01603319151928506, + "batch_tokens.mean": 0.7965437220913429, + "decode_batch_size.mean": 0.015822228472979205, + "graph_full_share": -0.00019338279244751622, + "graph_none_share": 0.00017580253858866246, + "graph_padding_fraction": -2.1621324535013326e-05, + "kv_usage_end_minus_start": -4.982561036370736e-05, + "kv_usage_max": 0.0, + "kv_usage_mean": 0.0010437773534030942, + "preemptions": 0.0, + "prefill_token_fraction": 4.236776766430861e-05, + "queue_running_mean": 0.07105187159999993, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -0.19999999999999574 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.24609375, + "request_hash": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp1_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp1_mns32/anchor-0.24609375/result.json" + }, + "target": { + "cell": "tp1_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp1_mns32/confirm-1-anchor-0.24609375/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.012605042016806678, + "batch_tokens.mean": 0.012605042016787138, + "decode_batch_size.mean": 0.012605042016806678, + "graph_full_share": 0.0, + "graph_none_share": 0.0, + "graph_padding_fraction": 0.00010847307953028382, + "kv_usage_end_minus_start": 4.982561036370736e-05, + "kv_usage_max": 0.0, + "kv_usage_mean": 0.00017122409391133464, + "preemptions": 0.0, + "prefill_token_fraction": -6.355303957206182e-05, + "queue_running_mean": 0.01412161399999956, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.0 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.25, + "request_hash": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp1_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp1_mns32/anchor-0.25/result.json" + }, + "target": { + "cell": "tp1_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp1_mns32/confirm-1-anchor-0.25/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.06534997517906493, + "batch_tokens.mean": -2.426725054960656, + "decode_batch_size.mean": -0.06471172257286728, + "graph_full_share": 0.0005850648890149968, + "graph_none_share": -0.000531877171831785, + "graph_padding_fraction": -0.00034778464236925486, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": -4.995504046367838e-05, + "kv_usage_mean": -0.0014804826259866288, + "preemptions": 0.0, + "prefill_token_fraction": -4.237146928010027e-05, + "queue_running_mean": -0.14321269680000004, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.5999999999999943 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.24609375, + "request_hash": "7e48c6bfc00eeadd6011111170c31123fb117c9fa75c18ccc8d8d505fd7fcff3", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp1_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp1_mns64/anchor-0.24609375/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp1_mns64/confirm-1-anchor-0.24609375/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.060654008438818074, + "batch_tokens.mean": -2.402267932489451, + "decode_batch_size.mean": -0.06002109704641434, + "graph_full_share": 0.0005801687763713481, + "graph_none_share": -0.0005274261603375532, + "graph_padding_fraction": -0.00017401096415257163, + "kv_usage_end_minus_start": 4.995504046356736e-05, + "kv_usage_max": -4.995504046367838e-05, + "kv_usage_mean": -0.0013647646656208823, + "preemptions": 0.0, + "prefill_token_fraction": -6.355303957206182e-05, + "queue_running_mean": -0.09758093800000012, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.6000000000000014 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.25, + "request_hash": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp1_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp1_mns64/anchor-0.25/result.json" + }, + "target": { + "cell": "tp1_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp1_mns64/confirm-1-anchor-0.25/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.9280844155844155, + "batch_tokens.mean": 100.79992784992785, + "decode_batch_size.mean": 0.9185245310245311, + "graph_full_share": -0.009559884559884546, + "graph_none_share": 0.02168109668109668, + "graph_padding_fraction": -0.18173061839089436, + "kv_usage_end_minus_start": 0.048360614955967995, + "kv_usage_max": 0.004527588437235774, + "kv_usage_mean": 0.004456117360644808, + "preemptions": 0.0, + "prefill_token_fraction": 0.8754134325195487, + "queue_running_mean": -0.277960427, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -31.799999999999997 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.2265625, + "request_hash": "a05848006335fc9f595432bab793a13d427244886e4f04e51ba74d495453ef90", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 0.9920634920634921, + "mns": 8, + "trial_id": "tp1_mns8/anchor-0.2265625/result.json" + }, + "target": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 0.9920634920634921, + "mns": 8, + "trial_id": "tp1_mns8/confirm-1-anchor-0.2265625/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.013264036958066772, + "batch_tokens.mean": 0.31208244491826065, + "decode_batch_size.mean": 0.013175195451314625, + "graph_full_share": -8.884150675192437e-05, + "graph_none_share": 8.88415067519556e-05, + "graph_padding_fraction": 5.628150611948664e-05, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": 0.0, + "kv_usage_mean": 6.337324494750524e-05, + "preemptions": 0.0, + "prefill_token_fraction": -2.779388529827731e-05, + "queue_running_mean": 0.007332840999999757, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": -0.20000000000000284 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.23046875, + "request_hash": "0f4c973153935c8abcd421eb631900670e31af0dc8a33bdf7e4d3d85e7ed7711", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 0.9923076923076923, + "mns": 8, + "trial_id": "tp1_mns8/anchor-0.23046875/result.json" + }, + "target": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 0.9923076923076923, + "mns": 8, + "trial_id": "tp1_mns8/confirm-1-anchor-0.23046875/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.01133760879523571, + "batch_tokens.mean": -0.4460948236371962, + "decode_batch_size.mean": -0.011211635364177575, + "graph_full_share": 0.00012597343105813508, + "graph_none_share": -0.0001259734310581767, + "graph_padding_fraction": 2.5032232616357952e-05, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": 0.0, + "kv_usage_mean": -2.595429314883324e-06, + "preemptions": 0.0, + "prefill_token_fraction": -2.465132225659694e-05, + "queue_running_mean": -0.0003634421999998416, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.20000000000000284 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.234375, + "request_hash": "14d2a74fffe25e7d9b3e7490c392eae0dc31bb1f3fe6566c9c0a567543bdd6ac", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 0.9924242424242424, + "mns": 8, + "trial_id": "tp1_mns8/anchor-0.234375/result.json" + }, + "target": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 0.9924242424242424, + "mns": 8, + "trial_id": "tp1_mns8/confirm-1-anchor-0.234375/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.03836578028285853, + "batch_tokens.mean": -2.3411943704830662, + "decode_batch_size.mean": -0.037743352121442975, + "graph_full_share": 0.0005705591479649996, + "graph_none_share": -0.0005705591479650066, + "graph_padding_fraction": 8.67237650413492e-05, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": 0.0, + "kv_usage_mean": -0.00012192636323195283, + "preemptions": 0.0, + "prefill_token_fraction": -0.00016953030936628366, + "queue_running_mean": -0.0205579666000002, + "queue_waiting_mean": -0.0408831314, + "scheduler_steps_per_s": 0.6000000000000014 + }, + "descriptive_full_outcome": { + "delta_pass_rate": -0.006993006993006978, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.25, + "request_hash": "26629995f38f2c013d5aa5e4b9d7344311ab1f951c9d83e68212c67ca8076fda", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 1 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 0.958041958041958, + "mns": 8, + "trial_id": "tp1_mns8/anchor-0.25/result.json" + }, + "target": { + "cell": "tp1_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 0.951048951048951, + "mns": 8, + "trial_id": "tp1_mns8/confirm-1-anchor-0.25/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.2758963500935341, + "batch_tokens.mean": -16.135882715276434, + "decode_batch_size.mean": -0.2723042461451284, + "graph_full_share": 0.0035921039484052564, + "graph_none_share": -0.0048721071863581, + "graph_padding_fraction": 0.10945908199689197, + "kv_usage_end_minus_start": -0.004480240988461115, + "kv_usage_max": 0.0, + "kv_usage_mean": -0.0010388280544342795, + "preemptions": 0.0, + "prefill_token_fraction": -0.7696270176402451, + "queue_running_mean": -0.36681181659999984, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 28.400000000000006 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.49609375, + "request_hash": "a78992ce59bb3b857a7ef8844e0a690ea9dd529a659cf5edd599b63fef8b6a80", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp2_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp2_mns16/anchor-0.49609375/result.json" + }, + "target": { + "cell": "tp2_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp2_mns16/confirm-1-anchor-0.49609375/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.006385050186775487, + "batch_tokens.mean": -0.006810792236799035, + "decode_batch_size.mean": -0.006341827643625564, + "graph_full_share": 4.322254314959029e-05, + "graph_none_share": 0.0, + "graph_padding_fraction": -0.0016266700040674653, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": 0.0, + "kv_usage_mean": -1.7558597020827724e-05, + "preemptions": 0.0, + "prefill_token_fraction": 3.0428082455902228e-05, + "queue_running_mean": -0.0059703380000000195, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.4000000000000057 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.5, + "request_hash": "3160017dbd8249f711eb115f1e72ddbb37eab3a3333a48762cc3104e94bd2f15", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp2_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp2_mns16/anchor-0.5/result.json" + }, + "target": { + "cell": "tp2_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp2_mns16/confirm-1-anchor-0.5/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.006361323155215537, + "batch_tokens.mean": -0.006361323155216425, + "decode_batch_size.mean": -0.006361323155215537, + "graph_full_share": 0.0, + "graph_none_share": 0.0, + "graph_padding_fraction": -0.0018169528035430205, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": 0.0, + "kv_usage_mean": -2.673005121892058e-05, + "preemptions": 0.0, + "prefill_token_fraction": 0.0001139335897381577, + "queue_running_mean": -0.008856016399999334, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.0 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.75, + "request_hash": "4f4c792e90f5c1bf33978fea26debcadfce5f8b872a31be86a88a1bf6b09a21c", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp2_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp2_mns32/anchor-0.75/result.json" + }, + "target": { + "cell": "tp2_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp2_mns32/confirm-1-anchor-0.75/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.36334379104298975, + "batch_tokens.mean": -99.4815933746712, + "decode_batch_size.mean": 0.38174161358586733, + "graph_full_share": 0.015964732518547464, + "graph_none_share": -0.019464720194647202, + "graph_padding_fraction": 0.26242965569766435, + "kv_usage_end_minus_start": -0.024225165055486664, + "kv_usage_max": -0.013153357937349175, + "kv_usage_mean": -0.006611009430699675, + "preemptions": 0.0, + "prefill_token_fraction": -0.8730961888774497, + "queue_running_mean": -2.3594406332, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 55.999999999999986 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.7385786802030457, + "feasibility_transition": "false->false" + }, + "group": { + "anchor": 0.75390625, + "request_hash": "2677a6a12ff0e104d6246dd121d0667ff4fe3d25f8d68699547906420246ebdd", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp2_mns32", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.20558375634517767, + "mns": 32, + "trial_id": "tp2_mns32/anchor-0.75390625/result.json" + }, + "target": { + "cell": "tp2_mns32", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.9441624365482234, + "mns": 32, + "trial_id": "tp2_mns32/confirm-1-anchor-0.75390625/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.008984926990108555, + "batch_tokens.mean": -0.009617608016100565, + "decode_batch_size.mean": -0.00892069541386542, + "graph_full_share": 6.42315762428014e-05, + "graph_none_share": 0.0, + "graph_padding_fraction": -0.00023488823078426924, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": 0.0, + "kv_usage_mean": -3.459821231118886e-05, + "preemptions": 0.0, + "prefill_token_fraction": 3.0382556995986465e-05, + "queue_running_mean": -0.01180635799999985, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.5999999999999943 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.5, + "request_hash": "3160017dbd8249f711eb115f1e72ddbb37eab3a3333a48762cc3104e94bd2f15", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp2_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp2_mns64/anchor-0.5/result.json" + }, + "target": { + "cell": "tp2_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp2_mns64/confirm-1-anchor-0.5/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.36810601285071254, + "batch_tokens.mean": -111.9216322174668, + "decode_batch_size.mean": -0.3386389524789051, + "graph_full_share": 0.025109761896862537, + "graph_none_share": -0.026143790849673203, + "graph_padding_fraction": 0.22131844466419573, + "kv_usage_end_minus_start": -0.03483866016958903, + "kv_usage_max": -0.018634334752970383, + "kv_usage_mean": -0.006551218139822743, + "preemptions": 0.0, + "prefill_token_fraction": -0.8766852234122235, + "queue_running_mean": -2.0965550011999996, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 65.8 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.6624040920716113, + "feasibility_transition": "false->false" + }, + "group": { + "anchor": 0.75, + "request_hash": "4f4c792e90f5c1bf33978fea26debcadfce5f8b872a31be86a88a1bf6b09a21c", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp2_mns64", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.23017902813299232, + "mns": 64, + "trial_id": "tp2_mns64/anchor-0.75/result.json" + }, + "target": { + "cell": "tp2_mns64", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.8925831202046036, + "mns": 64, + "trial_id": "tp2_mns64/confirm-1-anchor-0.75/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.017777364465908274, + "batch_tokens.mean": -0.019283121886095156, + "decode_batch_size.mean": -0.017624495692285524, + "graph_full_share": 0.0001528687736229717, + "graph_none_share": -4.586063208690105e-05, + "graph_padding_fraction": -0.0033762430254597187, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": 0.0, + "kv_usage_mean": -3.866481934330576e-05, + "preemptions": 0.0, + "prefill_token_fraction": -3.0428082455902228e-05, + "queue_running_mean": -0.01706728199999974, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 1.200000000000017 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.4921875, + "request_hash": "cd227aee3be472a35e8c35af60a44a1411c4dc689cbd98746b3fb8a8d329b44c", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp2_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 8, + "trial_id": "tp2_mns8/anchor-0.4921875/result.json" + }, + "target": { + "cell": "tp2_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 8, + "trial_id": "tp2_mns8/confirm-1-anchor-0.4921875/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.373100622158411, + "batch_tokens.mean": -19.752101280210574, + "decode_batch_size.mean": -0.36846434553720986, + "graph_full_share": 0.004636276621201252, + "graph_none_share": -0.006318796362766213, + "graph_padding_fraction": 0.13466522306023426, + "kv_usage_end_minus_start": -0.004491915827622761, + "kv_usage_max": 0.0, + "kv_usage_mean": -0.0010760794618353137, + "preemptions": 0.0, + "prefill_token_fraction": -0.7698627933477863, + "queue_running_mean": -0.3758000022000001, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 24.799999999999983 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.6190476190476191, + "feasibility_transition": "false->false" + }, + "group": { + "anchor": 0.49609375, + "request_hash": "a78992ce59bb3b857a7ef8844e0a690ea9dd529a659cf5edd599b63fef8b6a80", + "study_sha256": "9474f0d0b53579f1db852ca68abfb0b96ba43ae4e17738118bf8e3209eb09ece", + "tp": 2 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp2_mns8", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.27106227106227104, + "mns": 8, + "trial_id": "tp2_mns8/anchor-0.49609375/result.json" + }, + "target": { + "cell": "tp2_mns8", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.8901098901098901, + "mns": 8, + "trial_id": "tp2_mns8/confirm-1-anchor-0.49609375/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.035790047012957515, + "batch_tokens.mean": -0.037904196766425535, + "decode_batch_size.mean": -0.03550338263960562, + "graph_full_share": -0.0008241600733860377, + "graph_none_share": 0.0, + "graph_padding_fraction": -0.002233730654183863, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": -1.0202155715566263e-05, + "kv_usage_mean": -4.688019669552533e-05, + "preemptions": 0.0, + "prefill_token_fraction": 0.0, + "queue_running_mean": -0.04172862040000069, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 1.1999999999999886 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.2469565217391304, + "feasibility_transition": "false->true" + }, + "group": { + "anchor": 0.033182214016, + "request_hash": "e6b624a0212807df3733c54b4a2160fba24f19ca4776b14f34918d83428bba7f", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.7530434782608696, + "mns": 16, + "trial_id": "tp4_mns16/anchor-0.033182214016/result.json" + }, + "target": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp4_mns16/confirm-1-anchor-0.033182214016/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.31521561989075764, + "batch_tokens.mean": -113.69135049600723, + "decode_batch_size.mean": -0.2948443636019542, + "graph_full_share": 0.013747312308444393, + "graph_none_share": -0.02466367713004484, + "graph_padding_fraction": 0.2327910644295707, + "kv_usage_end_minus_start": -0.014762519320332457, + "kv_usage_max": -0.002734177731754772, + "kv_usage_mean": -0.003886883143649442, + "preemptions": 0.0, + "prefill_token_fraction": -0.8857303498661139, + "queue_running_mean": -3.9262134145999994, + "queue_waiting_mean": -1.5032174162, + "scheduler_steps_per_s": 81.39999999999999 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.34982935153583616, + "feasibility_transition": "false->false" + }, + "group": { + "anchor": 0.033717411016, + "request_hash": "472a874ecc0e844b5b36a715bbc309b63e73074d39327cd734869f76721d71db", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.06313993174061433, + "mns": 16, + "trial_id": "tp4_mns16/anchor-0.033717411016/result.json" + }, + "target": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.4129692832764505, + "mns": 16, + "trial_id": "tp4_mns16/confirm-1-anchor-0.033717411016/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.0565950637806516, + "batch_tokens.mean": -1.2259074371774261, + "decode_batch_size.mean": -0.05612473435097076, + "graph_full_share": -0.0006731434430105043, + "graph_none_share": -0.0011210762331838565, + "graph_padding_fraction": 0.03352610055520758, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": 0.0, + "kv_usage_mean": -8.342130161142786e-05, + "preemptions": 0.0, + "prefill_token_fraction": -0.1499865777764538, + "queue_running_mean": -0.07804562620000066, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 1.799999999999983 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.034252608017, + "request_hash": "21c1f3b7d972b19c952a894863a70d1e85bd0e24a7e7b6b0fcc088360d0e01bd", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp4_mns16/anchor-0.034252608017/result.json" + }, + "target": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp4_mns16/confirm-1-anchor-0.034252608017/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.017452685323813277, + "batch_tokens.mean": -0.018160486007140264, + "decode_batch_size.mean": -0.01735671234980174, + "graph_full_share": 8.877500095971769e-05, + "graph_none_share": 0.0, + "graph_padding_fraction": 0.0015546681503931792, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": -5.101077857783132e-06, + "kv_usage_mean": -2.2530897596876806e-05, + "preemptions": 0.0, + "prefill_token_fraction": 5.993471374250714e-05, + "queue_running_mean": -0.021152762000000713, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 0.4000000000000057 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.2469565217391304, + "feasibility_transition": "false->true" + }, + "group": { + "anchor": 0.033182214016, + "request_hash": "e6b624a0212807df3733c54b4a2160fba24f19ca4776b14f34918d83428bba7f", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns16", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.7530434782608696, + "mns": 16, + "trial_id": "tp4_mns16/anchor-0.033182214016/result.json" + }, + "target": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "trial_id": "tp4_mns16/confirm-2-anchor-0.033182214016/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.16723437281046838, + "batch_tokens.mean": -213.3770035894838, + "decode_batch_size.mean": 0.2293820267109341, + "graph_full_share": 0.029853079087214685, + "graph_none_share": -0.03201970443349754, + "graph_padding_fraction": 0.24386359763484056, + "kv_usage_end_minus_start": -0.02873466386876411, + "kv_usage_max": -0.01666487968508279, + "kv_usage_mean": -0.006565235690514186, + "preemptions": 0.0, + "prefill_token_fraction": -0.9129695381069503, + "queue_running_mean": -6.573865443400001, + "queue_waiting_mean": -0.105609193, + "scheduler_steps_per_s": 101.60000000000001 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.9692832764505119, + "feasibility_transition": "false->true" + }, + "group": { + "anchor": 0.033717411016, + "request_hash": "472a874ecc0e844b5b36a715bbc309b63e73074d39327cd734869f76721d71db", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns32", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.030716723549488054, + "mns": 32, + "trial_id": "tp4_mns32/anchor-0.033717411016/result.json" + }, + "target": { + "cell": "tp4_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp4_mns32/confirm-1-anchor-0.033717411016/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.06700882459564284, + "batch_tokens.mean": -1.237066033004087, + "decode_batch_size.mean": -0.06643525070889744, + "graph_full_share": -0.0005748153886646623, + "graph_none_share": -0.0011210762331838565, + "graph_padding_fraction": 0.032357274596769375, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": -5.1056616682254585e-06, + "kv_usage_mean": -0.00010377856500988665, + "preemptions": 0.0, + "prefill_token_fraction": -0.15007350008784556, + "queue_running_mean": -0.10319076419999984, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 2.1999999999999886 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.034252608017, + "request_hash": "21c1f3b7d972b19c952a894863a70d1e85bd0e24a7e7b6b0fcc088360d0e01bd", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp4_mns32/anchor-0.034252608017/result.json" + }, + "target": { + "cell": "tp4_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp4_mns32/confirm-1-anchor-0.034252608017/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.16017120245497818, + "batch_tokens.mean": -213.38441949985196, + "decode_batch_size.mean": 0.22236668550970418, + "graph_full_share": 0.029898516783762652, + "graph_none_share": -0.03201970443349754, + "graph_padding_fraction": 0.2419788945317641, + "kv_usage_end_minus_start": -0.02873466386876411, + "kv_usage_max": -0.016659774023414564, + "kv_usage_mean": -0.006562301403937485, + "preemptions": 0.0, + "prefill_token_fraction": -0.9129575557004137, + "queue_running_mean": -6.570182108200001, + "queue_waiting_mean": -0.105609193, + "scheduler_steps_per_s": 101.8 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.9692832764505119, + "feasibility_transition": "false->true" + }, + "group": { + "anchor": 0.033717411016, + "request_hash": "472a874ecc0e844b5b36a715bbc309b63e73074d39327cd734869f76721d71db", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns32", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.030716723549488054, + "mns": 32, + "trial_id": "tp4_mns32/anchor-0.033717411016/result.json" + }, + "target": { + "cell": "tp4_mns32", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 32, + "trial_id": "tp4_mns32/confirm-2-anchor-0.033717411016/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.5328616639627084, + "batch_tokens.mean": -211.42080067637045, + "decode_batch_size.mean": 0.5912671663277198, + "graph_full_share": 0.027680346411352086, + "graph_none_share": -0.03178484107579462, + "graph_padding_fraction": 0.23688813872052464, + "kv_usage_end_minus_start": -0.028772123554492213, + "kv_usage_max": -0.016661043117286778, + "kv_usage_mean": -0.006272675400456023, + "preemptions": 0.0, + "prefill_token_fraction": -0.912693798885484, + "queue_running_mean": -6.2424369058, + "queue_waiting_mean": -0.10551258, + "scheduler_steps_per_s": 89.39999999999999 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.8430034129692833, + "feasibility_transition": "false->true" + }, + "group": { + "anchor": 0.033717411016, + "request_hash": "472a874ecc0e844b5b36a715bbc309b63e73074d39327cd734869f76721d71db", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns64", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.11433447098976109, + "mns": 64, + "trial_id": "tp4_mns64/anchor-0.033717411016/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 0.9573378839590444, + "mns": 64, + "trial_id": "tp4_mns64/confirm-1-anchor-0.033717411016/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.08536723607223529, + "batch_tokens.mean": -1.2605052275442308, + "decode_batch_size.mean": -0.08463476949102322, + "graph_full_share": 0.0006975872202020783, + "graph_none_share": -0.0011248593925759281, + "graph_padding_fraction": 0.034272917448908424, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": 0.0, + "kv_usage_mean": -0.00010935706758585708, + "preemptions": 0.0, + "prefill_token_fraction": -0.15010635693287294, + "queue_running_mean": -0.10249850960000018, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 2.799999999999983 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.034252608017, + "request_hash": "21c1f3b7d972b19c952a894863a70d1e85bd0e24a7e7b6b0fcc088360d0e01bd", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp4_mns64/anchor-0.034252608017/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp4_mns64/confirm-1-anchor-0.034252608017/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": 0.1540235762873472, + "batch_tokens.mean": -211.8229141697429, + "decode_batch_size.mean": 0.21558506586552006, + "graph_full_share": 0.029510309964790093, + "graph_none_share": -0.03178484107579462, + "graph_padding_fraction": 0.24384653183386754, + "kv_usage_end_minus_start": -0.028772123554492213, + "kv_usage_max": -0.016681492387759067, + "kv_usage_mean": -0.006626073152505554, + "preemptions": 0.0, + "prefill_token_fraction": -0.9128498654266666, + "queue_running_mean": -6.5953128982, + "queue_waiting_mean": -0.10551258, + "scheduler_steps_per_s": 101.8 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.8856655290102389, + "feasibility_transition": "false->true" + }, + "group": { + "anchor": 0.033717411016, + "request_hash": "472a874ecc0e844b5b36a715bbc309b63e73074d39327cd734869f76721d71db", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns64", + "early_stopped": true, + "full_feasible": false, + "full_pass_rate": 0.11433447098976109, + "mns": 64, + "trial_id": "tp4_mns64/anchor-0.033717411016/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "trial_id": "tp4_mns64/confirm-2-anchor-0.033717411016/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.3497477888336098, + "batch_tokens.mean": -19.68794568822554, + "decode_batch_size.mean": -0.3465001381978996, + "graph_full_share": 0.004242675511332172, + "graph_none_share": -0.004954394693200663, + "graph_padding_fraction": 0.14768569288462857, + "kv_usage_end_minus_start": -0.0007955733482928329, + "kv_usage_max": 0.0, + "kv_usage_mean": -0.0004780084145793828, + "preemptions": 0.0, + "prefill_token_fraction": -0.8110422816759224, + "queue_running_mean": -0.41782893040000024, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 28.19999999999999 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.016591107009, + "request_hash": "2407501cda53bf9c761c448dc50ab4780b4a55958cc00ba1b19cd09de1cf73e1", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 8, + "trial_id": "tp4_mns8/anchor-0.016591107009/result.json" + }, + "target": { + "cell": "tp4_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 8, + "trial_id": "tp4_mns8/confirm-1-anchor-0.016591107009/result.json" + } + }, + { + "delta_state": { + "batch_size.mean": -0.060037523452157515, + "batch_tokens.mean": -3.365563298490128, + "decode_batch_size.mean": -0.05956848030018724, + "graph_full_share": 0.0004467077637808625, + "graph_none_share": -0.000982757080318056, + "graph_padding_fraction": 0.07118473038433726, + "kv_usage_end_minus_start": 0.0, + "kv_usage_max": 0.0, + "kv_usage_mean": -6.376841981488128e-05, + "preemptions": 0.0, + "prefill_token_fraction": -0.5189516489461117, + "queue_running_mean": -0.1074558633999998, + "queue_waiting_mean": 0.0, + "scheduler_steps_per_s": 5.200000000000017 + }, + "descriptive_full_outcome": { + "delta_pass_rate": 0.0, + "feasibility_transition": "true->true" + }, + "group": { + "anchor": 0.017126304009, + "request_hash": "93cd99ebf2a760cc440dce0cb7e51273686dcfbfb07650d27e9e16d2ff1d2eed", + "study_sha256": "76e9d642ba9570917cfeeea848ed92f27904e87760eedabd48805b84e82eca26", + "tp": 4 + }, + "kind": "same_config_repeat", + "source": { + "cell": "tp4_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 8, + "trial_id": "tp4_mns8/anchor-0.017126304009/result.json" + }, + "target": { + "cell": "tp4_mns8", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 8, + "trial_id": "tp4_mns8/confirm-1-anchor-0.017126304009/result.json" + } + } + ], + "sanity": { + "action_pairs": 17, + "distinct_action_vectors": 17, + "invariants": { + "all_pair_hashes_match": true, + "all_values_finite": true, + "expected_action_pair_count": true, + "nonnegative_counters": true, + "ratios_bounded": true, + "state_vectors_not_all_identical": true, + "sufficient_repeat_pair_count": true + }, + "red_flags": [], + "repeat_pairs": 29, + "trials": 66 + } + } + }, + "provenance": { + "analysis_script": "/home/gahow/phd/aituner/runs/intervention-response-v0/analyze_phase6.py", + "analysis_script_sha256": "f1b40c350f22e797b08b129df7b55386b407bd5ae7f466d36987e94d768ded2f", + "phase6_metrics": "/home/gahow/phd/aituner/runs/opprof-phase6/phase6/metrics.json", + "phase6_metrics_sha256": "290ba7fcb8727291166de7e4d47afdc84e230052495c81dd087db0ace9f93a16", + "raw_root": "/home/gahow/phd/aituner/runs/opprof-phase6/phase6/solo-authoritative/cells", + "streams": [ + { + "bytes": 12745297, + "path": "runs/opprof-phase6/phase6/solo-authoritative/cells/tp1_mns16/opprof/opprof-v1-dp0-pid2781473-1783876972093434233.jsonl", + "sha256": "265932302b079d1ad81febc4a345f87ac2a3e00118e63bf5873b92d61146d3b8" + }, + { + "bytes": 20018398, + "path": "runs/opprof-phase6/phase6/solo-authoritative/cells/tp1_mns32/opprof/opprof-v1-dp0-pid2786356-1783877278912197439.jsonl", + "sha256": "552bb3961893ab20760e95a6148a24387df544370ca2cb3af052fa87a7ef79f6" + }, + { + "bytes": 19825716, + "path": "runs/opprof-phase6/phase6/solo-authoritative/cells/tp1_mns64/opprof/opprof-v1-dp0-pid2795014-1783877830485564578.jsonl", + "sha256": "f07f2beed8ffcd695665edba931cd5824f10f4ef678064ac09d5a0d9b96acf50" + }, + { + "bytes": 38381207, + "path": "runs/opprof-phase6/phase6/solo-authoritative/cells/tp1_mns8/opprof/opprof-v1-dp0-pid2770271-1783876258461072221.jsonl", + "sha256": "f3758f599bd58952d039eae3cc7a1b6abe3372e82fe15aff38d8f27bedf0cbef" + }, + { + "bytes": 42681418, + "path": "runs/opprof-phase6/phase6/solo-authoritative/cells/tp2_mns16/opprof/opprof-v1-dp0-pid2755628-1783875389707263062.jsonl", + "sha256": "e56faa0ceb0f750ad781b04a0fa86388a8fd4acdff4503fb40064bbb43b8a4de" + }, + { + "bytes": 20701713, + "path": "runs/opprof-phase6/phase6/solo-authoritative/cells/tp2_mns32/opprof/opprof-v1-dp0-pid2730662-1783873935616837232.jsonl", + "sha256": "ab1070348c976491ff5ad4905ab172678f692598810ca72a0dbab824f3bb491a" + }, + { + "bytes": 24186851, + "path": "runs/opprof-phase6/phase6/solo-authoritative/cells/tp2_mns64/opprof/opprof-v1-dp0-pid2736153-1783874256706634444.jsonl", + "sha256": "c8d7582674a9ff3cba8363a93672b96977082f6f2eb595aee0ea41012e85978e" + }, + { + "bytes": 25542356, + "path": "runs/opprof-phase6/phase6/solo-authoritative/cells/tp2_mns8/opprof/opprof-v1-dp0-pid2750298-1783875076865595283.jsonl", + "sha256": "ad006affdccb0c9cf2ae97938810a7f232ed0517086d3e20772aa461930f8518" + }, + { + "bytes": 50625291, + "path": "runs/opprof-phase6/phase6/solo-authoritative/cells/tp4_mns16/opprof/opprof-v1-dp0-pid2741574-1783874579483097755.jsonl", + "sha256": "4443a0f159f23242445e48c846e45e29b3647012734edeff2678bcf75b008ada" + }, + { + "bytes": 34980126, + "path": "runs/opprof-phase6/phase6/solo-authoritative/cells/tp4_mns32/opprof/opprof-v1-dp0-pid2712385-1783873000008589576.jsonl", + "sha256": "445b190a5392df7aa05555de33a550d6401d739bc9000494af650671db4c63c9" + }, + { + "bytes": 34066572, + "path": "runs/opprof-phase6/phase6/solo-authoritative/cells/tp4_mns64/opprof/opprof-v1-dp0-pid2721119-1783873467764669604.jsonl", + "sha256": "ddc7bf89342fe7fececc68b6c6be988816abd1ff37dfc63c8c2a55f7b9a96f44" + }, + { + "bytes": 52957710, + "path": "runs/opprof-phase6/phase6/solo-authoritative/cells/tp4_mns8/opprof/opprof-v1-dp0-pid2762505-1783875811578126796.jsonl", + "sha256": "ffdbfe671cecc166ea718ff0c31f7a15f788dcc2da64d6b4ee10fc16d489a849" + } + ] + }, + "sanity": { + "red_flags": [], + "stream_bytes": { + "distinct_n": 12, + "max": 52957710.0, + "min": 12745297.0, + "n": 12 + }, + "stream_count": 12 + }, + "schema": "intervention-response-audit-v0", + "stable_qualifying_features": [], + "status": "COMPLETE" +} diff --git a/runs/intervention-response-v0/test_analysis.py b/runs/intervention-response-v0/test_analysis.py new file mode 100644 index 0000000..74ada8c --- /dev/null +++ b/runs/intervention-response-v0/test_analysis.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import importlib.util +import math +from pathlib import Path + + +HERE = Path(__file__).resolve().parent + + +def load_module(): + spec = importlib.util.spec_from_file_location( + "intervention_response_v0", HERE / "analyze_phase6.py" + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def pair(module, delta: dict[str, float]) -> dict[str, object]: + state = {feature: 0.0 for feature in module.ALL_FEATURES} + state.update(delta) + return {"delta_state": state} + + +def main() -> None: + module = load_module() + assert module.numeric([0.0, 1.0, 1.0]) == { + "n": 3, + "min": 0.0, + "max": 1.0, + "distinct_n": 2, + } + assert math.isclose(module.quantile([0.0, 10.0], 0.95), 9.5) + + actions = [ + pair(module, {"queue_waiting_mean": -1.0 - 0.1 * index}) + for index in range(8) + ] + repeats = [ + pair(module, {"queue_waiting_mean": 0.01 * ((index % 3) - 1)}) + for index in range(20) + ] + stats = module.response_statistics(actions, repeats) + waiting = stats["queue_waiting_mean"] + assert waiting["qualifies"] + assert waiting["action_signs"]["negative"] == 8 + assert waiting["action_signs"]["consistency"] == 1.0 + assert waiting["effect_to_repeat_median"] > 2.0 + assert not stats["kv_usage_mean"]["qualifies"] + print("intervention response v0 analysis: PASS") + + +if __name__ == "__main__": + main() diff --git a/runs/intervention-response-v0/test_p1_analysis.py b/runs/intervention-response-v0/test_p1_analysis.py new file mode 100644 index 0000000..8a99899 --- /dev/null +++ b/runs/intervention-response-v0/test_p1_analysis.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +HERE = Path(__file__).resolve().parent + + +def load_module(): + spec = importlib.util.spec_from_file_location( + "intervention_response_p1", HERE / "analyze_p1.py" + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def main() -> None: + module = load_module() + values = [-2.0, -1.0, 1.0, 2.0] + labels = [0, 0, 1, 1] + threshold, direction, balanced = module._fit_threshold(values, labels) + assert direction == 1 + assert -1.0 < threshold < 1.0 + assert balanced == 1.0 + assert module._balanced_accuracy(labels, labels) == 1.0 + assert module._balanced_accuracy(labels, [1, 1, 0, 0]) == 0.0 + print("intervention response P1 confirmation analysis: PASS") + + +if __name__ == "__main__": + main() From 0515ad8ecc09ec6255f0b7dc61913cb5df22289b Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 17:18:25 +0800 Subject: [PATCH 014/148] Make telemetry audit replay-phase aware --- ...sponse-v2-phase-aware-protocol-20260714.md | 84 + .../analyze_existing.py | 467 + .../intervention-response-v2/analyze_pilot.py | 449 + .../existing-phase-audit.json | 9915 +++++++++++++++++ .../pilot_controller.py | 471 + .../intervention-response-v2/prepare_pilot.py | 267 + .../intervention-response-v2/test_analysis.py | 107 + runs/opprof-phase6/opprof_phase6_client.py | 8 +- runs/opprof-phase6/test_phase6_tools.py | 13 + 9 files changed, 11780 insertions(+), 1 deletion(-) create mode 100644 docs/intervention-response-v2-phase-aware-protocol-20260714.md create mode 100644 runs/intervention-response-v2/analyze_existing.py create mode 100644 runs/intervention-response-v2/analyze_pilot.py create mode 100644 runs/intervention-response-v2/existing-phase-audit.json create mode 100644 runs/intervention-response-v2/pilot_controller.py create mode 100644 runs/intervention-response-v2/prepare_pilot.py create mode 100644 runs/intervention-response-v2/test_analysis.py diff --git a/docs/intervention-response-v2-phase-aware-protocol-20260714.md b/docs/intervention-response-v2-phase-aware-protocol-20260714.md new file mode 100644 index 0000000..d87025b --- /dev/null +++ b/docs/intervention-response-v2-phase-aware-protocol-20260714.md @@ -0,0 +1,84 @@ +# Phase-aware telemetry intervention-response v2 protocol + +Status: **FROZEN BEFORE THE SYSTEMATIC 10% DECILE AUDIT**. + +Date: 2026-07-14 (Asia/Singapore). + +## Correction to v0/v1 + +The 5/10-second analyses tested an ultra-early verifier. They did not test +whether telemetry observed after the engine has developed queue, batch, and KV +state can guide tuning. The P1 replay lasts 60 seconds after time scaling, and +the 5/10-second prefixes contain only a small fraction of its requests. + +V2 therefore replaces absolute cutoffs with replay phase. The old audits and +their negative decisions remain immutable, but their claim is narrowed to the +first 5/10 seconds. + +## Historical corrective audit + +The historical audit is development-only and cannot become confirmatory after +the horizon concern was observed. + +- Infer each trial's intended replay duration as selected requests divided by + offered requests per second. All trials must agree. +- Find every complete 10% replay decile supported by every trial. Analyze all + such deciles; selecting only the best horizon is forbidden. +- At each decile report both: + - cumulative state from replay start to the checkpoint; and + - the non-overlapping 10%-wide state block ending at the checkpoint. +- Report admitted/completed request coverage, response-versus-repeat statistics, + telemetry versus external-outcome efficacy, and per-feature trajectory drift. +- Reuse the frozen v1 action and repeat pairs and the frozen response and + incremental-efficacy thresholds. These thresholds are descriptive in V2; + passing one post-hoc horizon does not open a contribution claim. + +If early stopping prevents complete observation of the replay phases, the +historical decision is `REQUIRES_UNCENSORED_PHASE_AWARE_PILOT`, independent of +which early decile looks best. + +## Uncensored matched pilot + +The pilot is a mechanism gate, not paper evidence. + +- Hardware/engine/model: solo placement on dash0, 4 NVIDIA H20 GPUs, patched + vLLM `0.24.1.dev3+opprof`, Qwen3-30B-A3B, fixed `TP=4`. +- Action: `MNS 16 -> 64`; topology, model, engine build, workload, arrival + sequence, offered load, and all other settings remain fixed. +- Workload: `chat_w20260312_1000` at replay-time scale `0.5`, hence 300 seconds. +- Offered loads per GPU: `1.5`, `2.125`, and `3.125` requests/s. These supply a + low control and the two already-observed P1 pressure regimes. +- Repetitions: three disjoint session bands, exact request sequence matched + across action endpoints. Endpoint order alternates `A/B`, `B/A`, `A/B`; + load order is counter-rotated across repetitions. +- A fresh server receives the accepted long-request warm-up and a bounded + burn-in before each measured session. +- SLO-unrecoverable early stop is disabled. Every run must observe the full + 300-second arrival window; a separate 360-second safety deadline may mark a + run invalid but cannot manufacture a full-run label. +- Cumulative checkpoints: 10%, 25%, 50%, 75%, and 100%, or 30/75/150/225/300 + seconds. Quarter blocks are analyzed separately from cumulative means. +- Placement is serialized. Co-location remains forbidden because Phase 6 + observed material co-location-induced outcome shifts. +- Hard cap: 8 H20-hours including startup, warm-up, burn-in, invalid attempts, + and cleanup. + +## Gates + +Data validity requires complete 300-second Layer-1 coverage, zero dropped +records, exact request/arrival/length hashes across action endpoints, monotonic +timestamps, full request accounting, idle GPUs before and after each session, +and no co-resident GPU process. + +Mechanism evidence requires at least two telemetry features whose matched action +response exceeds same-config repeat noise at two consecutive checkpoints under +the unchanged v1 response thresholds. The same features must have a consistent +direction in at least two of the three load regimes. + +Decision evidence additionally requires both action-efficacy classes and an +instrumentation feature that reaches leave-one-repetition-out balanced accuracy +at least 0.75 and exceeds the best external prefix outcome by at least 0.15. +Without label balance the pilot can adjudicate mechanism evidence only. + +No H20 run is launched if the local analyzer/tests, manifest preflight, GPU +probe, command dry-run, projected cost, or cleanup plan fails. diff --git a/runs/intervention-response-v2/analyze_existing.py b/runs/intervention-response-v2/analyze_existing.py new file mode 100644 index 0000000..3d3fe85 --- /dev/null +++ b/runs/intervention-response-v2/analyze_existing.py @@ -0,0 +1,467 @@ +#!/usr/bin/env python3 +"""Audit telemetry responses over every uncensored replay decile. + +This corrective analysis keeps the frozen P1 pairs and thresholds, but replaces +the absolute 5/10-second cutoff with cumulative and non-overlapping 10%-of-trace +windows. It deliberately reports every common decile instead of selecting the +best-looking horizon. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import math +from pathlib import Path +from statistics import median +from typing import Any, Iterable, Mapping + + +HERE = Path(__file__).resolve().parent +P1_PATH = HERE.parent / "intervention-response-v0" / "analyze_p1.py" +SCHEMA = "intervention-response-phase-aware-existing-v2" +DECILE_FRACTION = 0.1 +MAX_DECILES = 10 + + +def _load_p1(): + spec = importlib.util.spec_from_file_location("intervention_response_p1", P1_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +P1 = _load_p1() + + +def numeric(values: Iterable[float | int]) -> dict[str, Any]: + finite = [float(value) for value in values] + result = P1.V0.numeric(finite) + result["median"] = median(finite) + return result + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def trial_directories(run_root: Path) -> list[Path]: + result = [] + for cell in sorted((run_root / "cells").iterdir()): + if not cell.is_dir(): + continue + for candidate in sorted(cell.iterdir()): + if candidate.is_dir() and P1.RUN_PATTERN.match(candidate.name): + result.append(candidate) + if not result: + raise ValueError("P1 run root contains no measured trial directories") + return result + + +def load_metadata(run_root: Path) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + metadata = [] + streams = [] + for cell in sorted((run_root / "cells").iterdir()): + if not cell.is_dir(): + continue + stream_paths = sorted((cell / "opprof").glob("*.jsonl")) + if len(stream_paths) != 1: + raise ValueError(f"{cell}: expected one Layer-1 stream") + stream_path = stream_paths[0] + streams.append( + { + "cell": cell.name, + "path": str(stream_path.resolve()), + "sha256": sha256_file(stream_path), + "bytes": stream_path.stat().st_size, + } + ) + for run_dir in trial_directories(run_root): + match = P1.RUN_PATTERN.match(run_dir.name) + assert match is not None + level, replicate_text = match.groups() + result_path = run_dir / "result.json" + requests_path = run_dir / "requests.jsonl" + result = json.loads(result_path.read_text(encoding="utf-8")) + selected = int(result["selection"]["count"]) + offered = float(result["selection"]["offered_req_s"]) + if selected <= 0 or offered <= 0.0: + raise ValueError(f"{result_path}: invalid selected count or offered rate") + metadata.append( + { + "trial_id": str(result_path.relative_to(run_root)), + "cell": str(result["cell"]), + "tp": int(result["tp"]), + "mns": int(result["mns"]), + "level": level, + "replicate": int(replicate_text), + "elapsed_s": float(result["interval"]["elapsed_s"]), + "trace_duration_s": round(selected / offered, 9), + "early_stopped": bool(result["early_stopped"]), + "request_count": selected, + "result_sha256": sha256_file(result_path), + "requests_sha256": sha256_file(requests_path), + } + ) + return metadata, streams + + +def common_decile_fractions( + *, trace_duration_s: float, minimum_elapsed_s: float +) -> tuple[float, ...]: + if trace_duration_s <= 0.0 or minimum_elapsed_s <= 0.0: + raise ValueError("trace duration and elapsed time must be positive") + supported = min( + MAX_DECILES, + int(math.floor((minimum_elapsed_s / trace_duration_s) * 10.0 + 1e-12)), + ) + return tuple( + round(index * DECILE_FRACTION, 10) for index in range(1, supported + 1) + ) + + +def _trial_record( + *, + run_root: Path, + run_dir: Path, + result: Mapping[str, Any], + state: dict[str, float], + outcome: dict[str, float], +) -> dict[str, Any]: + match = P1.RUN_PATTERN.match(run_dir.name) + assert match is not None + level, replicate_text = match.groups() + result_path = run_dir / "result.json" + requests_path = run_dir / "requests.jsonl" + return { + "trial_id": str(result_path.relative_to(run_root)), + "cell": str(result["cell"]), + "tp": int(result["tp"]), + "mns": int(result["mns"]), + "level": level, + "replicate": int(replicate_text), + "offered_rate_per_gpu": float( + result["selection"]["offered_req_s_per_gpu"] + ), + "request_hash": str(result["selection"]["request_id_order_sha256"]), + "request_count": int(result["selection"]["count"]), + "result_sha256": sha256_file(result_path), + "requests_sha256": sha256_file(requests_path), + "full_pass_rate": float(result["pass_rate"]), + "full_feasible": bool(result["feasible"]), + "early_stopped": bool(result["early_stopped"]), + "state": state, + "outcome": outcome, + } + + +def load_interval_trials( + run_root: Path, + intervals_s: tuple[tuple[float, float], ...], +) -> tuple[dict[tuple[float, float], list[dict[str, Any]]], list[dict[str, Any]]]: + by_interval = {interval: [] for interval in intervals_s} + stream_provenance = [] + for cell in sorted((run_root / "cells").iterdir()): + if not cell.is_dir(): + continue + stream_paths = sorted((cell / "opprof").glob("*.jsonl")) + if len(stream_paths) != 1: + raise ValueError(f"{cell}: expected one Layer-1 stream") + stream_path = stream_paths[0] + stream = P1.load_jsonl(stream_path) + stream_provenance.append( + { + "cell": cell.name, + "path": str(stream_path.resolve()), + "sha256": sha256_file(stream_path), + "bytes": stream_path.stat().st_size, + } + ) + for run_dir in sorted(cell.iterdir()): + if not run_dir.is_dir() or P1.RUN_PATTERN.match(run_dir.name) is None: + continue + result_path = run_dir / "result.json" + requests_path = run_dir / "requests.jsonl" + result = json.loads(result_path.read_text(encoding="utf-8")) + requests = P1.load_jsonl(requests_path) + start_ns = int(result["interval"]["start_mono_ns"]) + elapsed_s = float(result["interval"]["elapsed_s"]) + for interval in intervals_s: + start_s, end_s = interval + if start_s < 0.0 or end_s <= start_s: + raise ValueError(f"invalid analysis interval: {interval}") + if elapsed_s + 1e-9 < end_s: + raise ValueError( + f"{result_path}: elapsed {elapsed_s} shorter than {end_s}s" + ) + state = P1.V0.flatten_state( + P1.summarize_engine( + stream, + start_ns=start_ns + int(start_s * 1e9), + end_ns=start_ns + int(end_s * 1e9), + request_count=int(result["selection"]["count"]), + ) + ) + outcome = P1._prefix_outcome(result, requests, end_s) + by_interval[interval].append( + _trial_record( + run_root=run_root, + run_dir=run_dir, + result=result, + state=state, + outcome=outcome, + ) + ) + return by_interval, stream_provenance + + +def coverage(trials: list[dict[str, Any]]) -> dict[str, Any]: + admitted = [float(trial["outcome"]["admitted_fraction"]) for trial in trials] + completed = [ + float(trial["outcome"]["admitted_fraction"]) + * float(trial["outcome"]["completed_over_admitted"]) + for trial in trials + ] + return { + "admitted_fraction_of_total": numeric(admitted), + "completed_fraction_of_total": numeric(completed), + } + + +def slim_window_analysis( + trials: list[dict[str, Any]], *, start_s: float, end_s: float, fraction: float +) -> dict[str, Any]: + analysis = P1.analyze_horizon(trials, end_s) + return { + "start_s": start_s, + "end_s": end_s, + "end_fraction": fraction, + "coverage_at_end": coverage(trials), + "action_pairs": len(analysis["actions"]), + "repeat_pairs": len(analysis["repeats"]), + "response_statistics": analysis["response_statistics"], + "qualifying_response_features": analysis["qualifying_response_features"], + "efficacy": analysis["efficacy"], + "sanity": analysis["sanity"], + } + + +def _pearson(left: list[float], right: list[float]) -> float | None: + if len(left) != len(right) or not left: + raise ValueError("Pearson inputs must be non-empty and have equal length") + left_mean = sum(left) / len(left) + right_mean = sum(right) / len(right) + numerator = sum( + (x - left_mean) * (y - right_mean) + for x, y in zip(left, right, strict=True) + ) + left_ss = sum((x - left_mean) ** 2 for x in left) + right_ss = sum((y - right_mean) ** 2 for y in right) + if left_ss == 0.0 or right_ss == 0.0: + return None + return numerator / math.sqrt(left_ss * right_ss) + + +def trajectory_summary( + block_trials: list[tuple[tuple[float, float], list[dict[str, Any]]]] +) -> dict[str, Any]: + if not block_trials: + raise ValueError("trajectory requires at least one block") + identities = [] + states_by_block = [] + for interval, trials in block_trials: + ordered = sorted( + trials, + key=lambda trial: (trial["cell"], trial["level"], trial["replicate"]), + ) + current_identities = [ + (trial["cell"], trial["level"], trial["replicate"]) for trial in ordered + ] + if identities and current_identities != identities: + raise ValueError("trajectory blocks do not contain identical trials") + identities = current_identities + states_by_block.append((interval, [trial["state"] for trial in ordered])) + + features = {} + for feature in P1.V0.ALL_FEATURES: + block_values = [ + [float(state[feature]) for state in states] + for _interval, states in states_by_block + ] + first = block_values[0] + last = block_values[-1] + delta = [right - left for left, right in zip(first, last, strict=True)] + features[feature] = { + "block_medians": [median(values) for values in block_values], + "first_to_last_delta": numeric(delta), + "first_to_last_abs_delta": numeric(abs(value) for value in delta), + "first_to_last_pearson": _pearson(first, last), + "changed_trials": sum(abs(value) > 1e-12 for value in delta), + } + return { + "trial_count": len(identities), + "blocks": [ + {"start_s": interval[0], "end_s": interval[1]} + for interval, _states in states_by_block + ], + "features": features, + } + + +def audit(*, run_root: Path, manifest_path: Path, output_path: Path) -> dict[str, Any]: + metadata, metadata_streams = load_metadata(run_root) + durations = [float(item["trace_duration_s"]) for item in metadata] + elapsed = [float(item["elapsed_s"]) for item in metadata] + duration = median(durations) + deciles = common_decile_fractions( + trace_duration_s=duration, minimum_elapsed_s=min(elapsed) + ) + if not deciles: + raise ValueError("no complete replay decile is shared by all trials") + cumulative_intervals = tuple( + (0.0, round(duration * fraction, 9)) for fraction in deciles + ) + block_intervals = tuple( + ( + round(duration * (fraction - DECILE_FRACTION), 9), + round(duration * fraction, 9), + ) + for fraction in deciles + ) + all_intervals = tuple(dict.fromkeys([*cumulative_intervals, *block_intervals])) + trials_by_interval, streams = load_interval_trials(run_root, all_intervals) + manifest_validation = P1.validate_manifest( + trials_by_interval[cumulative_intervals[0]], manifest_path + ) + + cumulative = [] + blocks = [] + for fraction, cumulative_interval, block_interval in zip( + deciles, cumulative_intervals, block_intervals, strict=True + ): + cumulative.append( + slim_window_analysis( + trials_by_interval[cumulative_interval], + start_s=cumulative_interval[0], + end_s=cumulative_interval[1], + fraction=fraction, + ) + ) + blocks.append( + slim_window_analysis( + trials_by_interval[block_interval], + start_s=block_interval[0], + end_s=block_interval[1], + fraction=fraction, + ) + ) + + invariants = { + "expected_trial_count": len(metadata) == 36, + "trace_duration_consistent": max(durations) - min(durations) <= 1e-9, + "all_intervals_uncensored": all( + item["elapsed_s"] + 1e-9 >= cumulative_intervals[-1][1] + for item in metadata + ), + "stream_provenance_consistent": metadata_streams == streams, + "manifest_trials_match": ( + manifest_validation["expected_trials"] + == manifest_validation["matched_trials"] + == len(metadata) + ), + "all_window_sanity_pass": all( + not item["sanity"]["red_flags"] for item in [*cumulative, *blocks] + ), + } + red_flags = [name for name, passed in invariants.items() if not passed] + complete_full_trajectory = min(elapsed) + 1e-9 >= duration + if red_flags: + decision = "STOP_DATA_INVALID" + elif not complete_full_trajectory: + decision = "REQUIRES_UNCENSORED_PHASE_AWARE_PILOT" + else: + decision = "FULL_TRAJECTORY_AVAILABLE" + + payload = { + "schema": SCHEMA, + "status": "COMPLETE", + "decision": decision, + "claim_boundary": ( + "Post-hoc corrective audit over every common replay decile. It can " + "diagnose horizon sensitivity but cannot establish a held-out tuning claim." + ), + "design": { + "decile_fraction": DECILE_FRACTION, + "available_deciles": list(deciles), + "trace_duration_s": duration, + "maximum_common_end_s": cumulative_intervals[-1][1], + "maximum_common_fraction": deciles[-1], + "select_best_horizon": False, + "cumulative_and_nonoverlapping_blocks": True, + }, + "cumulative": cumulative, + "blocks": blocks, + "trajectory": trajectory_summary( + [(interval, trials_by_interval[interval]) for interval in block_intervals] + ), + "provenance": { + "analysis_script": str(Path(__file__).resolve()), + "analysis_script_sha256": sha256_file(Path(__file__).resolve()), + "p1_analysis_script": str(P1_PATH.resolve()), + "p1_analysis_script_sha256": sha256_file(P1_PATH), + "run_root": str(run_root.resolve()), + "manifest": str(manifest_path.resolve()), + "manifest_sha256": sha256_file(manifest_path), + "manifest_validation": manifest_validation, + "streams": streams, + "trial_inputs": metadata, + }, + "sanity": { + "trials": len(metadata), + "elapsed_s": numeric(elapsed), + "trace_duration_s": numeric(durations), + "early_stopped": sum(bool(item["early_stopped"]) for item in metadata), + "request_count": numeric(item["request_count"] for item in metadata), + "stream_bytes": numeric(item["bytes"] for item in streams), + "invariants": invariants, + "red_flags": red_flags, + }, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + return payload + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + payload = audit( + run_root=args.run_root, + manifest_path=args.manifest, + output_path=args.output, + ) + print( + json.dumps( + { + "decision": payload["decision"], + "design": payload["design"], + "sanity": payload["sanity"], + }, + indent=2, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/intervention-response-v2/analyze_pilot.py b/runs/intervention-response-v2/analyze_pilot.py new file mode 100644 index 0000000..edbfbe1 --- /dev/null +++ b/runs/intervention-response-v2/analyze_pilot.py @@ -0,0 +1,449 @@ +#!/usr/bin/env python3 +"""Analyze the uncensored 300-second phase-aware matched pilot.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import math +from collections import defaultdict +from pathlib import Path +from typing import Any, Iterable, Mapping + + +HERE = Path(__file__).resolve().parent +P1_PATH = HERE.parent / "intervention-response-v0" / "analyze_p1.py" +SCHEMA = "intervention-response-phase-aware-pilot-analysis-v2" +EXPECTED_ACTION_PAIRS = 9 +EXPECTED_REPEAT_PAIRS = 12 +MIN_EFFICACY_CLASS = 3 + + +def _load_p1(): + spec = importlib.util.spec_from_file_location("intervention_response_p1", P1_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +P1 = _load_p1() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def numeric(values: Iterable[float | int]) -> dict[str, Any]: + return P1.V0.numeric(values) + + +def _trial_record( + *, + run_root: Path, + session: Mapping[str, Any], + level: str, + result: Mapping[str, Any], + result_path: Path, + requests_path: Path, + state: dict[str, float], + outcome: dict[str, float], +) -> dict[str, Any]: + return { + "trial_id": str(result_path.relative_to(run_root)), + "cell": str(result["cell"]), + "tp": int(result["tp"]), + "mns": int(result["mns"]), + "level": level, + "replicate": int(session["replicate"]), + "offered_rate_per_gpu": float( + result["selection"]["offered_req_s_per_gpu"] + ), + "request_hash": str(result["selection"]["request_id_order_sha256"]), + "request_count": int(result["selection"]["count"]), + "result_sha256": sha256_file(result_path), + "requests_sha256": sha256_file(requests_path), + "full_pass_rate": float(result["pass_rate"]), + "full_feasible": bool(result["feasible"]), + "early_stopped": bool(result["early_stopped"]), + "state": state, + "outcome": outcome, + } + + +def validate_result_against_manifest( + *, + result: Mapping[str, Any], + selection: Mapping[str, Any], + session: Mapping[str, Any], + level: str, + expected_duration_s: float, +) -> None: + identity = f"{session['session']}:{level}" + if int(result["mns"]) != int(session["mns"]) or int(result["tp"]) != 4: + raise ValueError(f"config mismatch: {identity}") + if bool(result["early_stopped"]): + raise ValueError(f"early-stopped measured result: {identity}") + if result.get("slo_early_stop_disabled") is not True: + raise ValueError(f"SLO early stop was enabled: {identity}") + if float(result["interval"]["elapsed_s"]) + 1e-9 < expected_duration_s: + raise ValueError(f"result does not cover full arrival window: {identity}") + if int(result["selection"]["count"]) != int(selection["selected_count"]): + raise ValueError(f"selection count mismatch: {identity}") + for result_key, manifest_key in ( + ("request_id_order_sha256", "request_id_order_sha256"), + ("arrival_order_sha256", "arrival_order_sha256"), + ("raw_length_order_sha256", "input_length_order_sha256"), + ): + if result["selection"][result_key] != selection[manifest_key]: + raise ValueError(f"selection hash mismatch {result_key}: {identity}") + if int(result["observed_count"]) != int(selection["selected_count"]): + raise ValueError(f"request accounting mismatch: {identity}") + + +def load_interval_trials( + *, + run_root: Path, + manifest: Mapping[str, Any], + intervals_s: tuple[tuple[float, float], ...], +) -> tuple[dict[tuple[float, float], list[dict[str, Any]]], list[dict[str, Any]]]: + by_interval = {interval: [] for interval in intervals_s} + streams = [] + duration_s = float(manifest["engine"]["duration_s"]) + for session in manifest["sessions"]: + session_root = run_root / "sessions" / str(session["session"]) + stream_paths = sorted((session_root / "opprof").glob("*.jsonl")) + if len(stream_paths) != 1: + raise ValueError(f"{session_root}: expected one Layer-1 stream") + stream_path = stream_paths[0] + stream = P1.load_jsonl(stream_path) + streams.append( + { + "session": str(session["session"]), + "path": str(stream_path.resolve()), + "sha256": sha256_file(stream_path), + "bytes": stream_path.stat().st_size, + } + ) + repetition = manifest["repetitions"][str(session["replicate"])] + for level, selection in repetition["selections"].items(): + result_path = session_root / level / "result.json" + requests_path = session_root / level / "requests.jsonl" + result = json.loads(result_path.read_text(encoding="utf-8")) + requests = P1.load_jsonl(requests_path) + validate_result_against_manifest( + result=result, + selection=selection, + session=session, + level=level, + expected_duration_s=duration_s, + ) + start_ns = int(result["interval"]["start_mono_ns"]) + for interval in intervals_s: + start_s, end_s = interval + state = P1.V0.flatten_state( + P1.summarize_engine( + stream, + start_ns=start_ns + int(start_s * 1e9), + end_ns=start_ns + int(end_s * 1e9), + request_count=int(result["selection"]["count"]), + ) + ) + outcome = P1._prefix_outcome(result, requests, end_s) + by_interval[interval].append( + _trial_record( + run_root=run_root, + session=session, + level=level, + result=result, + result_path=result_path, + requests_path=requests_path, + state=state, + outcome=outcome, + ) + ) + return by_interval, streams + + +def analyze_window( + trials: list[dict[str, Any]], *, start_s: float, end_s: float, fraction: float +) -> dict[str, Any]: + actions, repeats = P1.build_pairs(trials) + response = P1.V0.response_statistics(actions, repeats) + response_qualifying = sorted( + feature for feature, item in response.items() if item["qualifies"] + ) + labels = [int(pair["full_action_efficacy"]) for pair in actions] + cross_validation_possible = all( + set( + int(pair["full_action_efficacy"]) + for pair in actions + if pair["group"]["replicate"] != held_out + ) + == {0, 1} + for held_out in (1, 2, 3) + ) + if cross_validation_possible: + outcome_cv = P1.one_feature_leave_repeat_out( + actions, delta_key="delta_outcome", features=P1.OUTCOME_FEATURES + ) + telemetry_cv = P1.one_feature_leave_repeat_out( + actions, delta_key="delta_state", features=P1.V0.GATE_FEATURES + ) + outcome_best = float(outcome_cv["best_balanced_accuracy"]) + efficacy_qualifying = sorted( + feature + for feature, item in telemetry_cv["features"].items() + if item["balanced_accuracy"] >= P1.MIN_EFFICACY_BALANCED_ACCURACY + and item["balanced_accuracy"] + >= outcome_best + P1.MIN_EFFICACY_DELTA_OVER_OUTCOME + ) + else: + unavailable = { + "status": "UNAVAILABLE", + "reason": "each leave-one-repetition-out train fold needs both classes", + } + outcome_cv = unavailable + telemetry_cv = unavailable + efficacy_qualifying = [] + transitions = defaultdict(int) + for pair in actions: + transitions[pair["full_feasibility_transition"]] += 1 + admitted = [float(trial["outcome"]["admitted_fraction"]) for trial in trials] + completed = [ + float(trial["outcome"]["admitted_fraction"]) + * float(trial["outcome"]["completed_over_admitted"]) + for trial in trials + ] + action_metadata = [ + { + "group": pair["group"], + "source": pair["source"], + "target": pair["target"], + "full_action_efficacy": pair["full_action_efficacy"], + "full_feasibility_transition": pair["full_feasibility_transition"], + "delta_state": pair["delta_state"], + "delta_outcome": pair["delta_outcome"], + } + for pair in actions + ] + invariants = { + "expected_action_pair_count": len(actions) == EXPECTED_ACTION_PAIRS, + "expected_repeat_pair_count": len(repeats) == EXPECTED_REPEAT_PAIRS, + "finite_deltas": all( + math.isfinite(value) + for pair in [*actions, *repeats] + for key in ("delta_state", "delta_outcome") + for value in pair[key].values() + ), + "all_results_uncensored": all(not trial["early_stopped"] for trial in trials), + } + return { + "start_s": start_s, + "end_s": end_s, + "end_fraction": fraction, + "coverage_at_end": { + "admitted_fraction_of_total": numeric(admitted), + "completed_fraction_of_total": numeric(completed), + }, + "action_pairs": len(actions), + "repeat_pairs": len(repeats), + "actions": action_metadata, + "response_statistics": response, + "qualifying_response_features": response_qualifying, + "efficacy": { + "labels": numeric(labels), + "positive": sum(labels), + "negative": len(labels) - sum(labels), + "label_balance_sufficient": ( + sum(labels) >= MIN_EFFICACY_CLASS + and len(labels) - sum(labels) >= MIN_EFFICACY_CLASS + ), + "cross_validation_possible": cross_validation_possible, + "transitions": dict(sorted(transitions.items())), + "outcome_delta": outcome_cv, + "telemetry_delta": telemetry_cv, + "telemetry_qualifying_features": efficacy_qualifying, + }, + "sanity": { + "trials": len(trials), + "invariants": invariants, + "red_flags": [name for name, passed in invariants.items() if not passed], + }, + } + + +def stable_adjacent_features(windows: list[dict[str, Any]]) -> dict[str, list[str]]: + result = {} + for left, right in zip(windows, windows[1:], strict=False): + key = f"{left['end_fraction']:.2f}->{right['end_fraction']:.2f}" + result[key] = sorted( + set(left["qualifying_response_features"]) + & set(right["qualifying_response_features"]) + ) + return result + + +def consistent_load_regimes( + windows: list[dict[str, Any]], stable: dict[str, list[str]] +) -> dict[str, Any]: + by_end = {float(window["end_fraction"]): window for window in windows} + result = {} + for transition, features in stable.items(): + _left_text, right_text = transition.split("->") + window = by_end[float(right_text)] + for feature in features: + deltas_by_level: dict[str, list[float]] = defaultdict(list) + all_deltas = [] + for action in window["actions"]: + value = float(action["delta_state"][feature]) + deltas_by_level[str(action["group"]["level"])].append(value) + all_deltas.append(value) + positive = sum(value > 1e-12 for value in all_deltas) + negative = sum(value < -1e-12 for value in all_deltas) + direction = 1 if positive >= negative else -1 + consistent = [] + for level, values in sorted(deltas_by_level.items()): + matching = sum(direction * value > 1e-12 for value in values) + nonzero = sum(abs(value) > 1e-12 for value in values) + if nonzero and matching / nonzero >= 2.0 / 3.0: + consistent.append(level) + result[f"{transition}:{feature}"] = { + "direction": direction, + "consistent_load_regimes": consistent, + "passes_two_regimes": len(consistent) >= 2, + } + return result + + +def audit(*, run_root: Path, manifest_path: Path, output_path: Path) -> dict[str, Any]: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest.get("schema") != "intervention-response-phase-aware-pilot-manifest-v2": + raise ValueError("unexpected phase-aware pilot manifest schema") + fractions = [float(value) for value in manifest["checkpoints"]["fractions"]] + seconds = [float(value) for value in manifest["checkpoints"]["seconds"]] + cumulative_intervals = tuple((0.0, end_s) for end_s in seconds) + quarter_intervals = ((0.0, 75.0), (75.0, 150.0), (150.0, 225.0), (225.0, 300.0)) + intervals = tuple(dict.fromkeys([*cumulative_intervals, *quarter_intervals])) + trials_by_interval, streams = load_interval_trials( + run_root=run_root, manifest=manifest, intervals_s=intervals + ) + cumulative = [ + analyze_window( + trials_by_interval[interval], + start_s=interval[0], + end_s=interval[1], + fraction=fraction, + ) + for fraction, interval in zip(fractions, cumulative_intervals, strict=True) + ] + quarter_blocks = [ + analyze_window( + trials_by_interval[interval], + start_s=interval[0], + end_s=interval[1], + fraction=interval[1] / 300.0, + ) + for interval in quarter_intervals + ] + stable = stable_adjacent_features(cumulative) + load_consistency = consistent_load_regimes(cumulative, stable) + mechanism_features = sorted( + { + key.split(":", 1)[1] + for key, item in load_consistency.items() + if item["passes_two_regimes"] + } + ) + full = cumulative[-1] + efficacy_features = sorted( + set.intersection( + *( + set(window["efficacy"]["telemetry_qualifying_features"]) + for window in cumulative + if window["end_fraction"] >= 0.25 + ) + ) + ) + red_flags = sorted( + { + flag + for window in [*cumulative, *quarter_blocks] + for flag in window["sanity"]["red_flags"] + } + ) + if red_flags: + decision = "STOP_DATA_INVALID" + elif not mechanism_features: + decision = "STOP_NO_PHASE_STABLE_RESPONSE" + elif not full["efficacy"]["label_balance_sufficient"]: + decision = "MECHANISM_ONLY_NO_LABEL_BALANCE" + elif not efficacy_features: + decision = "STOP_NO_INCREMENTAL_TUNING_SIGNAL" + else: + decision = "OPEN_E2E_POLICY_TEST" + payload = { + "schema": SCHEMA, + "status": "COMPLETE", + "decision": decision, + "claim_boundary": "Development mechanism pilot; not a held-out paper claim.", + "mechanism_features": mechanism_features, + "stable_adjacent_features": stable, + "load_consistency": load_consistency, + "stable_incremental_efficacy_features": efficacy_features, + "cumulative": cumulative, + "quarter_blocks": quarter_blocks, + "provenance": { + "analysis_script": str(Path(__file__).resolve()), + "analysis_script_sha256": sha256_file(Path(__file__).resolve()), + "manifest": str(manifest_path.resolve()), + "manifest_sha256": sha256_file(manifest_path), + "run_root": str(run_root.resolve()), + "streams": streams, + }, + "sanity": { + "streams": len(streams), + "stream_bytes": numeric(item["bytes"] for item in streams), + "red_flags": red_flags, + }, + } + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + return payload + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + payload = audit( + run_root=args.run_root, + manifest_path=args.manifest, + output_path=args.output, + ) + print( + json.dumps( + { + "decision": payload["decision"], + "mechanism_features": payload["mechanism_features"], + "stable_incremental_efficacy_features": payload[ + "stable_incremental_efficacy_features" + ], + "sanity": payload["sanity"], + }, + indent=2, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/intervention-response-v2/existing-phase-audit.json b/runs/intervention-response-v2/existing-phase-audit.json new file mode 100644 index 0000000..3884878 --- /dev/null +++ b/runs/intervention-response-v2/existing-phase-audit.json @@ -0,0 +1,9915 @@ +{ + "blocks": [ + { + "action_pairs": 12, + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 21, + "max": 0.12295081967213115, + "median": 0.08985507246376812, + "min": 0.04918032786885246, + "n": 36 + }, + "completed_fraction_of_total": { + "distinct_n": 30, + "max": 0.09361702127659574, + "median": 0.050437283852088316, + "min": 0.005586592178770949, + "n": 36 + } + }, + "efficacy": { + "feasibility_transitions": { + "false->false": 3, + "false->true": 6, + "true->true": 3 + }, + "minimum_balanced_accuracy": 0.75, + "minimum_delta_over_best_outcome": 0.15, + "outcome_delta": { + "best_accuracy": 0.75, + "best_balanced_accuracy": 0.75, + "best_feature": "ttft_max_over_slo_max", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_over_admitted": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.4166666666666667, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 0 + ], + "threshold": 0.051724137931034475, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.09339080459770116, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.018181818181818188, + "train_balanced_accuracy": 0.625 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.4166666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 0 + ], + "threshold": -0.051724137931034475, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.09339080459770113, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.01818181818181816, + "train_balanced_accuracy": 0.625 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.000601708027350864, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.000601708027350864, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.015623848188639142, + "train_balanced_accuracy": 0.75 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.0010320465723873684, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 0 + ], + "threshold": 0.009580791853053022, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.0002360795908454838, + "train_balanced_accuracy": 0.625 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.0072977563346891365, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": -0.021249613758603417, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.007412010668000825, + "train_balanced_accuracy": 0.875 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 1 + ], + "threshold": -0.0008523871757378475, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.001080823830574982, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.006279735096096474, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_delta": { + "best_accuracy": 0.6666666666666666, + "best_balanced_accuracy": 0.6666666666666666, + "best_feature": "queue_waiting_mean", + "features": { + "decode_batch_size.mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.5847120251326046, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.1503494070398912, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.759913509761732, + "train_balanced_accuracy": 0.75 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.0003962349313431858, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.0006270962078854048, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 0, + 1 + ], + "threshold": 0.003423657452078254, + "train_balanced_accuracy": 0.75 + } + ] + }, + "kv_usage_mean": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.00016329402239600305, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.00013516820682281123, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.00013516820682281123, + "train_balanced_accuracy": 0.625 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.3333333333333333, + "balanced_accuracy": 0.3333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 0 + ], + "threshold": -0.0003659931906908054, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.0003153433269442174, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 0 + ], + "threshold": 0.002846241748737599, + "train_balanced_accuracy": 1.0 + } + ] + }, + "queue_running_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 1.2024471918333335, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.18325025108333315, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 1.2024471918333335, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -1.6539956596666667, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.009266072583333333, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -1.6539956596666667, + "train_balanced_accuracy": 0.875 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.41666666666666663, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 0, + 1 + ], + "threshold": 1.5833333333333357, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 4.416666666666664, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -19.583333333333332, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_qualifying_features": [] + }, + "end_fraction": 0.1, + "end_s": 6.0, + "qualifying_response_features": [], + "repeat_pairs": 24, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.587816632458785, + "action_delta": { + "distinct_n": 12, + "max": 10.944163150492265, + "min": -0.9368137011239002, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.1464621283214008, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 4.013437734353163, + "repeat_abs_p95": 8.899704585700528, + "repeat_delta": { + "distinct_n": 24, + "max": 9.092698412698415, + "min": -9.933716475095787, + "n": 24 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 7.901570082144033, + "action_delta": { + "distinct_n": 12, + "max": 564.520698546648, + "min": -8.58386322115848, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.05293318721992956, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 149.27440604160446, + "repeat_abs_p95": 426.8141407023969, + "repeat_delta": { + "distinct_n": 24, + "max": 449.2116931216931, + "min": -584.6987867177522, + "n": 24 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.582158519407685, + "action_delta": { + "distinct_n": 12, + "max": 10.735314111579934, + "min": -0.9288831363334129, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.14646507698937813, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 3.9747257938484823, + "repeat_abs_p95": 8.770067452024344, + "repeat_delta": { + "distinct_n": 24, + "max": 8.954735449735448, + "min": -9.735983397190292, + "n": 24 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.005042559238173261, + "action_delta": { + "distinct_n": 12, + "max": 0.008374384236453203, + "min": -0.10829817158931077, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.12202178454078347, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.04132507369196753, + "repeat_abs_p95": 0.08139543508103343, + "repeat_delta": { + "distinct_n": 24, + "max": 0.12107279693486583, + "min": -0.0822222222222222, + "n": 24 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.008375994890060249, + "action_delta": { + "distinct_n": 12, + "max": 0.09274496015002344, + "min": -0.01986863711001642, + "n": 12 + }, + "action_signs": { + "consistency": 0.75, + "negative": 9, + "positive": 3, + "zero": 0 + }, + "effect_to_repeat_median": 0.22398385207837773, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.037395530134597704, + "repeat_abs_p95": 0.07586812708364432, + "repeat_delta": { + "distinct_n": 24, + "max": 0.07631077509373656, + "min": -0.10941890166028097, + "n": 24 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0011703901376772582, + "action_delta": { + "distinct_n": 12, + "max": 0.006703036588479568, + "min": -0.0015820428864207082, + "n": 12 + }, + "action_signs": { + "consistency": 0.75, + "negative": 3, + "positive": 9, + "zero": 0 + }, + "effect_to_repeat_median": 0.29946486376659237, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.003908271985422232, + "repeat_abs_p95": 0.008187883039017584, + "repeat_delta": { + "distinct_n": 24, + "max": 0.007141591578301508, + "min": -0.009201187757062074, + "n": 24 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.006820888350695331, + "action_delta": { + "distinct_n": 12, + "max": 0.140893666867539, + "min": -0.005054735620165451, + "n": 12 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 2, + "positive": 10, + "zero": 0 + }, + "effect_to_repeat_median": 0.6661896437498689, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.010238658638254572, + "repeat_abs_p95": 0.1277225497052652, + "repeat_delta": { + "distinct_n": 24, + "max": 0.13402937356379263, + "min": -0.1168947946847837, + "n": 24 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.007604372340016818, + "action_delta": { + "distinct_n": 12, + "max": 0.06684057734774362, + "min": 0.00016629145416469093, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 12, + "zero": 0 + }, + "effect_to_repeat_median": 0.3770466591391387, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.020168252802926, + "repeat_abs_p95": 0.11084274153262068, + "repeat_delta": { + "distinct_n": 24, + "max": 0.12803476870816266, + "min": -0.11234888600259774, + "n": 24 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0014831270777658032, + "action_delta": { + "distinct_n": 12, + "max": 0.02895016189128262, + "min": -0.00029784868681948656, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 8, + "zero": 0 + }, + "effect_to_repeat_median": 0.12120157645159224, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.01223686292857884, + "repeat_abs_p95": 0.07958153755957757, + "repeat_delta": { + "distinct_n": 24, + "max": 0.08820147130614941, + "min": -0.08208138515366167, + "n": 24 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 12 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 12 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 24 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0028294297587345696, + "action_delta": { + "distinct_n": 12, + "max": 0.013691652669458043, + "min": -0.002949832419021914, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.44362661948582977, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.006377953067861264, + "repeat_abs_p95": 0.031272369907012425, + "repeat_delta": { + "distinct_n": 24, + "max": 0.036461564751038456, + "min": -0.018166035027340932, + "n": 24 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.50741128125, + "action_delta": { + "distinct_n": 12, + "max": 15.614046583666667, + "min": -0.30207624649999953, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.12962585804860988, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 3.9144294887499997, + "repeat_abs_p95": 7.101594725466666, + "repeat_delta": { + "distinct_n": 24, + "max": 6.513803891666667, + "min": -13.5536487565, + "n": 24 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.24895167858333334, + "action_delta": { + "distinct_n": 10, + "max": 0.0, + "min": -8.040727011166666, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 9, + "positive": 0, + "zero": 3 + }, + "effect_to_repeat_median": 1.1132189691456902, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.22363226416666668, + "repeat_abs_p95": 2.296808863758333, + "repeat_delta": { + "distinct_n": 16, + "max": 2.3311656403333334, + "min": -5.974843045, + "n": 24 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 3.249999999999993, + "action_delta": { + "distinct_n": 12, + "max": 10.5, + "min": -30.999999999999993, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.1065573770491801, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 30.5, + "repeat_abs_p95": 59.92499999999999, + "repeat_delta": { + "distinct_n": 24, + "max": 59.5, + "min": -64.5, + "n": 24 + } + } + }, + "sanity": { + "action_pairs": 12, + "invariants": { + "efficacy_label_balance": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "matched_action_request_hashes": true, + "probabilities_bounded": true + }, + "red_flags": [], + "repeat_pairs": 24, + "trials": 36 + }, + "start_s": 0.0 + }, + { + "action_pairs": 12, + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 22, + "max": 0.20535714285714285, + "median": 0.18235294117647058, + "min": 0.1564245810055866, + "n": 36 + }, + "completed_fraction_of_total": { + "distinct_n": 31, + "max": 0.180327868852459, + "median": 0.14411764705882352, + "min": 0.08933333333333335, + "n": 36 + } + }, + "efficacy": { + "feasibility_transitions": { + "false->false": 3, + "false->true": 6, + "true->true": 3 + }, + "minimum_balanced_accuracy": 0.75, + "minimum_delta_over_best_outcome": 0.15, + "outcome_delta": { + "best_accuracy": 0.8333333333333334, + "best_balanced_accuracy": 0.8333333333333333, + "best_feature": "ttft_max_over_slo_max", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.012666666666666666, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.008666666666666666, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.008666666666666666, + "train_balanced_accuracy": 0.75 + } + ] + }, + "completed_over_admitted": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.03401360544217691, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 1 + ], + "threshold": 0.006172839506172811, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.04871948779511809, + "train_balanced_accuracy": 0.75 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.11875000000000002, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.07738095238095238, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.07738095238095238, + "train_balanced_accuracy": 0.75 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.03401360544217688, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 1 + ], + "threshold": -0.006172839506172839, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.04871948779511806, + "train_balanced_accuracy": 0.75 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.028440662518539606, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.02030446551608167, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": 0.008882265987138677, + "train_balanced_accuracy": 0.75 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.41666666666666663, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 0, + 1 + ], + "threshold": -0.0020290170113650557, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 0.04992359808650447, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.009324194656287296, + "train_balanced_accuracy": 0.75 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.01753702182516767, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.01753702182516767, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.018086205842943556, + "train_balanced_accuracy": 0.75 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.010556909058063094, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.003084423316020539, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.027367186208561683, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_delta": { + "best_accuracy": 0.75, + "best_balanced_accuracy": 0.75, + "best_feature": "queue_waiting_mean", + "features": { + "decode_batch_size.mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.22344269117369397, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.22344269117369397, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.5979430550644496, + "train_balanced_accuracy": 0.75 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.006014036688642874, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.003996367458148475, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 1 + ], + "threshold": 0.004817288488210095, + "train_balanced_accuracy": 0.875 + } + ] + }, + "kv_usage_mean": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 1 + ], + "threshold": 0.0009690245761390937, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 0.014778075272949286, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.00021729561459569828, + "train_balanced_accuracy": 0.75 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 0.0004249745076460587, + "train_balanced_accuracy": 0.875 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.0008306228170546492, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 1 + ], + "threshold": -0.005838187880452539, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_running_mean": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 1 + ], + "threshold": -0.014251387166666962, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 9.969867595499998, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.36342276333333334, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": -0.37299224008333337, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": -0.09157276941666666, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.08574448658333333, + "train_balanced_accuracy": 0.75 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": 3.75, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": 3.75, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 6.833333333333329, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_qualifying_features": [] + }, + "end_fraction": 0.2, + "end_s": 12.0, + "qualifying_response_features": [], + "repeat_pairs": 24, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.836467208595514, + "action_delta": { + "distinct_n": 12, + "max": 38.01660606060606, + "min": -3.7894183532722323, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.6942297063022367, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 1.2048853585521353, + "repeat_abs_p95": 9.90189234803657, + "repeat_delta": { + "distinct_n": 24, + "max": 6.213858257127487, + "min": -19.9836254925941, + "n": 24 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 68.40350288667949, + "action_delta": { + "distinct_n": 12, + "max": 1287.087393939394, + "min": -109.74922437881139, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 1.6828036858921454, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 40.648534026959354, + "repeat_abs_p95": 252.72760702584105, + "repeat_delta": { + "distinct_n": 24, + "max": 53.09534549057537, + "min": -841.3287810843865, + "n": 24 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.79935614604261, + "action_delta": { + "distinct_n": 12, + "max": 37.563515151515155, + "min": -3.7555281535670666, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.6727103099461282, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 1.1882620709449565, + "repeat_abs_p95": 9.833929419939027, + "repeat_delta": { + "distinct_n": 24, + "max": 6.226902904787521, + "min": -19.76212800652263, + "n": 24 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.023223408379118127, + "action_delta": { + "distinct_n": 12, + "max": 0.03622200353327565, + "min": -0.23381818181818181, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 2.5685579123999758, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.009041419026218855, + "repeat_abs_p95": 0.07343409055233649, + "repeat_delta": { + "distinct_n": 24, + "max": 0.08839516238619383, + "min": -0.027874161461947766, + "n": 24 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.0196424445317687, + "action_delta": { + "distinct_n": 12, + "max": 0.22624242424242422, + "min": -0.043130466400287726, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 7, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 1.9837484419794449, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.009901681138647232, + "repeat_abs_p95": 0.07568864235997094, + "repeat_delta": { + "distinct_n": 24, + "max": 0.02793199167244964, + "min": -0.10324092947411331, + "n": 24 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0019741621395090708, + "action_delta": { + "distinct_n": 12, + "max": 0.013535143477830873, + "min": -0.0002595396932579229, + "n": 12 + }, + "action_signs": { + "consistency": 0.9166666666666666, + "negative": 1, + "positive": 11, + "zero": 0 + }, + "effect_to_repeat_median": 0.6824854579006991, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.002892606892433317, + "repeat_abs_p95": 0.013935039231725855, + "repeat_delta": { + "distinct_n": 24, + "max": 0.016710257140637542, + "min": -0.0048630254047383765, + "n": 24 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.016292713617491184, + "action_delta": { + "distinct_n": 12, + "max": 0.06869972057736717, + "min": -0.13905774767384216, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 7, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 1.2163146605755397, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.013395146951350356, + "repeat_abs_p95": 0.10929110116632139, + "repeat_delta": { + "distinct_n": 24, + "max": 0.07238485363173153, + "min": -0.2557698071735438, + "n": 24 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.01447565921273003, + "action_delta": { + "distinct_n": 12, + "max": 0.10932093798799158, + "min": 0.0002683613903166071, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 12, + "zero": 0 + }, + "effect_to_repeat_median": 1.4049345895249048, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.010303439975540174, + "repeat_abs_p95": 0.07329653312019176, + "repeat_delta": { + "distinct_n": 24, + "max": 0.05015486062543706, + "min": -0.12688580277750028, + "n": 24 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.005335907884240179, + "action_delta": { + "distinct_n": 12, + "max": 0.06835893039525985, + "min": -0.0020715594873930866, + "n": 12 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 2, + "positive": 10, + "zero": 0 + }, + "effect_to_repeat_median": 1.4198198744789212, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.003758158327089537, + "repeat_abs_p95": 0.054511554710704666, + "repeat_delta": { + "distinct_n": 24, + "max": 0.05530066409194215, + "min": -0.1166293577215588, + "n": 24 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 12 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 12 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 24 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.00639806071748561, + "action_delta": { + "distinct_n": 12, + "max": 0.01743846124119386, + "min": -0.012883247545976029, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.5860208014218505, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.010917804797990316, + "repeat_abs_p95": 0.038061500221448455, + "repeat_delta": { + "distinct_n": 24, + "max": 0.006259881301210357, + "min": -0.03966667403951463, + "n": 24 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.8894945159166667, + "action_delta": { + "distinct_n": 12, + "max": 39.57822979983333, + "min": -2.9770223346666658, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.4702668338949774, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 1.8914676770833334, + "repeat_abs_p95": 10.096755181974991, + "repeat_delta": { + "distinct_n": 21, + "max": 6.583703286333334, + "min": -20.7125852155, + "n": 24 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.79726731775, + "action_delta": { + "distinct_n": 11, + "max": 0.0, + "min": -30.713414816833335, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 10, + "positive": 0, + "zero": 2 + }, + "effect_to_repeat_median": 17.026052234021265, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.04682631691666666, + "repeat_abs_p95": 3.5462989908416667, + "repeat_delta": { + "distinct_n": 17, + "max": 3.561578917166667, + "min": -17.805020274833332, + "n": 24 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 13.083333333333334, + "action_delta": { + "distinct_n": 12, + "max": 23.66666666666667, + "min": -40.5, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 7, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 1.2661290322580652, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 10.333333333333329, + "repeat_abs_p95": 55.25833333333328, + "repeat_delta": { + "distinct_n": 24, + "max": 71.83333333333333, + "min": -19.333333333333343, + "n": 24 + } + } + }, + "sanity": { + "action_pairs": 12, + "invariants": { + "efficacy_label_balance": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "matched_action_request_hashes": true, + "probabilities_bounded": true + }, + "red_flags": [], + "repeat_pairs": 24, + "trials": 36 + }, + "start_s": 6.0 + }, + { + "action_pairs": 12, + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 22, + "max": 0.30654761904761907, + "median": 0.2795403407264545, + "min": 0.24313725490196078, + "n": 36 + }, + "completed_fraction_of_total": { + "distinct_n": 30, + "max": 0.2681564245810056, + "median": 0.22518087736972248, + "min": 0.12849162011173185, + "n": 36 + } + }, + "efficacy": { + "feasibility_transitions": { + "false->false": 3, + "false->true": 6, + "true->true": 3 + }, + "minimum_balanced_accuracy": 0.75, + "minimum_delta_over_best_outcome": 0.15, + "outcome_delta": { + "best_accuracy": 0.75, + "best_balanced_accuracy": 0.75, + "best_feature": "completed_pass_rate", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.02825325884543762, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.02266666666666667, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.040253258845437616, + "train_balanced_accuracy": 0.75 + } + ] + }, + "completed_over_admitted": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.03272660415517559, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.022522522522522515, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.07358436332279389, + "train_balanced_accuracy": 0.75 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.14270516717325227, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.12142857142857144, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.2182462927143778, + "train_balanced_accuracy": 0.75 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.03272660415517557, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.022522522522522515, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.0735843633227939, + "train_balanced_accuracy": 0.75 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 0.37412821213021774, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.009857117327656123, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": 0.008882265987138677, + "train_balanced_accuracy": 0.625 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 1 + ], + "threshold": -0.004552157689988692, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.007167514027906635, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.006851562902858294, + "train_balanced_accuracy": 0.75 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": -0.017755539496041216, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.007733676747496546, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.5040019049192779, + "train_balanced_accuracy": 0.75 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": -0.007216190772090309, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.007216190772090309, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.24525856553394837, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_delta": { + "best_accuracy": 0.75, + "best_balanced_accuracy": 0.75, + "best_feature": "graph_padding_fraction", + "features": { + "decode_batch_size.mean": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.41666666666666663, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 1.367157621094477, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 0 + ], + "threshold": -0.45324800586584313, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 0.7719319083901095, + "train_balanced_accuracy": 0.75 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": 0.0006822319057190087, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": 0.0006574027141346845, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": 0.0007737011910822953, + "train_balanced_accuracy": 0.75 + } + ] + }, + "kv_usage_mean": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.41666666666666663, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 0.0014242600332016828, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 0 + ], + "threshold": -0.00020601061133531957, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 0.0014242600332016828, + "train_balanced_accuracy": 0.75 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": 0.0001890219308638419, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 1 + ], + "threshold": 6.874128372463817e-05, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 0 + ], + "threshold": 0.0012836451412676642, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_running_mean": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.41666666666666663, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 2.090120409833334, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 0 + ], + "threshold": -0.4494322117499996, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 1.383383161750001, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": -0.015561036916666665, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": -0.015561036916666665, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": -0.35703936641666667, + "train_balanced_accuracy": 0.75 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.41666666666666663, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": -5.666666666666664, + "train_balanced_accuracy": 0.875 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 0 + ], + "threshold": 3.0, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": -2.6666666666666607, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_qualifying_features": [] + }, + "end_fraction": 0.3, + "end_s": 18.0, + "qualifying_response_features": [], + "repeat_pairs": 24, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 1.4376624963208817, + "action_delta": { + "distinct_n": 12, + "max": 44.66904761904762, + "min": -3.1358226741876534, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 1.5185498088911062, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.9467338429752983, + "repeat_abs_p95": 8.569947817628815, + "repeat_delta": { + "distinct_n": 24, + "max": 4.8797843665768195, + "min": -24.305454429328734, + "n": 24 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 29.54235812446782, + "action_delta": { + "distinct_n": 12, + "max": 981.3419420915683, + "min": -40.25177564153491, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 8, + "zero": 0 + }, + "effect_to_repeat_median": 0.44186157650879665, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 66.85885285135149, + "repeat_abs_p95": 331.06991793371697, + "repeat_delta": { + "distinct_n": 24, + "max": 347.23329948092976, + "min": -494.75378424985627, + "n": 24 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 1.426379390321979, + "action_delta": { + "distinct_n": 12, + "max": 44.29932712215321, + "min": -3.1155318979899898, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 1.5217762546171796, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.9373121613602792, + "repeat_abs_p95": 8.519190735455247, + "repeat_delta": { + "distinct_n": 24, + "max": 4.8432704402515725, + "min": -24.101136871686787, + "n": 24 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 0.010376892235975155, + "action_delta": { + "distinct_n": 12, + "max": 0.020290776197664284, + "min": -0.19708796988318666, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.5862533874020509, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0177003535654775, + "repeat_abs_p95": 0.05622040303840937, + "repeat_delta": { + "distinct_n": 24, + "max": 0.14549402823018454, + "min": -0.05811329271044907, + "n": 24 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 0.015566450509702876, + "action_delta": { + "distinct_n": 12, + "max": 0.1830537562974035, + "min": -0.028434098673234293, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 7, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 0.9711084270914145, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.016029570000052673, + "repeat_abs_p95": 0.06874268847661695, + "repeat_delta": { + "distinct_n": 24, + "max": 0.07111825772963212, + "min": -0.1536373507057546, + "n": 24 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.0019204725324846257, + "action_delta": { + "distinct_n": 12, + "max": 0.011531473650189953, + "min": 1.5710198019351293e-05, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 12, + "zero": 0 + }, + "effect_to_repeat_median": 1.109916935840515, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0017302849163485332, + "repeat_abs_p95": 0.011201032721421313, + "repeat_delta": { + "distinct_n": 24, + "max": 0.012042080910355725, + "min": -0.011611080181771623, + "n": 24 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.012368563885621575, + "action_delta": { + "distinct_n": 12, + "max": 0.1693202894846253, + "min": -0.027238473582144707, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 8, + "zero": 0 + }, + "effect_to_repeat_median": 1.1201833133796462, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.01104155341174029, + "repeat_abs_p95": 0.14212743371007872, + "repeat_delta": { + "distinct_n": 24, + "max": 0.1535118393445899, + "min": -0.2053651713457888, + "n": 24 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.005959053027072225, + "action_delta": { + "distinct_n": 12, + "max": 0.22224967428314468, + "min": 3.590298430544703e-05, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 12, + "zero": 0 + }, + "effect_to_repeat_median": 0.49652693165856604, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.012001469904496409, + "repeat_abs_p95": 0.06026674766281165, + "repeat_delta": { + "distinct_n": 24, + "max": 0.06029573383954434, + "min": -0.267359376561095, + "n": 24 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.0057476210577813765, + "action_delta": { + "distinct_n": 12, + "max": 0.18534326857338257, + "min": -0.015438462146793161, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.6334301280804308, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.009073804359762887, + "repeat_abs_p95": 0.0458855138590948, + "repeat_delta": { + "distinct_n": 24, + "max": 0.033340684091908934, + "min": -0.24888103806759748, + "n": 24 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 12 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 12 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 24 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.003319188436037235, + "action_delta": { + "distinct_n": 12, + "max": 0.013152637562827363, + "min": -0.02492780421592411, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.1361252231813899, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.02438334614602866, + "repeat_abs_p95": 0.07693546674256392, + "repeat_delta": { + "distinct_n": 24, + "max": 0.08157979537248183, + "min": -0.04610629729936333, + "n": 24 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 1.8914151395833336, + "action_delta": { + "distinct_n": 12, + "max": 45.162770761333334, + "min": -2.3232337786666672, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 1.438574132498461, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 1.3147846168333333, + "repeat_abs_p95": 6.971346508883332, + "repeat_delta": { + "distinct_n": 21, + "max": 4.9605220578333356, + "min": -20.271844710166665, + "n": 24 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 1.12754089475, + "action_delta": { + "distinct_n": 10, + "max": 0.0, + "min": -47.07583785533333, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 9, + "positive": 0, + "zero": 3 + }, + "effect_to_repeat_median": 16.245384499464624, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.06940684566666666, + "repeat_abs_p95": 5.181811176816667, + "repeat_delta": { + "distinct_n": 18, + "max": 0.6568425336666667, + "min": -7.1538662623333344, + "n": 24 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 7.166666666666668, + "action_delta": { + "distinct_n": 12, + "max": 34.0, + "min": -47.33333333333333, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.31617647058823534, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 22.666666666666668, + "repeat_abs_p95": 55.666666666666664, + "repeat_delta": { + "distinct_n": 24, + "max": 85.33333333333333, + "min": -55.66666666666667, + "n": 24 + } + } + }, + "sanity": { + "action_pairs": 12, + "invariants": { + "efficacy_label_balance": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "matched_action_request_hashes": true, + "probabilities_bounded": true + }, + "red_flags": [], + "repeat_pairs": 24, + "trials": 36 + }, + "start_s": 12.0 + } + ], + "claim_boundary": "Post-hoc corrective audit over every common replay decile. It can diagnose horizon sensitivity but cannot establish a held-out tuning claim.", + "cumulative": [ + { + "action_pairs": 12, + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 21, + "max": 0.12295081967213115, + "median": 0.08985507246376812, + "min": 0.04918032786885246, + "n": 36 + }, + "completed_fraction_of_total": { + "distinct_n": 30, + "max": 0.09361702127659574, + "median": 0.050437283852088316, + "min": 0.005586592178770949, + "n": 36 + } + }, + "efficacy": { + "feasibility_transitions": { + "false->false": 3, + "false->true": 6, + "true->true": 3 + }, + "minimum_balanced_accuracy": 0.75, + "minimum_delta_over_best_outcome": 0.15, + "outcome_delta": { + "best_accuracy": 0.75, + "best_balanced_accuracy": 0.75, + "best_feature": "ttft_max_over_slo_max", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_over_admitted": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.4166666666666667, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 0 + ], + "threshold": 0.051724137931034475, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.09339080459770116, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.018181818181818188, + "train_balanced_accuracy": 0.625 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.4166666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 0 + ], + "threshold": -0.051724137931034475, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.09339080459770113, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.01818181818181816, + "train_balanced_accuracy": 0.625 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.000601708027350864, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.000601708027350864, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.015623848188639142, + "train_balanced_accuracy": 0.75 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.0010320465723873684, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 0 + ], + "threshold": 0.009580791853053022, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.0002360795908454838, + "train_balanced_accuracy": 0.625 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.0072977563346891365, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": -0.021249613758603417, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.007412010668000825, + "train_balanced_accuracy": 0.875 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 1 + ], + "threshold": -0.0008523871757378475, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.001080823830574982, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.006279735096096474, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_delta": { + "best_accuracy": 0.6666666666666666, + "best_balanced_accuracy": 0.6666666666666666, + "best_feature": "queue_waiting_mean", + "features": { + "decode_batch_size.mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.5847120251326046, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.1503494070398912, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.759913509761732, + "train_balanced_accuracy": 0.75 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.0003962349313431858, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.0006270962078854048, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 0, + 1 + ], + "threshold": 0.003423657452078254, + "train_balanced_accuracy": 0.75 + } + ] + }, + "kv_usage_mean": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.00016329402239600305, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.00013516820682281123, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.00013516820682281123, + "train_balanced_accuracy": 0.625 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.3333333333333333, + "balanced_accuracy": 0.3333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 0 + ], + "threshold": -0.0003659931906908054, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.0003153433269442174, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 0 + ], + "threshold": 0.002846241748737599, + "train_balanced_accuracy": 1.0 + } + ] + }, + "queue_running_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 1.2024471918333335, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.18325025108333315, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 1.2024471918333335, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -1.6539956596666667, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.009266072583333333, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -1.6539956596666667, + "train_balanced_accuracy": 0.875 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.41666666666666663, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 0, + 1 + ], + "threshold": 1.5833333333333357, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 4.416666666666664, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -19.583333333333332, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_qualifying_features": [] + }, + "end_fraction": 0.1, + "end_s": 6.0, + "qualifying_response_features": [], + "repeat_pairs": 24, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.587816632458785, + "action_delta": { + "distinct_n": 12, + "max": 10.944163150492265, + "min": -0.9368137011239002, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.1464621283214008, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 4.013437734353163, + "repeat_abs_p95": 8.899704585700528, + "repeat_delta": { + "distinct_n": 24, + "max": 9.092698412698415, + "min": -9.933716475095787, + "n": 24 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 7.901570082144033, + "action_delta": { + "distinct_n": 12, + "max": 564.520698546648, + "min": -8.58386322115848, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.05293318721992956, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 149.27440604160446, + "repeat_abs_p95": 426.8141407023969, + "repeat_delta": { + "distinct_n": 24, + "max": 449.2116931216931, + "min": -584.6987867177522, + "n": 24 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.582158519407685, + "action_delta": { + "distinct_n": 12, + "max": 10.735314111579934, + "min": -0.9288831363334129, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.14646507698937813, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 3.9747257938484823, + "repeat_abs_p95": 8.770067452024344, + "repeat_delta": { + "distinct_n": 24, + "max": 8.954735449735448, + "min": -9.735983397190292, + "n": 24 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.005042559238173261, + "action_delta": { + "distinct_n": 12, + "max": 0.008374384236453203, + "min": -0.10829817158931077, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.12202178454078347, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.04132507369196753, + "repeat_abs_p95": 0.08139543508103343, + "repeat_delta": { + "distinct_n": 24, + "max": 0.12107279693486583, + "min": -0.0822222222222222, + "n": 24 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.008375994890060249, + "action_delta": { + "distinct_n": 12, + "max": 0.09274496015002344, + "min": -0.01986863711001642, + "n": 12 + }, + "action_signs": { + "consistency": 0.75, + "negative": 9, + "positive": 3, + "zero": 0 + }, + "effect_to_repeat_median": 0.22398385207837773, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.037395530134597704, + "repeat_abs_p95": 0.07586812708364432, + "repeat_delta": { + "distinct_n": 24, + "max": 0.07631077509373656, + "min": -0.10941890166028097, + "n": 24 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0011703901376772582, + "action_delta": { + "distinct_n": 12, + "max": 0.006703036588479568, + "min": -0.0015820428864207082, + "n": 12 + }, + "action_signs": { + "consistency": 0.75, + "negative": 3, + "positive": 9, + "zero": 0 + }, + "effect_to_repeat_median": 0.29946486376659237, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.003908271985422232, + "repeat_abs_p95": 0.008187883039017584, + "repeat_delta": { + "distinct_n": 24, + "max": 0.007141591578301508, + "min": -0.009201187757062074, + "n": 24 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.006820888350695331, + "action_delta": { + "distinct_n": 12, + "max": 0.140893666867539, + "min": -0.005054735620165451, + "n": 12 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 2, + "positive": 10, + "zero": 0 + }, + "effect_to_repeat_median": 0.6661896437498689, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.010238658638254572, + "repeat_abs_p95": 0.1277225497052652, + "repeat_delta": { + "distinct_n": 24, + "max": 0.13402937356379263, + "min": -0.1168947946847837, + "n": 24 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.007604372340016818, + "action_delta": { + "distinct_n": 12, + "max": 0.06684057734774362, + "min": 0.00016629145416469093, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 12, + "zero": 0 + }, + "effect_to_repeat_median": 0.3770466591391387, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.020168252802926, + "repeat_abs_p95": 0.11084274153262068, + "repeat_delta": { + "distinct_n": 24, + "max": 0.12803476870816266, + "min": -0.11234888600259774, + "n": 24 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0014831270777658032, + "action_delta": { + "distinct_n": 12, + "max": 0.02895016189128262, + "min": -0.00029784868681948656, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 8, + "zero": 0 + }, + "effect_to_repeat_median": 0.12120157645159224, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.01223686292857884, + "repeat_abs_p95": 0.07958153755957757, + "repeat_delta": { + "distinct_n": 24, + "max": 0.08820147130614941, + "min": -0.08208138515366167, + "n": 24 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 12 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 12 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 24 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0028294297587345696, + "action_delta": { + "distinct_n": 12, + "max": 0.013691652669458043, + "min": -0.002949832419021914, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.44362661948582977, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.006377953067861264, + "repeat_abs_p95": 0.031272369907012425, + "repeat_delta": { + "distinct_n": 24, + "max": 0.036461564751038456, + "min": -0.018166035027340932, + "n": 24 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.50741128125, + "action_delta": { + "distinct_n": 12, + "max": 15.614046583666667, + "min": -0.30207624649999953, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.12962585804860988, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 3.9144294887499997, + "repeat_abs_p95": 7.101594725466666, + "repeat_delta": { + "distinct_n": 24, + "max": 6.513803891666667, + "min": -13.5536487565, + "n": 24 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.24895167858333334, + "action_delta": { + "distinct_n": 10, + "max": 0.0, + "min": -8.040727011166666, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 9, + "positive": 0, + "zero": 3 + }, + "effect_to_repeat_median": 1.1132189691456902, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.22363226416666668, + "repeat_abs_p95": 2.296808863758333, + "repeat_delta": { + "distinct_n": 16, + "max": 2.3311656403333334, + "min": -5.974843045, + "n": 24 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 3.249999999999993, + "action_delta": { + "distinct_n": 12, + "max": 10.5, + "min": -30.999999999999993, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.1065573770491801, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 30.5, + "repeat_abs_p95": 59.92499999999999, + "repeat_delta": { + "distinct_n": 24, + "max": 59.5, + "min": -64.5, + "n": 24 + } + } + }, + "sanity": { + "action_pairs": 12, + "invariants": { + "efficacy_label_balance": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "matched_action_request_hashes": true, + "probabilities_bounded": true + }, + "red_flags": [], + "repeat_pairs": 24, + "trials": 36 + }, + "start_s": 0.0 + }, + { + "action_pairs": 12, + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 22, + "max": 0.20535714285714285, + "median": 0.18235294117647058, + "min": 0.1564245810055866, + "n": 36 + }, + "completed_fraction_of_total": { + "distinct_n": 31, + "max": 0.180327868852459, + "median": 0.14411764705882352, + "min": 0.08933333333333335, + "n": 36 + } + }, + "efficacy": { + "feasibility_transitions": { + "false->false": 3, + "false->true": 6, + "true->true": 3 + }, + "minimum_balanced_accuracy": 0.75, + "minimum_delta_over_best_outcome": 0.15, + "outcome_delta": { + "best_accuracy": 0.8333333333333334, + "best_balanced_accuracy": 0.8333333333333333, + "best_feature": "ttft_max_over_slo_max", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.012666666666666666, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.008666666666666666, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.008666666666666666, + "train_balanced_accuracy": 0.75 + } + ] + }, + "completed_over_admitted": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.03401360544217691, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 1 + ], + "threshold": 0.006172839506172811, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.04871948779511809, + "train_balanced_accuracy": 0.75 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.11875000000000002, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.07738095238095238, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.07738095238095238, + "train_balanced_accuracy": 0.75 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.03401360544217688, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 1 + ], + "threshold": -0.006172839506172839, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.04871948779511806, + "train_balanced_accuracy": 0.75 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.028440662518539606, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.02030446551608167, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": 0.008882265987138677, + "train_balanced_accuracy": 0.75 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.4166666666666667, + "balanced_accuracy": 0.41666666666666663, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 0, + 1 + ], + "threshold": -0.0020290170113650557, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 0.04992359808650447, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.009324194656287296, + "train_balanced_accuracy": 0.75 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.01753702182516767, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.01753702182516767, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 1 + ], + "threshold": -0.018086205842943556, + "train_balanced_accuracy": 0.75 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.010556909058063094, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.003084423316020539, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.027367186208561683, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_delta": { + "best_accuracy": 0.6666666666666666, + "best_balanced_accuracy": 0.6666666666666666, + "best_feature": "scheduler_steps_per_s", + "features": { + "decode_batch_size.mean": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 1 + ], + "threshold": -0.17161348298791945, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.30619421565755833, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.6654132739143297, + "train_balanced_accuracy": 0.75 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.005113976430465315, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -5.156781096974641e-05, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 1 + ], + "threshold": 0.0022397228564162543, + "train_balanced_accuracy": 0.75 + } + ] + }, + "kv_usage_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -1.7899149433659497e-05, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -2.0776132762541973e-05, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.00013983916927168966, + "train_balanced_accuracy": 0.75 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 3.837762108915843e-05, + "train_balanced_accuracy": 0.875 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 1 + ], + "threshold": -0.00021675516061736566, + "train_balanced_accuracy": 0.625 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": 3.837762108915843e-05, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_running_mean": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 1 + ], + "threshold": -0.0830968510000003, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.12225680983333342, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.2734091407916659, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": -0.2763032289583333, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.08217400995833332, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -1.0689820661666667, + "train_balanced_accuracy": 0.75 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": 3.2916666666666643, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": 3.5833333333333286, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 6.291666666666657, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_qualifying_features": [] + }, + "end_fraction": 0.2, + "end_s": 12.0, + "qualifying_response_features": [], + "repeat_pairs": 24, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.6805455173736323, + "action_delta": { + "distinct_n": 12, + "max": 21.022824302134648, + "min": -1.6199557480463245, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.2914327913831549, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 2.335171392840622, + "repeat_abs_p95": 10.342426400114645, + "repeat_delta": { + "distinct_n": 24, + "max": 10.582610461920808, + "min": -14.50489117143556, + "n": 24 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 30.73978867286374, + "action_delta": { + "distinct_n": 12, + "max": 837.5693088520675, + "min": -38.259099896431565, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 8, + "zero": 0 + }, + "effect_to_repeat_median": 0.5085540691379722, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 60.44546792235762, + "repeat_abs_p95": 370.0653796793133, + "repeat_delta": { + "distinct_n": 24, + "max": 385.95091643367505, + "min": -701.2529783740349, + "n": 24 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.6733471921553615, + "action_delta": { + "distinct_n": 12, + "max": 20.720637408568443, + "min": -1.606656623670089, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.2892646285380025, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 2.327789593766041, + "repeat_abs_p95": 10.226231015894982, + "repeat_delta": { + "distinct_n": 24, + "max": 10.460062847993882, + "min": -14.289496209342136, + "n": 24 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.008498282584807226, + "action_delta": { + "distinct_n": 12, + "max": 0.024842293569343754, + "min": -0.15582922824302126, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 7, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 0.517404626039514, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.01642482915133081, + "repeat_abs_p95": 0.0799421671518786, + "repeat_delta": { + "distinct_n": 24, + "max": 0.11509974496034647, + "min": -0.08092575333954644, + "n": 24 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.01137367331008028, + "action_delta": { + "distinct_n": 12, + "max": 0.14248395282878046, + "min": -0.030463233217211186, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 8, + "positive": 4, + "zero": 0 + }, + "effect_to_repeat_median": 0.6710383267094245, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.016949364674672823, + "repeat_abs_p95": 0.07611571753021439, + "repeat_delta": { + "distinct_n": 24, + "max": 0.0761583865032141, + "min": -0.11305593403905953, + "n": 24 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.0022211647342633674, + "action_delta": { + "distinct_n": 12, + "max": 0.007755344475118115, + "min": -0.0006496983398026016, + "n": 12 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 2, + "positive": 10, + "zero": 0 + }, + "effect_to_repeat_median": 1.0130418206479903, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0021925696343341517, + "repeat_abs_p95": 0.006616310818721587, + "repeat_delta": { + "distinct_n": 24, + "max": 0.007943398472261458, + "min": -0.005259691066565701, + "n": 24 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.0021058619896323627, + "action_delta": { + "distinct_n": 12, + "max": 0.06906439560383515, + "min": -0.0026342182586551743, + "n": 12 + }, + "action_signs": { + "consistency": 0.75, + "negative": 3, + "positive": 9, + "zero": 0 + }, + "effect_to_repeat_median": 0.17774345954986717, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.011847760783802841, + "repeat_abs_p95": 0.0656650541454329, + "repeat_delta": { + "distinct_n": 24, + "max": 0.020281746428214564, + "min": -0.10385652912378862, + "n": 24 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.011216828476763308, + "action_delta": { + "distinct_n": 12, + "max": 0.08922043547542868, + "min": 0.0002683613903166071, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 12, + "zero": 0 + }, + "effect_to_repeat_median": 1.0844557892569866, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.010343278709820436, + "repeat_abs_p95": 0.07465434288403082, + "repeat_delta": { + "distinct_n": 24, + "max": 0.05015486062543706, + "min": -0.12688580277750028, + "n": 24 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0041456974144474384, + "action_delta": { + "distinct_n": 12, + "max": 0.047957700246711504, + "min": -0.00037567596362376825, + "n": 12 + }, + "action_signs": { + "consistency": 0.75, + "negative": 3, + "positive": 9, + "zero": 0 + }, + "effect_to_repeat_median": 0.8378282569881378, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.004948147045494235, + "repeat_abs_p95": 0.06813512586166338, + "repeat_delta": { + "distinct_n": 24, + "max": 0.07105816994418354, + "min": -0.09865864218773521, + "n": 24 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 12 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 12 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 24 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0003854997463232923, + "action_delta": { + "distinct_n": 12, + "max": 0.012211161699714035, + "min": -0.0029429188195312372, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.04836213965959768, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.007971106097386826, + "repeat_abs_p95": 0.020160934134987728, + "repeat_delta": { + "distinct_n": 24, + "max": 0.011632967170489006, + "min": -0.023705586156158254, + "n": 24 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.5689573993333328, + "action_delta": { + "distinct_n": 12, + "max": 27.59238343825, + "min": -0.6469868719999994, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.26512988706199775, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 2.1459572349166667, + "repeat_abs_p95": 7.856052630799997, + "repeat_delta": { + "distinct_n": 24, + "max": 6.557133349166667, + "min": -17.1293622325, + "n": 24 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 0.40246922141666663, + "action_delta": { + "distinct_n": 11, + "max": 0.0, + "min": -19.377070914, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 10, + "positive": 0, + "zero": 2 + }, + "effect_to_repeat_median": 4.226054832868273, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.09523521045833332, + "repeat_abs_p95": 2.510103133283333, + "repeat_delta": { + "distinct_n": 18, + "max": 2.268429387666668, + "min": -11.883906298916667, + "n": 24 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 7.958333333333329, + "action_delta": { + "distinct_n": 12, + "max": 11.75, + "min": -35.16666666666667, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 7, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 0.38047808764940205, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 20.91666666666667, + "repeat_abs_p95": 51.31666666666662, + "repeat_delta": { + "distinct_n": 24, + "max": 65.66666666666667, + "min": -34.58333333333333, + "n": 24 + } + } + }, + "sanity": { + "action_pairs": 12, + "invariants": { + "efficacy_label_balance": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "matched_action_request_hashes": true, + "probabilities_bounded": true + }, + "red_flags": [], + "repeat_pairs": 24, + "trials": 36 + }, + "start_s": 0.0 + }, + { + "action_pairs": 12, + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 22, + "max": 0.30654761904761907, + "median": 0.2795403407264545, + "min": 0.24313725490196078, + "n": 36 + }, + "completed_fraction_of_total": { + "distinct_n": 30, + "max": 0.2681564245810056, + "median": 0.22518087736972248, + "min": 0.12849162011173185, + "n": 36 + } + }, + "efficacy": { + "feasibility_transitions": { + "false->false": 3, + "false->true": 6, + "true->true": 3 + }, + "minimum_balanced_accuracy": 0.75, + "minimum_delta_over_best_outcome": 0.15, + "outcome_delta": { + "best_accuracy": 0.75, + "best_balanced_accuracy": 0.75, + "best_feature": "completed_pass_rate", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.02825325884543762, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.02266666666666667, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.040253258845437616, + "train_balanced_accuracy": 0.75 + } + ] + }, + "completed_over_admitted": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.03272660415517559, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.022522522522522515, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.07358436332279389, + "train_balanced_accuracy": 0.75 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.14270516717325227, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.12142857142857144, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 0.2182462927143778, + "train_balanced_accuracy": 0.75 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.03272660415517557, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": -0.022522522522522515, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.0735843633227939, + "train_balanced_accuracy": 0.75 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": 0.37412821213021774, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.009857117327656123, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": 0.008882265987138677, + "train_balanced_accuracy": 0.625 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666667, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 0, + 1 + ], + "threshold": -0.004552157689988692, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.007167514027906635, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.006851562902858294, + "train_balanced_accuracy": 0.75 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 0 + ], + "threshold": -0.017755539496041216, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.007733676747496546, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.5040019049192779, + "train_balanced_accuracy": 0.75 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": -0.007216190772090309, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.007216190772090309, + "train_balanced_accuracy": 0.875 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.24525856553394837, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_delta": { + "best_accuracy": 0.75, + "best_balanced_accuracy": 0.75, + "best_feature": "scheduler_steps_per_s", + "features": { + "decode_batch_size.mean": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.44507332514357945, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 1 + ], + "threshold": -0.24667983170250252, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.5943046676188155, + "train_balanced_accuracy": 0.75 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333334, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.004225031845271454, + "train_balanced_accuracy": 0.875 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": 0.0004360066659713927, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 1 + ], + "threshold": 0.0034270719814960106, + "train_balanced_accuracy": 0.875 + } + ] + }, + "kv_usage_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.00013357357159209972, + "train_balanced_accuracy": 0.625 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.00013492200088597965, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": -0.00018506887864057613, + "train_balanced_accuracy": 0.625 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.5833333333333334, + "balanced_accuracy": 0.5833333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 0, + 1 + ], + "threshold": 0.0008649236257326498, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": 0.0017369622931825424, + "train_balanced_accuracy": 0.875 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 1, + 1, + 1 + ], + "threshold": 0.0005824031404205732, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_running_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 11.957967036527776, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 0 + ], + "threshold": -0.23078758088888907, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 1, + 0 + ], + "threshold": 18.57988346463889, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": -0.18938916494444447, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 1, + 1, + 1 + ], + "threshold": -0.12479013208333334, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 1, + 0, + 1, + 1 + ], + "threshold": -0.23008102869444447, + "train_balanced_accuracy": 0.75 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.75, + "balanced_accuracy": 0.75, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": 4.277777777777775, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": 3.888888888888893, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 0, + 1, + 1 + ], + "test_predictions": [ + 0, + 0, + 0, + 1 + ], + "threshold": 4.277777777777775, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 12 + }, + "negative": 6, + "positive": 6 + }, + "telemetry_qualifying_features": [] + }, + "end_fraction": 0.3, + "end_s": 18.0, + "qualifying_response_features": [], + "repeat_pairs": 24, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.6066330085464959, + "action_delta": { + "distinct_n": 12, + "max": 28.73606563075747, + "min": -1.1485291194593508, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.4934987464973646, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 1.2292493402508278, + "repeat_abs_p95": 8.067582719584564, + "repeat_delta": { + "distinct_n": 24, + "max": 8.686808262759335, + "min": -14.918642465494425, + "n": 24 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 10.317307738579501, + "action_delta": { + "distinct_n": 12, + "max": 801.7642402753155, + "min": -13.613525901316592, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.17112975179373358, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 60.28938644763055, + "repeat_abs_p95": 340.3503302119946, + "repeat_delta": { + "distinct_n": 24, + "max": 374.64907917730886, + "min": -561.6272830402723, + "n": 24 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.6035604856985604, + "action_delta": { + "distinct_n": 12, + "max": 28.411725240727222, + "min": -1.1421295113155576, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.4962266942146829, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 1.2162999144045272, + "repeat_abs_p95": 7.9623825938693935, + "repeat_delta": { + "distinct_n": 24, + "max": 8.570025092978447, + "min": -14.737105312913595, + "n": 24 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.25, + "action_abs_median": 0.006690006870484877, + "action_delta": { + "distinct_n": 12, + "max": 0.013223015997172571, + "min": -0.1490562102409011, + "n": 12 + }, + "action_signs": { + "consistency": 0.5833333333333334, + "negative": 5, + "positive": 7, + "zero": 0 + }, + "effect_to_repeat_median": 0.6364709072202164, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.010511096099747042, + "repeat_abs_p95": 0.06858287350501925, + "repeat_delta": { + "distinct_n": 24, + "max": 0.08329551900170162, + "min": -0.0738002419680065, + "n": 24 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.01055037975803224, + "action_delta": { + "distinct_n": 12, + "max": 0.13907776271422118, + "min": -0.018167596600093858, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 8, + "positive": 4, + "zero": 0 + }, + "effect_to_repeat_median": 0.9080980815382388, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.01161810598714273, + "repeat_abs_p95": 0.06969738634332051, + "repeat_delta": { + "distinct_n": 24, + "max": 0.07468521754716137, + "min": -0.08558328606541879, + "n": 24 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.0024714849986972203, + "action_delta": { + "distinct_n": 12, + "max": 0.006494393749041705, + "min": 0.000227090832439927, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 12, + "zero": 0 + }, + "effect_to_repeat_median": 1.373420385875363, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0017995109320603211, + "repeat_abs_p95": 0.005547351274885883, + "repeat_delta": { + "distinct_n": 24, + "max": 0.005694038006766439, + "min": -0.0035699364880986415, + "n": 24 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.010270226283016048, + "action_delta": { + "distinct_n": 12, + "max": 0.23522057920397632, + "min": -0.003022494939389775, + "n": 12 + }, + "action_signs": { + "consistency": 0.9166666666666666, + "negative": 1, + "positive": 11, + "zero": 0 + }, + "effect_to_repeat_median": 1.07628737944467, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.009542271403679525, + "repeat_abs_p95": 0.08564791687481275, + "repeat_delta": { + "distinct_n": 24, + "max": 0.08662204016385266, + "min": -0.31416724947547214, + "n": 24 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.015846588112988946, + "action_delta": { + "distinct_n": 12, + "max": 0.22224967428314468, + "min": 0.0002683613903166071, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 12, + "zero": 0 + }, + "effect_to_repeat_median": 1.3242319261149558, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.011966625936500275, + "repeat_abs_p95": 0.10502297931861322, + "repeat_delta": { + "distinct_n": 24, + "max": 0.05015486062543706, + "min": -0.14511939254670791, + "n": 24 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.08333333333333333, + "action_abs_median": 0.004416664987538066, + "action_delta": { + "distinct_n": 12, + "max": 0.06671902028938567, + "min": -0.00038309683389285525, + "n": 12 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 2, + "positive": 10, + "zero": 0 + }, + "effect_to_repeat_median": 0.8499259269415633, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.005196529306302399, + "repeat_abs_p95": 0.03531051450496546, + "repeat_delta": { + "distinct_n": 24, + "max": 0.028898615902990022, + "min": -0.05744492686343953, + "n": 24 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 12 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 12 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 24 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.001539018865967745, + "action_delta": { + "distinct_n": 12, + "max": 0.003050797976300035, + "min": -0.0015887887661509836, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 8, + "zero": 0 + }, + "effect_to_repeat_median": 0.15591584638446954, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.009870830333516656, + "repeat_abs_p95": 0.021489430916148855, + "repeat_delta": { + "distinct_n": 24, + "max": 0.021795755533912664, + "min": -0.02021322592113628, + "n": 24 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 0.5940786813055561, + "action_delta": { + "distinct_n": 12, + "max": 33.446711043166665, + "min": -0.6256704651111118, + "n": 12 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 8, + "zero": 0 + }, + "effect_to_repeat_median": 0.30447578852990603, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 1.9511524518055556, + "repeat_abs_p95": 4.689088691175, + "repeat_delta": { + "distinct_n": 24, + "max": 3.8720077572777782, + "min": -11.837926215166668, + "n": 24 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.4166666666666667, + "action_abs_median": 0.9777240376944445, + "action_delta": { + "distinct_n": 11, + "max": 0.0, + "min": -28.609993227777778, + "n": 12 + }, + "action_signs": { + "consistency": 1.0, + "negative": 10, + "positive": 0, + "zero": 2 + }, + "effect_to_repeat_median": 8.318675069172114, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.11753362519444445, + "repeat_abs_p95": 2.0560520443777768, + "repeat_delta": { + "distinct_n": 20, + "max": 1.6669221998333335, + "min": -9.630657473833335, + "n": 24 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 5.527777777777779, + "action_delta": { + "distinct_n": 12, + "max": 9.166666666666671, + "min": -33.27777777777777, + "n": 12 + }, + "action_signs": { + "consistency": 0.5, + "negative": 6, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.3338926174496646, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 16.55555555555555, + "repeat_abs_p95": 37.27222222222221, + "repeat_delta": { + "distinct_n": 24, + "max": 31.888888888888886, + "min": -41.611111111111114, + "n": 24 + } + } + }, + "sanity": { + "action_pairs": 12, + "invariants": { + "efficacy_label_balance": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "matched_action_request_hashes": true, + "probabilities_bounded": true + }, + "red_flags": [], + "repeat_pairs": 24, + "trials": 36 + }, + "start_s": 0.0 + } + ], + "decision": "REQUIRES_UNCENSORED_PHASE_AWARE_PILOT", + "design": { + "available_deciles": [ + 0.1, + 0.2, + 0.3 + ], + "cumulative_and_nonoverlapping_blocks": true, + "decile_fraction": 0.1, + "maximum_common_end_s": 18.0, + "maximum_common_fraction": 0.3, + "select_best_horizon": false, + "trace_duration_s": 60.0 + }, + "provenance": { + "analysis_script": "/home/gahow/phd/aituner/runs/intervention-response-v2/analyze_existing.py", + "analysis_script_sha256": "c7eb2cd2d00f2b3afa62e22e7b2f1cce6b87ec51ebb7921ef7ffb9d13d28c50a", + "manifest": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/pilot-manifest.json", + "manifest_sha256": "86b249a087ab1bcd51cd8de3d1da6bf34afd41af977eb58c93229b941dbf5c8a", + "manifest_validation": { + "expected_trials": 36, + "matched_trials": 36, + "schema": "fidelity-prefix-pilot-manifest-v1" + }, + "p1_analysis_script": "/home/gahow/phd/aituner/runs/intervention-response-v0/analyze_p1.py", + "p1_analysis_script_sha256": "902adcba58bcb73f97c455fdea3710f6cb28fa8d8f4ea5bc4915143ed106b17a", + "run_root": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b", + "streams": [ + { + "bytes": 24501686, + "cell": "tp1_mns64", + "path": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/cells/tp1_mns64/opprof/opprof-v1-dp0-pid234511-1784005631788937266.jsonl", + "sha256": "a813076afe677e5c13010b2ec44f46998794ca44cdf30ee81be836e3623f0068" + }, + { + "bytes": 25524027, + "cell": "tp1_mns8", + "path": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/cells/tp1_mns8/opprof/opprof-v1-dp0-pid227734-1784005177936445360.jsonl", + "sha256": "1708759d2e77c42bff1940d43214fb2d927c3eebeda62d53a3fb7d04817e9b22" + }, + { + "bytes": 29431988, + "cell": "tp2_mns64", + "path": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/cells/tp2_mns64/opprof/opprof-v1-dp0-pid247791-1784006495467408663.jsonl", + "sha256": "65355de883f2eb5d31be161d82b26dddc53c41a0a35a0fdb807777bc5b21c5ec" + }, + { + "bytes": 26036993, + "cell": "tp2_mns8", + "path": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/cells/tp2_mns8/opprof/opprof-v1-dp0-pid241594-1784006096655191299.jsonl", + "sha256": "485e203d37d45fdd3c4de768b95da2050478f111f4d739b129d4f4ebd4737b70" + }, + { + "bytes": 17449143, + "cell": "tp4_mns16", + "path": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/cells/tp4_mns16/opprof/opprof-v1-dp0-pid256090-1784007028237607411.jsonl", + "sha256": "59e98014d01bfc68d9f97112ecd782cb03571ad437382ed67416c7b85ae48c56" + }, + { + "bytes": 22674678, + "cell": "tp4_mns64", + "path": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/cells/tp4_mns64/opprof/opprof-v1-dp0-pid262293-1784007376383569515.jsonl", + "sha256": "03e4f3cca3023a78e859b663703b9468dedf89e58320f0292e2973070e7616c8" + } + ], + "trial_inputs": [ + { + "cell": "tp1_mns64", + "early_stopped": true, + "elapsed_s": 19.448401608, + "level": "high", + "mns": 64, + "replicate": 1, + "request_count": 179, + "requests_sha256": "e970dcac20144cb1e13a35997377f6695a9c57d788a88bad7f18cf6c64e35cec", + "result_sha256": "3233686b60b920dcd6cc2f45193a81f3af2d46f781ef42c4d117630612084feb", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns64/high-rep1/result.json" + }, + { + "cell": "tp1_mns64", + "early_stopped": true, + "elapsed_s": 56.362951594, + "level": "high", + "mns": 64, + "replicate": 2, + "request_count": 179, + "requests_sha256": "ec06878e2aefdc4853d972197cedc29ab94df7147abb2701e0d88b5dabf29728", + "result_sha256": "8215477a014f061c55e94d879a25bbf1676ee2ef1c5e81efadd23d20d337e07a", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns64/high-rep2/result.json" + }, + { + "cell": "tp1_mns64", + "early_stopped": true, + "elapsed_s": 59.320339642, + "level": "high", + "mns": 64, + "replicate": 3, + "request_count": 179, + "requests_sha256": "18d72d1fe16136359e28a1909e395ffd553dbe3edd2c0fd93a9cc270a5ec6d60", + "result_sha256": "480a3b50acaf9b9c1902e2669cf3aa7723a6703a00778da269b5b9757990ceef", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns64/high-rep3/result.json" + }, + { + "cell": "tp1_mns64", + "early_stopped": false, + "elapsed_s": 60.653813767, + "level": "low", + "mns": 64, + "replicate": 1, + "request_count": 122, + "requests_sha256": "44d729c1ace9b09fe2ea4c4e110bd0f95fee69f83edc78ee0f65256922dcd2c2", + "result_sha256": "d2c08e71c5d4e0dd99662bab8457bd1ad0a4aba6bda57794dbfce62e36720f62", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns64/low-rep1/result.json" + }, + { + "cell": "tp1_mns64", + "early_stopped": false, + "elapsed_s": 61.821918535, + "level": "low", + "mns": 64, + "replicate": 2, + "request_count": 122, + "requests_sha256": "97de3cced64fd88a5c2dcd087ca6c29d8b09a0a43d7bf0e4e844ef246462465b", + "result_sha256": "f194db2f7abb995171cd24626e4e515c000481f81283c929b9a5b63f110f1506", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns64/low-rep2/result.json" + }, + { + "cell": "tp1_mns64", + "early_stopped": false, + "elapsed_s": 59.372640014, + "level": "low", + "mns": 64, + "replicate": 3, + "request_count": 122, + "requests_sha256": "30f4623100c58580cfa4efe9a1998c63581c8e44511c556ef39cf5bd777883bf", + "result_sha256": "0be457aa2fc9ce95bdb92a6eb25544f5f61a9739932ee5afa40535d26456e348", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns64/low-rep3/result.json" + }, + { + "cell": "tp1_mns8", + "early_stopped": true, + "elapsed_s": 24.256301395, + "level": "high", + "mns": 8, + "replicate": 1, + "request_count": 179, + "requests_sha256": "5f5bfa4a16846190f488d3369eed89ae895804145cea59197e6b544b703223fc", + "result_sha256": "f65e3dc91d17f5caab1f4aecf97d2c464b99f107231af2cfd96f5484880136ee", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns8/high-rep1/result.json" + }, + { + "cell": "tp1_mns8", + "early_stopped": true, + "elapsed_s": 57.560632316, + "level": "high", + "mns": 8, + "replicate": 2, + "request_count": 179, + "requests_sha256": "e802d09057de349e5164c18314fcad09dc88c9673834fd6e5b100a57dd12fbf4", + "result_sha256": "4835a985dd6bb8dc50b53737baa980ac95a49e22f0b8a7326e1fb416c9b4e798", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns8/high-rep2/result.json" + }, + { + "cell": "tp1_mns8", + "early_stopped": true, + "elapsed_s": 46.401123902, + "level": "high", + "mns": 8, + "replicate": 3, + "request_count": 179, + "requests_sha256": "cf1226185845cd8aa4e7ead0cb5c95c77eda62b84851a2ab579f984bbb88e8ac", + "result_sha256": "95d5103f2678eeb99c258870324566932e2f2853c3b6e0ee94bc6c538f6ed865", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns8/high-rep3/result.json" + }, + { + "cell": "tp1_mns8", + "early_stopped": false, + "elapsed_s": 60.653530104, + "level": "low", + "mns": 8, + "replicate": 1, + "request_count": 122, + "requests_sha256": "0edc6095b16e5361a88b2765cc5d0f267ed16ff1d883e02fc3c58f57e59c1018", + "result_sha256": "8bc433109d148c21b7d158df95ec5d1fa4fa619243114563068e1e98c4241a29", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns8/low-rep1/result.json" + }, + { + "cell": "tp1_mns8", + "early_stopped": false, + "elapsed_s": 62.156958699, + "level": "low", + "mns": 8, + "replicate": 2, + "request_count": 122, + "requests_sha256": "ca58a878401ac44cca0a9a595eab41659dafec2e923f7f2474e4f82ea30b2f3e", + "result_sha256": "42a56af136d1d3bdfc16d9cea092c2db0eaa4db2c85dceb21958c5e418122900", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns8/low-rep2/result.json" + }, + { + "cell": "tp1_mns8", + "early_stopped": false, + "elapsed_s": 59.499655201, + "level": "low", + "mns": 8, + "replicate": 3, + "request_count": 122, + "requests_sha256": "8a72240f84bbeff9eabd63e7d1f7a4c19910660fc5e4335013a1deaf89408127", + "result_sha256": "0214f32cabbcdc1bad6c331c3945c505cad136fcc9a8336aeb6314b27474e279", + "tp": 1, + "trace_duration_s": 60.0, + "trial_id": "cells/tp1_mns8/low-rep3/result.json" + }, + { + "cell": "tp2_mns64", + "early_stopped": false, + "elapsed_s": 60.842664607, + "level": "high", + "mns": 64, + "replicate": 1, + "request_count": 345, + "requests_sha256": "19537ec05f1e40543d850964db12234561954c209d55cd8cd0a1da716ab105fc", + "result_sha256": "49523b3269db649aa70072058666da3309ecabb93242cd504b6413f5de0f5d54", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns64/high-rep1/result.json" + }, + { + "cell": "tp2_mns64", + "early_stopped": false, + "elapsed_s": 61.33597184, + "level": "high", + "mns": 64, + "replicate": 2, + "request_count": 345, + "requests_sha256": "d95e0dcc3d3f25af11b4d3da0ca133f647fd602753b804337a7131cc56e03059", + "result_sha256": "f37a42d9ea734adcb24892f86cbb19e2c51cee0baf8a6a63c5d51c4820bd28f2", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns64/high-rep2/result.json" + }, + { + "cell": "tp2_mns64", + "early_stopped": false, + "elapsed_s": 60.925907661, + "level": "high", + "mns": 64, + "replicate": 3, + "request_count": 345, + "requests_sha256": "ccba3d0a18a7db8501b040855bda473ee55c519619d4b9fa2a105bda06300829", + "result_sha256": "55447419925f30397d19863e149890241c58863d0adaafd511a67757905cb58e", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns64/high-rep3/result.json" + }, + { + "cell": "tp2_mns64", + "early_stopped": false, + "elapsed_s": 60.73903317, + "level": "low", + "mns": 64, + "replicate": 1, + "request_count": 235, + "requests_sha256": "8685f475baf247f7ce52a6a2df1183cd77a6f580cd57a3a87c407b4e96c0895f", + "result_sha256": "7a8f4e462d496bd56b73b0a686653cbe499a238df62a8c9a50db687d051f1c68", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns64/low-rep1/result.json" + }, + { + "cell": "tp2_mns64", + "early_stopped": false, + "elapsed_s": 61.049384964, + "level": "low", + "mns": 64, + "replicate": 2, + "request_count": 235, + "requests_sha256": "016a76ab3dc015700f9ecb9f60a20fcc3b456c45270472fdd1d04e84ebd7021a", + "result_sha256": "368869326f44a0032985eff9d3abfa2306b93c25370caa05cee255a8c0208c0e", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns64/low-rep2/result.json" + }, + { + "cell": "tp2_mns64", + "early_stopped": false, + "elapsed_s": 60.611333104, + "level": "low", + "mns": 64, + "replicate": 3, + "request_count": 235, + "requests_sha256": "5455d1cdeabd6e2937130ed8fc8d86c1aaa6f04cbf26c3cb69ad064be5f67d73", + "result_sha256": "2de5c4524f57b487b94b4fdba35defc2a566236fc3e31f7260b7a9f85bb7007c", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns64/low-rep3/result.json" + }, + { + "cell": "tp2_mns8", + "early_stopped": true, + "elapsed_s": 25.452006333, + "level": "high", + "mns": 8, + "replicate": 1, + "request_count": 336, + "requests_sha256": "5d41a316f329e2efcc0f728cdd078377eb72fb9258738636db704bcfcca290aa", + "result_sha256": "466b3991190a232c984ba1b54fb6793cac1893dce196ee96f7238d70d49fcb74", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns8/high-rep1/result.json" + }, + { + "cell": "tp2_mns8", + "early_stopped": true, + "elapsed_s": 24.770382696, + "level": "high", + "mns": 8, + "replicate": 2, + "request_count": 336, + "requests_sha256": "1b7d2cdcd639c00c443ae1b98287a7cc57323eb80c7f0e1bfc72f1ab5bf48760", + "result_sha256": "e9156d6871cc8cb3a20dce629cc2443df7473f3836b46eb0944d491ea5c4d44c", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns8/high-rep2/result.json" + }, + { + "cell": "tp2_mns8", + "early_stopped": true, + "elapsed_s": 25.69871505, + "level": "high", + "mns": 8, + "replicate": 3, + "request_count": 336, + "requests_sha256": "5a68fda7c29477e5b3e66d70ad2b05b77be6c200cc1beea89d663a5729f31a2b", + "result_sha256": "f2116797b35057949acbf1f57898befac9ff18b94c808462f0969bdcf2446dd6", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns8/high-rep3/result.json" + }, + { + "cell": "tp2_mns8", + "early_stopped": false, + "elapsed_s": 60.605998599, + "level": "low", + "mns": 8, + "replicate": 1, + "request_count": 229, + "requests_sha256": "455a1c21388700f0e3234c49d15ba3b0426b0d3b551c096b297793afa0d4602f", + "result_sha256": "13de25a73cc8c752245f169cfca1d64220f697c15b835a646ba969a1005047fb", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns8/low-rep1/result.json" + }, + { + "cell": "tp2_mns8", + "early_stopped": false, + "elapsed_s": 61.925251312, + "level": "low", + "mns": 8, + "replicate": 2, + "request_count": 229, + "requests_sha256": "cae4b764ebb8714a6f5d2009bb79497420b388986a992c51410f1681810c14c6", + "result_sha256": "92f92a576e990946c4767a2107d2ce0e6ba1b066337911399360c199fae3f5e4", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns8/low-rep2/result.json" + }, + { + "cell": "tp2_mns8", + "early_stopped": false, + "elapsed_s": 60.612475588, + "level": "low", + "mns": 8, + "replicate": 3, + "request_count": 229, + "requests_sha256": "cc8c01e4560411cd5b84cdc9bebdda9a86b88615ee1c39a996affe669d26c167", + "result_sha256": "c9a7d026bd89cc027c45b8f1899d56e21d53d4234e95db365a9492f670a78d36", + "tp": 2, + "trace_duration_s": 60.0, + "trial_id": "cells/tp2_mns8/low-rep3/result.json" + }, + { + "cell": "tp4_mns16", + "early_stopped": true, + "elapsed_s": 25.468414615, + "level": "high", + "mns": 16, + "replicate": 1, + "request_count": 750, + "requests_sha256": "8695422a0f1e0e1c097a08a7f4c082f1ba2e3bb26b58caadae0afc7438aeca90", + "result_sha256": "c54a2bb5c1b78bdef3fa09c39f5974215370d6fc7c2b6d403a3d376d53a1d9eb", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns16/high-rep1/result.json" + }, + { + "cell": "tp4_mns16", + "early_stopped": true, + "elapsed_s": 24.508063881, + "level": "high", + "mns": 16, + "replicate": 2, + "request_count": 750, + "requests_sha256": "aa0482b10414d911af227e2df172c19ecca0add2e0024dc88f85ffbd063854bb", + "result_sha256": "91e2491de4ca0868b667e7d126b755f03bdd78378681b8070a2bbc0c38fb9055", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns16/high-rep2/result.json" + }, + { + "cell": "tp4_mns16", + "early_stopped": true, + "elapsed_s": 27.456202024, + "level": "high", + "mns": 16, + "replicate": 3, + "request_count": 750, + "requests_sha256": "3a23ffbe0e724648226152ef5433cc5825ad640c37cf47997c2f091210eceeec", + "result_sha256": "136dde0451f806841ff3996b081fe0bffac0a76f60c3d78b7516f2689e106008", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns16/high-rep3/result.json" + }, + { + "cell": "tp4_mns16", + "early_stopped": true, + "elapsed_s": 48.952560958, + "level": "low", + "mns": 16, + "replicate": 1, + "request_count": 510, + "requests_sha256": "d7eaaefd35cc906f759dcaa7ac3c8543249a1a02f28572bec953e41e4379a29f", + "result_sha256": "11d9ec9d2a3358aed988f7b4b1014af087d61c4b5a72eb9076ae19bb5cc481cd", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns16/low-rep1/result.json" + }, + { + "cell": "tp4_mns16", + "early_stopped": true, + "elapsed_s": 49.912113702, + "level": "low", + "mns": 16, + "replicate": 2, + "request_count": 510, + "requests_sha256": "7fdb90f679bba1747fbacc92fac2045c820e09965c755b20f6dafd4fd9e26a7a", + "result_sha256": "800e2aa6e9f9efef4be12ee778be6817364914d63ae0624775d02729089768d6", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns16/low-rep2/result.json" + }, + { + "cell": "tp4_mns16", + "early_stopped": true, + "elapsed_s": 48.923086251, + "level": "low", + "mns": 16, + "replicate": 3, + "request_count": 510, + "requests_sha256": "b592fe254476957265fc9139aefdea77be4b5540f3c1b8a429c10ea6ccab9a6a", + "result_sha256": "cc1b1f7f5280c2892d0c6deac9df9f34ce126b0284e6ad893ffbf0ddbf27969f", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns16/low-rep3/result.json" + }, + { + "cell": "tp4_mns64", + "early_stopped": false, + "elapsed_s": 61.434823743, + "level": "high", + "mns": 64, + "replicate": 1, + "request_count": 750, + "requests_sha256": "b4aeaf5b3175bd85fdac08a2ef657fb2278c6c4b36ea32ea417275ab6d55812d", + "result_sha256": "111a1042b614d05772888665a7cfbc86925b2fbeb52ffe19edb62a5bcfe2f513", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns64/high-rep1/result.json" + }, + { + "cell": "tp4_mns64", + "early_stopped": false, + "elapsed_s": 61.130504059, + "level": "high", + "mns": 64, + "replicate": 2, + "request_count": 750, + "requests_sha256": "5d82b6885f5c654ae1ae126696df4bafe04f5be1c1109db24c7856b0456d9443", + "result_sha256": "6ed574c18723e265ab1327c5e43d3c76d37b3fc5711d06c77a51718e4ba16601", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns64/high-rep2/result.json" + }, + { + "cell": "tp4_mns64", + "early_stopped": false, + "elapsed_s": 61.22613867, + "level": "high", + "mns": 64, + "replicate": 3, + "request_count": 750, + "requests_sha256": "5c5861d716181f06dc3f951ae0d6ee479b1b61fc21ebf24541c5b45053fcc167", + "result_sha256": "35b8f0465f6b0c197f71b5ecaf4e771e6613f86998ae44f8701ab9e4e1e440b4", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns64/high-rep3/result.json" + }, + { + "cell": "tp4_mns64", + "early_stopped": false, + "elapsed_s": 61.123849552, + "level": "low", + "mns": 64, + "replicate": 1, + "request_count": 510, + "requests_sha256": "931d372ded1e3aaa9c67f06653fac8f79b4ee4f369028c1148163580386681ce", + "result_sha256": "325ce57f33133670a6761b5dffb831ed3b2f464755ff743b56976b4a974c1215", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns64/low-rep1/result.json" + }, + { + "cell": "tp4_mns64", + "early_stopped": false, + "elapsed_s": 60.86041954, + "level": "low", + "mns": 64, + "replicate": 2, + "request_count": 510, + "requests_sha256": "72c8f70a240f79edcaa4eb0a2fe9bade8ac07093339c5b2d09c49ddd1cf27d16", + "result_sha256": "ff8b2283d02d5967893ca6225b561ae11011da6149d1f6cff7b4275c00170e1d", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns64/low-rep2/result.json" + }, + { + "cell": "tp4_mns64", + "early_stopped": false, + "elapsed_s": 60.47213624, + "level": "low", + "mns": 64, + "replicate": 3, + "request_count": 510, + "requests_sha256": "8b2c779d2097c5e4038e94e1ac9b986e29f55d9acfd67eeb366c4e0e9270f6e0", + "result_sha256": "f026b69356701894fab184e0f2f7504a30b537982e823373899c9ab123a33891", + "tp": 4, + "trace_duration_s": 60.0, + "trial_id": "cells/tp4_mns64/low-rep3/result.json" + } + ] + }, + "sanity": { + "early_stopped": 15, + "elapsed_s": { + "distinct_n": 36, + "max": 62.156958699, + "median": 60.5390674195, + "min": 19.448401608, + "n": 36 + }, + "invariants": { + "all_intervals_uncensored": true, + "all_window_sanity_pass": true, + "expected_trial_count": true, + "manifest_trials_match": true, + "stream_provenance_consistent": true, + "trace_duration_consistent": true + }, + "red_flags": [], + "request_count": { + "distinct_n": 8, + "max": 750.0, + "median": 285.5, + "min": 122.0, + "n": 36 + }, + "stream_bytes": { + "distinct_n": 6, + "max": 29431988.0, + "median": 25012856.5, + "min": 17449143.0, + "n": 6 + }, + "trace_duration_s": { + "distinct_n": 1, + "max": 60.0, + "median": 60.0, + "min": 60.0, + "n": 36 + }, + "trials": 36 + }, + "schema": "intervention-response-phase-aware-existing-v2", + "status": "COMPLETE", + "trajectory": { + "blocks": [ + { + "end_s": 6.0, + "start_s": 0.0 + }, + { + "end_s": 12.0, + "start_s": 6.0 + }, + { + "end_s": 18.0, + "start_s": 12.0 + } + ], + "features": { + "batch_size.mean": { + "block_medians": [ + 6.414992146729492, + 7.937718404518684, + 7.938679599969923 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 44.717667092379735, + "median": 2.4047228794539084, + "min": 0.12661717921526972, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 44.717667092379735, + "median": 1.715194194802789, + "min": -6.818300947246247, + "n": 36 + }, + "first_to_last_pearson": 0.7466021228318502 + }, + "batch_tokens.mean": { + "block_medians": [ + 182.10234474017744, + 172.7298136645963, + 202.28603044732077 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 908.0537249893571, + "median": 75.02808983931541, + "min": 7.692264317468613, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 908.0537249893571, + "median": 21.47667280168706, + "min": -316.4167321876521, + "n": 36 + }, + "first_to_last_pearson": 0.7254256632404311 + }, + "decode_batch_size.mean": { + "block_medians": [ + 6.354735422986181, + 7.873722548380083, + 7.875311697892343 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 44.34555129842486, + "median": 2.4157231102242664, + "min": 0.15471898197242862, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 44.34555129842486, + "median": 1.7131997250467412, + "min": -6.730278183383879, + "n": 36 + }, + "first_to_last_pearson": 0.7458594110784589 + }, + "graph_full_share": { + "block_medians": [ + 0.9440933791043857, + 0.9375353988798221, + 0.9361517143943923 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 0.20131971051511277, + "median": 0.027226282812290892, + "min": 0.00023443249531129862, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 0.08227563742558686, + "median": -0.0017697738257967388, + "min": -0.20131971051511277, + "n": 36 + }, + "first_to_last_pearson": 0.6826036350375567 + }, + "graph_none_share": { + "block_medians": [ + 0.05080178541907753, + 0.05845283871165627, + 0.06118872118872119 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 0.19429544487015749, + "median": 0.01965769475771631, + "min": 0.0028422907600285444, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 0.19429544487015749, + "median": 0.0040749575702726006, + "min": -0.0846718334643753, + "n": 36 + }, + "first_to_last_pearson": 0.65716602522967 + }, + "graph_padding_fraction": { + "block_medians": [ + 0.005497687293445758, + 0.0075248839288535735, + 0.006051201250183472 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 0.014587933688096199, + "median": 0.002744262881772697, + "min": 4.7975018144524526e-05, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 0.014587933688096199, + "median": -0.0005530753213689365, + "min": -0.008320005718079814, + "n": 36 + }, + "first_to_last_pearson": 0.3098612628425477 + }, + "kv_usage_end_minus_start": { + "block_medians": [ + 0.023398338975546773, + -0.001777370408002521, + 0.0038338466257342163 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 0.262813467878909, + "median": 0.017891096563429187, + "min": 0.0015150201237521532, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 0.07218503346987704, + "median": -0.01752170592552832, + "min": -0.262813467878909, + "n": 36 + }, + "first_to_last_pearson": -0.012340699297270389 + }, + "kv_usage_max": { + "block_medians": [ + 0.03336296364308622, + 0.04022944403944284, + 0.030052448221737305 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 0.29553401938255563, + "median": 0.011325832339947661, + "min": 0.0002397506593143639, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 0.29553401938255563, + "median": 0.00083657676867116, + "min": -0.09986012588670201, + "n": 36 + }, + "first_to_last_pearson": 0.4134682913789121 + }, + "kv_usage_mean": { + "block_medians": [ + 0.016731908775651918, + 0.02129250131291721, + 0.01914443130373699 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 0.271100737119201, + "median": 0.009396606492664827, + "min": 0.0007061033361193698, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 0.271100737119201, + "median": 0.002361521286958815, + "min": -0.06598177225454588, + "n": 36 + }, + "first_to_last_pearson": 0.10554346287202872 + }, + "preemptions": { + "block_medians": [ + 0.0, + 0.0, + 0.0 + ], + "changed_trials": 0, + "first_to_last_abs_delta": { + "distinct_n": 1, + "max": 0.0, + "median": 0.0, + "min": 0.0, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 1, + "max": 0.0, + "median": 0.0, + "min": 0.0, + "n": 36 + }, + "first_to_last_pearson": null + }, + "prefill_token_fraction": { + "block_medians": [ + 0.9617744444596128, + 0.955926466887596, + 0.9488462859261657 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 0.0574568246654229, + "median": 0.013436705979377572, + "min": 0.00024055263508171443, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 0.008129115874074722, + "median": -0.013436705979377572, + "min": -0.0574568246654229, + "n": 36 + }, + "first_to_last_pearson": 0.23303776424770067 + }, + "queue_running_mean": { + "block_medians": [ + 6.85322045575, + 8.0, + 8.0 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 44.04634992016667, + "median": 2.4438409930833327, + "min": 0.5856487165000002, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 44.04634992016667, + "median": 1.8893361798333332, + "min": -4.201522480666667, + "n": 36 + }, + "first_to_last_pearson": 0.8338308544566243 + }, + "queue_waiting_mean": { + "block_medians": [ + 0.0, + 0.020209624833333332, + 0.008993971666666666 + ], + "changed_trials": 23, + "first_to_last_abs_delta": { + "distinct_n": 24, + "max": 39.884282579499995, + "median": 0.008993971666666666, + "min": 0.0, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 24, + "max": 39.884282579499995, + "median": 0.00026988233333333, + "min": -0.7860144426666666, + "n": 36 + }, + "first_to_last_pearson": 0.8749679224022264 + }, + "scheduler_steps_per_s": { + "block_medians": [ + 75.83333333333334, + 78.16666666666667, + 73.58333333333334 + ], + "changed_trials": 36, + "first_to_last_abs_delta": { + "distinct_n": 36, + "max": 76.5, + "median": 28.250000000000004, + "min": 2.1666666666666714, + "n": 36 + }, + "first_to_last_delta": { + "distinct_n": 36, + "max": 73.33333333333333, + "median": -3.1666666666666714, + "min": -76.5, + "n": 36 + }, + "first_to_last_pearson": 0.40161538799446195 + } + }, + "trial_count": 36 + } +} diff --git a/runs/intervention-response-v2/pilot_controller.py b/runs/intervention-response-v2/pilot_controller.py new file mode 100644 index 0000000..98b5903 --- /dev/null +++ b/runs/intervention-response-v2/pilot_controller.py @@ -0,0 +1,471 @@ +#!/usr/bin/env python3 +"""Serialized, resumable controller for the 300-second phase-aware pilot.""" + +from __future__ import annotations + +import argparse +import json +import os +import shlex +import signal +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + + +HERE = Path(__file__).resolve().parent +PHASE6 = HERE.parent / "opprof-phase6" +sys.path.insert(0, str(PHASE6)) + +import opprof_phase6_controller as base # noqa: E402 + + +SCHEMA = "intervention-response-phase-aware-pilot-state-v2" +SESSION_ESTIMATE_H20_HOURS = 1.25 +SAFETY_H20_HOURS = 0.20 +CLIENT_TIMEOUT_S = 450.0 + + +def atomic_json(path: Path, payload: Any) -> None: + base.atomic_json(path, payload) + + +def wait_all_idle(timeout_s: float = 30.0) -> None: + deadline = time.monotonic() + timeout_s + last_error: Exception | None = None + while time.monotonic() < deadline: + try: + base.assert_all_idle() + return + except RuntimeError as error: + last_error = error + time.sleep(1.0) + raise last_error or RuntimeError("GPU idle timeout") + + +def configure(args: argparse.Namespace, manifest: dict[str, Any]) -> None: + base.WORKDIR = args.run_root.parent + base.RUN_ROOT = args.run_root + base.STATE = args.run_root / "controller-state.json" + base.SOURCE = args.vllm_source + base.VENV = args.venv + base.AITUNER = args.aituner_root + base.MODEL = args.model + base.CLIENT = args.client + base.GPU_LIMIT = float(manifest["budget"]["hard_cap_h20_hours"]) + base.MARKER = "intervention-response-phase-aware-v2" + base.CELLS = { + f"tp4_mns{mns}": {"tp": 4, "mns": int(mns)} + for mns in manifest["engine"]["mns_endpoints"] + } + + +def load_state(path: Path, hard_cap: float) -> dict[str, Any]: + if path.exists(): + return json.loads(path.read_text(encoding="utf-8")) + return { + "schema": SCHEMA, + "status": "initialized", + "hard_cap_h20_hours": hard_cap, + "gpu_hours_total": 0.0, + "completed_sessions": 0, + "sessions": {}, + "failures": [], + "started_at": time.time(), + } + + +def save_state(path: Path, state: dict[str, Any]) -> None: + atomic_json(path, state) + + +def append_echo(run_root: Path, line: str) -> None: + run_root.mkdir(parents=True, exist_ok=True) + with (run_root / "launch-echo.log").open("a", encoding="utf-8") as target: + target.write(line + "\n") + print(line, flush=True) + + +def remaining_projection(session_count: int, index: int) -> float: + return (session_count - index) * SESSION_ESTIMATE_H20_HOURS + SAFETY_H20_HOURS + + +def start_server( + *, session: dict[str, Any], index: int, run_root: Path +) -> dict[str, Any]: + cell = f"tp4_mns{int(session['mns'])}" + gpus = (0, 1, 2, 3) + session_root = run_root / "sessions" / str(session["session"]) + session_root.mkdir(parents=True, exist_ok=True) + port = 8950 + index + command = base.server_command(cell, gpus, port) + with (session_root / "commands.log").open("a", encoding="utf-8") as log: + log.write(f"SERVER {shlex.join(command)}\n") + server_log = (session_root / "server.log").open("ab", buffering=0) + environment = os.environ.copy() + environment.update( + { + "CUDA_VISIBLE_DEVICES": "0,1,2,3", + "VLLM_OPPROF_DIR": str(session_root / "opprof"), + "OPPROF_PHASE6_MARKER": base.MARKER, + "AITUNER_ROOT": str(base.AITUNER), + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "PYTHONUNBUFFERED": "1", + } + ) + server = subprocess.Popen( + command, + cwd=base.SOURCE, + env=environment, + stdout=server_log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + base.OWNED_PGIDS.add(server.pid) + return { + "cell": cell, + "gpus": gpus, + "port": port, + "dir": session_root, + "server": server, + "server_handle": server_log, + "spawned_at": time.time(), + "results": [], + } + + +def client_command( + entry: dict[str, Any], + *, + study: str, + anchor: float, + output: Path, + warmup: bool, +) -> list[str]: + config = base.CELLS[entry["cell"]] + command = [ + "taskset", + "-c", + base.cpu_mask(entry["gpus"]), + str(base.VENV / "bin/python"), + str(base.CLIENT), + "warmup" if warmup else "run-anchor", + "--study", + study, + "--cell", + entry["cell"], + "--anchor", + str(anchor), + "--tp", + str(config["tp"]), + "--mns", + str(config["mns"]), + "--base-url", + f"http://127.0.0.1:{entry['port']}", + "--result-dir", + str(output), + "--disable-slo-early-stop", + ] + return command + + +def run_client( + *, + entry: dict[str, Any], + role: str, + study: str, + selection: dict[str, Any], + output: Path, + state: dict[str, Any], + warmup: bool = False, +) -> dict[str, Any]: + command = client_command( + entry, + study=study, + anchor=float(selection["anchor"]), + output=output, + warmup=warmup, + ) + with (entry["dir"] / "commands.log").open("a", encoding="utf-8") as log: + log.write(f"CLIENT role={role} {shlex.join(command)}\n") + handle = (output.parent / f"{output.name}.log").open("ab", buffering=0) + environment = os.environ.copy() + environment.update({"AITUNER_ROOT": str(base.AITUNER), "PYTHONUNBUFFERED": "1"}) + process = subprocess.Popen( + command, + cwd=base.WORKDIR, + env=environment, + stdout=handle, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + deadline = time.monotonic() + (180.0 if warmup else CLIENT_TIMEOUT_S) + try: + while process.poll() is None: + if time.monotonic() > deadline: + raise TimeoutError(f"client timeout: {entry['cell']} {role}") + if entry["server"].poll() is not None: + raise RuntimeError(f"server exited during {entry['cell']} {role}") + base.assert_no_other_compute() + if state["gpu_hours_total"] + base.live_gpu_hours([entry]) >= base.GPU_LIMIT: + raise RuntimeError("phase-aware pilot H20-hour hard cap reached") + time.sleep(1.0) + except Exception: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + process.wait(timeout=10.0) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=10.0) + raise + finally: + handle.close() + if process.returncode: + raise RuntimeError( + f"client failed: cell={entry['cell']} role={role} rc={process.returncode}" + ) + result = json.loads((output / "result.json").read_text(encoding="utf-8")) + validate_result( + result=result, + selection=selection, + role=role, + warmup=warmup, + ) + entry["results"].append( + { + "anchor": float(selection["anchor"]), + "dir": str(output), + "kind": result["kind"], + } + ) + return result + + +def validate_result( + *, result: dict[str, Any], selection: dict[str, Any], role: str, warmup: bool +) -> None: + if result.get("slo_early_stop_disabled") is not True: + raise RuntimeError(f"SLO early stop was not disabled: {role}") + if warmup: + if result["kind"] != "warmup" or int(result["selection"]["count"]) != 16: + raise RuntimeError(f"invalid warmup result: {role}") + if not all( + result["invariants"].get(key, False) + for key in ("warmup_16", "warmup_exact_16", "warmup_long") + ): + raise RuntimeError(f"warmup invariant failed: {role}") + return + if bool(result["early_stopped"]): + raise RuntimeError(f"uncensored run early-stopped: {role}") + if int(result["selection"]["count"]) != int(selection["selected_count"]): + raise RuntimeError(f"selection count mismatch: {role}") + for key, manifest_key in ( + ("request_id_order_sha256", "request_id_order_sha256"), + ("arrival_order_sha256", "arrival_order_sha256"), + ("raw_length_order_sha256", "input_length_order_sha256"), + ): + if result["selection"][key] != selection[manifest_key]: + raise RuntimeError(f"selection hash mismatch {key}: {role}") + if int(result["observed_count"]) != int(selection["selected_count"]): + raise RuntimeError(f"request accounting mismatch: {role}") + + +def execute_session( + *, + index: int, + session: dict[str, Any], + manifest: dict[str, Any], + run_root: Path, + state_path: Path, + state: dict[str, Any], +) -> None: + name = str(session["session"]) + if state["sessions"].get(name, {}).get("status") == "complete": + return + projection = remaining_projection(len(manifest["sessions"]), index) + if state["gpu_hours_total"] + projection > base.GPU_LIMIT: + state["status"] = "budget_projection_stop" + state["budget_stop"] = { + "before_session": name, + "spent_h20_hours": state["gpu_hours_total"], + "remaining_projection_h20_hours": projection, + "hard_cap_h20_hours": base.GPU_LIMIT, + } + save_state(state_path, state) + raise RuntimeError(f"projected pilot cost exceeds hard cap before {name}") + + replicate = str(session["replicate"]) + repetition = manifest["repetitions"][replicate] + echo = ( + f"PHASE_PILOT_SESSION_ECHO session={name} tp=4 mns={session['mns']} " + f"gpus=0-3 workload={manifest['source']['window_id']} duration_s=300 " + f"loads={','.join(repetition['load_order'])} disable_slo_early_stop=true " + f"spent_h20h={state['gpu_hours_total']:.6f} " + f"remaining_projection_h20h={projection:.3f} cap_h20h={base.GPU_LIMIT:.1f} " + f"manifest={run_root / 'pilot-manifest.json'}" + ) + append_echo(run_root, echo) + wait_all_idle() + session_state = { + "status": "starting", + "replicate": int(replicate), + "mns": int(session["mns"]), + "started_at": time.time(), + "runs": [], + } + state["status"] = "running" + state["sessions"][name] = session_state + save_state(state_path, state) + entry = start_server(session=session, index=index, run_root=run_root) + failure: Exception | None = None + try: + base.wait_ready(entry) + high = repetition["selections"]["high"] + session_state["status"] = "warmup" + save_state(state_path, state) + run_client( + entry=entry, + role="warmup", + study=repetition["study"], + selection=high, + output=entry["dir"] / "warmup", + state=state, + warmup=True, + ) + session_state["status"] = "burnin" + save_state(state_path, state) + burnin = manifest["burnin"] + burnin_result = run_client( + entry=entry, + role="burnin", + study=burnin["study"], + selection=burnin, + output=entry["dir"] / "burnin", + state=state, + ) + session_state["burnin"] = { + "pass_rate": burnin_result["pass_rate"], + "feasible": burnin_result["feasible"], + "elapsed_s": burnin_result["interval"]["elapsed_s"], + } + session_state["status"] = "measured" + save_state(state_path, state) + for level in repetition["load_order"]: + selection = repetition["selections"][level] + result = run_client( + entry=entry, + role=level, + study=repetition["study"], + selection=selection, + output=entry["dir"] / level, + state=state, + ) + session_state["runs"].append( + { + "level": level, + "selected_count": selection["selected_count"], + "offered_req_s_per_gpu": selection["offered_req_s_per_gpu"], + "pass_rate": result["pass_rate"], + "feasible": result["feasible"], + "elapsed_s": result["interval"]["elapsed_s"], + "early_stopped": result["early_stopped"], + } + ) + save_state(state_path, state) + session_state["status"] = "stopping" + save_state(state_path, state) + except Exception as error: # noqa: BLE001 + failure = error + finally: + try: + base.stop_entry(entry) + except Exception as error: # noqa: BLE001 + failure = failure or error + time.sleep(2.0) + try: + wait_all_idle() + except Exception as error: # noqa: BLE001 + failure = failure or error + + session_hours = base.live_gpu_hours([entry]) + state["gpu_hours_total"] += session_hours + session_state["gpu_hours"] = session_hours + if failure is not None: + session_state["status"] = "failed" + session_state["failure"] = repr(failure) + state["status"] = "failed" + state["failures"].append({"session": name, "failure": repr(failure)}) + save_state(state_path, state) + raise failure + validation = base.validate_cell(entry) + session_state["validation"] = validation + session_state["status"] = "complete" + session_state["completed_at"] = time.time() + state["completed_sessions"] += 1 + save_state(state_path, state) + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser() + result.add_argument("--manifest", type=Path, required=True) + result.add_argument("--run-root", type=Path, required=True) + result.add_argument("--aituner-root", type=Path, required=True) + result.add_argument("--vllm-source", type=Path, required=True) + result.add_argument("--venv", type=Path, required=True) + result.add_argument("--model", type=Path, required=True) + result.add_argument("--client", type=Path, required=True) + return result + + +def main() -> None: + args = parser().parse_args() + manifest = json.loads(args.manifest.read_text(encoding="utf-8")) + if manifest.get("schema") != "intervention-response-phase-aware-pilot-manifest-v2": + raise RuntimeError("unexpected phase-aware pilot manifest schema") + if manifest["status"] != "PASS": + raise RuntimeError("phase-aware pilot manifest did not pass preflight") + args.run_root.mkdir(parents=True, exist_ok=True) + copied_manifest = args.run_root / "pilot-manifest.json" + if not copied_manifest.exists(): + atomic_json(copied_manifest, manifest) + configure(args, manifest) + state_path = args.run_root / "controller-state.json" + state = load_state(state_path, base.GPU_LIMIT) + state["status"] = "running" + save_state(state_path, state) + for index, session in enumerate(manifest["sessions"]): + execute_session( + index=index, + session=session, + manifest=manifest, + run_root=args.run_root, + state_path=state_path, + state=state, + ) + state["status"] = "complete" + state["completed_at"] = time.time() + save_state(state_path, state) + wait_all_idle() + print( + json.dumps( + { + "status": state["status"], + "completed_sessions": state["completed_sessions"], + "gpu_hours_total": state["gpu_hours_total"], + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/intervention-response-v2/prepare_pilot.py b/runs/intervention-response-v2/prepare_pilot.py new file mode 100644 index 0000000..2b735d5 --- /dev/null +++ b/runs/intervention-response-v2/prepare_pilot.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +"""Prepare the uncensored 300-second TP4 matched pilot on dash0.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import sys +from pathlib import Path +from typing import Any + + +AITUNER_ROOT = Path(os.environ.get("AITUNER_ROOT", Path(__file__).resolve().parents[2])) +sys.path.insert(0, str(AITUNER_ROOT / "src")) + +from aituner.spec import load_study_spec # noqa: E402 +from aituner.trace import load_trace_requests, select_requests_for_threshold # noqa: E402 + + +SCHEMA = "intervention-response-phase-aware-pilot-manifest-v2" +TP = 4 +MNS_ENDPOINTS = (16, 64) +REPLAY_TIME_SCALE = 0.5 +EXPECTED_DURATION_S = 300.0 +SAFETY_DEADLINE_S = 360.0 +LOADS_REQ_S_GPU = {"low": 1.5, "mid": 2.125, "high": 3.125} +REPLICATE_ROLES = ("high1", "high2", "high3") +LOAD_ORDERS = { + 1: ("low", "mid", "high"), + 2: ("high", "low", "mid"), + 3: ("mid", "high", "low"), +} +SESSION_ORDER = ( + (1, 16), + (1, 64), + (2, 64), + (2, 16), + (3, 16), + (3, 64), +) + + +def atomic_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + os.replace(temporary, path) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def order_hash(values: list[str]) -> str: + return hashlib.sha256("\n".join(values).encode()).hexdigest() + + +def attainable_anchor(requests: list[Any], target_count: int) -> tuple[float, list[Any]]: + ordered = sorted(float(request.sampling_u) for request in requests) + if not ordered: + raise ValueError("no requests remain after study filtering") + if target_count <= 0 or target_count > len(ordered): + raise ValueError( + f"target count {target_count} is outside available range 1..{len(ordered)}" + ) + indices = sorted( + { + max(0, min(len(ordered) - 1, target_count - 1)), + max(0, min(len(ordered) - 1, target_count)), + } + ) + candidates = [] + for index in indices: + anchor = ordered[index] + selected = select_requests_for_threshold(requests, threshold=anchor) + candidates.append((abs(len(selected) - target_count), len(selected), anchor, selected)) + _error, _count, anchor, selected = min( + candidates, key=lambda item: (item[0], item[1], item[2]) + ) + return anchor, selected + + +def selection_record(selected: list[Any], *, duration_s: float) -> dict[str, Any]: + return { + "anchor": max(float(request.sampling_u) for request in selected), + "selected_count": len(selected), + "offered_req_s": len(selected) / duration_s, + "offered_req_s_per_gpu": len(selected) / duration_s / TP, + "request_id_order_sha256": order_hash([request.row_id for request in selected]), + "arrival_order_sha256": order_hash( + [f"{request.arrival_s:.12f}" for request in selected] + ), + "input_length_order_sha256": order_hash( + [str(request.prompt_tokens_hint) for request in selected] + ), + } + + +def materialize_study(source: Path, target: Path, *, replicate: int) -> Path: + payload = json.loads(source.read_text(encoding="utf-8")) + payload["study_id"] = f"phase-aware-telemetry-v2-rep{replicate}" + payload["hardware"]["host_candidates"] = ["dash0"] + payload["engine"]["engine_version"] = "0.24.1.dev3+opprof" + trace = payload["trace"] + trace["replay_time_scale"] = REPLAY_TIME_SCALE + trace["early_stop_max_lag_s"] = None + trace["early_stop_max_elapsed_s"] = SAFETY_DEADLINE_S + trace["restart_engine_after_early_stop"] = False + trace["adaptive_stop"] = {"enabled": False} + atomic_json(target, payload) + return target + + +def build_manifest( + *, base_manifest_path: Path, private_root: Path +) -> dict[str, Any]: + base = json.loads(base_manifest_path.read_text(encoding="utf-8")) + if base.get("schema") != "fidelity-prefix-pilot-manifest-v1": + raise ValueError("unexpected base P1 manifest schema") + + repetitions = {} + selection_hashes = [] + selected_ids_by_replicate: dict[int, set[str]] = {} + for replicate, role in enumerate(REPLICATE_ROLES, start=1): + source_study = Path(base["private"]["studies"][role]["tp4"]) + target_study = private_root / "studies" / f"rep{replicate}-tp4.json" + materialize_study(source_study, target_study, replicate=replicate) + study = load_study_spec(target_study) + window, requests = load_trace_requests(study, study_spec_path=target_study) + duration_s = float(window.window_end - window.window_start) + if not math.isclose( + duration_s, EXPECTED_DURATION_S, rel_tol=0.0, abs_tol=1e-9 + ): + raise ValueError( + f"rep{replicate}: replay duration {duration_s} != {EXPECTED_DURATION_S}" + ) + selections = {} + selected_ids_by_level: dict[str, set[str]] = {} + for level, target_rate in LOADS_REQ_S_GPU.items(): + target_count = round(target_rate * duration_s * TP) + anchor, selected = attainable_anchor(requests, target_count) + record = selection_record(selected, duration_s=duration_s) + record.update( + { + "anchor": anchor, + "target_count": target_count, + "target_req_s_per_gpu": target_rate, + } + ) + selections[level] = record + selected_ids_by_level[level] = {request.row_id for request in selected} + selection_hashes.append(record["request_id_order_sha256"]) + selected_ids_by_replicate[replicate] = selected_ids_by_level["high"] + repetitions[str(replicate)] = { + "source_role": role, + "study": str(target_study), + "study_sha256": sha256_file(target_study), + "duration_s": duration_s, + "load_order": list(LOAD_ORDERS[replicate]), + "selections": selections, + } + + burnin = base["cells"]["tp4_mns16"]["targets"]["low"]["selections"][ + "burnin" + ] + burnin = dict(burnin) + burnin["study_sha256"] = sha256_file(Path(burnin["study"])) + sessions = [ + { + "session": f"rep{replicate}-mns{mns}", + "replicate": replicate, + "mns": mns, + } + for replicate, mns in SESSION_ORDER + ] + invariants = { + "three_repetitions": len(repetitions) == 3, + "six_sessions": len(sessions) == 6, + "load_levels_three": all( + len(item["selections"]) == 3 for item in repetitions.values() + ), + "selection_hashes_unique": len(selection_hashes) == len(set(selection_hashes)), + "selection_sets_disjoint_across_repetitions": all( + not selected_ids_by_replicate[left] & selected_ids_by_replicate[right] + for left in selected_ids_by_replicate + for right in selected_ids_by_replicate + if left < right + ), + "all_counts_positive": all( + selection["selected_count"] > 0 + for item in repetitions.values() + for selection in item["selections"].values() + ), + } + red_flags = [name for name, passed in invariants.items() if not passed] + return { + "schema": SCHEMA, + "status": "PASS" if not red_flags else "STOP", + "source": { + "base_manifest": str(base_manifest_path), + "base_manifest_sha256": sha256_file(base_manifest_path), + "window_id": base["source"]["window_id"], + "source_trace": base["source"]["trace"], + "source_trace_sha256": base["source"]["trace_sha256"], + }, + "engine": { + "tp": TP, + "mns_endpoints": list(MNS_ENDPOINTS), + "replay_time_scale": REPLAY_TIME_SCALE, + "duration_s": EXPECTED_DURATION_S, + "safety_deadline_s": SAFETY_DEADLINE_S, + "disable_slo_early_stop": True, + }, + "burnin": burnin, + "repetitions": repetitions, + "sessions": sessions, + "checkpoints": { + "fractions": [0.1, 0.25, 0.5, 0.75, 1.0], + "seconds": [30.0, 75.0, 150.0, 225.0, 300.0], + }, + "budget": { + "hard_cap_h20_hours": 8.0, + "expected_wall_minutes": [95, 120], + "expected_h20_hours": [6.3, 8.0], + }, + "sanity": { + "red_flags": red_flags, + "invariants": invariants, + "selected_sets": len(selection_hashes), + "distinct_selected_sets": len(set(selection_hashes)), + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-manifest", type=Path, required=True) + parser.add_argument("--private-root", type=Path, required=True) + parser.add_argument("--public-manifest", type=Path, required=True) + args = parser.parse_args() + manifest = build_manifest( + base_manifest_path=args.base_manifest, private_root=args.private_root + ) + atomic_json(args.public_manifest, manifest) + print( + json.dumps( + { + "status": manifest["status"], + "manifest": str(args.public_manifest), + "sanity": manifest["sanity"], + }, + sort_keys=True, + ) + ) + if manifest["status"] != "PASS": + raise RuntimeError(f"phase-aware pilot preflight failed: {manifest['sanity']}") + + +if __name__ == "__main__": + main() diff --git a/runs/intervention-response-v2/test_analysis.py b/runs/intervention-response-v2/test_analysis.py new file mode 100644 index 0000000..db13f56 --- /dev/null +++ b/runs/intervention-response-v2/test_analysis.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import importlib.util +import math +from pathlib import Path +from types import SimpleNamespace + + +HERE = Path(__file__).resolve().parent + + +def load_module(): + spec = importlib.util.spec_from_file_location( + "intervention_response_phase_aware_v2", HERE / "analyze_existing.py" + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def load_prepare_module(): + spec = importlib.util.spec_from_file_location( + "intervention_response_phase_aware_prepare", HERE / "prepare_pilot.py" + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def load_controller_module(): + spec = importlib.util.spec_from_file_location( + "intervention_response_phase_aware_controller", HERE / "pilot_controller.py" + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def load_pilot_analysis_module(): + spec = importlib.util.spec_from_file_location( + "intervention_response_phase_aware_pilot_analysis", HERE / "analyze_pilot.py" + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def main() -> None: + module = load_module() + assert module.common_decile_fractions( + trace_duration_s=60.0, minimum_elapsed_s=19.448 + ) == (0.1, 0.2, 0.3) + assert module.common_decile_fractions( + trace_duration_s=60.0, minimum_elapsed_s=60.0 + )[-1] == 1.0 + stats = module.numeric([0.0, 1.0, 2.0]) + assert stats == { + "n": 3, + "min": 0.0, + "max": 2.0, + "distinct_n": 3, + "median": 1.0, + } + assert math.isclose(module._pearson([1.0, 2.0], [2.0, 4.0]), 1.0) + assert module._pearson([1.0, 1.0], [2.0, 3.0]) is None + prepare = load_prepare_module() + requests = [ + SimpleNamespace( + sampling_u=index / 10.0, + row_id=f"r{index}", + arrival_s=float(index), + prompt_tokens_hint=100 + index, + ) + for index in range(1, 6) + ] + _anchor, selected = prepare.attainable_anchor(requests, 3) + assert len(selected) == 3 + record = prepare.selection_record(selected, duration_s=3.0) + assert record["selected_count"] == 3 + assert record["offered_req_s_per_gpu"] == 0.25 + assert len(prepare.SESSION_ORDER) == 6 + assert {mns for _replicate, mns in prepare.SESSION_ORDER} == {16, 64} + controller = load_controller_module() + assert math.isclose(controller.remaining_projection(6, 0), 7.7) + assert math.isclose(controller.remaining_projection(6, 5), 1.45) + pilot_analysis = load_pilot_analysis_module() + stable = pilot_analysis.stable_adjacent_features( + [ + {"end_fraction": 0.1, "qualifying_response_features": ["queue"]}, + { + "end_fraction": 0.25, + "qualifying_response_features": ["kv", "queue"], + }, + {"end_fraction": 0.5, "qualifying_response_features": ["queue"]}, + ] + ) + assert stable == {"0.10->0.25": ["queue"], "0.25->0.50": ["queue"]} + print("phase-aware intervention response v2 analysis: PASS") + + +if __name__ == "__main__": + main() diff --git a/runs/opprof-phase6/opprof_phase6_client.py b/runs/opprof-phase6/opprof_phase6_client.py index 1dde17b..600fb3a 100644 --- a/runs/opprof-phase6/opprof_phase6_client.py +++ b/runs/opprof-phase6/opprof_phase6_client.py @@ -95,7 +95,11 @@ def run_replay(args: argparse.Namespace, *, warmup: bool) -> dict[str, Any]: base_url=args.base_url, timeout_s=study.engine.request_timeout_s, max_concurrency=study.trace.max_concurrency, - target_pass_rate=(0.0 if warmup else study.slo.target_pass_rate), + target_pass_rate=( + 0.0 + if warmup or args.disable_slo_early_stop + else study.slo.target_pass_rate + ), max_lag_s=study.trace.early_stop_max_lag_s, max_elapsed_s=( 120.0 if warmup else _probe_drain_deadline( @@ -162,6 +166,7 @@ def run_replay(args: argparse.Namespace, *, warmup: bool) -> dict[str, Any]: "feasible": bool(slo_summary["feasible"]), "early_stopped": early_stopped, "early_stop_reason": early_stop_reason, + "slo_early_stop_disabled": bool(args.disable_slo_early_stop), "ttft_ms": numeric([item.ttft_ms for item in outcomes]), "tpot_ms": numeric([item.tpot_ms for item in outcomes]), "invariants": { @@ -240,6 +245,7 @@ def parser() -> argparse.ArgumentParser: q.add_argument("--mns", type=int, required=True) q.add_argument("--base-url", required=True) q.add_argument("--result-dir", required=True) + q.add_argument("--disable-slo-early-stop", action="store_true") return p diff --git a/runs/opprof-phase6/test_phase6_tools.py b/runs/opprof-phase6/test_phase6_tools.py index 0cc338d..6c71c47 100644 --- a/runs/opprof-phase6/test_phase6_tools.py +++ b/runs/opprof-phase6/test_phase6_tools.py @@ -20,6 +20,7 @@ def main() -> None: controller = load("p6controller", HERE / "opprof_phase6_controller.py") solo = load("p6solo", HERE / "opprof_phase6_solo_controller.py") analysis = load("p6analysis", HERE / "analyze_phase6.py") + client = load("p6client", HERE / "opprof_phase6_client.py") assert len(controller.CELLS) == 12 primary = sum(3 if item.get("trap") else 2 for item in controller.CELLS.values()) assert primary == 25 @@ -36,6 +37,18 @@ def main() -> None: assert sum(solo.CELL_ESTIMATE.values()) + solo.SAFETY_HOURS == 3.44 assert solo.next_below([.1, .2, .3], {.2, .3}) == .1 assert solo.next_above([.1, .2, .3], {.1, .2}) == .3 + parsed = client.parser().parse_args([ + "run-anchor", + "--study", "study.json", + "--cell", "tp4_mns16", + "--anchor", "0.5", + "--tp", "4", + "--mns", "16", + "--base-url", "http://127.0.0.1:8000", + "--result-dir", "out", + "--disable-slo-early-stop", + ]) + assert parsed.disable_slo_early_stop is True print("phase6 tools: PASS") From 650f54b35ef300fb5c9c245163300996bee2ab8a Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 17:21:30 +0800 Subject: [PATCH 015/148] Merge disjoint bands for long replay pilot --- .../intervention-response-v2/prepare_pilot.py | 63 +++++++++++++++++-- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/runs/intervention-response-v2/prepare_pilot.py b/runs/intervention-response-v2/prepare_pilot.py index 2b735d5..18f1b78 100644 --- a/runs/intervention-response-v2/prepare_pilot.py +++ b/runs/intervention-response-v2/prepare_pilot.py @@ -27,7 +27,7 @@ REPLAY_TIME_SCALE = 0.5 EXPECTED_DURATION_S = 300.0 SAFETY_DEADLINE_S = 360.0 LOADS_REQ_S_GPU = {"low": 1.5, "mid": 2.125, "high": 3.125} -REPLICATE_ROLES = ("high1", "high2", "high3") +REPLICATE_ROLE_PAIRS = (("low1", "high1"), ("low2", "high2"), ("low3", "high3")) LOAD_ORDERS = { 1: ("low", "mid", "high"), 2: ("high", "low", "mid"), @@ -103,12 +103,50 @@ def selection_record(selected: list[Any], *, duration_s: float) -> dict[str, Any } -def materialize_study(source: Path, target: Path, *, replicate: int) -> Path: +def resolve_role_trace(base: dict[str, Any], role: str) -> Path: + windows_path = Path(base["private"]["windows"]) + windows = json.loads(windows_path.read_text(encoding="utf-8"))["windows"] + window_id = f"fidelity_pilot_{role}" + record = next(item for item in windows if item["window_id"] == window_id) + trace = Path(record["trace_file"]) + if not trace.is_absolute(): + trace = (windows_path.parent / trace).resolve() + return trace + + +def merge_role_traces(sources: tuple[Path, Path], target: Path) -> dict[str, Any]: + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_suffix(target.suffix + ".tmp") + rows = 0 + with temporary.open("w", encoding="utf-8") as output: + for source in sources: + with source.open(encoding="utf-8") as input_file: + for line in input_file: + if not line.strip(): + continue + json.loads(line) + output.write(line if line.endswith("\n") else line + "\n") + rows += 1 + os.replace(temporary, target) + return { + "path": str(target), + "sha256": sha256_file(target), + "bytes": target.stat().st_size, + "rows": rows, + "sources": [str(source) for source in sources], + "source_sha256": [sha256_file(source) for source in sources], + } + + +def materialize_study( + source: Path, target: Path, *, replicate: int, trace_override: Path +) -> Path: payload = json.loads(source.read_text(encoding="utf-8")) payload["study_id"] = f"phase-aware-telemetry-v2-rep{replicate}" payload["hardware"]["host_candidates"] = ["dash0"] payload["engine"]["engine_version"] = "0.24.1.dev3+opprof" trace = payload["trace"] + trace["trace_file_override"] = str(trace_override) trace["replay_time_scale"] = REPLAY_TIME_SCALE trace["early_stop_max_lag_s"] = None trace["early_stop_max_elapsed_s"] = SAFETY_DEADLINE_S @@ -128,10 +166,21 @@ def build_manifest( repetitions = {} selection_hashes = [] selected_ids_by_replicate: dict[int, set[str]] = {} - for replicate, role in enumerate(REPLICATE_ROLES, start=1): - source_study = Path(base["private"]["studies"][role]["tp4"]) + merged_traces = {} + for replicate, roles in enumerate(REPLICATE_ROLE_PAIRS, start=1): + source_study = Path(base["private"]["studies"][roles[1]]["tp4"]) + source_traces = tuple(resolve_role_trace(base, role) for role in roles) + merged_trace = private_root / "traces" / f"rep{replicate}.jsonl" + merged_traces[str(replicate)] = merge_role_traces( + source_traces, merged_trace + ) target_study = private_root / "studies" / f"rep{replicate}-tp4.json" - materialize_study(source_study, target_study, replicate=replicate) + materialize_study( + source_study, + target_study, + replicate=replicate, + trace_override=merged_trace, + ) study = load_study_spec(target_study) window, requests = load_trace_requests(study, study_spec_path=target_study) duration_s = float(window.window_end - window.window_start) @@ -159,7 +208,8 @@ def build_manifest( selection_hashes.append(record["request_id_order_sha256"]) selected_ids_by_replicate[replicate] = selected_ids_by_level["high"] repetitions[str(replicate)] = { - "source_role": role, + "source_roles": list(roles), + "merged_trace": merged_traces[str(replicate)], "study": str(target_study), "study_sha256": sha256_file(target_study), "duration_s": duration_s, @@ -219,6 +269,7 @@ def build_manifest( "disable_slo_early_stop": True, }, "burnin": burnin, + "private": {"merged_traces": merged_traces}, "repetitions": repetitions, "sessions": sessions, "checkpoints": { From 52a9dc13ddddf54937a733d171892acc3a7171d7 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 17:22:56 +0800 Subject: [PATCH 016/148] Give long replay bands stable request identities --- .../intervention-response-v2/prepare_pilot.py | 23 ++++++++++++++++--- .../intervention-response-v2/test_analysis.py | 9 ++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/runs/intervention-response-v2/prepare_pilot.py b/runs/intervention-response-v2/prepare_pilot.py index 18f1b78..e2d0266 100644 --- a/runs/intervention-response-v2/prepare_pilot.py +++ b/runs/intervention-response-v2/prepare_pilot.py @@ -114,18 +114,34 @@ def resolve_role_trace(base: dict[str, Any], role: str) -> Path: return trace +def private_request_id( + *, source_sha256: str, line_number: int, original_id: str +) -> str: + payload = f"{source_sha256}:{line_number}:{original_id}".encode() + return f"phase-v2-{hashlib.sha256(payload).hexdigest()}" + + def merge_role_traces(sources: tuple[Path, Path], target: Path) -> dict[str, Any]: target.parent.mkdir(parents=True, exist_ok=True) temporary = target.with_suffix(target.suffix + ".tmp") rows = 0 with temporary.open("w", encoding="utf-8") as output: for source in sources: + source_digest = sha256_file(source) with source.open(encoding="utf-8") as input_file: - for line in input_file: + for line_number, line in enumerate(input_file, start=1): if not line.strip(): continue - json.loads(line) - output.write(line if line.endswith("\n") else line + "\n") + row = json.loads(line) + original_id = str( + row.get("request_id") or row.get("id") or line_number + ) + row["request_id"] = private_request_id( + source_sha256=source_digest, + line_number=line_number, + original_id=original_id, + ) + output.write(json.dumps(row, ensure_ascii=False) + "\n") rows += 1 os.replace(temporary, target) return { @@ -135,6 +151,7 @@ def merge_role_traces(sources: tuple[Path, Path], target: Path) -> dict[str, Any "rows": rows, "sources": [str(source) for source in sources], "source_sha256": [sha256_file(source) for source in sources], + "request_id_scheme": "sha256(source_sha256:line_number:original_id)", } diff --git a/runs/intervention-response-v2/test_analysis.py b/runs/intervention-response-v2/test_analysis.py index db13f56..ffd3a19 100644 --- a/runs/intervention-response-v2/test_analysis.py +++ b/runs/intervention-response-v2/test_analysis.py @@ -85,6 +85,15 @@ def main() -> None: assert record["offered_req_s_per_gpu"] == 0.25 assert len(prepare.SESSION_ORDER) == 6 assert {mns for _replicate, mns in prepare.SESSION_ORDER} == {16, 64} + first_id = prepare.private_request_id( + source_sha256="a" * 64, line_number=1, original_id="1" + ) + assert first_id == prepare.private_request_id( + source_sha256="a" * 64, line_number=1, original_id="1" + ) + assert first_id != prepare.private_request_id( + source_sha256="b" * 64, line_number=1, original_id="1" + ) controller = load_controller_module() assert math.isclose(controller.remaining_projection(6, 0), 7.7) assert math.isclose(controller.remaining_projection(6, 5), 1.45) From 2afc6eeb8dde91ae1280f78fef40c9f4cc33ba71 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 17:25:45 +0800 Subject: [PATCH 017/148] Add dry run gate for long telemetry pilot --- .../pilot_controller.py | 105 +++++++++++++++++- .../intervention-response-v2/test_analysis.py | 20 ++++ 2 files changed, 120 insertions(+), 5 deletions(-) diff --git a/runs/intervention-response-v2/pilot_controller.py b/runs/intervention-response-v2/pilot_controller.py index 98b5903..cb089ff 100644 --- a/runs/intervention-response-v2/pilot_controller.py +++ b/runs/intervention-response-v2/pilot_controller.py @@ -62,6 +62,100 @@ def configure(args: argparse.Namespace, manifest: dict[str, Any]) -> None: } +def validate_inputs(args: argparse.Namespace, manifest: dict[str, Any]) -> None: + if manifest.get("schema") != "intervention-response-phase-aware-pilot-manifest-v2": + raise RuntimeError("unexpected phase-aware pilot manifest schema") + if manifest.get("status") != "PASS": + raise RuntimeError("phase-aware pilot manifest did not pass preflight") + failed_invariants = [ + name + for name, passed in manifest.get("sanity", {}).get("invariants", {}).items() + if not passed + ] + if failed_invariants: + raise RuntimeError(f"phase-aware pilot invariants failed: {failed_invariants}") + + required = { + "manifest": args.manifest, + "aituner_root": args.aituner_root, + "vllm_source": args.vllm_source, + "venv_python": args.venv / "bin/python", + "venv_vllm": args.venv / "bin/vllm", + "model": args.model, + "client": args.client, + "burnin_study": Path(manifest["burnin"]["study"]), + } + for replicate, repetition in manifest["repetitions"].items(): + required[f"rep{replicate}_study"] = Path(repetition["study"]) + required[f"rep{replicate}_trace"] = Path( + repetition["merged_trace"]["path"] + ) + missing = {name: str(path) for name, path in required.items() if not path.exists()} + if missing: + raise RuntimeError(f"phase-aware pilot input paths missing: {missing}") + + +def dry_run_plan(args: argparse.Namespace, manifest: dict[str, Any]) -> dict[str, Any]: + sessions = [] + for index, session in enumerate(manifest["sessions"]): + cell = f"tp4_mns{int(session['mns'])}" + entry = {"cell": cell, "gpus": (0, 1, 2, 3), "port": 8950 + index} + repetition = manifest["repetitions"][str(session["replicate"])] + session_root = args.run_root / "sessions" / str(session["session"]) + high = repetition["selections"]["high"] + commands = { + "server": base.server_command(cell, entry["gpus"], entry["port"]), + "warmup": client_command( + entry, + study=repetition["study"], + anchor=float(high["anchor"]), + output=session_root / "warmup", + warmup=True, + ), + "burnin": client_command( + entry, + study=manifest["burnin"]["study"], + anchor=float(manifest["burnin"]["anchor"]), + output=session_root / "burnin", + warmup=False, + ), + } + for level in repetition["load_order"]: + selection = repetition["selections"][level] + commands[level] = client_command( + entry, + study=repetition["study"], + anchor=float(selection["anchor"]), + output=session_root / level, + warmup=False, + ) + sessions.append( + { + "session": session["session"], + "replicate": int(session["replicate"]), + "mns": int(session["mns"]), + "port": entry["port"], + "load_order": repetition["load_order"], + "remaining_projection_h20_hours": remaining_projection( + len(manifest["sessions"]), index + ), + "commands": { + role: shlex.join(command) for role, command in commands.items() + }, + } + ) + return { + "schema": "intervention-response-phase-aware-pilot-dry-run-v2", + "status": "PASS", + "manifest": str(args.manifest), + "run_root": str(args.run_root), + "session_count": len(sessions), + "projected_h20_hours": remaining_projection(len(sessions), 0), + "hard_cap_h20_hours": float(manifest["budget"]["hard_cap_h20_hours"]), + "sessions": sessions, + } + + def load_state(path: Path, hard_cap: float) -> dict[str, Any]: if path.exists(): return json.loads(path.read_text(encoding="utf-8")) @@ -423,21 +517,22 @@ def parser() -> argparse.ArgumentParser: result.add_argument("--venv", type=Path, required=True) result.add_argument("--model", type=Path, required=True) result.add_argument("--client", type=Path, required=True) + result.add_argument("--dry-run", action="store_true") return result def main() -> None: args = parser().parse_args() manifest = json.loads(args.manifest.read_text(encoding="utf-8")) - if manifest.get("schema") != "intervention-response-phase-aware-pilot-manifest-v2": - raise RuntimeError("unexpected phase-aware pilot manifest schema") - if manifest["status"] != "PASS": - raise RuntimeError("phase-aware pilot manifest did not pass preflight") + validate_inputs(args, manifest) + configure(args, manifest) + if args.dry_run: + print(json.dumps(dry_run_plan(args, manifest), indent=2, sort_keys=True)) + return args.run_root.mkdir(parents=True, exist_ok=True) copied_manifest = args.run_root / "pilot-manifest.json" if not copied_manifest.exists(): atomic_json(copied_manifest, manifest) - configure(args, manifest) state_path = args.run_root / "controller-state.json" state = load_state(state_path, base.GPU_LIMIT) state["status"] = "running" diff --git a/runs/intervention-response-v2/test_analysis.py b/runs/intervention-response-v2/test_analysis.py index ffd3a19..932dec4 100644 --- a/runs/intervention-response-v2/test_analysis.py +++ b/runs/intervention-response-v2/test_analysis.py @@ -97,6 +97,26 @@ def main() -> None: controller = load_controller_module() assert math.isclose(controller.remaining_projection(6, 0), 7.7) assert math.isclose(controller.remaining_projection(6, 5), 1.45) + parsed = controller.parser().parse_args( + [ + "--manifest", + "/tmp/manifest.json", + "--run-root", + "/tmp/run", + "--aituner-root", + "/tmp/aituner", + "--vllm-source", + "/tmp/vllm", + "--venv", + "/tmp/venv", + "--model", + "/tmp/model", + "--client", + "/tmp/client.py", + "--dry-run", + ] + ) + assert parsed.dry_run is True pilot_analysis = load_pilot_analysis_module() stable = pilot_analysis.stable_adjacent_features( [ From c0b40af24ff3a30c60f4bfb36ad4de14b1a02c67 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 17:35:58 +0800 Subject: [PATCH 018/148] Enforce phase-stable telemetry pilot gates --- ...sponse-v2-phase-aware-protocol-20260714.md | 19 +- .../intervention-response-v2/analyze_pilot.py | 230 ++++++++++++++++-- .../intervention-response-v2/test_analysis.py | 64 +++++ 3 files changed, 292 insertions(+), 21 deletions(-) diff --git a/docs/intervention-response-v2-phase-aware-protocol-20260714.md b/docs/intervention-response-v2-phase-aware-protocol-20260714.md index d87025b..a29e535 100644 --- a/docs/intervention-response-v2-phase-aware-protocol-20260714.md +++ b/docs/intervention-response-v2-phase-aware-protocol-20260714.md @@ -58,6 +58,9 @@ The pilot is a mechanism gate, not paper evidence. run invalid but cannot manufacture a full-run label. - Cumulative checkpoints: 10%, 25%, 50%, 75%, and 100%, or 30/75/150/225/300 seconds. Quarter blocks are analyzed separately from cumulative means. +- A measured Layer-1 interval is complete only when its start-boundary, + end-boundary, and maximum internal record gaps are each at most one second; + timestamps must be monotonic. - Placement is serialized. Co-location remains forbidden because Phase 6 observed material co-location-induced outcome shifts. - Hard cap: 8 H20-hours including startup, warm-up, burn-in, invalid attempts, @@ -71,14 +74,16 @@ timestamps, full request accounting, idle GPUs before and after each session, and no co-resident GPU process. Mechanism evidence requires at least two telemetry features whose matched action -response exceeds same-config repeat noise at two consecutive checkpoints under -the unchanged v1 response thresholds. The same features must have a consistent -direction in at least two of the three load regimes. +response exceeds same-config repeat noise at the same pair of consecutive +checkpoints under the unchanged v1 response thresholds. Those features must +also have a consistent direction in at least two of the three load regimes. -Decision evidence additionally requires both action-efficacy classes and an -instrumentation feature that reaches leave-one-repetition-out balanced accuracy -at least 0.75 and exceeds the best external prefix outcome by at least 0.15. -Without label balance the pilot can adjudicate mechanism evidence only. +Decision evidence additionally requires both action-efficacy classes and at +least one of the phase-stable mechanism features to reach +leave-one-repetition-out balanced accuracy at least 0.75 and exceed the best +external prefix outcome by at least 0.15 at two adjacent predeclared +checkpoints from 25% onward. Without label balance the pilot can adjudicate +mechanism evidence only. No H20 run is launched if the local analyzer/tests, manifest preflight, GPU probe, command dry-run, projected cost, or cleanup plan fails. diff --git a/runs/intervention-response-v2/analyze_pilot.py b/runs/intervention-response-v2/analyze_pilot.py index edbfbe1..7cd539a 100644 --- a/runs/intervention-response-v2/analyze_pilot.py +++ b/runs/intervention-response-v2/analyze_pilot.py @@ -19,6 +19,7 @@ SCHEMA = "intervention-response-phase-aware-pilot-analysis-v2" EXPECTED_ACTION_PAIRS = 9 EXPECTED_REPEAT_PAIRS = 12 MIN_EFFICACY_CLASS = 3 +MAX_LAYER1_GAP_S = 1.0 def _load_p1(): @@ -54,6 +55,7 @@ def _trial_record( requests_path: Path, state: dict[str, float], outcome: dict[str, float], + telemetry_coverage: dict[str, float], ) -> dict[str, Any]: return { "trial_id": str(result_path.relative_to(run_root)), @@ -74,9 +76,34 @@ def _trial_record( "early_stopped": bool(result["early_stopped"]), "state": state, "outcome": outcome, + "telemetry_coverage": telemetry_coverage, } +def telemetry_coverage( + records: list[dict[str, Any]], *, start_ns: int, end_ns: int +) -> dict[str, float]: + layer1 = [record for record in records if "step_index" in record] + timestamps = [int(record["submit_mono_ns"]) for record in layer1] + if timestamps != sorted(timestamps): + raise ValueError("Layer-1 timestamps are not monotonic") + selected = [timestamp for timestamp in timestamps if start_ns <= timestamp <= end_ns] + if not selected: + raise ValueError("Layer-1 coverage interval contains no records") + internal_gaps = [ + (right - left) / 1e9 + for left, right in zip(selected, selected[1:], strict=False) + ] + coverage = { + "start_gap_s": (selected[0] - start_ns) / 1e9, + "end_gap_s": (end_ns - selected[-1]) / 1e9, + "max_internal_gap_s": max(internal_gaps, default=0.0), + } + if any(value > MAX_LAYER1_GAP_S for value in coverage.values()): + raise ValueError(f"Layer-1 coverage gap exceeds {MAX_LAYER1_GAP_S}s: {coverage}") + return coverage + + def validate_result_against_manifest( *, result: Mapping[str, Any], @@ -147,11 +174,16 @@ def load_interval_trials( start_ns = int(result["interval"]["start_mono_ns"]) for interval in intervals_s: start_s, end_s = interval + interval_start_ns = start_ns + int(start_s * 1e9) + interval_end_ns = start_ns + int(end_s * 1e9) + coverage = telemetry_coverage( + stream, start_ns=interval_start_ns, end_ns=interval_end_ns + ) state = P1.V0.flatten_state( P1.summarize_engine( stream, - start_ns=start_ns + int(start_s * 1e9), - end_ns=start_ns + int(end_s * 1e9), + start_ns=interval_start_ns, + end_ns=interval_end_ns, request_count=int(result["selection"]["count"]), ) ) @@ -166,6 +198,7 @@ def load_interval_trials( requests_path=requests_path, state=state, outcome=outcome, + telemetry_coverage=coverage, ) ) return by_interval, streams @@ -221,6 +254,31 @@ def analyze_window( * float(trial["outcome"]["completed_over_admitted"]) for trial in trials ] + state_vectors = { + tuple(round(float(trial["state"][feature]), 12) for feature in P1.V0.ALL_FEATURES) + for trial in trials + } + per_cell_vectors: dict[str, set[tuple[float, ...]]] = defaultdict(set) + for trial in trials: + per_cell_vectors[str(trial["cell"])].add( + tuple( + round(float(trial["state"][feature]), 12) + for feature in P1.V0.ALL_FEATURES + ) + ) + ratio_features = ( + "prefill_token_fraction", + "kv_usage_mean", + "kv_usage_max", + "graph_none_share", + "graph_full_share", + "graph_padding_fraction", + ) + nonnegative_features = tuple( + feature + for feature in P1.V0.ALL_FEATURES + if feature != "kv_usage_end_minus_start" + ) action_metadata = [ { "group": pair["group"], @@ -243,6 +301,25 @@ def analyze_window( for value in pair[key].values() ), "all_results_uncensored": all(not trial["early_stopped"] for trial in trials), + "state_vectors_not_all_identical": len(state_vectors) > 1, + "per_cell_state_vectors_not_all_identical": all( + len(vectors) > 1 for vectors in per_cell_vectors.values() + ), + "ratios_bounded": all( + 0.0 <= float(trial["state"][feature]) <= 1.0 + for trial in trials + for feature in ratio_features + ), + "nonnegative_counters": all( + float(trial["state"][feature]) >= 0.0 + for trial in trials + for feature in nonnegative_features + ), + "layer1_boundary_and_internal_gaps_bounded": all( + value <= MAX_LAYER1_GAP_S + for trial in trials + for value in trial["telemetry_coverage"].values() + ), } return { "start_s": start_s, @@ -255,6 +332,21 @@ def analyze_window( "action_pairs": len(actions), "repeat_pairs": len(repeats), "actions": action_metadata, + "trial_sanity": [ + { + "trial_id": trial["trial_id"], + "cell": trial["cell"], + "level": trial["level"], + "replicate": trial["replicate"], + "admitted_fraction": trial["outcome"]["admitted_fraction"], + "completed_fraction": ( + trial["outcome"]["admitted_fraction"] + * trial["outcome"]["completed_over_admitted"] + ), + "telemetry_coverage": trial["telemetry_coverage"], + } + for trial in trials + ], "response_statistics": response, "qualifying_response_features": response_qualifying, "efficacy": { @@ -290,6 +382,20 @@ def stable_adjacent_features(windows: list[dict[str, Any]]) -> dict[str, list[st return result +def stable_adjacent_efficacy_features( + windows: list[dict[str, Any]], +) -> dict[str, list[str]]: + eligible = [window for window in windows if window["end_fraction"] >= 0.25] + result = {} + for left, right in zip(eligible, eligible[1:], strict=False): + key = f"{left['end_fraction']:.2f}->{right['end_fraction']:.2f}" + result[key] = sorted( + set(left["efficacy"]["telemetry_qualifying_features"]) + & set(right["efficacy"]["telemetry_qualifying_features"]) + ) + return result + + def consistent_load_regimes( windows: list[dict[str, Any]], stable: dict[str, list[str]] ) -> dict[str, Any]: @@ -322,6 +428,96 @@ def consistent_load_regimes( return result +def mechanism_gate( + stable: Mapping[str, list[str]], load_consistency: Mapping[str, Mapping[str, Any]] +) -> dict[str, Any]: + by_transition = {} + for transition, features in stable.items(): + qualifying = sorted( + feature + for feature in features + if load_consistency[f"{transition}:{feature}"]["passes_two_regimes"] + ) + by_transition[transition] = qualifying + passing_transitions = sorted( + transition + for transition, features in by_transition.items() + if len(features) >= 2 + ) + return { + "minimum_features": 2, + "by_transition": by_transition, + "passing_transitions": passing_transitions, + "passes": bool(passing_transitions), + } + + +def controller_gate( + run_root: Path, manifest: Mapping[str, Any] +) -> dict[str, Any]: + path = run_root / "controller-state.json" + state = json.loads(path.read_text(encoding="utf-8")) + expected_sessions = {str(session["session"]) for session in manifest["sessions"]} + actual_sessions = set(state.get("sessions", {})) + session_invariants = [ + passed + for session in state.get("sessions", {}).values() + for passed in session.get("validation", {}).get("invariants", {}).values() + ] + invariants = { + "controller_complete": state.get("status") == "complete", + "completed_session_count": int(state.get("completed_sessions", -1)) + == len(expected_sessions), + "exact_session_set": actual_sessions == expected_sessions, + "all_sessions_complete": all( + session.get("status") == "complete" + for session in state.get("sessions", {}).values() + ), + "no_controller_failures": not state.get("failures"), + "under_h20_hour_cap": float(state.get("gpu_hours_total", math.inf)) + <= float(manifest["budget"]["hard_cap_h20_hours"]), + "all_stream_validation_invariants_pass": bool(session_invariants) + and all(session_invariants), + } + return { + "path": str(path.resolve()), + "sha256": sha256_file(path), + "gpu_hours_total": float(state.get("gpu_hours_total", math.nan)), + "invariants": invariants, + "red_flags": [name for name, passed in invariants.items() if not passed], + } + + +def cumulative_coverage_gate(windows: list[dict[str, Any]]) -> dict[str, Any]: + trajectories: dict[str, list[tuple[float, float]]] = defaultdict(list) + trial_sets = [] + for window in windows: + trial_sets.append({str(trial["trial_id"]) for trial in window["trial_sanity"]}) + for trial in window["trial_sanity"]: + trajectories[str(trial["trial_id"])].append( + ( + float(trial["admitted_fraction"]), + float(trial["completed_fraction"]), + ) + ) + invariants = { + "same_trials_at_every_checkpoint": bool(trial_sets) + and all(trial_set == trial_sets[0] for trial_set in trial_sets[1:]), + "admitted_fraction_monotonic": all( + all(right[0] + 1e-12 >= left[0] for left, right in zip(values, values[1:])) + for values in trajectories.values() + ), + "completed_fraction_monotonic": all( + all(right[1] + 1e-12 >= left[1] for left, right in zip(values, values[1:])) + for values in trajectories.values() + ), + } + return { + "invariants": invariants, + "red_flags": [name for name, passed in invariants.items() if not passed], + } + + def audit(*, run_root: Path, manifest_path: Path, output_path: Path) -> dict[str, Any]: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) if manifest.get("schema") != "intervention-response-phase-aware-pilot-manifest-v2": @@ -354,33 +550,34 @@ def audit(*, run_root: Path, manifest_path: Path, output_path: Path) -> dict[str ] stable = stable_adjacent_features(cumulative) load_consistency = consistent_load_regimes(cumulative, stable) + mechanism = mechanism_gate(stable, load_consistency) mechanism_features = sorted( { - key.split(":", 1)[1] - for key, item in load_consistency.items() - if item["passes_two_regimes"] + feature + for transition in mechanism["passing_transitions"] + for feature in mechanism["by_transition"][transition] } ) full = cumulative[-1] - efficacy_features = sorted( - set.intersection( - *( - set(window["efficacy"]["telemetry_qualifying_features"]) - for window in cumulative - if window["end_fraction"] >= 0.25 - ) - ) + efficacy_stable = stable_adjacent_efficacy_features(cumulative) + efficacy_candidates = sorted( + {feature for features in efficacy_stable.values() for feature in features} ) + efficacy_features = sorted(set(efficacy_candidates) & set(mechanism_features)) + controller = controller_gate(run_root, manifest) + coverage = cumulative_coverage_gate(cumulative) red_flags = sorted( { flag for window in [*cumulative, *quarter_blocks] for flag in window["sanity"]["red_flags"] } + | set(controller["red_flags"]) + | set(coverage["red_flags"]) ) if red_flags: decision = "STOP_DATA_INVALID" - elif not mechanism_features: + elif not mechanism["passes"]: decision = "STOP_NO_PHASE_STABLE_RESPONSE" elif not full["efficacy"]["label_balance_sufficient"]: decision = "MECHANISM_ONLY_NO_LABEL_BALANCE" @@ -394,9 +591,12 @@ def audit(*, run_root: Path, manifest_path: Path, output_path: Path) -> dict[str "decision": decision, "claim_boundary": "Development mechanism pilot; not a held-out paper claim.", "mechanism_features": mechanism_features, + "mechanism_gate": mechanism, "stable_adjacent_features": stable, "load_consistency": load_consistency, "stable_incremental_efficacy_features": efficacy_features, + "stable_incremental_efficacy_candidates": efficacy_candidates, + "stable_adjacent_efficacy_features": efficacy_stable, "cumulative": cumulative, "quarter_blocks": quarter_blocks, "provenance": { @@ -410,6 +610,8 @@ def audit(*, run_root: Path, manifest_path: Path, output_path: Path) -> dict[str "sanity": { "streams": len(streams), "stream_bytes": numeric(item["bytes"] for item in streams), + "controller": controller, + "cumulative_coverage": coverage, "red_flags": red_flags, }, } diff --git a/runs/intervention-response-v2/test_analysis.py b/runs/intervention-response-v2/test_analysis.py index 932dec4..03ef524 100644 --- a/runs/intervention-response-v2/test_analysis.py +++ b/runs/intervention-response-v2/test_analysis.py @@ -129,6 +129,70 @@ def main() -> None: ] ) assert stable == {"0.10->0.25": ["queue"], "0.25->0.50": ["queue"]} + load_consistency = { + "0.10->0.25:queue": {"passes_two_regimes": True}, + "0.25->0.50:queue": {"passes_two_regimes": True}, + } + mechanism = pilot_analysis.mechanism_gate(stable, load_consistency) + assert mechanism["passes"] is False + stable["0.25->0.50"].append("kv") + load_consistency["0.25->0.50:kv"] = {"passes_two_regimes": True} + mechanism = pilot_analysis.mechanism_gate(stable, load_consistency) + assert mechanism["passes"] is True + assert mechanism["passing_transitions"] == ["0.25->0.50"] + efficacy = pilot_analysis.stable_adjacent_efficacy_features( + [ + { + "end_fraction": 0.1, + "efficacy": {"telemetry_qualifying_features": ["early"]}, + }, + { + "end_fraction": 0.25, + "efficacy": {"telemetry_qualifying_features": ["queue"]}, + }, + { + "end_fraction": 0.5, + "efficacy": {"telemetry_qualifying_features": ["kv", "queue"]}, + }, + ] + ) + assert efficacy == {"0.25->0.50": ["queue"]} + coverage = pilot_analysis.telemetry_coverage( + [ + {"step_index": 1, "submit_mono_ns": 100_000_000}, + {"step_index": 2, "submit_mono_ns": 200_000_000}, + ], + start_ns=0, + end_ns=300_000_000, + ) + assert coverage == { + "start_gap_s": 0.1, + "end_gap_s": 0.1, + "max_internal_gap_s": 0.1, + } + coverage_gate = pilot_analysis.cumulative_coverage_gate( + [ + { + "trial_sanity": [ + { + "trial_id": "a", + "admitted_fraction": 0.25, + "completed_fraction": 0.2, + } + ] + }, + { + "trial_sanity": [ + { + "trial_id": "a", + "admitted_fraction": 0.5, + "completed_fraction": 0.4, + } + ] + }, + ] + ) + assert coverage_gate["red_flags"] == [] print("phase-aware intervention response v2 analysis: PASS") From 7fd956355072399b1b56946ecd8007da8f9d2fb0 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 17:52:21 +0800 Subject: [PATCH 019/148] Replace undrainable telemetry load with fresh rerun --- ...sponse-v2-phase-aware-protocol-20260714.md | 7 +- ...-response-v3-two-load-protocol-20260714.md | 73 +++++++++++++++++++ .../intervention-response-v2/analyze_pilot.py | 10 +-- .../pilot_controller.py | 32 +++++--- .../intervention-response-v2/prepare_pilot.py | 49 +++++++++---- .../intervention-response-v2/test_analysis.py | 14 +++- 6 files changed, 150 insertions(+), 35 deletions(-) create mode 100644 docs/intervention-response-v3-two-load-protocol-20260714.md diff --git a/docs/intervention-response-v2-phase-aware-protocol-20260714.md b/docs/intervention-response-v2-phase-aware-protocol-20260714.md index a29e535..4c72d37 100644 --- a/docs/intervention-response-v2-phase-aware-protocol-20260714.md +++ b/docs/intervention-response-v2-phase-aware-protocol-20260714.md @@ -1,9 +1,14 @@ # Phase-aware telemetry intervention-response v2 protocol -Status: **FROZEN BEFORE THE SYSTEMATIC 10% DECILE AUDIT**. +Status: **INVALID OPERATIONAL ATTEMPT; SUPERSEDED BY V3**. Date: 2026-07-14 (Asia/Singapore). +The first MNS=16 session timed out while draining the 3.125 requests/s/GPU +workload after its 300-second arrival window. It produced no high-load result, +and no MNS=64 endpoint was run. No comparative conclusion is drawn from this +attempt; see `intervention-response-v3-two-load-protocol-20260714.md`. + ## Correction to v0/v1 The 5/10-second analyses tested an ultra-early verifier. They did not test diff --git a/docs/intervention-response-v3-two-load-protocol-20260714.md b/docs/intervention-response-v3-two-load-protocol-20260714.md new file mode 100644 index 0000000..f5f83b7 --- /dev/null +++ b/docs/intervention-response-v3-two-load-protocol-20260714.md @@ -0,0 +1,73 @@ +# Phase-aware telemetry intervention-response v3 protocol + +Status: **FROZEN AFTER A NON-COMPARATIVE OPERATIONAL FAILURE AND BEFORE V3 RUNS**. + +Date: 2026-07-14 (Asia/Singapore). + +## Why v2 was invalid + +The first v2 session completed the 300-second arrival windows at 1.5 and 2.125 +requests/s/GPU. At 3.125 requests/s/GPU, MNS=16 could not drain the admitted +requests before the 450-second client timeout. The session produced no result +and no MNS=64 action endpoint was run. V2 is therefore an invalid operational +attempt, not evidence for or against the telemetry hypothesis. + +This failure was observed before any MNS action comparison. V3 excludes only +the unmeasurable overload point and reruns every retained point on fresh +servers; it does not reuse the completed v2 low/mid results. + +## Question and hypothesis + +Question: after enough replay time for queue, batch, and KV state to develop, +does an MNS intervention create telemetry responses that exceed workload-repeat +noise, and does any such response predict whether the intervention repairs the +full-run SLO outcome better than external prefix outcomes alone? + +Hypothesis: increasing MNS from 16 to 64 has little value at the 1.5 +requests/s/GPU control load but can repair the 2.125 requests/s/GPU pressure +load. Queue, running-set, batch, or KV telemetry should expose the difference +at stable replay phases. Label balance is an assumption to test, not a fact. + +## Frozen setup + +- Solo placement on dash0 GPU 0-3: 4 NVIDIA H20 GPUs, Qwen3-30B-A3B, patched + vLLM `0.24.1.dev3+opprof`, fixed TP=4. +- Action: MNS `16 -> 64`; all other engine and workload parameters fixed. +- Workload: `chat_w20260312_1000`, replay-time scale 0.5, hence 300 seconds. +- Loads per GPU: 1.5 control and 2.125 pressure requests/s. The failed 3.125 + overload point is excluded from V3 and retained only as a failure artifact. +- Three disjoint request bands. Each MNS action pair has exact request, + arrival, and input-length hashes. Endpoint order is A/B, B/A, A/B; load + order is low/mid, mid/low, low/mid. +- Every session starts a fresh server, then runs the accepted 16-request long + warm-up and bounded burn-in before measured runs. +- SLO-unrecoverable early stop is disabled. Measured results must cover the + full 300-second arrival window and must not be early-stopped. +- Cumulative checkpoints are 10%, 25%, 50%, 75%, and 100%; non-overlapping + quarter blocks are also reported. +- A Layer-1 interval is complete only if timestamps are monotonic and its + start, end, and maximum internal record gaps are each at most one second. +- Incremental V3 cap is the unused portion of the original 8 H20-hour cap. + The exact prior cost and V3 cap are machine-recorded in the manifest. + +## Frozen gates + +Data validity requires six uncensored sessions, six action pairs, eight +same-config repeat pairs, exact action-pair hashes, full request accounting, +zero Layer-1 drops, continuous coverage, all stream/footer invariants, no +co-resident compute process, idle GPUs before and after sessions, nonnegative +counters, bounded ratios, non-identical per-config state, and monotonic request +coverage across checkpoints. Any red flag stops analysis. + +Mechanism evidence requires at least two telemetry features to exceed the +unchanged v1 repeat-noise thresholds at the same pair of adjacent checkpoints. +Those features must have a direction consistent in both retained load regimes. + +Decision evidence additionally requires at least two positive and two negative +full-run action-efficacy labels, valid leave-one-repetition-out folds, and at +least one phase-stable mechanism feature whose balanced accuracy is at least +0.75 and at least 0.15 above the best external prefix-outcome feature at two +adjacent predeclared checkpoints from 25% onward. + +V3 remains a development mechanism pilot. Even `OPEN_E2E_POLICY_TEST` opens a +held-out tuning-policy experiment; it is not itself a paper performance claim. diff --git a/runs/intervention-response-v2/analyze_pilot.py b/runs/intervention-response-v2/analyze_pilot.py index 7cd539a..0586680 100644 --- a/runs/intervention-response-v2/analyze_pilot.py +++ b/runs/intervention-response-v2/analyze_pilot.py @@ -15,10 +15,10 @@ from typing import Any, Iterable, Mapping HERE = Path(__file__).resolve().parent P1_PATH = HERE.parent / "intervention-response-v0" / "analyze_p1.py" -SCHEMA = "intervention-response-phase-aware-pilot-analysis-v2" -EXPECTED_ACTION_PAIRS = 9 -EXPECTED_REPEAT_PAIRS = 12 -MIN_EFFICACY_CLASS = 3 +SCHEMA = "intervention-response-phase-aware-pilot-analysis-v3" +EXPECTED_ACTION_PAIRS = 6 +EXPECTED_REPEAT_PAIRS = 8 +MIN_EFFICACY_CLASS = 2 MAX_LAYER1_GAP_S = 1.0 @@ -520,7 +520,7 @@ def cumulative_coverage_gate(windows: list[dict[str, Any]]) -> dict[str, Any]: def audit(*, run_root: Path, manifest_path: Path, output_path: Path) -> dict[str, Any]: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - if manifest.get("schema") != "intervention-response-phase-aware-pilot-manifest-v2": + if manifest.get("schema") != "intervention-response-phase-aware-pilot-manifest-v3": raise ValueError("unexpected phase-aware pilot manifest schema") fractions = [float(value) for value in manifest["checkpoints"]["fractions"]] seconds = [float(value) for value in manifest["checkpoints"]["seconds"]] diff --git a/runs/intervention-response-v2/pilot_controller.py b/runs/intervention-response-v2/pilot_controller.py index cb089ff..fe41848 100644 --- a/runs/intervention-response-v2/pilot_controller.py +++ b/runs/intervention-response-v2/pilot_controller.py @@ -12,7 +12,7 @@ import subprocess import sys import time from pathlib import Path -from typing import Any +from typing import Any, Mapping HERE = Path(__file__).resolve().parent @@ -22,8 +22,8 @@ sys.path.insert(0, str(PHASE6)) import opprof_phase6_controller as base # noqa: E402 -SCHEMA = "intervention-response-phase-aware-pilot-state-v2" -SESSION_ESTIMATE_H20_HOURS = 1.25 +SCHEMA = "intervention-response-phase-aware-pilot-state-v3" +SESSION_ESTIMATE_H20_HOURS = 1.0 SAFETY_H20_HOURS = 0.20 CLIENT_TIMEOUT_S = 450.0 @@ -55,7 +55,7 @@ def configure(args: argparse.Namespace, manifest: dict[str, Any]) -> None: base.MODEL = args.model base.CLIENT = args.client base.GPU_LIMIT = float(manifest["budget"]["hard_cap_h20_hours"]) - base.MARKER = "intervention-response-phase-aware-v2" + base.MARKER = "intervention-response-phase-aware-v3" base.CELLS = { f"tp4_mns{mns}": {"tp": 4, "mns": int(mns)} for mns in manifest["engine"]["mns_endpoints"] @@ -63,10 +63,15 @@ def configure(args: argparse.Namespace, manifest: dict[str, Any]) -> None: def validate_inputs(args: argparse.Namespace, manifest: dict[str, Any]) -> None: - if manifest.get("schema") != "intervention-response-phase-aware-pilot-manifest-v2": + if manifest.get("schema") != "intervention-response-phase-aware-pilot-manifest-v3": raise RuntimeError("unexpected phase-aware pilot manifest schema") if manifest.get("status") != "PASS": raise RuntimeError("phase-aware pilot manifest did not pass preflight") + if abs( + float(manifest["budget"]["session_estimate_h20_hours"]) + - SESSION_ESTIMATE_H20_HOURS + ) > 1e-12: + raise RuntimeError("controller and manifest session-cost estimates disagree") failed_invariants = [ name for name, passed in manifest.get("sanity", {}).get("invariants", {}).items() @@ -95,6 +100,13 @@ def validate_inputs(args: argparse.Namespace, manifest: dict[str, Any]) -> None: raise RuntimeError(f"phase-aware pilot input paths missing: {missing}") +def warmup_selection(repetition: Mapping[str, Any]) -> Mapping[str, Any]: + return max( + repetition["selections"].values(), + key=lambda selection: float(selection["offered_req_s_per_gpu"]), + ) + + def dry_run_plan(args: argparse.Namespace, manifest: dict[str, Any]) -> dict[str, Any]: sessions = [] for index, session in enumerate(manifest["sessions"]): @@ -102,13 +114,13 @@ def dry_run_plan(args: argparse.Namespace, manifest: dict[str, Any]) -> dict[str entry = {"cell": cell, "gpus": (0, 1, 2, 3), "port": 8950 + index} repetition = manifest["repetitions"][str(session["replicate"])] session_root = args.run_root / "sessions" / str(session["session"]) - high = repetition["selections"]["high"] + warmup = warmup_selection(repetition) commands = { "server": base.server_command(cell, entry["gpus"], entry["port"]), "warmup": client_command( entry, study=repetition["study"], - anchor=float(high["anchor"]), + anchor=float(warmup["anchor"]), output=session_root / "warmup", warmup=True, ), @@ -145,7 +157,7 @@ def dry_run_plan(args: argparse.Namespace, manifest: dict[str, Any]) -> dict[str } ) return { - "schema": "intervention-response-phase-aware-pilot-dry-run-v2", + "schema": "intervention-response-phase-aware-pilot-dry-run-v3", "status": "PASS", "manifest": str(args.manifest), "run_root": str(args.run_root), @@ -423,14 +435,14 @@ def execute_session( failure: Exception | None = None try: base.wait_ready(entry) - high = repetition["selections"]["high"] + warmup = warmup_selection(repetition) session_state["status"] = "warmup" save_state(state_path, state) run_client( entry=entry, role="warmup", study=repetition["study"], - selection=high, + selection=warmup, output=entry["dir"] / "warmup", state=state, warmup=True, diff --git a/runs/intervention-response-v2/prepare_pilot.py b/runs/intervention-response-v2/prepare_pilot.py index e2d0266..14c5a00 100644 --- a/runs/intervention-response-v2/prepare_pilot.py +++ b/runs/intervention-response-v2/prepare_pilot.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Prepare the uncensored 300-second TP4 matched pilot on dash0.""" +"""Prepare the uncensored 300-second TP4 two-load matched pilot on dash0.""" from __future__ import annotations @@ -20,18 +20,18 @@ from aituner.spec import load_study_spec # noqa: E402 from aituner.trace import load_trace_requests, select_requests_for_threshold # noqa: E402 -SCHEMA = "intervention-response-phase-aware-pilot-manifest-v2" +SCHEMA = "intervention-response-phase-aware-pilot-manifest-v3" TP = 4 MNS_ENDPOINTS = (16, 64) REPLAY_TIME_SCALE = 0.5 EXPECTED_DURATION_S = 300.0 SAFETY_DEADLINE_S = 360.0 -LOADS_REQ_S_GPU = {"low": 1.5, "mid": 2.125, "high": 3.125} +LOADS_REQ_S_GPU = {"low": 1.5, "mid": 2.125} REPLICATE_ROLE_PAIRS = (("low1", "high1"), ("low2", "high2"), ("low3", "high3")) LOAD_ORDERS = { - 1: ("low", "mid", "high"), - 2: ("high", "low", "mid"), - 3: ("mid", "high", "low"), + 1: ("low", "mid"), + 2: ("mid", "low"), + 3: ("low", "mid"), } SESSION_ORDER = ( (1, 16), @@ -118,7 +118,7 @@ def private_request_id( *, source_sha256: str, line_number: int, original_id: str ) -> str: payload = f"{source_sha256}:{line_number}:{original_id}".encode() - return f"phase-v2-{hashlib.sha256(payload).hexdigest()}" + return f"phase-v3-{hashlib.sha256(payload).hexdigest()}" def merge_role_traces(sources: tuple[Path, Path], target: Path) -> dict[str, Any]: @@ -159,7 +159,7 @@ def materialize_study( source: Path, target: Path, *, replicate: int, trace_override: Path ) -> Path: payload = json.loads(source.read_text(encoding="utf-8")) - payload["study_id"] = f"phase-aware-telemetry-v2-rep{replicate}" + payload["study_id"] = f"phase-aware-telemetry-v3-rep{replicate}" payload["hardware"]["host_candidates"] = ["dash0"] payload["engine"]["engine_version"] = "0.24.1.dev3+opprof" trace = payload["trace"] @@ -174,7 +174,7 @@ def materialize_study( def build_manifest( - *, base_manifest_path: Path, private_root: Path + *, base_manifest_path: Path, private_root: Path, prior_attempt_state: Path ) -> dict[str, Any]: base = json.loads(base_manifest_path.read_text(encoding="utf-8")) if base.get("schema") != "fidelity-prefix-pilot-manifest-v1": @@ -223,7 +223,7 @@ def build_manifest( selections[level] = record selected_ids_by_level[level] = {request.row_id for request in selected} selection_hashes.append(record["request_id_order_sha256"]) - selected_ids_by_replicate[replicate] = selected_ids_by_level["high"] + selected_ids_by_replicate[replicate] = selected_ids_by_level["mid"] repetitions[str(replicate)] = { "source_roles": list(roles), "merged_trace": merged_traces[str(replicate)], @@ -250,8 +250,8 @@ def build_manifest( invariants = { "three_repetitions": len(repetitions) == 3, "six_sessions": len(sessions) == 6, - "load_levels_three": all( - len(item["selections"]) == 3 for item in repetitions.values() + "load_levels_two": all( + len(item["selections"]) == 2 for item in repetitions.values() ), "selection_hashes_unique": len(selection_hashes) == len(set(selection_hashes)), "selection_sets_disjoint_across_repetitions": all( @@ -267,6 +267,13 @@ def build_manifest( ), } red_flags = [name for name, passed in invariants.items() if not passed] + prior_state = json.loads(prior_attempt_state.read_text(encoding="utf-8")) + if prior_state.get("status") != "failed": + raise ValueError("prior three-load attempt was not recorded as failed") + prior_h20_hours = float(prior_state["gpu_hours_total"]) + incremental_cap_h20_hours = 8.0 - prior_h20_hours + if incremental_cap_h20_hours < 6.5: + raise ValueError("insufficient global H20-hour budget for the two-load rerun") return { "schema": SCHEMA, "status": "PASS" if not red_flags else "STOP", @@ -276,6 +283,10 @@ def build_manifest( "window_id": base["source"]["window_id"], "source_trace": base["source"]["trace"], "source_trace_sha256": base["source"]["trace_sha256"], + "prior_attempt_state": str(prior_attempt_state), + "prior_attempt_state_sha256": sha256_file(prior_attempt_state), + "prior_attempt_h20_hours": prior_h20_hours, + "prior_attempt_failure": prior_state["failures"], }, "engine": { "tp": TP, @@ -294,9 +305,12 @@ def build_manifest( "seconds": [30.0, 75.0, 150.0, 225.0, 300.0], }, "budget": { - "hard_cap_h20_hours": 8.0, - "expected_wall_minutes": [95, 120], - "expected_h20_hours": [6.3, 8.0], + "global_hard_cap_h20_hours": 8.0, + "prior_attempt_h20_hours": prior_h20_hours, + "hard_cap_h20_hours": incremental_cap_h20_hours, + "session_estimate_h20_hours": 1.0, + "expected_wall_minutes": [75, 95], + "expected_h20_hours": [4.8, incremental_cap_h20_hours], }, "sanity": { "red_flags": red_flags, @@ -312,9 +326,12 @@ def main() -> None: parser.add_argument("--base-manifest", type=Path, required=True) parser.add_argument("--private-root", type=Path, required=True) parser.add_argument("--public-manifest", type=Path, required=True) + parser.add_argument("--prior-attempt-state", type=Path, required=True) args = parser.parse_args() manifest = build_manifest( - base_manifest_path=args.base_manifest, private_root=args.private_root + base_manifest_path=args.base_manifest, + private_root=args.private_root, + prior_attempt_state=args.prior_attempt_state, ) atomic_json(args.public_manifest, manifest) print( diff --git a/runs/intervention-response-v2/test_analysis.py b/runs/intervention-response-v2/test_analysis.py index 03ef524..5a0c816 100644 --- a/runs/intervention-response-v2/test_analysis.py +++ b/runs/intervention-response-v2/test_analysis.py @@ -95,8 +95,8 @@ def main() -> None: source_sha256="b" * 64, line_number=1, original_id="1" ) controller = load_controller_module() - assert math.isclose(controller.remaining_projection(6, 0), 7.7) - assert math.isclose(controller.remaining_projection(6, 5), 1.45) + assert math.isclose(controller.remaining_projection(6, 0), 6.2) + assert math.isclose(controller.remaining_projection(6, 5), 1.2) parsed = controller.parser().parse_args( [ "--manifest", @@ -193,7 +193,15 @@ def main() -> None: ] ) assert coverage_gate["red_flags"] == [] - print("phase-aware intervention response v2 analysis: PASS") + assert controller.warmup_selection( + { + "selections": { + "low": {"offered_req_s_per_gpu": 1.5}, + "mid": {"offered_req_s_per_gpu": 2.125}, + } + } + )["offered_req_s_per_gpu"] == 2.125 + print("phase-aware intervention response v3 analysis: PASS") if __name__ == "__main__": From 26c2cdab2b7e4685fcaeffc3fe46e54430e83f02 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 19:18:57 +0800 Subject: [PATCH 020/148] Record phase-aware telemetry pilot result --- ...tervention-response-v3-results-20260714.md | 160 + ...failed-three-load-controller-state-v2.json | 49 + .../pilot-audit-v3.json | 16451 ++++++++++++++++ .../pilot-controller-state-v3.json | 342 + .../pilot-manifest-v3.json | 324 + .../intervention-response-v2/platform-v3.json | 46 + 6 files changed, 17372 insertions(+) create mode 100644 docs/intervention-response-v3-results-20260714.md create mode 100644 runs/intervention-response-v2/failed-three-load-controller-state-v2.json create mode 100644 runs/intervention-response-v2/pilot-audit-v3.json create mode 100644 runs/intervention-response-v2/pilot-controller-state-v3.json create mode 100644 runs/intervention-response-v2/pilot-manifest-v3.json create mode 100644 runs/intervention-response-v2/platform-v3.json diff --git a/docs/intervention-response-v3-results-20260714.md b/docs/intervention-response-v3-results-20260714.md new file mode 100644 index 0000000..75ac723 --- /dev/null +++ b/docs/intervention-response-v3-results-20260714.md @@ -0,0 +1,160 @@ +# Phase-aware telemetry intervention-response v3 results + +Date: 2026-07-14 (Asia/Singapore). + +Decision: **`STOP_NO_INCREMENTAL_TUNING_SIGNAL`**. + +## Claim tested + +After the replay has developed queue, batch, and KV state, does increasing MNS +from 16 to 64 create telemetry responses that exceed workload-repeat noise, and +does any such response identify whether the action repairs the full-run SLO +better than external prefix outcomes alone? + +The first clause passed. The second clause failed. Long-window telemetry is +mechanistically informative, but this pilot does not support its necessity for +tuning this action. + +## Setup + +- dash0 GPU 0-3: four NVIDIA H20 GPUs; Qwen3-30B-A3B; patched vLLM + `0.24.1.dev3+opprof`; TP=4. +- Action: MNS `16 -> 64` with exact request, arrival, and input-length hashes. +- Trace: `chat_w20260312_1000`, replay-time scale 0.5, 300 seconds. +- Loads: 1.5 and 2.125 requests/s/GPU; three disjoint bands; endpoint order + A/B, B/A, A/B; load order low/mid, mid/low, low/mid. +- Five cumulative checkpoints: 30, 75, 150, 225, and 300 seconds; four + non-overlapping quarter blocks. +- Six fresh-server sessions, 12 measured runs, six action pairs, and eight + same-config repeat pairs. + +The prior three-load attempt is not part of the result. Its MNS=16 workload at +3.125 requests/s/GPU could not drain by the 450-second client timeout and +produced no high-load result. V3 reran every retained point from scratch. + +## End-to-end outcome + +All three low-load pairs remained feasible (`true->true`, label 0). All three +pressure-load pairs changed from infeasible to feasible (`false->true`, label +1). MNS=16 pressure pass rates were 0.5604, 0.3145, and 0.2635; all three +MNS=64 pressure runs reached 1.0. This yielded a balanced 3/3 action label set. + +## Mechanism result + +No telemetry feature passed the action-versus-repeat gate at 10% or 25%. At +50%, graph padding first passed. At both 75% and 100%, graph padding and queue +waiting passed, satisfying the requirement for two features at the same pair +of adjacent checkpoints and with consistent directions in both load regimes. + +| Feature | Direction for MNS 16->64 | 75% effect/repeat median | 75% above repeat p95 | 100% effect/repeat median | 100% above repeat p95 | +|---|---:|---:|---:|---:|---:| +| `queue_waiting_mean` | lower | 1346.22 | 3/6 | 898.70 | 3/6 | +| `graph_padding_fraction` | higher | 5.00 | 4/6 | 5.74 | 5/6 | + +The very large queue effect/median-repeat ratios should not be read alone: its +repeat p95 was much larger than its repeat median, so the independent p95 +coverage criterion remained binding. Full-window queue-waiting deltas were +-0.19 to -0.32 at low load and -20.99 to -32.16 at pressure load. Graph +padding increased in every pair, by 0.00133-0.00217 at low load and +0.00705-0.00873 at pressure load. + +The mechanism is therefore a real tradeoff: larger MNS reduces queueing, +especially under pressure, while increasing CUDA-graph padding. + +## Tuning-signal result + +Leave-one-repetition-out balanced accuracy was evaluated against the best +external prefix-outcome feature at every predeclared checkpoint. + +| Replay phase | Best external BA | Best telemetry BA | Incremental telemetry gate | +|---:|---:|---:|---| +| 10% | 0.833 | 0.833 | fail | +| 25% | 1.000 | 1.000 | fail | +| 50% | 0.833 | 1.000 | pass at this checkpoint only | +| 75% | 1.000 | 1.000 | fail | +| 100% | 1.000 | 1.000 | fail | + +At 50%, several telemetry features exceeded the external baseline by at least +0.15, including the phase-stable mechanism feature +`graph_padding_fraction`. The advantage did not hold at either adjacent +checkpoint. Consequently no feature passed the frozen two-adjacent-phase +requirement. + +The important ordering is that external TTFT already classified the action +perfectly at 25%, whereas the robust two-feature mechanism response did not +emerge until 75%-100%. In this setup telemetry explains *why* MNS helps, but it +does not provide earlier or more reliable action selection than direct prefix +outcomes. + +## Research conclusion + +The 5/10-second negative result was indeed too narrow. It only ruled out an +ultra-early telemetry verifier; it did not rule out engine-state information. +The 300-second pilot finds a clear and reproducible queueing-versus-padding +response. + +However, this does not rescue the direct telemetry-guided tuning claim. For +this action and workload, the external signal is already as good or better +before the telemetry mechanism becomes stable. The project should therefore +not claim that engine instrumentation is necessary for tuning on this evidence, +and should not open an E2E policy test from this pilot. + +The narrower simulator-residual route remains logically open: telemetry may +explain why a simulator misranks real configurations even when direct online +outcomes can guide a tuner. That is a different hypothesis and was not tested +here. + +## Change and verification + +Change: absolute 5/10-second prefixes were replaced by phase-aware 30/75/150/ +225/300-second analysis; SLO early stop was disabled; full Layer-1 coverage, +hash, request-accounting, controller, and stream/footer gates were added. The +mechanism gate was corrected to require two features at the same adjacent +phase pair. + +Expected effect: distinguish “telemetry has not developed yet” from “telemetry +does not identify or improve the action decision.” + +Verification: five local analysis/controller test suites passed; remote +manifest preflight and command dry-run passed; six serialized sessions passed +all stream invariants; the analyzer was rerun and produced byte-identical +output. + +Result: mechanism evidence passed; incremental tuning evidence failed. Audit +SHA256: `45f6f248712f9cbd3ed72036837ff6dc5b5c14c0f2eb6ba5cd5daceb1aa4ddb7`. + +Remaining risk: this is a development pilot with one model, one TP, one action, +two retained loads, three request bands, and six action labels. It is adequate +to reject opening the next direct-policy stage, not to establish a universal +negative claim about telemetry. + +## Research-validity audit + +| Check | Verdict | Evidence / boundary | +|---|---|---| +| Real system and E2E outcome | PASS | Real H20/vLLM replay; full SLO outcome accompanies mechanism telemetry. | +| Matched action baseline | PASS | Exact request/arrival/length hashes for MNS 16 and 64; external prefix outcome is the decision baseline. | +| Repeats and order effects | PASS for pilot | Three disjoint bands; A/B, B/A, A/B endpoint order; counter-rotated load order. | +| Selective load removal | PASS with narrowed claim | The 3.125 load produced no result before any action comparison; the failure and cost are retained, and all kept points were freshly rerun. | +| Significance/generalization | NEEDS EVIDENCE for a paper claim | Only three bands, one model, one TP, one action, and six labels. This is explicitly a stage gate. | +| Calibration versus evaluation | NEEDS EVIDENCE for a positive policy claim | Frozen gates and leave-one-band-out folds reduce leakage, but a new workload/model hold-out is still required. | +| Platform/reproducibility | PASS | Commit, commands, manifest, controller state, platform fingerprint, raw remote paths, and audit hashes are recorded. | + +## Data sanity + +- Measured runs: n=12; elapsed 300.346-317.012 seconds; 12 distinct; pass rate + 0.2635-1.0 with 4 distinct values; selected requests 1800-2550 with 2 + distinct values. +- Sessions: n=6; 0.8413-0.8631 H20-hours; 6 distinct; Layer-1 records + 58,465-64,776; 6 distinct. V3 cost was 5.0924 H20-hours; total including + the invalid attempt was 6.4505, below the 8.0 cap. +- Labels: n=6; min/max 0/1; 2 distinct. Action pairs were 6 and repeat pairs + were 8 at every checkpoint. +- Coverage-gap observations: n=60; start gaps 0.0427-0.1247 seconds; end gaps + 0.00014-0.1705; maximum internal gaps 0.1695-0.6528, all below one second. +- Checked invariants: exact pair hashes and counts, all runs uncensored, full + request accounting, monotonic admitted/completed coverage, monotonic Layer-1 + timestamps, nonnegative counters, bounded ratios, non-identical per-config + states, contiguous step indices, zero drops, footer/sidecar agreement, no + controller failures, all sessions complete, GPU idle after completion. No + red flags were found. diff --git a/runs/intervention-response-v2/failed-three-load-controller-state-v2.json b/runs/intervention-response-v2/failed-three-load-controller-state-v2.json new file mode 100644 index 0000000..94a6dd4 --- /dev/null +++ b/runs/intervention-response-v2/failed-three-load-controller-state-v2.json @@ -0,0 +1,49 @@ +{ + "completed_sessions": 0, + "failures": [ + { + "failure": "TimeoutError('client timeout: tp4_mns16 high')", + "session": "rep1-mns16" + } + ], + "gpu_hours_total": 1.3580546813540988, + "hard_cap_h20_hours": 8.0, + "schema": "intervention-response-phase-aware-pilot-state-v2", + "sessions": { + "rep1-mns16": { + "burnin": { + "elapsed_s": 64.469448031, + "feasible": false, + "pass_rate": 0.5137254901960784 + }, + "failure": "TimeoutError('client timeout: tp4_mns16 high')", + "gpu_hours": 1.3580546813540988, + "mns": 16, + "replicate": 1, + "runs": [ + { + "early_stopped": false, + "elapsed_s": 301.114789132, + "feasible": true, + "level": "low", + "offered_req_s_per_gpu": 1.5, + "pass_rate": 1.0, + "selected_count": 1800 + }, + { + "early_stopped": false, + "elapsed_s": 313.040655133, + "feasible": false, + "level": "mid", + "offered_req_s_per_gpu": 2.125, + "pass_rate": 0.5631372549019608, + "selected_count": 2550 + } + ], + "started_at": 1784021277.1191542, + "status": "failed" + } + }, + "started_at": 1784021276.8224819, + "status": "failed" +} diff --git a/runs/intervention-response-v2/pilot-audit-v3.json b/runs/intervention-response-v2/pilot-audit-v3.json new file mode 100644 index 0000000..8a775e7 --- /dev/null +++ b/runs/intervention-response-v2/pilot-audit-v3.json @@ -0,0 +1,16451 @@ +{ + "claim_boundary": "Development mechanism pilot; not a held-out paper claim.", + "cumulative": [ + { + "action_pairs": 6, + "actions": [ + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.01629688095037396, + "tpot_mean_over_slo": -0.00616777464948251, + "ttft_max_over_slo_max": -0.04632652748841792, + "ttft_mean_over_slo_max": -0.002870656998408553 + }, + "delta_state": { + "batch_size.mean": -0.07015726376607656, + "batch_tokens.mean": -1.516200383603163, + "decode_batch_size.mean": -0.06904842848992132, + "graph_full_share": 0.0002648406614842447, + "graph_none_share": -0.002304071470614101, + "graph_padding_fraction": 0.0016774700470183774, + "kv_usage_end_minus_start": 1.8935114667439912e-05, + "kv_usage_max": 0.000835313152050321, + "kv_usage_mean": -0.0002023458881830597, + "preemptions": 0.0, + "prefill_token_fraction": -6.6475857588566e-06, + "queue_running_mean": -0.2196172835333332, + "queue_waiting_mean": -0.06376780166666667, + "scheduler_steps_per_s": 1.6666666666666714 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 1, + "request_hash": "af50f0c3edfc6f9fb74734b2501378fc8ba6fdd3dd3fe84f2bb1d68540eee69b", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "5856cc9ff24c9a984a2abecc59ab92c1f87b02b2a9dc0a47eae7a08462169373", + "result_sha256": "4e788371d6f24cab1be2bd4aed4bfe0354b5a5c23973bac9bf69ad0aa4c1b7d9", + "trial_id": "sessions/rep1-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "a91536c628053d3cf530bf0b8ebe51628cc41e71af31b90310d3307d1badc473", + "result_sha256": "2dd1d5ae610db6d738291032ab07a3ad73c46235b30e4ff38b87ce2715204bc6", + "trial_id": "sessions/rep1-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.000784313725490196, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.009216589861751112, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.29719366945920317, + "tpot_mean_over_slo": 0.03216772663644496, + "ttft_max_over_slo_max": -0.26695654933185625, + "ttft_mean_over_slo_max": -0.0567957568681583 + }, + "delta_state": { + "batch_size.mean": -1.148782542917619, + "batch_tokens.mean": -26.133196704277452, + "decode_batch_size.mean": -1.1386565766713446, + "graph_full_share": 0.012748972070391162, + "graph_none_share": -0.017569378163986613, + "graph_padding_fraction": 0.0029836610976256795, + "kv_usage_end_minus_start": 6.887268460797902e-05, + "kv_usage_max": 0.023285407613873432, + "kv_usage_mean": 0.0014582718656800941, + "preemptions": 0.0, + "prefill_token_fraction": -0.00016554148424863246, + "queue_running_mean": 1.499029866199999, + "queue_waiting_mean": -2.5166945028333334, + "scheduler_steps_per_s": 9.266666666666666 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 1, + "request_hash": "0bb61dbc9c26875e991d0d4f984134910d37463e5063f86ee960cf4f8aafb771", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.5603921568627451, + "mns": 16, + "requests_sha256": "8dcab1a24d009309a6dd54676e762ee4e74717ed530d19b3965f30feba32f193", + "result_sha256": "ae7cbca9785fa548acf8bd82646dfe53694ffd0b58d2ef6953e3d531d4d7ec4f", + "trial_id": "sessions/rep1-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "e7bf606cb77817e67a3e64e60f035a5fb344977a7991be741a81db8b3a8d28fe", + "result_sha256": "9604c8a0fe449d3798c6de566e4c804665d66b859a3b43c716dd5ecbb60bee1c", + "trial_id": "sessions/rep1-mns64/mid/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.051139209137158914, + "tpot_mean_over_slo": -0.01140034356948863, + "ttft_max_over_slo_max": -0.016116498833677426, + "ttft_mean_over_slo_max": -0.0032859918483368646 + }, + "delta_state": { + "batch_size.mean": -0.38049431384111454, + "batch_tokens.mean": -9.40007307911111, + "decode_batch_size.mean": -0.37729230010560766, + "graph_full_share": 0.003814274093207559, + "graph_none_share": -0.008561445619684624, + "graph_padding_fraction": 0.0018195892219365715, + "kv_usage_end_minus_start": 3.2185675094664745e-05, + "kv_usage_max": 0.0022799714777140823, + "kv_usage_mean": -0.00029136261801412065, + "preemptions": 0.0, + "prefill_token_fraction": 1.739291503155549e-06, + "queue_running_mean": -0.4335179698999996, + "queue_waiting_mean": -0.0903444594, + "scheduler_steps_per_s": 5.733333333333334 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 2, + "request_hash": "adc728f189a1ce0301609cdbae7d1dc2318609b13eafb94bcb90903bb7998eb1", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "a5ce6cd64890663ef30486701236af3de6169f86b72349928b07adafcb406bcb", + "result_sha256": "289f8f99f5056b26822bab5c1feb4252042c7852b80c5399271ddb1b6db25fbb", + "trial_id": "sessions/rep2-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "1c6931ea4a0913e2460e18c59f5a07e9c21cb2650454f2dd2f2997c416f63209", + "result_sha256": "9988f2b6c9e8c45144eb7b53aa1bea62942ae8fd36aba10c9e6bf5c8cbd65a50", + "trial_id": "sessions/rep2-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.007450980392156863, + "completed_over_admitted": 0.09411764705882353, + "completed_pass_rate": 0.0875576036866359, + "outstanding_over_admitted": -0.09411764705882353, + "tpot_max_over_slo": 0.23737520567053472, + "tpot_mean_over_slo": 0.05593989653787507, + "ttft_max_over_slo_max": -0.35981755234145874, + "ttft_mean_over_slo_max": -0.14934518008110995 + }, + "delta_state": { + "batch_size.mean": 1.556060197394805, + "batch_tokens.mean": 36.05112475298853, + "decode_batch_size.mean": 1.538767142928517, + "graph_full_share": -0.0035723945897613296, + "graph_none_share": -0.005640562586105874, + "graph_padding_fraction": 0.005841856698433652, + "kv_usage_end_minus_start": 0.0004291753466407755, + "kv_usage_max": 0.022672590909591506, + "kv_usage_mean": 0.004255843424978539, + "preemptions": 0.0, + "prefill_token_fraction": 0.00010886369563634091, + "queue_running_mean": 4.158758860866666, + "queue_waiting_mean": -8.736845286133335, + "scheduler_steps_per_s": -2.8666666666666742 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 2, + "request_hash": "56a0616b6b54abafd37875c7cb25f8639afef2706ccc55dfbe568f45859ea382", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.31450980392156863, + "mns": 16, + "requests_sha256": "e00c0d9acfb0c22624fdde79f4e8a0300fc2aba09ad4851908d68c1f46be30d3", + "result_sha256": "d081e1ac0143b4e8a63f4c60b58905320ca6e6163fa858e78f5643221fd87c2b", + "trial_id": "sessions/rep2-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "e1543f873745dbd1d3a1461e63549747b6e1517d63e281e6cdfa974d2e2798e4", + "result_sha256": "8556783dc3ba3274269500add337c75c493e3eece4f66260449549ee86506035", + "trial_id": "sessions/rep2-mns64/mid/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.006756756756756799, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.006756756756756757, + "tpot_max_over_slo": -0.041394997635621655, + "tpot_mean_over_slo": -0.01348315882501705, + "ttft_max_over_slo_max": -0.017940445327743276, + "ttft_mean_over_slo_max": -0.0005297892694696611 + }, + "delta_state": { + "batch_size.mean": -0.31669151594137457, + "batch_tokens.mean": -6.478151733400836, + "decode_batch_size.mean": -0.31462821691596865, + "graph_full_share": 0.0030818985907783425, + "graph_none_share": -0.003947353186013504, + "graph_padding_fraction": 0.0005071155621097952, + "kv_usage_end_minus_start": 1.1754955765441721e-05, + "kv_usage_max": -0.0005344961965740858, + "kv_usage_mean": -0.00033430782337439107, + "preemptions": 0.0, + "prefill_token_fraction": -1.9980466968050337e-05, + "queue_running_mean": -0.4079099217666666, + "queue_waiting_mean": -0.0088438692, + "scheduler_steps_per_s": 8.866666666666674 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 3, + "request_hash": "4993c26442fc6b5a4c09b6f576fd8b3cb1be7ce83b7a771e4a57ee7600de688d", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "de93d4364969393741a920ea72831b69003cbd9f10bcfd1809848fcf4e948549", + "result_sha256": "3b32b5ff3aea07cb7b7cf252ba23852542cb8429c83123fe772b2c8093c408a6", + "trial_id": "sessions/rep3-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "3623a99013725d7b42845bce09678368b3dc8d82e0ab26fad549c7eab8a6b6fe", + "result_sha256": "77766d9ec36ff11893feb73626768fa593160394f2b4720ff9253d1dfd5fa950", + "trial_id": "sessions/rep3-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.00462962962962965, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.004629629629629629, + "tpot_max_over_slo": 0.08779462519483128, + "tpot_mean_over_slo": -0.008887102176446698, + "ttft_max_over_slo_max": -0.1334900961664971, + "ttft_mean_over_slo_max": -0.024482643441387678 + }, + "delta_state": { + "batch_size.mean": -0.9935947147364157, + "batch_tokens.mean": -21.599624508861467, + "decode_batch_size.mean": -0.9860765047065438, + "graph_full_share": 0.01304670412975728, + "graph_none_share": -0.016958392743085346, + "graph_padding_fraction": 0.0017066418645361164, + "kv_usage_end_minus_start": -0.0004999534145386786, + "kv_usage_max": 0.01138222251747023, + "kv_usage_mean": -0.0002145679209778744, + "preemptions": 0.0, + "prefill_token_fraction": 4.875021747730024e-06, + "queue_running_mean": -0.33937291983333395, + "queue_waiting_mean": -1.0697946031666667, + "scheduler_steps_per_s": 9.5 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 3, + "request_hash": "2a2fabe2c4cf176aeb7e0d32fb8e7dbb1f27429a2e7a0cd18d7d186f23096f19", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.2635294117647059, + "mns": 16, + "requests_sha256": "c7fca9b2484c8ea68c6c11438b8e59db74fa60e6189dbb81d8834038156dbf68", + "result_sha256": "bb45674fa5e19ef18d8c13d08493f1a08a62fb682b5d5e8b7d8a1db2fa3b2ea6", + "trial_id": "sessions/rep3-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "d75e1711b8c0485cf1aceaa9e26ae079fa2adc5487d37f28006c01e83c5a782b", + "result_sha256": "1654b044d815beb3a5b8913fde4283b11507464a2de9f41b5ddc3cdb9f2d150b", + "trial_id": "sessions/rep3-mns64/mid/result.json" + } + } + ], + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 6, + "max": 0.10055555555555555, + "min": 0.08222222222222222, + "n": 12 + }, + "completed_fraction_of_total": { + "distinct_n": 9, + "max": 0.09450980392156863, + "min": 0.08055555555555556, + "n": 12 + } + }, + "efficacy": { + "cross_validation_possible": true, + "label_balance_sufficient": true, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "outcome_delta": { + "best_accuracy": 0.8333333333333334, + "best_balanced_accuracy": 0.8333333333333333, + "best_feature": "ttft_mean_over_slo_max", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.0037254901960784314, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.000392156862745098, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.000392156862745098, + "train_balanced_accuracy": 1.0 + } + ] + }, + "completed_over_admitted": { + "accuracy": 0.3333333333333333, + "balanced_accuracy": 0.3333333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.002314814814814825, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 0 + ], + "threshold": 0.0056931931931932245, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.047058823529411764, + "train_balanced_accuracy": 0.75 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.04377880184331795, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.004608294930875556, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.004608294930875556, + "train_balanced_accuracy": 1.0 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 0.3333333333333333, + "balanced_accuracy": 0.3333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.0023148148148148147, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 0 + ], + "threshold": -0.005693193193193193, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.047058823529411764, + "train_balanced_accuracy": 0.75 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0694669171659951, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.05204575307260262, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.14425720740384682, + "train_balanced_accuracy": 1.0 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": -0.010143722872967664, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.011185130500731874, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.012999975993481225, + "train_balanced_accuracy": 1.0 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.07571527074712019, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.08990831182745751, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.1566415384101371, + "train_balanced_accuracy": 1.0 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.013884317644862271, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.013676650219898116, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.03004087435824758, + "train_balanced_accuracy": 1.0 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "positive": 3 + }, + "positive": 3, + "telemetry_delta": { + "best_accuracy": 0.8333333333333334, + "best_balanced_accuracy": 0.8333333333333333, + "best_feature": "queue_waiting_mean", + "features": { + "decode_batch_size.mean": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.6120694630062742, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.6503523608112562, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.7348593572192978, + "train_balanced_accuracy": 0.75 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0011068787133229558, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0016920559557772469, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.0024016251597811255, + "train_balanced_accuracy": 1.0 + } + ] + }, + "kv_usage_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": -0.00025296526949599753, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.00027443787217613274, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.0006279629887485172, + "train_balanced_accuracy": 1.0 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 3.3071566254427864e-06, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": -8.862820055632881e-07, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 5.530149356974823e-05, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_running_mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": -0.3736414208000003, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.3736414208000003, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.6397062913333329, + "train_balanced_accuracy": 1.0 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5800695312833334, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5667812024166667, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -1.3035194811166666, + "train_balanced_accuracy": 1.0 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 1.43333333333333, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 9.06666666666667, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.6000000000000014, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "positive": 3 + }, + "telemetry_qualifying_features": [], + "transitions": { + "false->true": 3, + "true->true": 3 + } + }, + "end_fraction": 0.1, + "end_s": 30.0, + "qualifying_response_features": [], + "repeat_pairs": 8, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.6870445142887651, + "action_delta": { + "distinct_n": 6, + "max": 1.556060197394805, + "min": -1.148782542917619, + "n": 6 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 5, + "positive": 1, + "zero": 0 + }, + "effect_to_repeat_median": 0.3035211077938369, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 2.263580675764507, + "repeat_abs_p95": 4.395096149234066, + "repeat_delta": { + "distinct_n": 8, + "max": 3.5883262763875994, + "min": -4.829510696151395, + "n": 8 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 15.499848793986288, + "action_delta": { + "distinct_n": 6, + "max": 36.05112475298853, + "min": -26.133196704277452, + "n": 6 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 5, + "positive": 1, + "zero": 0 + }, + "effect_to_repeat_median": 0.22257219949412319, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 69.63964425573081, + "repeat_abs_p95": 109.27564134554108, + "repeat_delta": { + "distinct_n": 8, + "max": 90.40754990562004, + "min": -119.43538289011397, + "n": 8 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.6816844024060758, + "action_delta": { + "distinct_n": 6, + "max": 1.538767142928517, + "min": -1.1386565766713446, + "n": 6 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 5, + "positive": 1, + "zero": 0 + }, + "effect_to_repeat_median": 0.3037217743723986, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 2.2444370470793134, + "repeat_abs_p95": 4.356596129476966, + "repeat_delta": { + "distinct_n": 8, + "max": 3.5576095550798925, + "min": -4.7868196695369285, + "n": 8 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0036933343414844444, + "action_delta": { + "distinct_n": 6, + "max": 0.01304670412975728, + "min": -0.0035723945897613296, + "n": 6 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 1, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 0.20058285425846942, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.01841301119748373, + "repeat_abs_p95": 0.03154665462224765, + "repeat_delta": { + "distinct_n": 8, + "max": 0.035311983128397895, + "min": -0.024553901682254353, + "n": 8 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.007101004102895249, + "action_delta": { + "distinct_n": 6, + "max": -0.002304071470614101, + "min": -0.017569378163986613, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 0.43631188200171145, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.01627506468610771, + "repeat_abs_p95": 0.023219157547472946, + "repeat_delta": { + "distinct_n": 8, + "max": 0.01731224262368622, + "min": -0.024172434833284892, + "n": 8 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.001763115543236344, + "action_delta": { + "distinct_n": 6, + "max": 0.005841856698433652, + "min": 0.0005071155621097952, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 1.3596998263590112, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0012966946888252496, + "repeat_abs_p95": 0.004289292768312434, + "repeat_delta": { + "distinct_n": 8, + "max": 0.005183048738787855, + "min": -0.0026294602517152265, + "n": 8 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 5.052917985132188e-05, + "action_delta": { + "distinct_n": 6, + "max": 0.0004291753466407755, + "min": -0.0004999534145386786, + "n": 6 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 1, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 0.005664167631863443, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.008920848240273282, + "repeat_abs_p95": 0.01385760527350104, + "repeat_delta": { + "distinct_n": 8, + "max": 0.007292667470716885, + "min": -0.013864756025266267, + "n": 8 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.0068310969975921565, + "action_delta": { + "distinct_n": 6, + "max": 0.023285407613873432, + "min": -0.0005344961965740858, + "n": 6 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 1, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 2.5270151274703703, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0027032275839323217, + "repeat_abs_p95": 0.00993555673664627, + "repeat_delta": { + "distinct_n": 8, + "max": 0.006112144565506239, + "min": -0.011994317136490906, + "n": 8 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.00031283522069425586, + "action_delta": { + "distinct_n": 6, + "max": 0.004255843424978539, + "min": -0.00033430782337439107, + "n": 6 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 2, + "zero": 0 + }, + "effect_to_repeat_median": 0.09690654635110718, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0032282155589448643, + "repeat_abs_p95": 0.007211323545202779, + "repeat_delta": { + "distinct_n": 8, + "max": 0.005822121070260689, + "min": -0.007959355647094676, + "n": 8 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 6 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 6 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 8 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 1.3314026363453468e-05, + "action_delta": { + "distinct_n": 6, + "max": 0.00010886369563634091, + "min": -0.00016554148424863246, + "n": 6 + }, + "action_signs": { + "consistency": 0.5, + "negative": 3, + "positive": 3, + "zero": 0 + }, + "effect_to_repeat_median": 0.003330724358187011, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.003997336594584067, + "repeat_abs_p95": 0.008763072555415058, + "repeat_delta": { + "distinct_n": 8, + "max": 0.00594070169016081, + "min": -0.00877067447087998, + "n": 8 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.4207139458333331, + "action_delta": { + "distinct_n": 6, + "max": 4.158758860866666, + "min": -0.4335179698999996, + "n": 6 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 2, + "zero": 0 + }, + "effect_to_repeat_median": 0.15311155459778686, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 2.7477609181, + "repeat_abs_p95": 6.218551537058331, + "repeat_delta": { + "distinct_n": 8, + "max": 4.289906706066667, + "min": -7.2570525999, + "n": 8 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.5800695312833334, + "action_delta": { + "distinct_n": 6, + "max": -0.0088438692, + "min": -8.736845286133335, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 41.55250132555395, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.013959918483333333, + "repeat_abs_p95": 7.160673893926665, + "repeat_delta": { + "distinct_n": 8, + "max": 6.219170523933334, + "min": -7.667637247, + "n": 8 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 7.300000000000004, + "action_delta": { + "distinct_n": 6, + "max": 9.5, + "min": -2.8666666666666742, + "n": 6 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 1, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 0.3724489795918368, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 19.60000000000001, + "repeat_abs_p95": 26.37000000000001, + "repeat_delta": { + "distinct_n": 8, + "max": 27.466666666666683, + "min": -21.63333333333334, + "n": 8 + } + } + }, + "sanity": { + "invariants": { + "all_results_uncensored": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "layer1_boundary_and_internal_gaps_bounded": true, + "nonnegative_counters": true, + "per_cell_state_vectors_not_all_identical": true, + "ratios_bounded": true, + "state_vectors_not_all_identical": true + }, + "red_flags": [], + "trials": 12 + }, + "start_s": 0.0, + "trial_sanity": [ + { + "admitted_fraction": 0.08944444444444444, + "cell": "tp4_mns16", + "completed_fraction": 0.08499999999999999, + "level": "low", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.060456432, + "max_internal_gap_s": 0.648316718, + "start_gap_s": 0.042673865 + }, + "trial_id": "sessions/rep1-mns16/low/result.json" + }, + { + "admitted_fraction": 0.09058823529411765, + "cell": "tp4_mns16", + "completed_fraction": 0.08509803921568629, + "level": "mid", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.057208547, + "max_internal_gap_s": 0.183419024, + "start_gap_s": 0.042930986 + }, + "trial_id": "sessions/rep1-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.08944444444444444, + "cell": "tp4_mns64", + "completed_fraction": 0.08499999999999999, + "level": "low", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.058728784, + "max_internal_gap_s": 0.637319097, + "start_gap_s": 0.043186851 + }, + "trial_id": "sessions/rep1-mns64/low/result.json" + }, + { + "admitted_fraction": 0.09058823529411765, + "cell": "tp4_mns64", + "completed_fraction": 0.08509803921568629, + "level": "mid", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.05828864, + "max_internal_gap_s": 0.184687818, + "start_gap_s": 0.043187905 + }, + "trial_id": "sessions/rep1-mns64/mid/result.json" + }, + { + "admitted_fraction": 0.10055555555555555, + "cell": "tp4_mns64", + "completed_fraction": 0.09444444444444444, + "level": "low", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.003924877, + "max_internal_gap_s": 0.185686919, + "start_gap_s": 0.107508086 + }, + "trial_id": "sessions/rep2-mns64/low/result.json" + }, + { + "admitted_fraction": 0.1, + "cell": "tp4_mns64", + "completed_fraction": 0.09450980392156863, + "level": "mid", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.00433309, + "max_internal_gap_s": 0.17460317, + "start_gap_s": 0.107570476 + }, + "trial_id": "sessions/rep2-mns64/mid/result.json" + }, + { + "admitted_fraction": 0.10055555555555555, + "cell": "tp4_mns16", + "completed_fraction": 0.09444444444444444, + "level": "low", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.005845828, + "max_internal_gap_s": 0.174851445, + "start_gap_s": 0.107276851 + }, + "trial_id": "sessions/rep2-mns16/low/result.json" + }, + { + "admitted_fraction": 0.1, + "cell": "tp4_mns16", + "completed_fraction": 0.08509803921568627, + "level": "mid", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.00492273, + "max_internal_gap_s": 0.184443946, + "start_gap_s": 0.10804117 + }, + "trial_id": "sessions/rep2-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.08222222222222222, + "cell": "tp4_mns16", + "completed_fraction": 0.08055555555555556, + "level": "low", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.00040958, + "max_internal_gap_s": 0.611972913, + "start_gap_s": 0.120533656 + }, + "trial_id": "sessions/rep3-mns16/low/result.json" + }, + { + "admitted_fraction": 0.08470588235294117, + "cell": "tp4_mns16", + "completed_fraction": 0.08156862745098038, + "level": "mid", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.008592672, + "max_internal_gap_s": 0.169483992, + "start_gap_s": 0.124699964 + }, + "trial_id": "sessions/rep3-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.08222222222222222, + "cell": "tp4_mns64", + "completed_fraction": 0.08111111111111112, + "level": "low", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.000396531, + "max_internal_gap_s": 0.612067602, + "start_gap_s": 0.123493588 + }, + "trial_id": "sessions/rep3-mns64/low/result.json" + }, + { + "admitted_fraction": 0.08470588235294117, + "cell": "tp4_mns64", + "completed_fraction": 0.08196078431372548, + "level": "mid", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.009495956, + "max_internal_gap_s": 0.18072421, + "start_gap_s": 0.122478295 + }, + "trial_id": "sessions/rep3-mns64/mid/result.json" + } + ] + }, + { + "action_pairs": 6, + "actions": [ + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.07082632707989239, + "tpot_mean_over_slo": -0.00490180963817316, + "ttft_max_over_slo_max": -0.04153848900265682, + "ttft_mean_over_slo_max": -0.006891098180069163 + }, + "delta_state": { + "batch_size.mean": -0.2638104929979219, + "batch_tokens.mean": -5.845648825359461, + "decode_batch_size.mean": -0.26164909366742517, + "graph_full_share": 0.002180359902912321, + "graph_none_share": -0.006323059607193614, + "graph_padding_fraction": 0.0018335461795120685, + "kv_usage_end_minus_start": 1.026766711520466e-05, + "kv_usage_max": 0.004459034805309736, + "kv_usage_mean": -0.00014393587383104527, + "preemptions": 0.0, + "prefill_token_fraction": -2.725586323304441e-05, + "queue_running_mean": -0.18631020918666685, + "queue_waiting_mean": -0.20975605228, + "scheduler_steps_per_s": 4.9466666666666725 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 1, + "request_hash": "af50f0c3edfc6f9fb74734b2501378fc8ba6fdd3dd3fe84f2bb1d68540eee69b", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "5856cc9ff24c9a984a2abecc59ab92c1f87b02b2a9dc0a47eae7a08462169373", + "result_sha256": "4e788371d6f24cab1be2bd4aed4bfe0354b5a5c23973bac9bf69ad0aa4c1b7d9", + "trial_id": "sessions/rep1-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "a91536c628053d3cf530bf0b8ebe51628cc41e71af31b90310d3307d1badc473", + "result_sha256": "2dd1d5ae610db6d738291032ab07a3ad73c46235b30e4ff38b87ce2715204bc6", + "trial_id": "sessions/rep1-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.00196078431372549, + "completed_over_admitted": 0.01983471074380172, + "completed_pass_rate": 0.008665511265164683, + "outstanding_over_admitted": -0.01983471074380165, + "tpot_max_over_slo": 0.2753294892902449, + "tpot_mean_over_slo": 0.04011748202807386, + "ttft_max_over_slo_max": -0.2852395010025551, + "ttft_mean_over_slo_max": -0.06592333248276636 + }, + "delta_state": { + "batch_size.mean": -0.8231037548470876, + "batch_tokens.mean": -19.74330932592642, + "decode_batch_size.mean": -0.8152070406366647, + "graph_full_share": 0.00839719842479858, + "graph_none_share": -0.018146448159773024, + "graph_padding_fraction": 0.005340401585168801, + "kv_usage_end_minus_start": -0.004438767356134465, + "kv_usage_max": 0.027975123151637415, + "kv_usage_mean": 0.002385274257794787, + "preemptions": 0.0, + "prefill_token_fraction": -0.00025035500653580733, + "queue_running_mean": 2.133982110813333, + "queue_waiting_mean": -3.33810440732, + "scheduler_steps_per_s": 6.320000000000007 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 1, + "request_hash": "0bb61dbc9c26875e991d0d4f984134910d37463e5063f86ee960cf4f8aafb771", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.5603921568627451, + "mns": 16, + "requests_sha256": "8dcab1a24d009309a6dd54676e762ee4e74717ed530d19b3965f30feba32f193", + "result_sha256": "ae7cbca9785fa548acf8bd82646dfe53694ffd0b58d2ef6953e3d531d4d7ec4f", + "trial_id": "sessions/rep1-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "e7bf606cb77817e67a3e64e60f035a5fb344977a7991be741a81db8b3a8d28fe", + "result_sha256": "9604c8a0fe449d3798c6de566e4c804665d66b859a3b43c716dd5ecbb60bee1c", + "trial_id": "sessions/rep1-mns64/mid/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.004796163069544446, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.004796163069544366, + "tpot_max_over_slo": 0.051139209137158914, + "tpot_mean_over_slo": -0.0074211636478002385, + "ttft_max_over_slo_max": -0.03542824399967988, + "ttft_mean_over_slo_max": -0.0033018737683347885 + }, + "delta_state": { + "batch_size.mean": -0.2685691864197386, + "batch_tokens.mean": -6.959429182398708, + "decode_batch_size.mean": -0.26598172047692614, + "graph_full_share": 0.0028314394428595735, + "graph_none_share": -0.007209322700149463, + "graph_padding_fraction": 0.0012877200176958283, + "kv_usage_end_minus_start": -0.0011333814702583211, + "kv_usage_max": 0.0022799714777140823, + "kv_usage_mean": -0.0001951815494633533, + "preemptions": 0.0, + "prefill_token_fraction": -0.00014483809619469934, + "queue_running_mean": -0.2747586459066662, + "queue_waiting_mean": -0.08017956545333334, + "scheduler_steps_per_s": 5.586666666666659 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 2, + "request_hash": "adc728f189a1ce0301609cdbae7d1dc2318609b13eafb94bcb90903bb7998eb1", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "a5ce6cd64890663ef30486701236af3de6169f86b72349928b07adafcb406bcb", + "result_sha256": "289f8f99f5056b26822bab5c1feb4252042c7852b80c5399271ddb1b6db25fbb", + "trial_id": "sessions/rep2-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "1c6931ea4a0913e2460e18c59f5a07e9c21cb2650454f2dd2f2997c416f63209", + "result_sha256": "9988f2b6c9e8c45144eb7b53aa1bea62942ae8fd36aba10c9e6bf5c8cbd65a50", + "trial_id": "sessions/rep2-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.02980392156862745, + "completed_over_admitted": 0.018394648829431426, + "completed_pass_rate": 0.13356766256590513, + "outstanding_over_admitted": -0.01839464882943144, + "tpot_max_over_slo": 0.23737520567053472, + "tpot_mean_over_slo": 0.010490282370595683, + "ttft_max_over_slo_max": -0.38453158099825185, + "ttft_mean_over_slo_max": -0.19098879755079737 + }, + "delta_state": { + "batch_size.mean": -1.413753104794269, + "batch_tokens.mean": -30.311859850911844, + "decode_batch_size.mean": -1.4061952013981784, + "graph_full_share": 0.021360307900858633, + "graph_none_share": -0.03011422871811248, + "graph_padding_fraction": 0.007512227776197788, + "kv_usage_end_minus_start": 0.0021360977816522464, + "kv_usage_max": 0.022672590909591506, + "kv_usage_mean": 0.0010254846876152635, + "preemptions": 0.0, + "prefill_token_fraction": 0.0003313271687591657, + "queue_running_mean": 0.7838954335199997, + "queue_waiting_mean": -9.07047515872, + "scheduler_steps_per_s": 8.959999999999994 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 2, + "request_hash": "56a0616b6b54abafd37875c7cb25f8639afef2706ccc55dfbe568f45859ea382", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.31450980392156863, + "mns": 16, + "requests_sha256": "e00c0d9acfb0c22624fdde79f4e8a0300fc2aba09ad4851908d68c1f46be30d3", + "result_sha256": "d081e1ac0143b4e8a63f4c60b58905320ca6e6163fa858e78f5643221fd87c2b", + "trial_id": "sessions/rep2-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "e1543f873745dbd1d3a1461e63549747b6e1517d63e281e6cdfa974d2e2798e4", + "result_sha256": "8556783dc3ba3274269500add337c75c493e3eece4f66260449549ee86506035", + "trial_id": "sessions/rep2-mns64/mid/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0024154589371980784, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.0024154589371980714, + "tpot_max_over_slo": -0.006984168036628269, + "tpot_mean_over_slo": -0.01111313281352852, + "ttft_max_over_slo_max": -0.017940445327743276, + "ttft_mean_over_slo_max": -0.0018007220652844075 + }, + "delta_state": { + "batch_size.mean": -0.3420664768336481, + "batch_tokens.mean": -6.950334498514508, + "decode_batch_size.mean": -0.3395672760130166, + "graph_full_share": 0.0029764291623558536, + "graph_none_share": -0.005728618501492012, + "graph_padding_fraction": 0.0005459147599019987, + "kv_usage_end_minus_start": -0.003587734700701506, + "kv_usage_max": 3.276805657914661e-05, + "kv_usage_mean": -0.0002979372353175526, + "preemptions": 0.0, + "prefill_token_fraction": -6.130606240972725e-05, + "queue_running_mean": -0.4039724217999998, + "queue_waiting_mean": -0.030365675946666667, + "scheduler_steps_per_s": 7.826666666666668 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 3, + "request_hash": "4993c26442fc6b5a4c09b6f576fd8b3cb1be7ce83b7a771e4a57ee7600de688d", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "de93d4364969393741a920ea72831b69003cbd9f10bcfd1809848fcf4e948549", + "result_sha256": "3b32b5ff3aea07cb7b7cf252ba23852542cb8429c83123fe772b2c8093c408a6", + "trial_id": "sessions/rep3-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "3623a99013725d7b42845bce09678368b3dc8d82e0ab26fad549c7eab8a6b6fe", + "result_sha256": "77766d9ec36ff11893feb73626768fa593160394f2b4720ff9253d1dfd5fa950", + "trial_id": "sessions/rep3-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.013628620102214661, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.013628620102214647, + "tpot_max_over_slo": 0.08779462519483128, + "tpot_mean_over_slo": -0.005083324937856265, + "ttft_max_over_slo_max": -0.17233173300822574, + "ttft_mean_over_slo_max": -0.03508650576710582 + }, + "delta_state": { + "batch_size.mean": -0.5655282844513536, + "batch_tokens.mean": -11.412062377336781, + "decode_batch_size.mean": -0.562525695471205, + "graph_full_share": 0.009956297228293054, + "graph_none_share": -0.01796699078506772, + "graph_padding_fraction": 0.0038365149210898572, + "kv_usage_end_minus_start": 0.005666174682015579, + "kv_usage_max": 0.01138222251747023, + "kv_usage_mean": 0.0003053257977473668, + "preemptions": 0.0, + "prefill_token_fraction": -1.837494489209668e-05, + "queue_running_mean": 0.1581681952266667, + "queue_waiting_mean": -2.0748791364133337, + "scheduler_steps_per_s": 7.079999999999998 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 3, + "request_hash": "2a2fabe2c4cf176aeb7e0d32fb8e7dbb1f27429a2e7a0cd18d7d186f23096f19", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.2635294117647059, + "mns": 16, + "requests_sha256": "c7fca9b2484c8ea68c6c11438b8e59db74fa60e6189dbb81d8834038156dbf68", + "result_sha256": "bb45674fa5e19ef18d8c13d08493f1a08a62fb682b5d5e8b7d8a1db2fa3b2ea6", + "trial_id": "sessions/rep3-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "d75e1711b8c0485cf1aceaa9e26ae079fa2adc5487d37f28006c01e83c5a782b", + "result_sha256": "1654b044d815beb3a5b8913fde4283b11507464a2de9f41b5ddc3cdb9f2d150b", + "trial_id": "sessions/rep3-mns64/mid/result.json" + } + } + ], + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 6, + "max": 0.23777777777777778, + "min": 0.23, + "n": 12 + }, + "completed_fraction_of_total": { + "distinct_n": 11, + "max": 0.23555555555555557, + "min": 0.2152941176470588, + "n": 12 + } + }, + "efficacy": { + "cross_validation_possible": true, + "label_balance_sufficient": true, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "outcome_delta": { + "best_accuracy": 1.0, + "best_balanced_accuracy": 1.0, + "best_feature": "ttft_max_over_slo_max", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.014901960784313726, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.000980392156862745, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.000980392156862745, + "train_balanced_accuracy": 1.0 + } + ] + }, + "completed_over_admitted": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.009212391585879554, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.00802203951970637, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.011595405949487936, + "train_balanced_accuracy": 1.0 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.06678383128295257, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.004332755632582341, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.004332755632582341, + "train_balanced_accuracy": 1.0 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.009212391585879507, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.00802203951970636, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.011595405949487903, + "train_balanced_accuracy": 1.0 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0694669171659951, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.07931047613736183, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.15410076637521355, + "train_balanced_accuracy": 1.0 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": -0.006252244292828252, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": -0.008098228875692393, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.0027942363662112613, + "train_balanced_accuracy": 1.0 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.1038799885039528, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.10693511100544129, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.16338899500260595, + "train_balanced_accuracy": 1.0 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.019194189767720304, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.02098880197358749, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.03640721533141776, + "train_balanced_accuracy": 1.0 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "positive": 3 + }, + "positive": 3, + "telemetry_delta": { + "best_accuracy": 1.0, + "best_balanced_accuracy": 1.0, + "best_feature": "queue_waiting_mean", + "features": { + "decode_batch_size.mean": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.4510464857421108, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.4510464857421108, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5405943805567954, + "train_balanced_accuracy": 1.0 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0025621174693928428, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.002835030550300963, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.003586973882340435, + "train_balanced_accuracy": 1.0 + } + ] + }, + "kv_usage_mean": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 5.5072124142006764e-05, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 8.069496195816078e-05, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.0004407744068921091, + "train_balanced_accuracy": 1.0 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 0 + ], + "threshold": -3.984050365091196e-05, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -2.2815404062570543e-05, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.00015203565276306064, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_running_mean": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.05829522533999976, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.014071006980000078, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.2987926121666664, + "train_balanced_accuracy": 1.0 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -1.0775293509333335, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -1.1423175943466668, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -1.7739302298, + "train_balanced_accuracy": 1.0 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 6.333333333333329, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 5.63333333333334, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 5.953333333333333, + "train_balanced_accuracy": 1.0 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "positive": 3 + }, + "telemetry_qualifying_features": [], + "transitions": { + "false->true": 3, + "true->true": 3 + } + }, + "end_fraction": 0.25, + "end_s": 75.0, + "qualifying_response_features": [], + "repeat_pairs": 8, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.45379738064250086, + "action_delta": { + "distinct_n": 6, + "max": -0.2638104929979219, + "min": -1.413753104794269, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 0.9705757741391393, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.467554819246339, + "repeat_abs_p95": 2.634518964255186, + "repeat_delta": { + "distinct_n": 8, + "max": 0.7001454483303338, + "min": -2.9313976513752067, + "n": 8 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 9.185745779867744, + "action_delta": { + "distinct_n": 6, + "max": -5.845648825359461, + "min": -30.311859850911844, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 0.298783186954296, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 30.743850996116663, + "repeat_abs_p95": 93.908100721683, + "repeat_delta": { + "distinct_n": 8, + "max": 27.153830102554593, + "min": -100.52302983743428, + "n": 8 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.4510464857421108, + "action_delta": { + "distinct_n": 6, + "max": -0.26164909366742517, + "min": -1.4061952013981784, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 0.9732742893789799, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.4634320362350386, + "repeat_abs_p95": 2.613462716433853, + "repeat_delta": { + "distinct_n": 8, + "max": 0.6993897150480155, + "min": -2.908747043508294, + "n": 8 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.005686813793577217, + "action_delta": { + "distinct_n": 6, + "max": 0.021360307900858633, + "min": 0.002180359902912321, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 1.4307684434738428, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0039746569890581895, + "repeat_abs_p95": 0.021315647296768075, + "repeat_delta": { + "distinct_n": 8, + "max": 0.024883252660879762, + "min": -0.01469009447770353, + "n": 8 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.012588156742608592, + "action_delta": { + "distinct_n": 6, + "max": -0.005728618501492012, + "min": -0.03011422871811248, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 2.4566361209710017, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.005124143797752612, + "repeat_abs_p95": 0.018876919314381714, + "repeat_delta": { + "distinct_n": 8, + "max": 0.011349421052443798, + "min": -0.022930187609271374, + "n": 8 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 0.002835030550300963, + "action_delta": { + "distinct_n": 6, + "max": 0.007512227776197788, + "min": 0.0005459147599019987, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 1.9399840028075197, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0014613680041681495, + "repeat_abs_p95": 0.004511807887601192, + "repeat_delta": { + "distinct_n": 8, + "max": 0.005553318782013202, + "min": -0.0010848582943063204, + "n": 8 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.002861916241176876, + "action_delta": { + "distinct_n": 6, + "max": 0.005666174682015579, + "min": -0.004438767356134465, + "n": 6 + }, + "action_signs": { + "consistency": 0.5, + "negative": 3, + "positive": 3, + "zero": 0 + }, + "effect_to_repeat_median": 0.547685400888532, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.00522547476440649, + "repeat_abs_p95": 0.007084887036630881, + "repeat_delta": { + "distinct_n": 8, + "max": 0.007595504930191788, + "min": -0.001265067308722334, + "n": 8 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.007920628661389983, + "action_delta": { + "distinct_n": 6, + "max": 0.027975123151637415, + "min": 3.276805657914661e-05, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 5.977407604410328, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0013250942859486248, + "repeat_abs_p95": 0.0092433410331364, + "repeat_delta": { + "distinct_n": 8, + "max": 0.0011681468294250097, + "min": -0.011994317136490906, + "n": 8 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0003016315165324597, + "action_delta": { + "distinct_n": 6, + "max": 0.002385274257794787, + "min": -0.0002979372353175526, + "n": 6 + }, + "action_signs": { + "consistency": 0.5, + "negative": 3, + "positive": 3, + "zero": 0 + }, + "effect_to_repeat_median": 0.1832373452641962, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.001646124681066296, + "repeat_abs_p95": 0.004755246907733842, + "repeat_delta": { + "distinct_n": 8, + "max": 0.001880248703786131, + "min": -0.005007302519187606, + "n": 8 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 6 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 6 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 8 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.00010307207930221329, + "action_delta": { + "distinct_n": 6, + "max": 0.0003313271687591657, + "min": -0.00025035500653580733, + "n": 6 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 5, + "positive": 1, + "zero": 0 + }, + "effect_to_repeat_median": 0.0193424286695313, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.005328807517567591, + "repeat_abs_p95": 0.008606069879593642, + "repeat_delta": { + "distinct_n": 8, + "max": 0.004217023010198662, + "min": -0.008635306091418382, + "n": 8 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.339365533853333, + "action_delta": { + "distinct_n": 6, + "max": 2.133982110813333, + "min": -0.4039724217999998, + "n": 6 + }, + "action_signs": { + "consistency": 0.5, + "negative": 3, + "positive": 3, + "zero": 0 + }, + "effect_to_repeat_median": 0.4388333493005985, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.7733357877066664, + "repeat_abs_p95": 3.1493800733973325, + "repeat_delta": { + "distinct_n": 8, + "max": 1.0551951064133327, + "min": -3.3683846067999994, + "n": 8 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 1.1423175943466668, + "action_delta": { + "distinct_n": 6, + "max": -0.030365675946666667, + "min": -9.07047515872, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 42.90527668627242, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.02662417498666667, + "repeat_abs_p95": 6.55139452659, + "repeat_delta": { + "distinct_n": 8, + "max": 5.732538843706667, + "min": -6.992316817373333, + "n": 8 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 6.700000000000003, + "action_delta": { + "distinct_n": 6, + "max": 8.959999999999994, + "min": 4.9466666666666725, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 1.0959651035986926, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 6.11333333333333, + "repeat_abs_p95": 14.715333333333335, + "repeat_delta": { + "distinct_n": 8, + "max": 15.373333333333335, + "min": -4.8799999999999955, + "n": 8 + } + } + }, + "sanity": { + "invariants": { + "all_results_uncensored": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "layer1_boundary_and_internal_gaps_bounded": true, + "nonnegative_counters": true, + "per_cell_state_vectors_not_all_identical": true, + "ratios_bounded": true, + "state_vectors_not_all_identical": true + }, + "red_flags": [], + "trials": 12 + }, + "start_s": 0.0, + "trial_sanity": [ + { + "admitted_fraction": 0.23777777777777778, + "cell": "tp4_mns16", + "completed_fraction": 0.23555555555555557, + "level": "low", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.001928339, + "max_internal_gap_s": 0.648316718, + "start_gap_s": 0.042673865 + }, + "trial_id": "sessions/rep1-mns16/low/result.json" + }, + { + "admitted_fraction": 0.2372549019607843, + "cell": "tp4_mns16", + "completed_fraction": 0.22627450980392155, + "level": "mid", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.022443998, + "max_internal_gap_s": 0.186336261, + "start_gap_s": 0.042930986 + }, + "trial_id": "sessions/rep1-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.23777777777777778, + "cell": "tp4_mns64", + "completed_fraction": 0.23555555555555557, + "level": "low", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.004213731, + "max_internal_gap_s": 0.637319097, + "start_gap_s": 0.043186851 + }, + "trial_id": "sessions/rep1-mns64/low/result.json" + }, + { + "admitted_fraction": 0.2372549019607843, + "cell": "tp4_mns64", + "completed_fraction": 0.23098039215686275, + "level": "mid", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.003542116, + "max_internal_gap_s": 0.184687818, + "start_gap_s": 0.043187905 + }, + "trial_id": "sessions/rep1-mns64/mid/result.json" + }, + { + "admitted_fraction": 0.23166666666666666, + "cell": "tp4_mns64", + "completed_fraction": 0.22611111111111112, + "level": "low", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.007493238, + "max_internal_gap_s": 0.185686919, + "start_gap_s": 0.107508086 + }, + "trial_id": "sessions/rep2-mns64/low/result.json" + }, + { + "admitted_fraction": 0.23450980392156862, + "cell": "tp4_mns64", + "completed_fraction": 0.22745098039215686, + "level": "mid", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.106828533, + "max_internal_gap_s": 0.177241403, + "start_gap_s": 0.107570476 + }, + "trial_id": "sessions/rep2-mns64/mid/result.json" + }, + { + "admitted_fraction": 0.23166666666666666, + "cell": "tp4_mns16", + "completed_fraction": 0.22499999999999998, + "level": "low", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.01771671, + "max_internal_gap_s": 0.174851445, + "start_gap_s": 0.107276851 + }, + "trial_id": "sessions/rep2-mns16/low/result.json" + }, + { + "admitted_fraction": 0.23450980392156862, + "cell": "tp4_mns16", + "completed_fraction": 0.2231372549019608, + "level": "mid", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.049603839, + "max_internal_gap_s": 0.184443946, + "start_gap_s": 0.10804117 + }, + "trial_id": "sessions/rep2-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.23, + "cell": "tp4_mns16", + "completed_fraction": 0.2205555555555556, + "level": "low", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.001863681, + "max_internal_gap_s": 0.6478406, + "start_gap_s": 0.120533656 + }, + "trial_id": "sessions/rep3-mns16/low/result.json" + }, + { + "admitted_fraction": 0.23019607843137255, + "cell": "tp4_mns16", + "completed_fraction": 0.2152941176470588, + "level": "mid", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.17047515, + "max_internal_gap_s": 0.182092338, + "start_gap_s": 0.124699964 + }, + "trial_id": "sessions/rep3-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.23, + "cell": "tp4_mns64", + "completed_fraction": 0.22111111111111112, + "level": "low", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.000487667, + "max_internal_gap_s": 0.652764433, + "start_gap_s": 0.123493588 + }, + "trial_id": "sessions/rep3-mns64/low/result.json" + }, + { + "admitted_fraction": 0.23019607843137255, + "cell": "tp4_mns64", + "completed_fraction": 0.2184313725490196, + "level": "mid", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.006237065, + "max_internal_gap_s": 0.32072392, + "start_gap_s": 0.122478295 + }, + "trial_id": "sessions/rep3-mns64/mid/result.json" + } + ] + }, + { + "action_pairs": 6, + "actions": [ + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.07082632707989239, + "tpot_mean_over_slo": -0.002512563182247657, + "ttft_max_over_slo_max": -0.0679067323411194, + "ttft_mean_over_slo_max": -0.007600825142914866 + }, + "delta_state": { + "batch_size.mean": -0.20807041709602903, + "batch_tokens.mean": -4.353835378889755, + "decode_batch_size.mean": -0.20629266327798934, + "graph_full_share": 0.0016140185178266098, + "graph_none_share": -0.005313603464127993, + "graph_padding_fraction": 0.001957578860938683, + "kv_usage_end_minus_start": -1.4782427070203497e-06, + "kv_usage_max": 0.004459034805309736, + "kv_usage_mean": -3.034391627598776e-05, + "preemptions": 0.0, + "prefill_token_fraction": 8.331880587686413e-07, + "queue_running_mean": -0.09870830548666643, + "queue_waiting_mean": -0.24030856458, + "scheduler_steps_per_s": 4.060000000000002 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 1, + "request_hash": "af50f0c3edfc6f9fb74734b2501378fc8ba6fdd3dd3fe84f2bb1d68540eee69b", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "5856cc9ff24c9a984a2abecc59ab92c1f87b02b2a9dc0a47eae7a08462169373", + "result_sha256": "4e788371d6f24cab1be2bd4aed4bfe0354b5a5c23973bac9bf69ad0aa4c1b7d9", + "trial_id": "sessions/rep1-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "a91536c628053d3cf530bf0b8ebe51628cc41e71af31b90310d3307d1badc473", + "result_sha256": "2dd1d5ae610db6d738291032ab07a3ad73c46235b30e4ff38b87ce2715204bc6", + "trial_id": "sessions/rep1-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.0027450980392156863, + "completed_over_admitted": 0.0008250825082508628, + "completed_pass_rate": 0.005833333333333357, + "outstanding_over_admitted": -0.0008250825082508247, + "tpot_max_over_slo": 0.2753294892902449, + "tpot_mean_over_slo": 0.018526148273117438, + "ttft_max_over_slo_max": -0.2765812376649895, + "ttft_mean_over_slo_max": -0.08009172416899431 + }, + "delta_state": { + "batch_size.mean": -1.5321059258942888, + "batch_tokens.mean": -32.485419814050886, + "decode_batch_size.mean": -1.5198458327871904, + "graph_full_share": 0.015038490257038628, + "graph_none_share": -0.024038730314888165, + "graph_padding_fraction": 0.006587055652816478, + "kv_usage_end_minus_start": -0.00022317841283692186, + "kv_usage_max": 0.025909186619248126, + "kv_usage_mean": 0.0012400736021554327, + "preemptions": 0.0, + "prefill_token_fraction": -2.8364032274130757e-05, + "queue_running_mean": 0.8917809416333338, + "queue_waiting_mean": -3.7602262873733334, + "scheduler_steps_per_s": 9.026666666666657 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 1, + "request_hash": "0bb61dbc9c26875e991d0d4f984134910d37463e5063f86ee960cf4f8aafb771", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.5603921568627451, + "mns": 16, + "requests_sha256": "8dcab1a24d009309a6dd54676e762ee4e74717ed530d19b3965f30feba32f193", + "result_sha256": "ae7cbca9785fa548acf8bd82646dfe53694ffd0b58d2ef6953e3d531d4d7ec4f", + "trial_id": "sessions/rep1-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "e7bf606cb77817e67a3e64e60f035a5fb344977a7991be741a81db8b3a8d28fe", + "result_sha256": "9604c8a0fe449d3798c6de566e4c804665d66b859a3b43c716dd5ecbb60bee1c", + "trial_id": "sessions/rep1-mns64/mid/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.051139209137158914, + "tpot_mean_over_slo": -0.009262480213267976, + "ttft_max_over_slo_max": -0.03542824399967988, + "ttft_mean_over_slo_max": -0.003943418327769104 + }, + "delta_state": { + "batch_size.mean": -0.37986757946740823, + "batch_tokens.mean": -8.534190946122862, + "decode_batch_size.mean": -0.3768320500033102, + "graph_full_share": 0.0035430578279135316, + "graph_none_share": -0.008383916364329017, + "graph_padding_fraction": 0.001202567354812955, + "kv_usage_end_minus_start": 8.450631670098296e-06, + "kv_usage_max": 0.003782455175254884, + "kv_usage_mean": -0.0002369648399766014, + "preemptions": 0.0, + "prefill_token_fraction": 1.1258245187217852e-06, + "queue_running_mean": -0.3704274981866664, + "queue_waiting_mean": -0.11654787507333333, + "scheduler_steps_per_s": 6.200000000000003 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 2, + "request_hash": "adc728f189a1ce0301609cdbae7d1dc2318609b13eafb94bcb90903bb7998eb1", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "a5ce6cd64890663ef30486701236af3de6169f86b72349928b07adafcb406bcb", + "result_sha256": "289f8f99f5056b26822bab5c1feb4252042c7852b80c5399271ddb1b6db25fbb", + "trial_id": "sessions/rep2-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "1c6931ea4a0913e2460e18c59f5a07e9c21cb2650454f2dd2f2997c416f63209", + "result_sha256": "9988f2b6c9e8c45144eb7b53aa1bea62942ae8fd36aba10c9e6bf5c8cbd65a50", + "trial_id": "sessions/rep2-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.14941176470588236, + "completed_over_admitted": 0.044764188649080716, + "completed_pass_rate": 0.32206255283178364, + "outstanding_over_admitted": -0.04476418864908073, + "tpot_max_over_slo": 0.2416662007889263, + "tpot_mean_over_slo": 0.010014700010439026, + "ttft_max_over_slo_max": -0.9367796959995758, + "ttft_mean_over_slo_max": -0.3197808858475722 + }, + "delta_state": { + "batch_size.mean": -0.8883519907951474, + "batch_tokens.mean": -20.195851559300422, + "decode_batch_size.mean": -0.8851481700211252, + "graph_full_share": 0.021219737900431856, + "graph_none_share": -0.03165408628562745, + "graph_padding_fraction": 0.008308104332878177, + "kv_usage_end_minus_start": -0.0007503119935704516, + "kv_usage_max": 0.022672590909591506, + "kv_usage_mean": 0.0015858745872947538, + "preemptions": 0.0, + "prefill_token_fraction": -0.00012884333456097963, + "queue_running_mean": 1.2845878308199996, + "queue_waiting_mean": -16.806316893553333, + "scheduler_steps_per_s": 7.573333333333338 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 2, + "request_hash": "56a0616b6b54abafd37875c7cb25f8639afef2706ccc55dfbe568f45859ea382", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.31450980392156863, + "mns": 16, + "requests_sha256": "e00c0d9acfb0c22624fdde79f4e8a0300fc2aba09ad4851908d68c1f46be30d3", + "result_sha256": "d081e1ac0143b4e8a63f4c60b58905320ca6e6163fa858e78f5643221fd87c2b", + "trial_id": "sessions/rep2-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "e1543f873745dbd1d3a1461e63549747b6e1517d63e281e6cdfa974d2e2798e4", + "result_sha256": "8556783dc3ba3274269500add337c75c493e3eece4f66260449549ee86506035", + "trial_id": "sessions/rep2-mns64/mid/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.001179245283018937, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.0011792452830188694, + "tpot_max_over_slo": 0.04993343196889949, + "tpot_mean_over_slo": -0.008105257443938751, + "ttft_max_over_slo_max": -0.026339164838039622, + "ttft_mean_over_slo_max": -0.0029500323901768717 + }, + "delta_state": { + "batch_size.mean": -0.28903355041794665, + "batch_tokens.mean": -5.944731148030087, + "decode_batch_size.mean": -0.2869327863683875, + "graph_full_share": 0.0018351555779636541, + "graph_none_share": -0.005482404566122215, + "graph_padding_fraction": 0.000721905257347253, + "kv_usage_end_minus_start": -0.0025560168080752677, + "kv_usage_max": 0.005265908554136223, + "kv_usage_mean": -0.00021185230120904664, + "preemptions": 0.0, + "prefill_token_fraction": -6.655514854192468e-05, + "queue_running_mean": -0.30910058838666643, + "queue_waiting_mean": -0.06934922306666667, + "scheduler_steps_per_s": 6.219999999999999 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 3, + "request_hash": "4993c26442fc6b5a4c09b6f576fd8b3cb1be7ce83b7a771e4a57ee7600de688d", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "de93d4364969393741a920ea72831b69003cbd9f10bcfd1809848fcf4e948549", + "result_sha256": "3b32b5ff3aea07cb7b7cf252ba23852542cb8429c83123fe772b2c8093c408a6", + "trial_id": "sessions/rep3-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "3623a99013725d7b42845bce09678368b3dc8d82e0ab26fad549c7eab8a6b6fe", + "result_sha256": "77766d9ec36ff11893feb73626768fa593160394f2b4720ff9253d1dfd5fa950", + "trial_id": "sessions/rep3-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.19137254901960785, + "completed_over_admitted": 0.034146341463414664, + "completed_pass_rate": 0.42068965517241375, + "outstanding_over_admitted": -0.03414634146341463, + "tpot_max_over_slo": 0.1493312743232093, + "tpot_mean_over_slo": 0.00256767021710691, + "ttft_max_over_slo_max": -0.9516305106711418, + "ttft_mean_over_slo_max": -0.3356434934483016 + }, + "delta_state": { + "batch_size.mean": -0.7507094405947292, + "batch_tokens.mean": -14.968838436325854, + "decode_batch_size.mean": -0.7471714629717603, + "graph_full_share": 0.016020663089875264, + "graph_none_share": -0.02573950202606222, + "graph_padding_fraction": 0.006257606960140573, + "kv_usage_end_minus_start": 0.005366359584320524, + "kv_usage_max": 0.012606264387448385, + "kv_usage_mean": 0.0010847467256849628, + "preemptions": 0.0, + "prefill_token_fraction": 7.198152666310076e-05, + "queue_running_mean": 0.8715743724000014, + "queue_waiting_mean": -17.05635223198667, + "scheduler_steps_per_s": 7.959999999999994 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 3, + "request_hash": "2a2fabe2c4cf176aeb7e0d32fb8e7dbb1f27429a2e7a0cd18d7d186f23096f19", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.2635294117647059, + "mns": 16, + "requests_sha256": "c7fca9b2484c8ea68c6c11438b8e59db74fa60e6189dbb81d8834038156dbf68", + "result_sha256": "bb45674fa5e19ef18d8c13d08493f1a08a62fb682b5d5e8b7d8a1db2fa3b2ea6", + "trial_id": "sessions/rep3-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "d75e1711b8c0485cf1aceaa9e26ae079fa2adc5487d37f28006c01e83c5a782b", + "result_sha256": "1654b044d815beb3a5b8913fde4283b11507464a2de9f41b5ddc3cdb9f2d150b", + "trial_id": "sessions/rep3-mns64/mid/result.json" + } + } + ], + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 6, + "max": 0.49444444444444446, + "min": 0.4711111111111111, + "n": 12 + }, + "completed_fraction_of_total": { + "distinct_n": 10, + "max": 0.49111111111111116, + "min": 0.4549019607843137, + "n": 12 + } + }, + "efficacy": { + "cross_validation_possible": true, + "label_balance_sufficient": true, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "outcome_delta": { + "best_accuracy": 0.8333333333333334, + "best_balanced_accuracy": 0.8333333333333333, + "best_feature": "ttft_mean_over_slo_max", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.07470588235294118, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.0013725490196078432, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.0013725490196078432, + "train_balanced_accuracy": 1.0 + } + ] + }, + "completed_over_admitted": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.0176627933732168, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0004125412541254314, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0004125412541254314, + "train_balanced_accuracy": 1.0 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.16103127641589182, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0029166666666666785, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0029166666666666785, + "train_balanced_accuracy": 1.0 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.01766279337321675, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.00041254125412541233, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": -0.00041254125412541233, + "train_balanced_accuracy": 1.0 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.10023524173018411, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.11007880070155085, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.15624626393440935, + "train_balanced_accuracy": 1.0 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": -0.002768793613415921, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 2.755351742962653e-05, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.0037510684140956846, + "train_balanced_accuracy": 1.0 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.48610396999962785, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.17224398500305443, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.17224398500305443, + "train_balanced_accuracy": 1.0 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.16186215208767066, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.04384627465595459, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.04384627465595459, + "train_balanced_accuracy": 1.0 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "positive": 3 + }, + "positive": 3, + "telemetry_delta": { + "best_accuracy": 1.0, + "best_balanced_accuracy": 1.0, + "best_feature": "scheduler_steps_per_s", + "features": { + "decode_batch_size.mean": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5620017564875353, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5170521246700739, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.6309901100122177, + "train_balanced_accuracy": 1.0 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.003730087157476764, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.004107592910539628, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.00427231725687758, + "train_balanced_accuracy": 1.0 + } + ] + }, + "kv_usage_mean": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0004364472122379581, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0005272014047044875, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0006048648429397225, + "train_balanced_accuracy": 1.0 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.3333333333333333, + "balanced_accuracy": 0.3333333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 3.655367559091127e-05, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 3.64073573609347e-05, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 0 + ], + "threshold": -1.3765422107681058e-05, + "train_balanced_accuracy": 1.0 + } + ] + }, + "queue_running_mean": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.28123689200666746, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.38643303345666746, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.39653631807333367, + "train_balanced_accuracy": 1.0 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -8.461432384313333, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -2.000267425976667, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -2.000267425976667, + "train_balanced_accuracy": 1.0 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 6.896666666666668, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 7.089999999999996, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 6.88666666666667, + "train_balanced_accuracy": 1.0 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "positive": 3 + }, + "telemetry_qualifying_features": [ + "decode_batch_size.mean", + "graph_padding_fraction", + "kv_usage_mean", + "queue_running_mean", + "scheduler_steps_per_s" + ], + "transitions": { + "false->true": 3, + "true->true": 3 + } + }, + "end_fraction": 0.5, + "end_s": 150.0, + "qualifying_response_features": [ + "graph_padding_fraction" + ], + "repeat_pairs": 8, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.5652885100310687, + "action_delta": { + "distinct_n": 6, + "max": -0.20807041709602903, + "min": -1.5321059258942888, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 0.600949368258532, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.9406591301846219, + "repeat_abs_p95": 1.7016197794945327, + "repeat_delta": { + "distinct_n": 8, + "max": 1.5391576560173217, + "min": -1.749794672064679, + "n": 8 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 11.751514691224358, + "action_delta": { + "distinct_n": 6, + "max": -4.353835378889755, + "min": -32.485419814050886, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 0.3300117500902243, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 35.60938265989476, + "repeat_abs_p95": 53.31315307889063, + "repeat_delta": { + "distinct_n": 8, + "max": 39.85703951344101, + "min": -55.14260767193173, + "n": 8 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.5620017564875353, + "action_delta": { + "distinct_n": 6, + "max": -0.20629266327798934, + "min": -1.5198458327871904, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 0.6013807777033149, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.9345189891732675, + "repeat_abs_p95": 1.6881894328027403, + "repeat_delta": { + "distinct_n": 8, + "max": 1.5259244681432733, + "min": -1.7364812802700182, + "n": 8 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.00929077404247608, + "action_delta": { + "distinct_n": 6, + "max": 0.021219737900431856, + "min": 0.0016140185178266098, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 1.067941220289058, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.008699705438808103, + "repeat_abs_p95": 0.016177686020265352, + "repeat_delta": { + "distinct_n": 8, + "max": 0.014828021569913696, + "min": -0.016904428416608552, + "n": 8 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.01621132333960859, + "action_delta": { + "distinct_n": 6, + "max": -0.005313603464127993, + "min": -0.03165408628562745, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 2.171920932023271, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.007464048575887519, + "repeat_abs_p95": 0.014440194586440296, + "repeat_delta": { + "distinct_n": 8, + "max": 0.014932938342374844, + "min": -0.01352509903970471, + "n": 8 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.004107592910539628, + "action_delta": { + "distinct_n": 6, + "max": 0.008308104332878177, + "min": 0.000721905257347253, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 5.563724343406081, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": true, + "repeat_abs_median": 0.0007382811686937355, + "repeat_abs_p95": 0.0022186303657579878, + "repeat_delta": { + "distinct_n": 8, + "max": 0.0025539048662290243, + "min": -0.0015959777220260637, + "n": 8 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.00048674520320368675, + "action_delta": { + "distinct_n": 6, + "max": 0.005366359584320524, + "min": -0.0025560168080752677, + "n": 6 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 2, + "zero": 0 + }, + "effect_to_repeat_median": 0.19573347569321622, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0024867754556537336, + "repeat_abs_p95": 0.0054480365502156845, + "repeat_delta": { + "distinct_n": 8, + "max": 0.0058004047507104195, + "min": -0.004793638463582606, + "n": 8 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.008936086470792304, + "action_delta": { + "distinct_n": 6, + "max": 0.025909186619248126, + "min": 0.003782455175254884, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 6.294267207393665, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0014197183208706776, + "repeat_abs_p95": 0.007519317647539844, + "repeat_delta": { + "distinct_n": 8, + "max": 0.0022486150575440345, + "min": -0.009341973466342357, + "n": 8 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0006608557828307821, + "action_delta": { + "distinct_n": 6, + "max": 0.0015858745872947538, + "min": -0.0002369648399766014, + "n": 6 + }, + "action_signs": { + "consistency": 0.5, + "negative": 3, + "positive": 3, + "zero": 0 + }, + "effect_to_repeat_median": 0.449805118176904, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0014692046758144578, + "repeat_abs_p95": 0.002327639544032804, + "repeat_delta": { + "distinct_n": 8, + "max": 0.0016405734067875804, + "min": -0.002503034295596231, + "n": 8 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 6 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 6 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 8 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 4.745959040802772e-05, + "action_delta": { + "distinct_n": 6, + "max": 7.198152666310076e-05, + "min": -0.00012884333456097963, + "n": 6 + }, + "action_signs": { + "consistency": 0.5, + "negative": 3, + "positive": 3, + "zero": 0 + }, + "effect_to_repeat_median": 0.015433261304563437, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.003075149799608101, + "repeat_abs_p95": 0.005455117882940929, + "repeat_delta": { + "distinct_n": 8, + "max": 0.0032221068728307545, + "min": -0.005478806223512156, + "n": 8 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.6210009352933339, + "action_delta": { + "distinct_n": 6, + "max": 1.2845878308199996, + "min": -0.3704274981866664, + "n": 6 + }, + "action_signs": { + "consistency": 0.5, + "negative": 3, + "positive": 3, + "zero": 0 + }, + "effect_to_repeat_median": 0.620669066679828, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 1.0005346949466665, + "repeat_abs_p95": 1.639787432226332, + "repeat_delta": { + "distinct_n": 8, + "max": 1.3395429762466655, + "min": -1.7843421426733315, + "n": 8 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 2.000267425976667, + "action_delta": { + "distinct_n": 6, + "max": -0.06934922306666667, + "min": -17.05635223198667, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 82.33375700160043, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.02429462105, + "repeat_abs_p95": 8.566731735707661, + "repeat_delta": { + "distinct_n": 8, + "max": 13.045614636299998, + "min": -0.12448899892666668, + "n": 8 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 6.896666666666668, + "action_delta": { + "distinct_n": 6, + "max": 9.026666666666657, + "min": 4.060000000000002, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.8987836663770638, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 7.673333333333332, + "repeat_abs_p95": 11.486333333333327, + "repeat_delta": { + "distinct_n": 8, + "max": 11.493333333333325, + "min": -9.206666666666663, + "n": 8 + } + } + }, + "sanity": { + "invariants": { + "all_results_uncensored": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "layer1_boundary_and_internal_gaps_bounded": true, + "nonnegative_counters": true, + "per_cell_state_vectors_not_all_identical": true, + "ratios_bounded": true, + "state_vectors_not_all_identical": true + }, + "red_flags": [], + "trials": 12 + }, + "start_s": 0.0, + "trial_sanity": [ + { + "admitted_fraction": 0.47555555555555556, + "cell": "tp4_mns16", + "completed_fraction": 0.4733333333333334, + "level": "low", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.001282354, + "max_internal_gap_s": 0.648316718, + "start_gap_s": 0.042673865 + }, + "trial_id": "sessions/rep1-mns16/low/result.json" + }, + { + "admitted_fraction": 0.4752941176470588, + "cell": "tp4_mns16", + "completed_fraction": 0.47058823529411764, + "level": "mid", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.004049563, + "max_internal_gap_s": 0.186336261, + "start_gap_s": 0.042930986 + }, + "trial_id": "sessions/rep1-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.47555555555555556, + "cell": "tp4_mns64", + "completed_fraction": 0.4733333333333334, + "level": "low", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.003246004, + "max_internal_gap_s": 0.637319097, + "start_gap_s": 0.043186851 + }, + "trial_id": "sessions/rep1-mns64/low/result.json" + }, + { + "admitted_fraction": 0.4752941176470588, + "cell": "tp4_mns64", + "completed_fraction": 0.47098039215686277, + "level": "mid", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.00327497, + "max_internal_gap_s": 0.184687818, + "start_gap_s": 0.043187905 + }, + "trial_id": "sessions/rep1-mns64/mid/result.json" + }, + { + "admitted_fraction": 0.49444444444444446, + "cell": "tp4_mns64", + "completed_fraction": 0.49111111111111116, + "level": "low", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.003493587, + "max_internal_gap_s": 0.444552932, + "start_gap_s": 0.107508086 + }, + "trial_id": "sessions/rep2-mns64/low/result.json" + }, + { + "admitted_fraction": 0.49058823529411766, + "cell": "tp4_mns64", + "completed_fraction": 0.48588235294117643, + "level": "mid", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.0063142, + "max_internal_gap_s": 0.183290846, + "start_gap_s": 0.107570476 + }, + "trial_id": "sessions/rep2-mns64/mid/result.json" + }, + { + "admitted_fraction": 0.49444444444444446, + "cell": "tp4_mns16", + "completed_fraction": 0.49111111111111116, + "level": "low", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.012013597, + "max_internal_gap_s": 0.445602102, + "start_gap_s": 0.107276851 + }, + "trial_id": "sessions/rep2-mns16/low/result.json" + }, + { + "admitted_fraction": 0.49058823529411766, + "cell": "tp4_mns16", + "completed_fraction": 0.463921568627451, + "level": "mid", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.02999801, + "max_internal_gap_s": 0.184443946, + "start_gap_s": 0.10804117 + }, + "trial_id": "sessions/rep2-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.4711111111111111, + "cell": "tp4_mns16", + "completed_fraction": 0.4655555555555555, + "level": "low", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.026096497, + "max_internal_gap_s": 0.6478406, + "start_gap_s": 0.120533656 + }, + "trial_id": "sessions/rep3-mns16/low/result.json" + }, + { + "admitted_fraction": 0.4823529411764706, + "cell": "tp4_mns16", + "completed_fraction": 0.4549019607843137, + "level": "mid", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.001098192, + "max_internal_gap_s": 0.182092338, + "start_gap_s": 0.124699964 + }, + "trial_id": "sessions/rep3-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.4711111111111111, + "cell": "tp4_mns64", + "completed_fraction": 0.4661111111111111, + "level": "low", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.001270441, + "max_internal_gap_s": 0.652764433, + "start_gap_s": 0.123493588 + }, + "trial_id": "sessions/rep3-mns64/low/result.json" + }, + { + "admitted_fraction": 0.4823529411764706, + "cell": "tp4_mns64", + "completed_fraction": 0.4713725490196079, + "level": "mid", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.048916774, + "max_internal_gap_s": 0.32072392, + "start_gap_s": 0.122478295 + }, + "trial_id": "sessions/rep3-mns64/mid/result.json" + } + ] + }, + { + "action_pairs": 6, + "actions": [ + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.13955613873224326, + "tpot_mean_over_slo": 0.00021561369396999597, + "ttft_max_over_slo_max": -0.162763388332678, + "ttft_mean_over_slo_max": -0.0113370356308976 + }, + "delta_state": { + "batch_size.mean": -0.2315625922329474, + "batch_tokens.mean": -4.866425400698262, + "decode_batch_size.mean": -0.2298593364511925, + "graph_full_share": 0.0022019142911884337, + "graph_none_share": -0.0061464601290289606, + "graph_padding_fraction": 0.002459307837583828, + "kv_usage_end_minus_start": 8.281600267778799e-07, + "kv_usage_max": 0.012148644893025695, + "kv_usage_mean": 0.00010369316025509931, + "preemptions": 0.0, + "prefill_token_fraction": -3.204953873736649e-06, + "queue_running_mean": -0.004656362764444921, + "queue_waiting_mean": -0.38515557383555554, + "scheduler_steps_per_s": 4.035555555555547 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 1, + "request_hash": "af50f0c3edfc6f9fb74734b2501378fc8ba6fdd3dd3fe84f2bb1d68540eee69b", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "5856cc9ff24c9a984a2abecc59ab92c1f87b02b2a9dc0a47eae7a08462169373", + "result_sha256": "4e788371d6f24cab1be2bd4aed4bfe0354b5a5c23973bac9bf69ad0aa4c1b7d9", + "trial_id": "sessions/rep1-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "a91536c628053d3cf530bf0b8ebe51628cc41e71af31b90310d3307d1badc473", + "result_sha256": "2dd1d5ae610db6d738291032ab07a3ad73c46235b30e4ff38b87ce2715204bc6", + "trial_id": "sessions/rep1-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.14980392156862746, + "completed_over_admitted": 0.03292618162506633, + "completed_pass_rate": 0.21093318608503586, + "outstanding_over_admitted": -0.032926181625066386, + "tpot_max_over_slo": 0.3362407738628878, + "tpot_mean_over_slo": 0.035823037105198796, + "ttft_max_over_slo_max": -1.0155032691739811, + "ttft_mean_over_slo_max": -0.23418190946398737 + }, + "delta_state": { + "batch_size.mean": -0.8757080766407306, + "batch_tokens.mean": -21.903077459843473, + "decode_batch_size.mean": -0.8701416261315948, + "graph_full_share": 0.015083297248042693, + "graph_none_share": -0.024665889172553218, + "graph_padding_fraction": 0.006691320837900951, + "kv_usage_end_minus_start": -0.007731540433626116, + "kv_usage_max": 0.031275425686576486, + "kv_usage_mean": 0.0024751040544861297, + "preemptions": 0.0, + "prefill_token_fraction": -0.0005203792320278566, + "queue_running_mean": 2.356088796968889, + "queue_waiting_mean": -12.268314980844444, + "scheduler_steps_per_s": 7.142222222222216 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 1, + "request_hash": "0bb61dbc9c26875e991d0d4f984134910d37463e5063f86ee960cf4f8aafb771", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.5603921568627451, + "mns": 16, + "requests_sha256": "8dcab1a24d009309a6dd54676e762ee4e74717ed530d19b3965f30feba32f193", + "result_sha256": "ae7cbca9785fa548acf8bd82646dfe53694ffd0b58d2ef6953e3d531d4d7ec4f", + "trial_id": "sessions/rep1-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "e7bf606cb77817e67a3e64e60f035a5fb344977a7991be741a81db8b3a8d28fe", + "result_sha256": "9604c8a0fe449d3798c6de566e4c804665d66b859a3b43c716dd5ecbb60bee1c", + "trial_id": "sessions/rep1-mns64/mid/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0007401924500369805, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.00074019245003701, + "tpot_max_over_slo": 0.17641319103142994, + "tpot_mean_over_slo": -0.008167534216044714, + "ttft_max_over_slo_max": -0.055667730843803526, + "ttft_mean_over_slo_max": -0.004452474842343956 + }, + "delta_state": { + "batch_size.mean": -0.391971250222225, + "batch_tokens.mean": -8.51800794109036, + "decode_batch_size.mean": -0.3888263577646134, + "graph_full_share": 0.0034883439917251513, + "graph_none_share": -0.00892717433146234, + "graph_padding_fraction": 0.0013420770820979835, + "kv_usage_end_minus_start": -0.0014785130164154392, + "kv_usage_max": 0.012914898602144231, + "kv_usage_mean": -0.00017462402054690008, + "preemptions": 0.0, + "prefill_token_fraction": -1.381071014561197e-05, + "queue_running_mean": -0.33874452614666684, + "queue_waiting_mean": -0.12769319111555558, + "scheduler_steps_per_s": 6.342222222222233 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 2, + "request_hash": "adc728f189a1ce0301609cdbae7d1dc2318609b13eafb94bcb90903bb7998eb1", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "a5ce6cd64890663ef30486701236af3de6169f86b72349928b07adafcb406bcb", + "result_sha256": "289f8f99f5056b26822bab5c1feb4252042c7852b80c5399271ddb1b6db25fbb", + "trial_id": "sessions/rep2-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "1c6931ea4a0913e2460e18c59f5a07e9c21cb2650454f2dd2f2997c416f63209", + "result_sha256": "9988f2b6c9e8c45144eb7b53aa1bea62942ae8fd36aba10c9e6bf5c8cbd65a50", + "trial_id": "sessions/rep2-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.3898039215686275, + "completed_over_admitted": 0.047995780590717296, + "completed_pass_rate": 0.5534521158129175, + "outstanding_over_admitted": -0.0479957805907173, + "tpot_max_over_slo": 0.485827943940801, + "tpot_mean_over_slo": 0.015614815582012831, + "ttft_max_over_slo_max": -0.9946015174985708, + "ttft_mean_over_slo_max": -0.5387282368223152 + }, + "delta_state": { + "batch_size.mean": -1.1276789336012651, + "batch_tokens.mean": -25.023441704969116, + "decode_batch_size.mean": -1.1232562014682532, + "graph_full_share": 0.024054684043115837, + "graph_none_share": -0.03557238862010634, + "graph_padding_fraction": 0.008753688501931332, + "kv_usage_end_minus_start": -0.01018813309553912, + "kv_usage_max": 0.04505142122092243, + "kv_usage_mean": 0.0021208568963759205, + "preemptions": 0.0, + "prefill_token_fraction": -0.0001673196203758609, + "queue_running_mean": 1.6553335670222218, + "queue_waiting_mean": -26.90367780516, + "scheduler_steps_per_s": 8.879999999999995 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 2, + "request_hash": "56a0616b6b54abafd37875c7cb25f8639afef2706ccc55dfbe568f45859ea382", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.31450980392156863, + "mns": 16, + "requests_sha256": "e00c0d9acfb0c22624fdde79f4e8a0300fc2aba09ad4851908d68c1f46be30d3", + "result_sha256": "d081e1ac0143b4e8a63f4c60b58905320ca6e6163fa858e78f5643221fd87c2b", + "trial_id": "sessions/rep2-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "e1543f873745dbd1d3a1461e63549747b6e1517d63e281e6cdfa974d2e2798e4", + "result_sha256": "8556783dc3ba3274269500add337c75c493e3eece4f66260449549ee86506035", + "trial_id": "sessions/rep2-mns64/mid/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.06666873637623494, + "tpot_mean_over_slo": -0.009318131681115394, + "ttft_max_over_slo_max": -0.07750733549861857, + "ttft_mean_over_slo_max": -0.004705775197540927 + }, + "delta_state": { + "batch_size.mean": -0.370851789404929, + "batch_tokens.mean": -7.256413232949271, + "decode_batch_size.mean": -0.3681028118073941, + "graph_full_share": 0.0028280490692854654, + "graph_none_share": -0.00702385128031266, + "graph_padding_fraction": 0.0009512228076606274, + "kv_usage_end_minus_start": 3.1056000993068267e-07, + "kv_usage_max": 0.005265908554136223, + "kv_usage_mean": -0.00023540705993194654, + "preemptions": 0.0, + "prefill_token_fraction": 5.342859896728669e-07, + "queue_running_mean": -0.37729419491111127, + "queue_waiting_mean": -0.12941095815111112, + "scheduler_steps_per_s": 7.137777777777785 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 3, + "request_hash": "4993c26442fc6b5a4c09b6f576fd8b3cb1be7ce83b7a771e4a57ee7600de688d", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "de93d4364969393741a920ea72831b69003cbd9f10bcfd1809848fcf4e948549", + "result_sha256": "3b32b5ff3aea07cb7b7cf252ba23852542cb8429c83123fe772b2c8093c408a6", + "trial_id": "sessions/rep3-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "3623a99013725d7b42845bce09678368b3dc8d82e0ab26fad549c7eab8a6b6fe", + "result_sha256": "77766d9ec36ff11893feb73626768fa593160394f2b4720ff9253d1dfd5fa950", + "trial_id": "sessions/rep3-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.43333333333333335, + "completed_over_admitted": 0.05807814149947199, + "completed_pass_rate": 0.6218345526167699, + "outstanding_over_admitted": -0.05807814149947202, + "tpot_max_over_slo": 0.30513300693329926, + "tpot_mean_over_slo": 0.006471198446666382, + "ttft_max_over_slo_max": -0.9754032485070638, + "ttft_mean_over_slo_max": -0.5503063221993506 + }, + "delta_state": { + "batch_size.mean": -0.6183604933300941, + "batch_tokens.mean": -14.376542002465783, + "decode_batch_size.mean": -0.6159981002912982, + "graph_full_share": 0.01869117884155347, + "graph_none_share": -0.028652094681839838, + "graph_padding_fraction": 0.007072597788281806, + "kv_usage_end_minus_start": 0.0004982293932893622, + "kv_usage_max": 0.02549546289093474, + "kv_usage_mean": 0.0013450068711313955, + "preemptions": 0.0, + "prefill_token_fraction": -0.00032277984328055176, + "queue_running_mean": 1.3239998894977774, + "queue_waiting_mean": -27.13096374471111, + "scheduler_steps_per_s": 7.786666666666676 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 3, + "request_hash": "2a2fabe2c4cf176aeb7e0d32fb8e7dbb1f27429a2e7a0cd18d7d186f23096f19", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.2635294117647059, + "mns": 16, + "requests_sha256": "c7fca9b2484c8ea68c6c11438b8e59db74fa60e6189dbb81d8834038156dbf68", + "result_sha256": "bb45674fa5e19ef18d8c13d08493f1a08a62fb682b5d5e8b7d8a1db2fa3b2ea6", + "trial_id": "sessions/rep3-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "d75e1711b8c0485cf1aceaa9e26ae079fa2adc5487d37f28006c01e83c5a782b", + "result_sha256": "1654b044d815beb3a5b8913fde4283b11507464a2de9f41b5ddc3cdb9f2d150b", + "trial_id": "sessions/rep3-mns64/mid/result.json" + } + } + ], + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 6, + "max": 0.7505555555555555, + "min": 0.7316666666666667, + "n": 12 + }, + "completed_fraction_of_total": { + "distinct_n": 9, + "max": 0.7472222222222221, + "min": 0.6968627450980392, + "n": 12 + } + }, + "efficacy": { + "cross_validation_possible": true, + "label_balance_sufficient": true, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "outcome_delta": { + "best_accuracy": 1.0, + "best_balanced_accuracy": 1.0, + "best_feature": "ttft_max_over_slo_max", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.19490196078431374, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.07490196078431373, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.07490196078431373, + "train_balanced_accuracy": 1.0 + } + ] + }, + "completed_over_admitted": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.02436798652037714, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.016463090812533165, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.016833187037551656, + "train_balanced_accuracy": 1.0 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.27672605790645877, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.10546659304251793, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.10546659304251793, + "train_balanced_accuracy": 1.0 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.024367986520377156, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.016463090812533193, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.016833187037551697, + "train_balanced_accuracy": 1.0 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.2407730989823646, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.22234457283277126, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.2563269824471589, + "train_balanced_accuracy": 1.0 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": -0.000848167884689166, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.003343406070318189, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.007915214637991413, + "train_balanced_accuracy": 1.0 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5264552920028412, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5690833184198709, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5786824529156245, + "train_balanced_accuracy": 1.0 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.27171700600992804, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.12275947254744249, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.12275947254744249, + "train_balanced_accuracy": 1.0 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "positive": 3 + }, + "positive": 3, + "telemetry_delta": { + "best_accuracy": 1.0, + "best_balanced_accuracy": 1.0, + "best_feature": "queue_running_mean", + "features": { + "decode_batch_size.mean": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5024122290279558, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.49205045604934616, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.6294839919481041, + "train_balanced_accuracy": 1.0 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.004207337435189895, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.004575314337742389, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.004575314337742389, + "train_balanced_accuracy": 1.0 + } + ] + }, + "kv_usage_mean": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0005851914252922477, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0007243500156932474, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.00111227502831551, + "train_balanced_accuracy": 1.0 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -9.056516526073644e-05, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.0001629923985771442, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -9.056516526073644e-05, + "train_balanced_accuracy": 1.0 + } + ] + }, + "queue_running_mean": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.49262768167555526, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.6596717633666662, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.8253386021288884, + "train_balanced_accuracy": 1.0 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -13.516544381655557, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -6.32673527734, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -6.32673527734, + "train_balanced_accuracy": 1.0 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 7.4622222222222305, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 7.140000000000001, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 6.742222222222225, + "train_balanced_accuracy": 1.0 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "positive": 3 + }, + "telemetry_qualifying_features": [], + "transitions": { + "false->true": 3, + "true->true": 3 + } + }, + "end_fraction": 0.75, + "end_s": 225.0, + "qualifying_response_features": [ + "graph_padding_fraction", + "queue_waiting_mean" + ], + "repeat_pairs": 8, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.5051658717761596, + "action_delta": { + "distinct_n": 6, + "max": -0.2315625922329474, + "min": -1.1276789336012651, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 0.7593203575357039, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.6652868802512071, + "repeat_abs_p95": 1.054027495945265, + "repeat_delta": { + "distinct_n": 8, + "max": 0.7189335327348694, + "min": -1.232288950040175, + "n": 8 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 11.447274971778072, + "action_delta": { + "distinct_n": 6, + "max": -4.866425400698262, + "min": -25.023441704969116, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 0.575768911929723, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 19.88171770756407, + "repeat_abs_p95": 34.7043135482259, + "repeat_delta": { + "distinct_n": 8, + "max": 13.173463929732975, + "min": -38.394254431520324, + "n": 8 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.5024122290279558, + "action_delta": { + "distinct_n": 6, + "max": -0.2298593364511925, + "min": -1.1232562014682532, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 0.7617660157398797, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.6595361549963323, + "repeat_abs_p95": 1.045411693203953, + "repeat_delta": { + "distinct_n": 8, + "max": 0.7159576534104133, + "min": -1.2228100223235518, + "n": 8 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.009285820619883922, + "action_delta": { + "distinct_n": 6, + "max": 0.024054684043115837, + "min": 0.0022019142911884337, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 1.846168814253066, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0050297787223975154, + "repeat_abs_p95": 0.012718990273054656, + "repeat_delta": { + "distinct_n": 8, + "max": 0.010581275778524746, + "min": -0.013870067308570766, + "n": 8 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.01679653175200778, + "action_delta": { + "distinct_n": 6, + "max": -0.0061464601290289606, + "min": -0.03557238862010634, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 4.06293763524991, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.004134085545956116, + "repeat_abs_p95": 0.011398711005122675, + "repeat_delta": { + "distinct_n": 8, + "max": 0.01240808302992423, + "min": -0.009524162959062651, + "n": 8 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.6666666666666666, + "action_abs_median": 0.004575314337742389, + "action_delta": { + "distinct_n": 6, + "max": 0.008753688501931332, + "min": 0.0009512228076606274, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 5.003367170865857, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": true, + "repeat_abs_median": 0.0009144470476570299, + "repeat_abs_p95": 0.0019536589959716906, + "repeat_delta": { + "distinct_n": 8, + "max": 0.0021029671629594227, + "min": -0.0012192099761899004, + "n": 8 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 0.0009883712048524007, + "action_delta": { + "distinct_n": 6, + "max": 0.0004982293932893622, + "min": -0.01018813309553912, + "n": 6 + }, + "action_signs": { + "consistency": 0.5, + "negative": 3, + "positive": 3, + "zero": 0 + }, + "effect_to_repeat_median": 0.2973800031975333, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.003323596725486211, + "repeat_abs_p95": 0.006865285634854577, + "repeat_delta": { + "distinct_n": 8, + "max": 0.004233894621933687, + "min": -0.008131118105255575, + "n": 8 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.019205180746539485, + "action_delta": { + "distinct_n": 6, + "max": 0.04505142122092243, + "min": 0.005265908554136223, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 7.501452126191231, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0025601950693632802, + "repeat_abs_p95": 0.016512755779962775, + "repeat_delta": { + "distinct_n": 8, + "max": 0.012755779962795644, + "min": -0.01853574275843739, + "n": 8 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 0.000790206965531671, + "action_delta": { + "distinct_n": 6, + "max": 0.0024751040544861297, + "min": -0.00023540705993194654, + "n": 6 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 2, + "positive": 4, + "zero": 0 + }, + "effect_to_repeat_median": 1.0014106568857974, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.000789093824894044, + "repeat_abs_p95": 0.0017615365202587913, + "repeat_delta": { + "distinct_n": 8, + "max": 0.0004699285473880651, + "min": -0.0020330840290943755, + "n": 8 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 6 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 6 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 8 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 9.056516526073644e-05, + "action_delta": { + "distinct_n": 6, + "max": 5.342859896728669e-07, + "min": -0.0005203792320278566, + "n": 6 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 5, + "positive": 1, + "zero": 0 + }, + "effect_to_repeat_median": 0.05219845322135223, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0017350162633495425, + "repeat_abs_p95": 0.004831444067139573, + "repeat_delta": { + "distinct_n": 8, + "max": 0.0014404459696625294, + "min": -0.004836464815786923, + "n": 8 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.8506470422044443, + "action_delta": { + "distinct_n": 6, + "max": 2.356088796968889, + "min": -0.37729419491111127, + "n": 6 + }, + "action_signs": { + "consistency": 0.5, + "negative": 3, + "positive": 3, + "zero": 0 + }, + "effect_to_repeat_median": 1.3604877117547791, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.6252515438800001, + "repeat_abs_p95": 1.1295860132264428, + "repeat_delta": { + "distinct_n": 8, + "max": 0.7170174645377774, + "min": -1.2455528003599987, + "n": 8 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 6.32673527734, + "action_delta": { + "distinct_n": 6, + "max": -0.12769319111555558, + "min": -27.13096374471111, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 1346.2237705715072, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": true, + "repeat_abs_median": 0.004699616375555556, + "repeat_abs_p95": 9.605207488385105, + "repeat_delta": { + "distinct_n": 8, + "max": 14.638355337377778, + "min": -0.25793291168444443, + "n": 8 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 7.140000000000001, + "action_delta": { + "distinct_n": 6, + "max": 8.879999999999995, + "min": 4.035555555555547, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 1.6227272727272708, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 4.400000000000006, + "repeat_abs_p95": 9.05933333333333, + "repeat_delta": { + "distinct_n": 8, + "max": 9.337777777777774, + "min": -4.684444444444452, + "n": 8 + } + } + }, + "sanity": { + "invariants": { + "all_results_uncensored": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "layer1_boundary_and_internal_gaps_bounded": true, + "nonnegative_counters": true, + "per_cell_state_vectors_not_all_identical": true, + "ratios_bounded": true, + "state_vectors_not_all_identical": true + }, + "red_flags": [], + "trials": 12 + }, + "start_s": 0.0, + "trial_sanity": [ + { + "admitted_fraction": 0.7388888888888889, + "cell": "tp4_mns16", + "completed_fraction": 0.7377777777777779, + "level": "low", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.002047043, + "max_internal_gap_s": 0.648316718, + "start_gap_s": 0.042673865 + }, + "trial_id": "sessions/rep1-mns16/low/result.json" + }, + { + "admitted_fraction": 0.7384313725490196, + "cell": "tp4_mns16", + "completed_fraction": 0.7101960784313726, + "level": "mid", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.000359161, + "max_internal_gap_s": 0.188948337, + "start_gap_s": 0.042930986 + }, + "trial_id": "sessions/rep1-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.7388888888888889, + "cell": "tp4_mns64", + "completed_fraction": 0.7377777777777779, + "level": "low", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.003650633, + "max_internal_gap_s": 0.637319097, + "start_gap_s": 0.043186851 + }, + "trial_id": "sessions/rep1-mns64/low/result.json" + }, + { + "admitted_fraction": 0.7384313725490196, + "cell": "tp4_mns64", + "completed_fraction": 0.7345098039215686, + "level": "mid", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.004944353, + "max_internal_gap_s": 0.219527251, + "start_gap_s": 0.043187905 + }, + "trial_id": "sessions/rep1-mns64/mid/result.json" + }, + { + "admitted_fraction": 0.7505555555555555, + "cell": "tp4_mns64", + "completed_fraction": 0.7472222222222221, + "level": "low", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.002633243, + "max_internal_gap_s": 0.646724237, + "start_gap_s": 0.107508086 + }, + "trial_id": "sessions/rep2-mns64/low/result.json" + }, + { + "admitted_fraction": 0.7435294117647059, + "cell": "tp4_mns64", + "completed_fraction": 0.74, + "level": "mid", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.000558705, + "max_internal_gap_s": 0.183301269, + "start_gap_s": 0.107570476 + }, + "trial_id": "sessions/rep2-mns64/mid/result.json" + }, + { + "admitted_fraction": 0.7505555555555555, + "cell": "tp4_mns16", + "completed_fraction": 0.7466666666666666, + "level": "low", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.001410558, + "max_internal_gap_s": 0.583053185, + "start_gap_s": 0.107276851 + }, + "trial_id": "sessions/rep2-mns16/low/result.json" + }, + { + "admitted_fraction": 0.7435294117647059, + "cell": "tp4_mns16", + "completed_fraction": 0.7043137254901961, + "level": "mid", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.042708099, + "max_internal_gap_s": 0.184443946, + "start_gap_s": 0.10804117 + }, + "trial_id": "sessions/rep2-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.7316666666666667, + "cell": "tp4_mns16", + "completed_fraction": 0.7311111111111112, + "level": "low", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.000142513, + "max_internal_gap_s": 0.6478406, + "start_gap_s": 0.120533656 + }, + "trial_id": "sessions/rep3-mns16/low/result.json" + }, + { + "admitted_fraction": 0.7427450980392157, + "cell": "tp4_mns16", + "completed_fraction": 0.6968627450980392, + "level": "mid", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.049578949, + "max_internal_gap_s": 0.182904479, + "start_gap_s": 0.124699964 + }, + "trial_id": "sessions/rep3-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.7316666666666667, + "cell": "tp4_mns64", + "completed_fraction": 0.7311111111111112, + "level": "low", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.00141287, + "max_internal_gap_s": 0.652764433, + "start_gap_s": 0.123493588 + }, + "trial_id": "sessions/rep3-mns64/low/result.json" + }, + { + "admitted_fraction": 0.7427450980392157, + "cell": "tp4_mns64", + "completed_fraction": 0.74, + "level": "mid", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.009657729, + "max_internal_gap_s": 0.32072392, + "start_gap_s": 0.122478295 + }, + "trial_id": "sessions/rep3-mns64/mid/result.json" + } + ] + }, + { + "action_pairs": 6, + "actions": [ + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0011111111111111738, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.0011111111111111113, + "tpot_max_over_slo": 0.13955613873224326, + "tpot_mean_over_slo": -0.0010090653746841205, + "ttft_max_over_slo_max": -0.162763388332678, + "ttft_mean_over_slo_max": -0.00961120104762887 + }, + "delta_state": { + "batch_size.mean": -0.26379559738229474, + "batch_tokens.mean": -5.348391834808808, + "decode_batch_size.mean": -0.2617666450994136, + "graph_full_share": 0.0022388856413185776, + "graph_none_share": -0.006502634438630757, + "graph_padding_fraction": 0.0021671821218428883, + "kv_usage_end_minus_start": 0.0006863000045611578, + "kv_usage_max": 0.012148644893025695, + "kv_usage_mean": 4.226955534929189e-05, + "preemptions": 0.0, + "prefill_token_fraction": 5.3820087398737115e-05, + "queue_running_mean": -0.06008688729666822, + "queue_waiting_mean": -0.31836611265666664, + "scheduler_steps_per_s": 4.469999999999999 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 1, + "request_hash": "af50f0c3edfc6f9fb74734b2501378fc8ba6fdd3dd3fe84f2bb1d68540eee69b", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "5856cc9ff24c9a984a2abecc59ab92c1f87b02b2a9dc0a47eae7a08462169373", + "result_sha256": "4e788371d6f24cab1be2bd4aed4bfe0354b5a5c23973bac9bf69ad0aa4c1b7d9", + "trial_id": "sessions/rep1-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "a91536c628053d3cf530bf0b8ebe51628cc41e71af31b90310d3307d1badc473", + "result_sha256": "2dd1d5ae610db6d738291032ab07a3ad73c46235b30e4ff38b87ce2715204bc6", + "trial_id": "sessions/rep1-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.3949019607843137, + "completed_over_admitted": 0.03529411764705881, + "completed_pass_rate": 0.41338259441707714, + "outstanding_over_admitted": -0.03529411764705882, + "tpot_max_over_slo": 0.3362407738628878, + "tpot_mean_over_slo": 0.031421107704212525, + "ttft_max_over_slo_max": -1.0149181675127088, + "ttft_mean_over_slo_max": -0.4177688425802161 + }, + "delta_state": { + "batch_size.mean": -0.6909306729068057, + "batch_tokens.mean": -12.641524933835228, + "decode_batch_size.mean": -0.6881317239210976, + "graph_full_share": 0.01650801205775032, + "graph_none_share": -0.026570069621619105, + "graph_padding_fraction": 0.007045814940879979, + "kv_usage_end_minus_start": 0.009741050899909953, + "kv_usage_max": 0.031275425686576486, + "kv_usage_mean": 0.0024717436230722416, + "preemptions": 0.0, + "prefill_token_fraction": 0.00029854606973478415, + "queue_running_mean": 2.260168520989998, + "queue_waiting_mean": -20.98940505023, + "scheduler_steps_per_s": 6.183333333333337 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 1, + "request_hash": "0bb61dbc9c26875e991d0d4f984134910d37463e5063f86ee960cf4f8aafb771", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.5603921568627451, + "mns": 16, + "requests_sha256": "8dcab1a24d009309a6dd54676e762ee4e74717ed530d19b3965f30feba32f193", + "result_sha256": "ae7cbca9785fa548acf8bd82646dfe53694ffd0b58d2ef6953e3d531d4d7ec4f", + "trial_id": "sessions/rep1-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "e7bf606cb77817e67a3e64e60f035a5fb344977a7991be741a81db8b3a8d28fe", + "result_sha256": "9604c8a0fe449d3798c6de566e4c804665d66b859a3b43c716dd5ecbb60bee1c", + "trial_id": "sessions/rep1-mns64/mid/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.17641319103142994, + "tpot_mean_over_slo": -0.004863679616933292, + "ttft_max_over_slo_max": -0.16410399084755528, + "ttft_mean_over_slo_max": -0.005948038735831104 + }, + "delta_state": { + "batch_size.mean": -0.3452936706369112, + "batch_tokens.mean": -7.480013628048113, + "decode_batch_size.mean": -0.34254059469005593, + "graph_full_share": 0.002915066846803871, + "graph_none_share": -0.007828081051085262, + "graph_padding_fraction": 0.0013323004677419665, + "kv_usage_end_minus_start": 9.758748075849866e-06, + "kv_usage_max": 0.01567456253640287, + "kv_usage_mean": -7.388307997985809e-05, + "preemptions": 0.0, + "prefill_token_fraction": 2.1031949494210167e-06, + "queue_running_mean": -0.2080717158233334, + "queue_waiting_mean": -0.18922616308333332, + "scheduler_steps_per_s": 5.61666666666666 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 2, + "request_hash": "adc728f189a1ce0301609cdbae7d1dc2318609b13eafb94bcb90903bb7998eb1", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "a5ce6cd64890663ef30486701236af3de6169f86b72349928b07adafcb406bcb", + "result_sha256": "289f8f99f5056b26822bab5c1feb4252042c7852b80c5399271ddb1b6db25fbb", + "trial_id": "sessions/rep2-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "1c6931ea4a0913e2460e18c59f5a07e9c21cb2650454f2dd2f2997c416f63209", + "result_sha256": "9988f2b6c9e8c45144eb7b53aa1bea62942ae8fd36aba10c9e6bf5c8cbd65a50", + "trial_id": "sessions/rep2-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.6321568627450981, + "completed_over_admitted": 0.04980392156862745, + "completed_pass_rate": 0.667771333885667, + "outstanding_over_admitted": -0.04980392156862745, + "tpot_max_over_slo": 0.485827943940801, + "tpot_mean_over_slo": 0.017498560395543927, + "ttft_max_over_slo_max": -0.9946015174985708, + "ttft_mean_over_slo_max": -0.6463312081620042 + }, + "delta_state": { + "batch_size.mean": -0.9812671529958905, + "batch_tokens.mean": -20.775938819398107, + "decode_batch_size.mean": -0.9782503281970616, + "graph_full_share": 0.024736354046330344, + "graph_none_share": -0.03589813374533629, + "graph_padding_fraction": 0.00872979044141211, + "kv_usage_end_minus_start": -0.0018192978173303676, + "kv_usage_max": 0.04505142122092243, + "kv_usage_mean": 0.002220219555318084, + "preemptions": 0.0, + "prefill_token_fraction": 1.1463365605646203e-06, + "queue_running_mean": 1.8554246239900003, + "queue_waiting_mean": -31.984563914093336, + "scheduler_steps_per_s": 8.263333333333335 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 2, + "request_hash": "56a0616b6b54abafd37875c7cb25f8639afef2706ccc55dfbe568f45859ea382", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.31450980392156863, + "mns": 16, + "requests_sha256": "e00c0d9acfb0c22624fdde79f4e8a0300fc2aba09ad4851908d68c1f46be30d3", + "result_sha256": "d081e1ac0143b4e8a63f4c60b58905320ca6e6163fa858e78f5643221fd87c2b", + "trial_id": "sessions/rep2-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "e1543f873745dbd1d3a1461e63549747b6e1517d63e281e6cdfa974d2e2798e4", + "result_sha256": "8556783dc3ba3274269500add337c75c493e3eece4f66260449549ee86506035", + "trial_id": "sessions/rep2-mns64/mid/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0005555555555556424, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.0005555555555555552, + "tpot_max_over_slo": 0.18505735732578327, + "tpot_mean_over_slo": -0.007144351590753079, + "ttft_max_over_slo_max": -0.12056507783321042, + "ttft_mean_over_slo_max": -0.006789902099118429 + }, + "delta_state": { + "batch_size.mean": -0.3723911494360035, + "batch_tokens.mean": -7.297855407175092, + "decode_batch_size.mean": -0.3696269360813078, + "graph_full_share": 0.002705515086113608, + "graph_none_share": -0.007253399469024875, + "graph_padding_fraction": 0.001586911205060191, + "kv_usage_end_minus_start": -4.616416023950354e-05, + "kv_usage_max": 0.015583985375003384, + "kv_usage_mean": -0.0001345075402517243, + "preemptions": 0.0, + "prefill_token_fraction": 1.2342954064381573e-06, + "queue_running_mean": -0.3007637260933329, + "queue_waiting_mean": -0.20904505781333332, + "scheduler_steps_per_s": 6.739999999999995 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 3, + "request_hash": "4993c26442fc6b5a4c09b6f576fd8b3cb1be7ce83b7a771e4a57ee7600de688d", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "de93d4364969393741a920ea72831b69003cbd9f10bcfd1809848fcf4e948549", + "result_sha256": "3b32b5ff3aea07cb7b7cf252ba23852542cb8429c83123fe772b2c8093c408a6", + "trial_id": "sessions/rep3-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "3623a99013725d7b42845bce09678368b3dc8d82e0ab26fad549c7eab8a6b6fe", + "result_sha256": "77766d9ec36ff11893feb73626768fa593160394f2b4720ff9253d1dfd5fa950", + "trial_id": "sessions/rep3-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.6756862745098039, + "completed_over_admitted": 0.05372549019607842, + "completed_pass_rate": 0.7194154488517746, + "outstanding_over_admitted": -0.05372549019607843, + "tpot_max_over_slo": 0.37831041669495047, + "tpot_mean_over_slo": 0.007493808043833983, + "ttft_max_over_slo_max": -0.9795807876701778, + "ttft_mean_over_slo_max": -0.6540993991122069 + }, + "delta_state": { + "batch_size.mean": -0.8431446284631257, + "batch_tokens.mean": -19.08621547820553, + "decode_batch_size.mean": -0.839780585287075, + "graph_full_share": 0.021245526407298043, + "graph_none_share": -0.03191300655092645, + "graph_padding_fraction": 0.00783703914940752, + "kv_usage_end_minus_start": 0.000710563518775853, + "kv_usage_max": 0.033983984733163775, + "kv_usage_mean": 0.0015253876505999867, + "preemptions": 0.0, + "prefill_token_fraction": -0.00035310442232838746, + "queue_running_mean": 1.397843253986668, + "queue_waiting_mean": -32.16364194347333, + "scheduler_steps_per_s": 8.763333333333335 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 3, + "request_hash": "2a2fabe2c4cf176aeb7e0d32fb8e7dbb1f27429a2e7a0cd18d7d186f23096f19", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.2635294117647059, + "mns": 16, + "requests_sha256": "c7fca9b2484c8ea68c6c11438b8e59db74fa60e6189dbb81d8834038156dbf68", + "result_sha256": "bb45674fa5e19ef18d8c13d08493f1a08a62fb682b5d5e8b7d8a1db2fa3b2ea6", + "trial_id": "sessions/rep3-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "d75e1711b8c0485cf1aceaa9e26ae079fa2adc5487d37f28006c01e83c5a782b", + "result_sha256": "1654b044d815beb3a5b8913fde4283b11507464a2de9f41b5ddc3cdb9f2d150b", + "trial_id": "sessions/rep3-mns64/mid/result.json" + } + } + ], + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 1, + "max": 1.0, + "min": 1.0, + "n": 12 + }, + "completed_fraction_of_total": { + "distinct_n": 10, + "max": 0.9966666666666667, + "min": 0.9392156862745098, + "n": 12 + } + }, + "efficacy": { + "cross_validation_possible": true, + "label_balance_sufficient": true, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "outcome_delta": { + "best_accuracy": 1.0, + "best_balanced_accuracy": 1.0, + "best_feature": "ttft_mean_over_slo_max", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.31607843137254904, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.19745098039215686, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.19745098039215686, + "train_balanced_accuracy": 1.0 + } + ] + }, + "completed_over_admitted": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.025179738562091547, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.01820261437908499, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.01820261437908499, + "train_balanced_accuracy": 1.0 + } + ] + }, + "completed_pass_rate": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.3338856669428335, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.20669129720853857, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.20669129720853857, + "train_balanced_accuracy": 1.0 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.025179738562091502, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.018202614379084967, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.018202614379084967, + "train_balanced_accuracy": 1.0 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.28168388701036684, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.26064906559433554, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.2563269824471589, + "train_balanced_accuracy": 1.0 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0013150642134503454, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.003242371334574931, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.008244747510429903, + "train_balanced_accuracy": 1.0 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5718423892588665, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5711720880014279, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5793527541730631, + "train_balanced_accuracy": 1.0 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.3265605551305613, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.2136900218139225, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.2136900218139225, + "train_balanced_accuracy": 1.0 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "positive": 3 + }, + "positive": 3, + "telemetry_delta": { + "best_accuracy": 1.0, + "best_balanced_accuracy": 1.0, + "best_feature": "queue_waiting_mean", + "features": { + "decode_batch_size.mean": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.6047037606841914, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5288793300012027, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5153361593055767, + "train_balanced_accuracy": 1.0 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.004711975177233855, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.004606498531361434, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.004606498531361434, + "train_balanced_accuracy": 1.0 + } + ] + }, + "kv_usage_mean": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0007257522853100643, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0007838286029746393, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0011312445553336879, + "train_balanced_accuracy": 1.0 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 1.1903159835013888e-06, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.00017593506346097465, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 1.6247657549928185e-06, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_running_mean": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.5948857690816673, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.6688781833449999, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.897668868346666, + "train_balanced_accuracy": 1.0 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -16.096804485953335, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -10.653885581443333, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -10.653885581443333, + "train_balanced_accuracy": 1.0 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 7.501666666666665, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 5.326666666666668, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 5.899999999999999, + "train_balanced_accuracy": 1.0 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "positive": 3 + }, + "telemetry_qualifying_features": [], + "transitions": { + "false->true": 3, + "true->true": 3 + } + }, + "end_fraction": 1.0, + "end_s": 300.0, + "qualifying_response_features": [ + "graph_padding_fraction", + "queue_waiting_mean" + ], + "repeat_pairs": 8, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.5316609111714046, + "action_delta": { + "distinct_n": 6, + "max": -0.26379559738229474, + "min": -0.9812671529958905, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 1.4818088309881625, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.358791836067585, + "repeat_abs_p95": 0.9062470454242428, + "repeat_delta": { + "distinct_n": 8, + "max": 0.5513066192162643, + "min": -0.9545899290107105, + "n": 8 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 10.06076928094167, + "action_delta": { + "distinct_n": 6, + "max": -5.348391834808808, + "min": -20.775938819398107, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 0.573954119692525, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 17.528873712643374, + "repeat_abs_p95": 31.853379856357762, + "repeat_delta": { + "distinct_n": 8, + "max": 13.785312549619732, + "min": -32.444783025775166, + "n": 8 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.5288793300012027, + "action_delta": { + "distinct_n": 6, + "max": -0.2617666450994136, + "min": -0.9782503281970616, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 1.4866497181990304, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.3557524839421502, + "repeat_abs_p95": 0.8989199315256944, + "repeat_delta": { + "distinct_n": 8, + "max": 0.5489012995649425, + "min": -0.9473843415441898, + "n": 8 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.009711539452277096, + "action_delta": { + "distinct_n": 6, + "max": 0.024736354046330344, + "min": 0.0022388856413185776, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 3.945566969995893, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0024613799553089843, + "repeat_abs_p95": 0.009783132290188977, + "repeat_delta": { + "distinct_n": 8, + "max": 0.008076086279964478, + "min": -0.010702310911079094, + "n": 8 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.017199075336352183, + "action_delta": { + "distinct_n": 6, + "max": -0.006502634438630757, + "min": -0.03589813374533629, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 6.047036959442635, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.002844215349055424, + "repeat_abs_p95": 0.008425467057080636, + "repeat_delta": { + "distinct_n": 8, + "max": 0.00925018771871168, + "min": -0.006893842971194414, + "n": 8 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.8333333333333334, + "action_abs_median": 0.004606498531361434, + "action_delta": { + "distinct_n": 6, + "max": 0.00872979044141211, + "min": 0.0013323004677419665, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 5.74258050189957, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": true, + "repeat_abs_median": 0.0008021652512903675, + "repeat_abs_p95": 0.0013850876756400163, + "repeat_delta": { + "distinct_n": 8, + "max": 0.001451018727788989, + "min": -0.0009098361529306493, + "n": 8 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0006984317616685054, + "action_delta": { + "distinct_n": 6, + "max": 0.009741050899909953, + "min": -0.0018192978173303676, + "n": 6 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 2, + "positive": 4, + "zero": 0 + }, + "effect_to_repeat_median": 0.1291097203463754, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.005409598594085352, + "repeat_abs_p95": 0.012010926224984144, + "repeat_delta": { + "distinct_n": 8, + "max": 0.002621680737545762, + "min": -0.01455978249759804, + "n": 8 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 1.0, + "action_abs_median": 0.023474994111489678, + "action_delta": { + "distinct_n": 6, + "max": 0.04505142122092243, + "min": 0.012148644893025695, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 23.009837103169843, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.001020215571550298, + "repeat_abs_p95": 0.011807784296490091, + "repeat_delta": { + "distinct_n": 8, + "max": 0.012755779962795644, + "min": -0.010047220916208355, + "n": 8 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 0.0008299475954258555, + "action_delta": { + "distinct_n": 6, + "max": 0.0024717436230722416, + "min": -0.0001345075402517243, + "n": 6 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 2, + "positive": 4, + "zero": 0 + }, + "effect_to_repeat_median": 1.2357530752548687, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.000671612810071043, + "repeat_abs_p95": 0.0016041738838027961, + "repeat_delta": { + "distinct_n": 8, + "max": 0.0005327474947367915, + "min": -0.0018473650504541306, + "n": 8 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 6 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 6 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 8 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 2.7961641174079066e-05, + "action_delta": { + "distinct_n": 6, + "max": 0.00029854606973478415, + "min": -0.00035310442232838746, + "n": 6 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 1, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 0.014748067700673472, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0018959528625436262, + "repeat_abs_p95": 0.0048750185359022645, + "repeat_delta": { + "distinct_n": 8, + "max": 0.001808064432466061, + "min": -0.004875322650742309, + "n": 8 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.8493034900400005, + "action_delta": { + "distinct_n": 6, + "max": 2.260168520989998, + "min": -0.3007637260933329, + "n": 6 + }, + "action_signs": { + "consistency": 0.5, + "negative": 3, + "positive": 3, + "zero": 0 + }, + "effect_to_repeat_median": 2.784700761391589, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.30498913987999954, + "repeat_abs_p95": 0.9830922326288332, + "repeat_delta": { + "distinct_n": 8, + "max": 0.5377630984033335, + "min": -1.1432457121299997, + "n": 8 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 10.653885581443333, + "action_delta": { + "distinct_n": 6, + "max": -0.18922616308333332, + "min": -32.16364194347333, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 898.7004216601898, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": true, + "repeat_abs_median": 0.01185476864666667, + "repeat_abs_p95": 7.209323996711327, + "repeat_delta": { + "distinct_n": 8, + "max": 10.997039040103335, + "min": -0.12967447423333334, + "n": 8 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.6666666666666666, + "action_abs_median": 6.461666666666666, + "action_delta": { + "distinct_n": 6, + "max": 8.763333333333335, + "min": 4.469999999999999, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 1.8488316642823093, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 3.4949999999999974, + "repeat_abs_p95": 6.136833333333333, + "repeat_delta": { + "distinct_n": 8, + "max": 6.530000000000001, + "min": -3.1566666666666663, + "n": 8 + } + } + }, + "sanity": { + "invariants": { + "all_results_uncensored": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "layer1_boundary_and_internal_gaps_bounded": true, + "nonnegative_counters": true, + "per_cell_state_vectors_not_all_identical": true, + "ratios_bounded": true, + "state_vectors_not_all_identical": true + }, + "red_flags": [], + "trials": 12 + }, + "start_s": 0.0, + "trial_sanity": [ + { + "admitted_fraction": 1.0, + "cell": "tp4_mns16", + "completed_fraction": 0.99, + "level": "low", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.00637513, + "max_internal_gap_s": 0.648316718, + "start_gap_s": 0.042673865 + }, + "trial_id": "sessions/rep1-mns16/low/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns16", + "completed_fraction": 0.9552941176470588, + "level": "mid", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.00415122, + "max_internal_gap_s": 0.188948337, + "start_gap_s": 0.042930986 + }, + "trial_id": "sessions/rep1-mns16/mid/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns64", + "completed_fraction": 0.9911111111111112, + "level": "low", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.03110111, + "max_internal_gap_s": 0.637319097, + "start_gap_s": 0.043186851 + }, + "trial_id": "sessions/rep1-mns64/low/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns64", + "completed_fraction": 0.9905882352941177, + "level": "mid", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.033263413, + "max_internal_gap_s": 0.325952494, + "start_gap_s": 0.043187905 + }, + "trial_id": "sessions/rep1-mns64/mid/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns64", + "completed_fraction": 0.9966666666666667, + "level": "low", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.002571552, + "max_internal_gap_s": 0.646724237, + "start_gap_s": 0.107508086 + }, + "trial_id": "sessions/rep2-mns64/low/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns64", + "completed_fraction": 0.9964705882352941, + "level": "mid", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.002910387, + "max_internal_gap_s": 0.26729158, + "start_gap_s": 0.107570476 + }, + "trial_id": "sessions/rep2-mns64/mid/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns16", + "completed_fraction": 0.9966666666666667, + "level": "low", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.005037876, + "max_internal_gap_s": 0.583053185, + "start_gap_s": 0.107276851 + }, + "trial_id": "sessions/rep2-mns16/low/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns16", + "completed_fraction": 0.9466666666666667, + "level": "mid", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.04911377, + "max_internal_gap_s": 0.184443946, + "start_gap_s": 0.10804117 + }, + "trial_id": "sessions/rep2-mns16/mid/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns16", + "completed_fraction": 0.9961111111111111, + "level": "low", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.001459145, + "max_internal_gap_s": 0.6478406, + "start_gap_s": 0.120533656 + }, + "trial_id": "sessions/rep3-mns16/low/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns16", + "completed_fraction": 0.9392156862745098, + "level": "mid", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.024192492, + "max_internal_gap_s": 0.182904479, + "start_gap_s": 0.124699964 + }, + "trial_id": "sessions/rep3-mns16/mid/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns64", + "completed_fraction": 0.9966666666666667, + "level": "low", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.004830812, + "max_internal_gap_s": 0.652764433, + "start_gap_s": 0.123493588 + }, + "trial_id": "sessions/rep3-mns64/low/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns64", + "completed_fraction": 0.9929411764705882, + "level": "mid", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.021026794, + "max_internal_gap_s": 0.32072392, + "start_gap_s": 0.122478295 + }, + "trial_id": "sessions/rep3-mns64/mid/result.json" + } + ] + } + ], + "decision": "STOP_NO_INCREMENTAL_TUNING_SIGNAL", + "load_consistency": { + "0.50->0.75:graph_padding_fraction": { + "consistent_load_regimes": [ + "low", + "mid" + ], + "direction": 1, + "passes_two_regimes": true + }, + "0.75->1.00:graph_padding_fraction": { + "consistent_load_regimes": [ + "low", + "mid" + ], + "direction": 1, + "passes_two_regimes": true + }, + "0.75->1.00:queue_waiting_mean": { + "consistent_load_regimes": [ + "low", + "mid" + ], + "direction": -1, + "passes_two_regimes": true + } + }, + "mechanism_features": [ + "graph_padding_fraction", + "queue_waiting_mean" + ], + "mechanism_gate": { + "by_transition": { + "0.10->0.25": [], + "0.25->0.50": [], + "0.50->0.75": [ + "graph_padding_fraction" + ], + "0.75->1.00": [ + "graph_padding_fraction", + "queue_waiting_mean" + ] + }, + "minimum_features": 2, + "passes": true, + "passing_transitions": [ + "0.75->1.00" + ] + }, + "provenance": { + "analysis_script": "/home/admin/cpfs/wjh/aituner/aituner-phase-aware-20260714/runs/intervention-response-v2/analyze_pilot.py", + "analysis_script_sha256": "4467aa8e3d1f3c3308847a9bc5b8846daa0b4381966fd8ec3643a8a56e017b08", + "manifest": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/pilot-manifest.json", + "manifest_sha256": "273db1181dcc9d6b64439650d0642ebe553b12e6aa9adebfbe3758a7977e5611", + "run_root": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/runs/pilot", + "streams": [ + { + "bytes": 54326667, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/runs/pilot/sessions/rep1-mns16/opprof/opprof-v1-dp0-pid458427-1784022917661752826.jsonl", + "session": "rep1-mns16", + "sha256": "1c1418223dbc4e21585a9e642e95ff60be53a5762483a8fdf56ea2cf300cc3fb" + }, + { + "bytes": 56460016, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/runs/pilot/sessions/rep1-mns64/opprof/opprof-v1-dp0-pid471212-1784023695315485142.jsonl", + "session": "rep1-mns64", + "sha256": "db4c76070c5d005922fa164545766c84e25ace254793bc64ffeeae8edc8e7a11" + }, + { + "bytes": 55778303, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/runs/pilot/sessions/rep2-mns64/opprof/opprof-v1-dp0-pid483758-1784024456752184540.jsonl", + "session": "rep2-mns64", + "sha256": "7c24a7059ff6ce171fc38237c0a7ccd5dc7dcc4ccefde5e07d9c8a276969e004" + }, + { + "bytes": 52931532, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/runs/pilot/sessions/rep2-mns16/opprof/opprof-v1-dp0-pid496100-1784025206790752505.jsonl", + "session": "rep2-mns16", + "sha256": "db3a43cecf82adfe71acec9e47e45445944277076a54ee5e14f91381c26aa6ca" + }, + { + "bytes": 55604208, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/runs/pilot/sessions/rep3-mns16/opprof/opprof-v1-dp0-pid508913-1784025983046041167.jsonl", + "session": "rep3-mns16", + "sha256": "5465eb5ec08568152bf854597953ada99956b74a3cf8377c4a5c5a544eb8a5b7" + }, + { + "bytes": 58550249, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/runs/pilot/sessions/rep3-mns64/opprof/opprof-v1-dp0-pid521797-1784026767359409666.jsonl", + "session": "rep3-mns64", + "sha256": "7ef67bcd4f9e1ab6b597a7eae03da466677eeba756d1206d02575c497dfd788e" + } + ] + }, + "quarter_blocks": [ + { + "action_pairs": 6, + "actions": [ + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.07082632707989239, + "tpot_mean_over_slo": -0.00490180963817316, + "ttft_max_over_slo_max": -0.04153848900265682, + "ttft_mean_over_slo_max": -0.006891098180069163 + }, + "delta_state": { + "batch_size.mean": -0.2638104929979219, + "batch_tokens.mean": -5.845648825359461, + "decode_batch_size.mean": -0.26164909366742517, + "graph_full_share": 0.002180359902912321, + "graph_none_share": -0.006323059607193614, + "graph_padding_fraction": 0.0018335461795120685, + "kv_usage_end_minus_start": 1.026766711520466e-05, + "kv_usage_max": 0.004459034805309736, + "kv_usage_mean": -0.00014393587383104527, + "preemptions": 0.0, + "prefill_token_fraction": -2.725586323304441e-05, + "queue_running_mean": -0.18631020918666685, + "queue_waiting_mean": -0.20975605228, + "scheduler_steps_per_s": 4.9466666666666725 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 1, + "request_hash": "af50f0c3edfc6f9fb74734b2501378fc8ba6fdd3dd3fe84f2bb1d68540eee69b", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "5856cc9ff24c9a984a2abecc59ab92c1f87b02b2a9dc0a47eae7a08462169373", + "result_sha256": "4e788371d6f24cab1be2bd4aed4bfe0354b5a5c23973bac9bf69ad0aa4c1b7d9", + "trial_id": "sessions/rep1-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "a91536c628053d3cf530bf0b8ebe51628cc41e71af31b90310d3307d1badc473", + "result_sha256": "2dd1d5ae610db6d738291032ab07a3ad73c46235b30e4ff38b87ce2715204bc6", + "trial_id": "sessions/rep1-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.00196078431372549, + "completed_over_admitted": 0.01983471074380172, + "completed_pass_rate": 0.008665511265164683, + "outstanding_over_admitted": -0.01983471074380165, + "tpot_max_over_slo": 0.2753294892902449, + "tpot_mean_over_slo": 0.04011748202807386, + "ttft_max_over_slo_max": -0.2852395010025551, + "ttft_mean_over_slo_max": -0.06592333248276636 + }, + "delta_state": { + "batch_size.mean": -0.8231037548470876, + "batch_tokens.mean": -19.74330932592642, + "decode_batch_size.mean": -0.8152070406366647, + "graph_full_share": 0.00839719842479858, + "graph_none_share": -0.018146448159773024, + "graph_padding_fraction": 0.005340401585168801, + "kv_usage_end_minus_start": -0.004438767356134465, + "kv_usage_max": 0.027975123151637415, + "kv_usage_mean": 0.002385274257794787, + "preemptions": 0.0, + "prefill_token_fraction": -0.00025035500653580733, + "queue_running_mean": 2.133982110813333, + "queue_waiting_mean": -3.33810440732, + "scheduler_steps_per_s": 6.320000000000007 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 1, + "request_hash": "0bb61dbc9c26875e991d0d4f984134910d37463e5063f86ee960cf4f8aafb771", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.5603921568627451, + "mns": 16, + "requests_sha256": "8dcab1a24d009309a6dd54676e762ee4e74717ed530d19b3965f30feba32f193", + "result_sha256": "ae7cbca9785fa548acf8bd82646dfe53694ffd0b58d2ef6953e3d531d4d7ec4f", + "trial_id": "sessions/rep1-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "e7bf606cb77817e67a3e64e60f035a5fb344977a7991be741a81db8b3a8d28fe", + "result_sha256": "9604c8a0fe449d3798c6de566e4c804665d66b859a3b43c716dd5ecbb60bee1c", + "trial_id": "sessions/rep1-mns64/mid/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.004796163069544446, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.004796163069544366, + "tpot_max_over_slo": 0.051139209137158914, + "tpot_mean_over_slo": -0.0074211636478002385, + "ttft_max_over_slo_max": -0.03542824399967988, + "ttft_mean_over_slo_max": -0.0033018737683347885 + }, + "delta_state": { + "batch_size.mean": -0.2685691864197386, + "batch_tokens.mean": -6.959429182398708, + "decode_batch_size.mean": -0.26598172047692614, + "graph_full_share": 0.0028314394428595735, + "graph_none_share": -0.007209322700149463, + "graph_padding_fraction": 0.0012877200176958283, + "kv_usage_end_minus_start": -0.0011333814702583211, + "kv_usage_max": 0.0022799714777140823, + "kv_usage_mean": -0.0001951815494633533, + "preemptions": 0.0, + "prefill_token_fraction": -0.00014483809619469934, + "queue_running_mean": -0.2747586459066662, + "queue_waiting_mean": -0.08017956545333334, + "scheduler_steps_per_s": 5.586666666666659 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 2, + "request_hash": "adc728f189a1ce0301609cdbae7d1dc2318609b13eafb94bcb90903bb7998eb1", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "a5ce6cd64890663ef30486701236af3de6169f86b72349928b07adafcb406bcb", + "result_sha256": "289f8f99f5056b26822bab5c1feb4252042c7852b80c5399271ddb1b6db25fbb", + "trial_id": "sessions/rep2-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "1c6931ea4a0913e2460e18c59f5a07e9c21cb2650454f2dd2f2997c416f63209", + "result_sha256": "9988f2b6c9e8c45144eb7b53aa1bea62942ae8fd36aba10c9e6bf5c8cbd65a50", + "trial_id": "sessions/rep2-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.02980392156862745, + "completed_over_admitted": 0.018394648829431426, + "completed_pass_rate": 0.13356766256590513, + "outstanding_over_admitted": -0.01839464882943144, + "tpot_max_over_slo": 0.23737520567053472, + "tpot_mean_over_slo": 0.010490282370595683, + "ttft_max_over_slo_max": -0.38453158099825185, + "ttft_mean_over_slo_max": -0.19098879755079737 + }, + "delta_state": { + "batch_size.mean": -1.413753104794269, + "batch_tokens.mean": -30.311859850911844, + "decode_batch_size.mean": -1.4061952013981784, + "graph_full_share": 0.021360307900858633, + "graph_none_share": -0.03011422871811248, + "graph_padding_fraction": 0.007512227776197788, + "kv_usage_end_minus_start": 0.0021360977816522464, + "kv_usage_max": 0.022672590909591506, + "kv_usage_mean": 0.0010254846876152635, + "preemptions": 0.0, + "prefill_token_fraction": 0.0003313271687591657, + "queue_running_mean": 0.7838954335199997, + "queue_waiting_mean": -9.07047515872, + "scheduler_steps_per_s": 8.959999999999994 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 2, + "request_hash": "56a0616b6b54abafd37875c7cb25f8639afef2706ccc55dfbe568f45859ea382", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.31450980392156863, + "mns": 16, + "requests_sha256": "e00c0d9acfb0c22624fdde79f4e8a0300fc2aba09ad4851908d68c1f46be30d3", + "result_sha256": "d081e1ac0143b4e8a63f4c60b58905320ca6e6163fa858e78f5643221fd87c2b", + "trial_id": "sessions/rep2-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "e1543f873745dbd1d3a1461e63549747b6e1517d63e281e6cdfa974d2e2798e4", + "result_sha256": "8556783dc3ba3274269500add337c75c493e3eece4f66260449549ee86506035", + "trial_id": "sessions/rep2-mns64/mid/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0024154589371980784, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.0024154589371980714, + "tpot_max_over_slo": -0.006984168036628269, + "tpot_mean_over_slo": -0.01111313281352852, + "ttft_max_over_slo_max": -0.017940445327743276, + "ttft_mean_over_slo_max": -0.0018007220652844075 + }, + "delta_state": { + "batch_size.mean": -0.3420664768336481, + "batch_tokens.mean": -6.950334498514508, + "decode_batch_size.mean": -0.3395672760130166, + "graph_full_share": 0.0029764291623558536, + "graph_none_share": -0.005728618501492012, + "graph_padding_fraction": 0.0005459147599019987, + "kv_usage_end_minus_start": -0.003587734700701506, + "kv_usage_max": 3.276805657914661e-05, + "kv_usage_mean": -0.0002979372353175526, + "preemptions": 0.0, + "prefill_token_fraction": -6.130606240972725e-05, + "queue_running_mean": -0.4039724217999998, + "queue_waiting_mean": -0.030365675946666667, + "scheduler_steps_per_s": 7.826666666666668 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 3, + "request_hash": "4993c26442fc6b5a4c09b6f576fd8b3cb1be7ce83b7a771e4a57ee7600de688d", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "de93d4364969393741a920ea72831b69003cbd9f10bcfd1809848fcf4e948549", + "result_sha256": "3b32b5ff3aea07cb7b7cf252ba23852542cb8429c83123fe772b2c8093c408a6", + "trial_id": "sessions/rep3-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "3623a99013725d7b42845bce09678368b3dc8d82e0ab26fad549c7eab8a6b6fe", + "result_sha256": "77766d9ec36ff11893feb73626768fa593160394f2b4720ff9253d1dfd5fa950", + "trial_id": "sessions/rep3-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.013628620102214661, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.013628620102214647, + "tpot_max_over_slo": 0.08779462519483128, + "tpot_mean_over_slo": -0.005083324937856265, + "ttft_max_over_slo_max": -0.17233173300822574, + "ttft_mean_over_slo_max": -0.03508650576710582 + }, + "delta_state": { + "batch_size.mean": -0.5655282844513536, + "batch_tokens.mean": -11.412062377336781, + "decode_batch_size.mean": -0.562525695471205, + "graph_full_share": 0.009956297228293054, + "graph_none_share": -0.01796699078506772, + "graph_padding_fraction": 0.0038365149210898572, + "kv_usage_end_minus_start": 0.005666174682015579, + "kv_usage_max": 0.01138222251747023, + "kv_usage_mean": 0.0003053257977473668, + "preemptions": 0.0, + "prefill_token_fraction": -1.837494489209668e-05, + "queue_running_mean": 0.1581681952266667, + "queue_waiting_mean": -2.0748791364133337, + "scheduler_steps_per_s": 7.079999999999998 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 3, + "request_hash": "2a2fabe2c4cf176aeb7e0d32fb8e7dbb1f27429a2e7a0cd18d7d186f23096f19", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.2635294117647059, + "mns": 16, + "requests_sha256": "c7fca9b2484c8ea68c6c11438b8e59db74fa60e6189dbb81d8834038156dbf68", + "result_sha256": "bb45674fa5e19ef18d8c13d08493f1a08a62fb682b5d5e8b7d8a1db2fa3b2ea6", + "trial_id": "sessions/rep3-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "d75e1711b8c0485cf1aceaa9e26ae079fa2adc5487d37f28006c01e83c5a782b", + "result_sha256": "1654b044d815beb3a5b8913fde4283b11507464a2de9f41b5ddc3cdb9f2d150b", + "trial_id": "sessions/rep3-mns64/mid/result.json" + } + } + ], + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 6, + "max": 0.23777777777777778, + "min": 0.23, + "n": 12 + }, + "completed_fraction_of_total": { + "distinct_n": 11, + "max": 0.23555555555555557, + "min": 0.2152941176470588, + "n": 12 + } + }, + "efficacy": { + "cross_validation_possible": true, + "label_balance_sufficient": true, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "outcome_delta": { + "best_accuracy": 1.0, + "best_balanced_accuracy": 1.0, + "best_feature": "ttft_max_over_slo_max", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.014901960784313726, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.000980392156862745, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.000980392156862745, + "train_balanced_accuracy": 1.0 + } + ] + }, + "completed_over_admitted": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.009212391585879554, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.00802203951970637, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.011595405949487936, + "train_balanced_accuracy": 1.0 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.06678383128295257, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.004332755632582341, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.004332755632582341, + "train_balanced_accuracy": 1.0 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.009212391585879507, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.00802203951970636, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.011595405949487903, + "train_balanced_accuracy": 1.0 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0694669171659951, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.07931047613736183, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.15410076637521355, + "train_balanced_accuracy": 1.0 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": -0.006252244292828252, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": -0.008098228875692393, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.0027942363662112613, + "train_balanced_accuracy": 1.0 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.1038799885039528, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.10693511100544129, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.16338899500260595, + "train_balanced_accuracy": 1.0 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.019194189767720304, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.02098880197358749, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.03640721533141776, + "train_balanced_accuracy": 1.0 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "positive": 3 + }, + "positive": 3, + "telemetry_delta": { + "best_accuracy": 1.0, + "best_balanced_accuracy": 1.0, + "best_feature": "queue_waiting_mean", + "features": { + "decode_batch_size.mean": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.4510464857421108, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.4510464857421108, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5405943805567954, + "train_balanced_accuracy": 1.0 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0025621174693928428, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.002835030550300963, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.003586973882340435, + "train_balanced_accuracy": 1.0 + } + ] + }, + "kv_usage_mean": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 5.5072124142006764e-05, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 8.069496195816078e-05, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.0004407744068921091, + "train_balanced_accuracy": 1.0 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 0 + ], + "threshold": -3.984050365091196e-05, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -2.2815404062570543e-05, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.00015203565276306064, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_running_mean": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.05829522533999976, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.014071006980000078, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.2987926121666664, + "train_balanced_accuracy": 1.0 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -1.0775293509333335, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -1.1423175943466668, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -1.7739302298, + "train_balanced_accuracy": 1.0 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 6.333333333333329, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 5.63333333333334, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 5.953333333333333, + "train_balanced_accuracy": 1.0 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "positive": 3 + }, + "telemetry_qualifying_features": [], + "transitions": { + "false->true": 3, + "true->true": 3 + } + }, + "end_fraction": 0.25, + "end_s": 75.0, + "qualifying_response_features": [], + "repeat_pairs": 8, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.45379738064250086, + "action_delta": { + "distinct_n": 6, + "max": -0.2638104929979219, + "min": -1.413753104794269, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 0.9705757741391393, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.467554819246339, + "repeat_abs_p95": 2.634518964255186, + "repeat_delta": { + "distinct_n": 8, + "max": 0.7001454483303338, + "min": -2.9313976513752067, + "n": 8 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 9.185745779867744, + "action_delta": { + "distinct_n": 6, + "max": -5.845648825359461, + "min": -30.311859850911844, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 0.298783186954296, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 30.743850996116663, + "repeat_abs_p95": 93.908100721683, + "repeat_delta": { + "distinct_n": 8, + "max": 27.153830102554593, + "min": -100.52302983743428, + "n": 8 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.4510464857421108, + "action_delta": { + "distinct_n": 6, + "max": -0.26164909366742517, + "min": -1.4061952013981784, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 0.9732742893789799, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.4634320362350386, + "repeat_abs_p95": 2.613462716433853, + "repeat_delta": { + "distinct_n": 8, + "max": 0.6993897150480155, + "min": -2.908747043508294, + "n": 8 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.005686813793577217, + "action_delta": { + "distinct_n": 6, + "max": 0.021360307900858633, + "min": 0.002180359902912321, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 1.4307684434738428, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0039746569890581895, + "repeat_abs_p95": 0.021315647296768075, + "repeat_delta": { + "distinct_n": 8, + "max": 0.024883252660879762, + "min": -0.01469009447770353, + "n": 8 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.012588156742608592, + "action_delta": { + "distinct_n": 6, + "max": -0.005728618501492012, + "min": -0.03011422871811248, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 2.4566361209710017, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.005124143797752612, + "repeat_abs_p95": 0.018876919314381714, + "repeat_delta": { + "distinct_n": 8, + "max": 0.011349421052443798, + "min": -0.022930187609271374, + "n": 8 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 0.002835030550300963, + "action_delta": { + "distinct_n": 6, + "max": 0.007512227776197788, + "min": 0.0005459147599019987, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 1.9399840028075197, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0014613680041681495, + "repeat_abs_p95": 0.004511807887601192, + "repeat_delta": { + "distinct_n": 8, + "max": 0.005553318782013202, + "min": -0.0010848582943063204, + "n": 8 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.002861916241176876, + "action_delta": { + "distinct_n": 6, + "max": 0.005666174682015579, + "min": -0.004438767356134465, + "n": 6 + }, + "action_signs": { + "consistency": 0.5, + "negative": 3, + "positive": 3, + "zero": 0 + }, + "effect_to_repeat_median": 0.547685400888532, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.00522547476440649, + "repeat_abs_p95": 0.007084887036630881, + "repeat_delta": { + "distinct_n": 8, + "max": 0.007595504930191788, + "min": -0.001265067308722334, + "n": 8 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.007920628661389983, + "action_delta": { + "distinct_n": 6, + "max": 0.027975123151637415, + "min": 3.276805657914661e-05, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 5.977407604410328, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0013250942859486248, + "repeat_abs_p95": 0.0092433410331364, + "repeat_delta": { + "distinct_n": 8, + "max": 0.0011681468294250097, + "min": -0.011994317136490906, + "n": 8 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0003016315165324597, + "action_delta": { + "distinct_n": 6, + "max": 0.002385274257794787, + "min": -0.0002979372353175526, + "n": 6 + }, + "action_signs": { + "consistency": 0.5, + "negative": 3, + "positive": 3, + "zero": 0 + }, + "effect_to_repeat_median": 0.1832373452641962, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.001646124681066296, + "repeat_abs_p95": 0.004755246907733842, + "repeat_delta": { + "distinct_n": 8, + "max": 0.001880248703786131, + "min": -0.005007302519187606, + "n": 8 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 6 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 6 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 8 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.00010307207930221329, + "action_delta": { + "distinct_n": 6, + "max": 0.0003313271687591657, + "min": -0.00025035500653580733, + "n": 6 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 5, + "positive": 1, + "zero": 0 + }, + "effect_to_repeat_median": 0.0193424286695313, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.005328807517567591, + "repeat_abs_p95": 0.008606069879593642, + "repeat_delta": { + "distinct_n": 8, + "max": 0.004217023010198662, + "min": -0.008635306091418382, + "n": 8 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.339365533853333, + "action_delta": { + "distinct_n": 6, + "max": 2.133982110813333, + "min": -0.4039724217999998, + "n": 6 + }, + "action_signs": { + "consistency": 0.5, + "negative": 3, + "positive": 3, + "zero": 0 + }, + "effect_to_repeat_median": 0.4388333493005985, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.7733357877066664, + "repeat_abs_p95": 3.1493800733973325, + "repeat_delta": { + "distinct_n": 8, + "max": 1.0551951064133327, + "min": -3.3683846067999994, + "n": 8 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 1.1423175943466668, + "action_delta": { + "distinct_n": 6, + "max": -0.030365675946666667, + "min": -9.07047515872, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 42.90527668627242, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.02662417498666667, + "repeat_abs_p95": 6.55139452659, + "repeat_delta": { + "distinct_n": 8, + "max": 5.732538843706667, + "min": -6.992316817373333, + "n": 8 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 6.700000000000003, + "action_delta": { + "distinct_n": 6, + "max": 8.959999999999994, + "min": 4.9466666666666725, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 1.0959651035986926, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 6.11333333333333, + "repeat_abs_p95": 14.715333333333335, + "repeat_delta": { + "distinct_n": 8, + "max": 15.373333333333335, + "min": -4.8799999999999955, + "n": 8 + } + } + }, + "sanity": { + "invariants": { + "all_results_uncensored": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "layer1_boundary_and_internal_gaps_bounded": true, + "nonnegative_counters": true, + "per_cell_state_vectors_not_all_identical": true, + "ratios_bounded": true, + "state_vectors_not_all_identical": true + }, + "red_flags": [], + "trials": 12 + }, + "start_s": 0.0, + "trial_sanity": [ + { + "admitted_fraction": 0.23777777777777778, + "cell": "tp4_mns16", + "completed_fraction": 0.23555555555555557, + "level": "low", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.001928339, + "max_internal_gap_s": 0.648316718, + "start_gap_s": 0.042673865 + }, + "trial_id": "sessions/rep1-mns16/low/result.json" + }, + { + "admitted_fraction": 0.2372549019607843, + "cell": "tp4_mns16", + "completed_fraction": 0.22627450980392155, + "level": "mid", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.022443998, + "max_internal_gap_s": 0.186336261, + "start_gap_s": 0.042930986 + }, + "trial_id": "sessions/rep1-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.23777777777777778, + "cell": "tp4_mns64", + "completed_fraction": 0.23555555555555557, + "level": "low", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.004213731, + "max_internal_gap_s": 0.637319097, + "start_gap_s": 0.043186851 + }, + "trial_id": "sessions/rep1-mns64/low/result.json" + }, + { + "admitted_fraction": 0.2372549019607843, + "cell": "tp4_mns64", + "completed_fraction": 0.23098039215686275, + "level": "mid", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.003542116, + "max_internal_gap_s": 0.184687818, + "start_gap_s": 0.043187905 + }, + "trial_id": "sessions/rep1-mns64/mid/result.json" + }, + { + "admitted_fraction": 0.23166666666666666, + "cell": "tp4_mns64", + "completed_fraction": 0.22611111111111112, + "level": "low", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.007493238, + "max_internal_gap_s": 0.185686919, + "start_gap_s": 0.107508086 + }, + "trial_id": "sessions/rep2-mns64/low/result.json" + }, + { + "admitted_fraction": 0.23450980392156862, + "cell": "tp4_mns64", + "completed_fraction": 0.22745098039215686, + "level": "mid", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.106828533, + "max_internal_gap_s": 0.177241403, + "start_gap_s": 0.107570476 + }, + "trial_id": "sessions/rep2-mns64/mid/result.json" + }, + { + "admitted_fraction": 0.23166666666666666, + "cell": "tp4_mns16", + "completed_fraction": 0.22499999999999998, + "level": "low", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.01771671, + "max_internal_gap_s": 0.174851445, + "start_gap_s": 0.107276851 + }, + "trial_id": "sessions/rep2-mns16/low/result.json" + }, + { + "admitted_fraction": 0.23450980392156862, + "cell": "tp4_mns16", + "completed_fraction": 0.2231372549019608, + "level": "mid", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.049603839, + "max_internal_gap_s": 0.184443946, + "start_gap_s": 0.10804117 + }, + "trial_id": "sessions/rep2-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.23, + "cell": "tp4_mns16", + "completed_fraction": 0.2205555555555556, + "level": "low", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.001863681, + "max_internal_gap_s": 0.6478406, + "start_gap_s": 0.120533656 + }, + "trial_id": "sessions/rep3-mns16/low/result.json" + }, + { + "admitted_fraction": 0.23019607843137255, + "cell": "tp4_mns16", + "completed_fraction": 0.2152941176470588, + "level": "mid", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.17047515, + "max_internal_gap_s": 0.182092338, + "start_gap_s": 0.124699964 + }, + "trial_id": "sessions/rep3-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.23, + "cell": "tp4_mns64", + "completed_fraction": 0.22111111111111112, + "level": "low", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.000487667, + "max_internal_gap_s": 0.652764433, + "start_gap_s": 0.123493588 + }, + "trial_id": "sessions/rep3-mns64/low/result.json" + }, + { + "admitted_fraction": 0.23019607843137255, + "cell": "tp4_mns64", + "completed_fraction": 0.2184313725490196, + "level": "mid", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.006237065, + "max_internal_gap_s": 0.32072392, + "start_gap_s": 0.122478295 + }, + "trial_id": "sessions/rep3-mns64/mid/result.json" + } + ] + }, + { + "action_pairs": 6, + "actions": [ + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.07082632707989239, + "tpot_mean_over_slo": -0.002512563182247657, + "ttft_max_over_slo_max": -0.0679067323411194, + "ttft_mean_over_slo_max": -0.007600825142914866 + }, + "delta_state": { + "batch_size.mean": -0.15804783275966194, + "batch_tokens.mean": -3.0894547775551757, + "decode_batch_size.mean": -0.15660557052023627, + "graph_full_share": 0.001103652377702713, + "graph_none_share": -0.0043728510978748095, + "graph_padding_fraction": 0.0020921453623694904, + "kv_usage_end_minus_start": -1.1745909822225009e-05, + "kv_usage_max": 0.003994488881572833, + "kv_usage_mean": 8.324804127910184e-05, + "preemptions": 0.0, + "prefill_token_fraction": 3.1223127688440044e-05, + "queue_running_mean": -0.011106401786666886, + "queue_waiting_mean": -0.27086107688, + "scheduler_steps_per_s": 3.173333333333332 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 1, + "request_hash": "af50f0c3edfc6f9fb74734b2501378fc8ba6fdd3dd3fe84f2bb1d68540eee69b", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "5856cc9ff24c9a984a2abecc59ab92c1f87b02b2a9dc0a47eae7a08462169373", + "result_sha256": "4e788371d6f24cab1be2bd4aed4bfe0354b5a5c23973bac9bf69ad0aa4c1b7d9", + "trial_id": "sessions/rep1-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "a91536c628053d3cf530bf0b8ebe51628cc41e71af31b90310d3307d1badc473", + "result_sha256": "2dd1d5ae610db6d738291032ab07a3ad73c46235b30e4ff38b87ce2715204bc6", + "trial_id": "sessions/rep1-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.0027450980392156863, + "completed_over_admitted": 0.0008250825082508628, + "completed_pass_rate": 0.005833333333333357, + "outstanding_over_admitted": -0.0008250825082508247, + "tpot_max_over_slo": 0.2753294892902449, + "tpot_mean_over_slo": 0.018526148273117438, + "ttft_max_over_slo_max": -0.2765812376649895, + "ttft_mean_over_slo_max": -0.08009172416899431 + }, + "delta_state": { + "batch_size.mean": -2.2691156081486135, + "batch_tokens.mean": -44.89764761415685, + "decode_batch_size.mean": -2.2524677264815143, + "graph_full_share": 0.021894194879364726, + "graph_none_share": -0.030171302893482743, + "graph_padding_fraction": 0.007904176391026585, + "kv_usage_end_minus_start": 0.002935218401001971, + "kv_usage_max": 0.01698116274304451, + "kv_usage_mean": 9.548656867379608e-05, + "preemptions": 0.0, + "prefill_token_fraction": 0.00015029680675249235, + "queue_running_mean": -0.35042022754666746, + "queue_waiting_mean": -4.181868913853333, + "scheduler_steps_per_s": 11.733333333333334 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 1, + "request_hash": "0bb61dbc9c26875e991d0d4f984134910d37463e5063f86ee960cf4f8aafb771", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.5603921568627451, + "mns": 16, + "requests_sha256": "8dcab1a24d009309a6dd54676e762ee4e74717ed530d19b3965f30feba32f193", + "result_sha256": "ae7cbca9785fa548acf8bd82646dfe53694ffd0b58d2ef6953e3d531d4d7ec4f", + "trial_id": "sessions/rep1-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "e7bf606cb77817e67a3e64e60f035a5fb344977a7991be741a81db8b3a8d28fe", + "result_sha256": "9604c8a0fe449d3798c6de566e4c804665d66b859a3b43c716dd5ecbb60bee1c", + "trial_id": "sessions/rep1-mns64/mid/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.051139209137158914, + "tpot_mean_over_slo": -0.009262480213267976, + "ttft_max_over_slo_max": -0.03542824399967988, + "ttft_mean_over_slo_max": -0.003943418327769104 + }, + "delta_state": { + "batch_size.mean": -0.5154516769784294, + "batch_tokens.mean": -10.397397526512549, + "decode_batch_size.mean": -0.511830034908046, + "graph_full_share": 0.004410718283113013, + "graph_none_share": -0.009755038550069609, + "graph_padding_fraction": 0.0011193731937274784, + "kv_usage_end_minus_start": 0.0011418132801096492, + "kv_usage_max": 0.003853870265263404, + "kv_usage_mean": -0.0002787530429213137, + "preemptions": 0.0, + "prefill_token_fraction": 0.00014412819519771247, + "queue_running_mean": -0.46609635046666753, + "queue_waiting_mean": -0.15291618469333335, + "scheduler_steps_per_s": 6.813333333333333 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 2, + "request_hash": "adc728f189a1ce0301609cdbae7d1dc2318609b13eafb94bcb90903bb7998eb1", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "a5ce6cd64890663ef30486701236af3de6169f86b72349928b07adafcb406bcb", + "result_sha256": "289f8f99f5056b26822bab5c1feb4252042c7852b80c5399271ddb1b6db25fbb", + "trial_id": "sessions/rep2-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "1c6931ea4a0913e2460e18c59f5a07e9c21cb2650454f2dd2f2997c416f63209", + "result_sha256": "9988f2b6c9e8c45144eb7b53aa1bea62942ae8fd36aba10c9e6bf5c8cbd65a50", + "trial_id": "sessions/rep2-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.14941176470588236, + "completed_over_admitted": 0.044764188649080716, + "completed_pass_rate": 0.32206255283178364, + "outstanding_over_admitted": -0.04476418864908073, + "tpot_max_over_slo": 0.2416662007889263, + "tpot_mean_over_slo": 0.010014700010439026, + "ttft_max_over_slo_max": -0.9367796959995758, + "ttft_mean_over_slo_max": -0.3197808858475722 + }, + "delta_state": { + "batch_size.mean": -0.2846481455922074, + "batch_tokens.mean": -9.105088784407371, + "decode_batch_size.mean": -0.28638908954680886, + "graph_full_share": 0.02080155419765417, + "graph_none_share": -0.03302550516502446, + "graph_padding_fraction": 0.009148827020083142, + "kv_usage_end_minus_start": -0.0029587377234424617, + "kv_usage_max": 0.021920351504425173, + "kv_usage_mean": 0.0021466714075248396, + "preemptions": 0.0, + "prefill_token_fraction": -0.0004908425008621764, + "queue_running_mean": 1.7869539801733332, + "queue_waiting_mean": -24.542158628386666, + "scheduler_steps_per_s": 6.186666666666667 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 2, + "request_hash": "56a0616b6b54abafd37875c7cb25f8639afef2706ccc55dfbe568f45859ea382", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.31450980392156863, + "mns": 16, + "requests_sha256": "e00c0d9acfb0c22624fdde79f4e8a0300fc2aba09ad4851908d68c1f46be30d3", + "result_sha256": "d081e1ac0143b4e8a63f4c60b58905320ca6e6163fa858e78f5643221fd87c2b", + "trial_id": "sessions/rep2-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "e1543f873745dbd1d3a1461e63549747b6e1517d63e281e6cdfa974d2e2798e4", + "result_sha256": "8556783dc3ba3274269500add337c75c493e3eece4f66260449549ee86506035", + "trial_id": "sessions/rep2-mns64/mid/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.001179245283018937, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.0011792452830188694, + "tpot_max_over_slo": 0.04993343196889949, + "tpot_mean_over_slo": -0.008105257443938751, + "ttft_max_over_slo_max": -0.026339164838039622, + "ttft_mean_over_slo_max": -0.0029500323901768717 + }, + "delta_state": { + "batch_size.mean": -0.22671705086071015, + "batch_tokens.mean": -4.740617496917622, + "decode_batch_size.mean": -0.22507516605851396, + "graph_full_share": 0.0006042607986376236, + "graph_none_share": -0.005169444156433126, + "graph_padding_fraction": 0.0008846012466042618, + "kv_usage_end_minus_start": 0.003668730096473505, + "kv_usage_max": 0.008719338263833865, + "kv_usage_mean": -0.00012598990958833517, + "preemptions": 0.0, + "prefill_token_fraction": -7.139411620438185e-05, + "queue_running_mean": -0.21431314688000036, + "queue_waiting_mean": -0.10833277018666666, + "scheduler_steps_per_s": 4.613333333333344 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 3, + "request_hash": "4993c26442fc6b5a4c09b6f576fd8b3cb1be7ce83b7a771e4a57ee7600de688d", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "de93d4364969393741a920ea72831b69003cbd9f10bcfd1809848fcf4e948549", + "result_sha256": "3b32b5ff3aea07cb7b7cf252ba23852542cb8429c83123fe772b2c8093c408a6", + "trial_id": "sessions/rep3-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "3623a99013725d7b42845bce09678368b3dc8d82e0ab26fad549c7eab8a6b6fe", + "result_sha256": "77766d9ec36ff11893feb73626768fa593160394f2b4720ff9253d1dfd5fa950", + "trial_id": "sessions/rep3-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.19137254901960785, + "completed_over_admitted": 0.034146341463414664, + "completed_pass_rate": 0.42068965517241375, + "outstanding_over_admitted": -0.03414634146341463, + "tpot_max_over_slo": 0.1493312743232093, + "tpot_mean_over_slo": 0.00256767021710691, + "ttft_max_over_slo_max": -0.9516305106711418, + "ttft_mean_over_slo_max": -0.3356434934483016 + }, + "delta_state": { + "batch_size.mean": -1.1031807575648234, + "batch_tokens.mean": -22.138099877318325, + "decode_batch_size.mean": -1.098093622958313, + "graph_full_share": 0.024621184785722505, + "graph_none_share": -0.036507530380663095, + "graph_padding_fraction": 0.008444758995033443, + "kv_usage_end_minus_start": -0.0009272853178361462, + "kv_usage_max": 0.012606264387448385, + "kv_usage_mean": 0.0018642037638938545, + "preemptions": 0.0, + "prefill_token_fraction": 0.0001249562235060564, + "queue_running_mean": 1.5849805495733342, + "queue_waiting_mean": -32.03782532756, + "scheduler_steps_per_s": 8.840000000000003 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 3, + "request_hash": "2a2fabe2c4cf176aeb7e0d32fb8e7dbb1f27429a2e7a0cd18d7d186f23096f19", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.2635294117647059, + "mns": 16, + "requests_sha256": "c7fca9b2484c8ea68c6c11438b8e59db74fa60e6189dbb81d8834038156dbf68", + "result_sha256": "bb45674fa5e19ef18d8c13d08493f1a08a62fb682b5d5e8b7d8a1db2fa3b2ea6", + "trial_id": "sessions/rep3-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "d75e1711b8c0485cf1aceaa9e26ae079fa2adc5487d37f28006c01e83c5a782b", + "result_sha256": "1654b044d815beb3a5b8913fde4283b11507464a2de9f41b5ddc3cdb9f2d150b", + "trial_id": "sessions/rep3-mns64/mid/result.json" + } + } + ], + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 6, + "max": 0.49444444444444446, + "min": 0.4711111111111111, + "n": 12 + }, + "completed_fraction_of_total": { + "distinct_n": 10, + "max": 0.49111111111111116, + "min": 0.4549019607843137, + "n": 12 + } + }, + "efficacy": { + "cross_validation_possible": true, + "label_balance_sufficient": true, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "outcome_delta": { + "best_accuracy": 0.8333333333333334, + "best_balanced_accuracy": 0.8333333333333333, + "best_feature": "ttft_mean_over_slo_max", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.07470588235294118, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.0013725490196078432, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.0013725490196078432, + "train_balanced_accuracy": 1.0 + } + ] + }, + "completed_over_admitted": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.0176627933732168, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0004125412541254314, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0004125412541254314, + "train_balanced_accuracy": 1.0 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.16103127641589182, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0029166666666666785, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0029166666666666785, + "train_balanced_accuracy": 1.0 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.01766279337321675, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.00041254125412541233, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": -0.00041254125412541233, + "train_balanced_accuracy": 1.0 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.10023524173018411, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.11007880070155085, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.15624626393440935, + "train_balanced_accuracy": 1.0 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": -0.002768793613415921, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 2.755351742962653e-05, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.0037510684140956846, + "train_balanced_accuracy": 1.0 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.48610396999962785, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.17224398500305443, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.17224398500305443, + "train_balanced_accuracy": 1.0 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.16186215208767066, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.04384627465595459, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.04384627465595459, + "train_balanced_accuracy": 1.0 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "positive": 3 + }, + "positive": 3, + "telemetry_delta": { + "best_accuracy": 1.0, + "best_balanced_accuracy": 1.0, + "best_feature": "graph_padding_fraction", + "features": { + "decode_batch_size.mean": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.2557321278026614, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.6615843945084134, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": -0.22149733003352257, + "train_balanced_accuracy": 0.75 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.004782066094380461, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.004998160876698038, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.004998160876698038, + "train_balanced_accuracy": 1.0 + } + ] + }, + "kv_usage_mean": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.0008691069271527596, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 8.936730497644896e-05, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 8.936730497644896e-05, + "train_balanced_accuracy": 1.0 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.16666666666666666, + "balanced_accuracy": 0.16666666666666666, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 0 + ], + "threshold": 0.00013454220935188443, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 0 + ], + "threshold": 7.808967559724822e-05, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.0001472125009751024, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_running_mean": { + "accuracy": 0.3333333333333333, + "balanced_accuracy": 0.3333333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.6853337013466669, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 0 + ], + "threshold": -0.2823666872133339, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": -0.4082582890066675, + "train_balanced_accuracy": 0.75 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -12.347537406539999, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -2.2263649953666667, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -2.2263649953666667, + "train_balanced_accuracy": 1.0 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 5.400000000000006, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 0 + ], + "threshold": 6.726666666666674, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 4.68, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "positive": 3 + }, + "telemetry_qualifying_features": [ + "graph_padding_fraction" + ], + "transitions": { + "false->true": 3, + "true->true": 3 + } + }, + "end_fraction": 0.5, + "end_s": 150.0, + "qualifying_response_features": [ + "graph_padding_fraction" + ], + "repeat_pairs": 8, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.4000499112853184, + "action_delta": { + "distinct_n": 6, + "max": -0.15804783275966194, + "min": -2.2691156081486135, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 0.2802630070966653, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 1.4274089021935654, + "repeat_abs_p95": 2.672248195457046, + "repeat_delta": { + "distinct_n": 8, + "max": 3.099204835224729, + "min": -1.621627553120839, + "n": 8 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 9.75124315545996, + "action_delta": { + "distinct_n": 6, + "max": -3.0894547775551757, + "min": -44.89764761415685, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 0.2665496795389639, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 36.58321095086717, + "repeat_abs_p95": 57.539092487805476, + "repeat_delta": { + "distinct_n": 8, + "max": 63.80774773212434, + "min": -40.23384050335905, + "n": 8 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.39910956222742744, + "action_delta": { + "distinct_n": 6, + "max": -0.15660557052023627, + "min": -2.2524677264815143, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 0.2818240506833578, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 1.4161657291479544, + "repeat_abs_p95": 2.6502790089519457, + "repeat_delta": { + "distinct_n": 8, + "max": 3.0729743073386615, + "min": -1.6090373367280613, + "n": 8 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.012606136240383592, + "action_delta": { + "distinct_n": 6, + "max": 0.024621184785722505, + "min": 0.0006042607986376236, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 1.1046254041741963, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.011412136813753415, + "repeat_abs_p95": 0.020065382031526628, + "repeat_delta": { + "distinct_n": 8, + "max": 0.012164469199423045, + "min": -0.020447806270125324, + "n": 8 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.019963170721776176, + "action_delta": { + "distinct_n": 6, + "max": -0.0043728510978748095, + "min": -0.036507530380663095, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 1.8698607257091873, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.010676287515587392, + "repeat_abs_p95": 0.017837185208830555, + "repeat_delta": { + "distinct_n": 8, + "max": 0.01883615600387016, + "min": -0.012662463335994696, + "n": 8 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.004998160876698038, + "action_delta": { + "distinct_n": 6, + "max": 0.009148827020083142, + "min": 0.0008846012466042618, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 5.841347922729203, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": true, + "repeat_abs_median": 0.0008556519732799599, + "repeat_abs_p95": 0.002114872464387871, + "repeat_delta": { + "distinct_n": 8, + "max": 0.002067321151258763, + "min": -0.0021404770176112367, + "n": 8 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.00203851584055581, + "action_delta": { + "distinct_n": 6, + "max": 0.003668730096473505, + "min": -0.0029587377234424617, + "n": 6 + }, + "action_signs": { + "consistency": 0.5, + "negative": 3, + "positive": 3, + "zero": 0 + }, + "effect_to_repeat_median": 0.689091144297968, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0029582673604557908, + "repeat_abs_p95": 0.008930052912586221, + "repeat_delta": { + "distinct_n": 8, + "max": 0.0019231063523722502, + "min": -0.009814473798313572, + "n": 8 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.6666666666666666, + "action_abs_median": 0.010662801325641125, + "action_delta": { + "distinct_n": 6, + "max": 0.021920351504425173, + "min": 0.003853870265263404, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 5.783811852453453, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0018435595067149424, + "repeat_abs_p95": 0.006457102557288577, + "repeat_delta": { + "distinct_n": 8, + "max": 0.0035006848054948136, + "min": -0.008049019808254454, + "n": 8 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.00020237147625482444, + "action_delta": { + "distinct_n": 6, + "max": 0.0021466714075248396, + "min": -0.0002787530429213137, + "n": 6 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 2, + "positive": 4, + "zero": 0 + }, + "effect_to_repeat_median": 0.1687685271313482, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0011991067273895441, + "repeat_abs_p95": 0.0023351857861895597, + "repeat_delta": { + "distinct_n": 8, + "max": 0.0027611025339853053, + "min": -0.001368780132029305, + "n": 8 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 6 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 6 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 8 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.00013454220935188443, + "action_delta": { + "distinct_n": 6, + "max": 0.00015029680675249235, + "min": -0.0004908425008621764, + "n": 6 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 2, + "positive": 4, + "zero": 0 + }, + "effect_to_repeat_median": 0.0774352591631262, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0017374799388022444, + "repeat_abs_p95": 0.0025224968780584887, + "repeat_delta": { + "distinct_n": 8, + "max": 0.0025533385639567863, + "min": -0.0024652194613902223, + "n": 8 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.4082582890066675, + "action_delta": { + "distinct_n": 6, + "max": 1.7869539801733332, + "min": -0.46609635046666753, + "n": 6 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 4, + "positive": 2, + "zero": 0 + }, + "effect_to_repeat_median": 0.38701557093710687, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 1.0548885359266662, + "repeat_abs_p95": 2.4461823174926662, + "repeat_delta": { + "distinct_n": 8, + "max": 2.9756512754266673, + "min": -1.3536673999333333, + "n": 8 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 2.2263649953666667, + "action_delta": { + "distinct_n": 6, + "max": -0.10833277018666666, + "min": -32.03782532756, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 88.19287762590257, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.025244272046666672, + "repeat_abs_p95": 15.854935623909324, + "repeat_delta": { + "distinct_n": 8, + "max": 20.359169682466664, + "min": -0.11883915477333334, + "n": 8 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 6.5, + "action_delta": { + "distinct_n": 6, + "max": 11.733333333333334, + "min": 3.173333333333332, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 0.5247578040904196, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 12.38666666666667, + "repeat_abs_p95": 17.40266666666666, + "repeat_delta": { + "distinct_n": 8, + "max": 15.599999999999994, + "min": -18.373333333333335, + "n": 8 + } + } + }, + "sanity": { + "invariants": { + "all_results_uncensored": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "layer1_boundary_and_internal_gaps_bounded": true, + "nonnegative_counters": true, + "per_cell_state_vectors_not_all_identical": true, + "ratios_bounded": true, + "state_vectors_not_all_identical": true + }, + "red_flags": [], + "trials": 12 + }, + "start_s": 75.0, + "trial_sanity": [ + { + "admitted_fraction": 0.47555555555555556, + "cell": "tp4_mns16", + "completed_fraction": 0.4733333333333334, + "level": "low", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.001282354, + "max_internal_gap_s": 0.471713815, + "start_gap_s": 0.002667743 + }, + "trial_id": "sessions/rep1-mns16/low/result.json" + }, + { + "admitted_fraction": 0.4752941176470588, + "cell": "tp4_mns16", + "completed_fraction": 0.47058823529411764, + "level": "mid", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.004049563, + "max_internal_gap_s": 0.179027071, + "start_gap_s": 0.035944018 + }, + "trial_id": "sessions/rep1-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.47555555555555556, + "cell": "tp4_mns64", + "completed_fraction": 0.4733333333333334, + "level": "low", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.003246004, + "max_internal_gap_s": 0.48099535, + "start_gap_s": 0.000371266 + }, + "trial_id": "sessions/rep1-mns64/low/result.json" + }, + { + "admitted_fraction": 0.4752941176470588, + "cell": "tp4_mns64", + "completed_fraction": 0.47098039215686277, + "level": "mid", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.00327497, + "max_internal_gap_s": 0.178353557, + "start_gap_s": 0.002613787 + }, + "trial_id": "sessions/rep1-mns64/mid/result.json" + }, + { + "admitted_fraction": 0.49444444444444446, + "cell": "tp4_mns64", + "completed_fraction": 0.49111111111111116, + "level": "low", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.003493587, + "max_internal_gap_s": 0.444552932, + "start_gap_s": 0.00122792 + }, + "trial_id": "sessions/rep2-mns64/low/result.json" + }, + { + "admitted_fraction": 0.49058823529411766, + "cell": "tp4_mns64", + "completed_fraction": 0.48588235294117643, + "level": "mid", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.0063142, + "max_internal_gap_s": 0.183290846, + "start_gap_s": 0.062765702 + }, + "trial_id": "sessions/rep2-mns64/mid/result.json" + }, + { + "admitted_fraction": 0.49444444444444446, + "cell": "tp4_mns16", + "completed_fraction": 0.49111111111111116, + "level": "low", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.012013597, + "max_internal_gap_s": 0.445602102, + "start_gap_s": 0.037343372 + }, + "trial_id": "sessions/rep2-mns16/low/result.json" + }, + { + "admitted_fraction": 0.49058823529411766, + "cell": "tp4_mns16", + "completed_fraction": 0.463921568627451, + "level": "mid", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.02999801, + "max_internal_gap_s": 0.176673609, + "start_gap_s": 0.001405349 + }, + "trial_id": "sessions/rep2-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.4711111111111111, + "cell": "tp4_mns16", + "completed_fraction": 0.4655555555555555, + "level": "low", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.026096497, + "max_internal_gap_s": 0.24359672, + "start_gap_s": 0.004691362 + }, + "trial_id": "sessions/rep3-mns16/low/result.json" + }, + { + "admitted_fraction": 0.4823529411764706, + "cell": "tp4_mns16", + "completed_fraction": 0.4549019607843137, + "level": "mid", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.001098192, + "max_internal_gap_s": 0.177840927, + "start_gap_s": 0.00439926 + }, + "trial_id": "sessions/rep3-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.4711111111111111, + "cell": "tp4_mns64", + "completed_fraction": 0.4661111111111111, + "level": "low", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.001270441, + "max_internal_gap_s": 0.272099734, + "start_gap_s": 0.006329393 + }, + "trial_id": "sessions/rep3-mns64/low/result.json" + }, + { + "admitted_fraction": 0.4823529411764706, + "cell": "tp4_mns64", + "completed_fraction": 0.4713725490196079, + "level": "mid", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.048916774, + "max_internal_gap_s": 0.184986457, + "start_gap_s": 0.001848903 + }, + "trial_id": "sessions/rep3-mns64/mid/result.json" + } + ] + }, + { + "action_pairs": 6, + "actions": [ + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.13955613873224326, + "tpot_mean_over_slo": 0.00021561369396999597, + "ttft_max_over_slo_max": -0.162763388332678, + "ttft_mean_over_slo_max": -0.0113370356308976 + }, + "delta_state": { + "batch_size.mean": -0.2888291061573778, + "batch_tokens.mean": -6.113142522726264, + "decode_batch_size.mean": -0.28726153067008475, + "graph_full_share": 0.003556405337845958, + "graph_none_share": -0.008054518886335793, + "graph_padding_fraction": 0.003354803792462677, + "kv_usage_end_minus_start": -7.914574800538254e-06, + "kv_usage_max": 0.012148644893025695, + "kv_usage_mean": 0.0003717674926776663, + "preemptions": 0.0, + "prefill_token_fraction": -1.0418331766737232e-05, + "queue_running_mean": 0.18344752267999986, + "queue_waiting_mean": -0.6748495923466666, + "scheduler_steps_per_s": 3.9866666666666646 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 1, + "request_hash": "af50f0c3edfc6f9fb74734b2501378fc8ba6fdd3dd3fe84f2bb1d68540eee69b", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "5856cc9ff24c9a984a2abecc59ab92c1f87b02b2a9dc0a47eae7a08462169373", + "result_sha256": "4e788371d6f24cab1be2bd4aed4bfe0354b5a5c23973bac9bf69ad0aa4c1b7d9", + "trial_id": "sessions/rep1-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "a91536c628053d3cf530bf0b8ebe51628cc41e71af31b90310d3307d1badc473", + "result_sha256": "2dd1d5ae610db6d738291032ab07a3ad73c46235b30e4ff38b87ce2715204bc6", + "trial_id": "sessions/rep1-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.14980392156862746, + "completed_over_admitted": 0.03292618162506633, + "completed_pass_rate": 0.21093318608503586, + "outstanding_over_admitted": -0.032926181625066386, + "tpot_max_over_slo": 0.3362407738628878, + "tpot_mean_over_slo": 0.035823037105198796, + "ttft_max_over_slo_max": -1.0155032691739811, + "ttft_mean_over_slo_max": -0.23418190946398737 + }, + "delta_state": { + "batch_size.mean": 0.7340835489721993, + "batch_tokens.mean": 4.870249914593785, + "decode_batch_size.mean": 0.7235214534720207, + "graph_full_share": 0.01432646120104708, + "graph_none_share": -0.025264419492621093, + "graph_padding_fraction": 0.006957260165806216, + "kv_usage_end_minus_start": -0.007513472509556363, + "kv_usage_max": 0.031275425686576486, + "kv_usage_mean": 0.0049451651951918, + "preemptions": 0.0, + "prefill_token_fraction": -0.0014595808679910283, + "queue_running_mean": 5.284704507639999, + "queue_waiting_mean": -29.284492367786665, + "scheduler_steps_per_s": 3.373333333333335 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 1, + "request_hash": "0bb61dbc9c26875e991d0d4f984134910d37463e5063f86ee960cf4f8aafb771", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.5603921568627451, + "mns": 16, + "requests_sha256": "8dcab1a24d009309a6dd54676e762ee4e74717ed530d19b3965f30feba32f193", + "result_sha256": "ae7cbca9785fa548acf8bd82646dfe53694ffd0b58d2ef6953e3d531d4d7ec4f", + "trial_id": "sessions/rep1-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "e7bf606cb77817e67a3e64e60f035a5fb344977a7991be741a81db8b3a8d28fe", + "result_sha256": "9604c8a0fe449d3798c6de566e4c804665d66b859a3b43c716dd5ecbb60bee1c", + "trial_id": "sessions/rep1-mns64/mid/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0007401924500369805, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.00074019245003701, + "tpot_max_over_slo": 0.17641319103142994, + "tpot_mean_over_slo": -0.008167534216044714, + "ttft_max_over_slo_max": -0.055667730843803526, + "ttft_mean_over_slo_max": -0.004452474842343956 + }, + "delta_state": { + "batch_size.mean": -0.4169380613868432, + "batch_tokens.mean": -8.458043870097356, + "decode_batch_size.mean": -0.4135692006454068, + "graph_full_share": 0.003382014563382052, + "graph_none_share": -0.01001403055152162, + "graph_padding_fraction": 0.0016407136927011953, + "kv_usage_end_minus_start": -0.0014920741368527057, + "kv_usage_max": 0.013006718003583773, + "kv_usage_mean": -4.9942287421177005e-05, + "preemptions": 0.0, + "prefill_token_fraction": -4.5771229986280026e-05, + "queue_running_mean": -0.27537858206666677, + "queue_waiting_mean": -0.14998382319999998, + "scheduler_steps_per_s": 6.626666666666665 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 2, + "request_hash": "adc728f189a1ce0301609cdbae7d1dc2318609b13eafb94bcb90903bb7998eb1", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "a5ce6cd64890663ef30486701236af3de6169f86b72349928b07adafcb406bcb", + "result_sha256": "289f8f99f5056b26822bab5c1feb4252042c7852b80c5399271ddb1b6db25fbb", + "trial_id": "sessions/rep2-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "1c6931ea4a0913e2460e18c59f5a07e9c21cb2650454f2dd2f2997c416f63209", + "result_sha256": "9988f2b6c9e8c45144eb7b53aa1bea62942ae8fd36aba10c9e6bf5c8cbd65a50", + "trial_id": "sessions/rep2-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.3898039215686275, + "completed_over_admitted": 0.047995780590717296, + "completed_pass_rate": 0.5534521158129175, + "outstanding_over_admitted": -0.0479957805907173, + "tpot_max_over_slo": 0.485827943940801, + "tpot_mean_over_slo": 0.015614815582012831, + "ttft_max_over_slo_max": -0.9946015174985708, + "ttft_mean_over_slo_max": -0.5387282368223152 + }, + "delta_state": { + "batch_size.mean": -1.6273974153257296, + "batch_tokens.mean": -34.29456683018134, + "decode_batch_size.mean": -1.6204010215371003, + "graph_full_share": 0.02986827684967852, + "graph_none_share": -0.04355833918802347, + "graph_padding_fraction": 0.009679698181700893, + "kv_usage_end_minus_start": -0.0096678119183089, + "kv_usage_max": 0.04533198050309872, + "kv_usage_mean": 0.003191336087705077, + "preemptions": 0.0, + "prefill_token_fraction": -0.00023339635256569569, + "queue_running_mean": 2.3989840409999985, + "queue_waiting_mean": -47.09879207850667, + "scheduler_steps_per_s": 11.493333333333325 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 2, + "request_hash": "56a0616b6b54abafd37875c7cb25f8639afef2706ccc55dfbe568f45859ea382", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.31450980392156863, + "mns": 16, + "requests_sha256": "e00c0d9acfb0c22624fdde79f4e8a0300fc2aba09ad4851908d68c1f46be30d3", + "result_sha256": "d081e1ac0143b4e8a63f4c60b58905320ca6e6163fa858e78f5643221fd87c2b", + "trial_id": "sessions/rep2-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "e1543f873745dbd1d3a1461e63549747b6e1517d63e281e6cdfa974d2e2798e4", + "result_sha256": "8556783dc3ba3274269500add337c75c493e3eece4f66260449549ee86506035", + "trial_id": "sessions/rep2-mns64/mid/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.06666873637623494, + "tpot_mean_over_slo": -0.009318131681115394, + "ttft_max_over_slo_max": -0.07750733549861857, + "ttft_mean_over_slo_max": -0.004705775197540927 + }, + "delta_state": { + "batch_size.mean": -0.5672759530887257, + "batch_tokens.mean": -10.329959059848179, + "decode_batch_size.mean": -0.562993756337625, + "graph_full_share": 0.005089834536355586, + "graph_none_share": -0.01044980089531864, + "graph_padding_fraction": 0.0013844048532868167, + "kv_usage_end_minus_start": 0.002908292329360629, + "kv_usage_max": 0.00626947513206999, + "kv_usage_mean": -0.0002826822094093009, + "preemptions": 0.0, + "prefill_token_fraction": 0.00012710629028711562, + "queue_running_mean": -0.5141460273999989, + "queue_waiting_mean": -0.24953442832, + "scheduler_steps_per_s": 8.97333333333333 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 3, + "request_hash": "4993c26442fc6b5a4c09b6f576fd8b3cb1be7ce83b7a771e4a57ee7600de688d", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "de93d4364969393741a920ea72831b69003cbd9f10bcfd1809848fcf4e948549", + "result_sha256": "3b32b5ff3aea07cb7b7cf252ba23852542cb8429c83123fe772b2c8093c408a6", + "trial_id": "sessions/rep3-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "3623a99013725d7b42845bce09678368b3dc8d82e0ab26fad549c7eab8a6b6fe", + "result_sha256": "77766d9ec36ff11893feb73626768fa593160394f2b4720ff9253d1dfd5fa950", + "trial_id": "sessions/rep3-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.43333333333333335, + "completed_over_admitted": 0.05807814149947199, + "completed_pass_rate": 0.6218345526167699, + "outstanding_over_admitted": -0.05807814149947202, + "tpot_max_over_slo": 0.30513300693329926, + "tpot_mean_over_slo": 0.006471198446666382, + "ttft_max_over_slo_max": -0.9754032485070638, + "ttft_mean_over_slo_max": -0.5503063221993506 + }, + "delta_state": { + "batch_size.mean": -0.33070896537783234, + "batch_tokens.mean": -13.234057192010994, + "decode_batch_size.mean": -0.33092189282726103, + "graph_full_share": 0.024782576956253966, + "graph_none_share": -0.03529189804944581, + "graph_padding_fraction": 0.00864361242258734, + "kv_usage_end_minus_start": -0.004863029113173378, + "kv_usage_max": 0.02549546289093474, + "kv_usage_mean": 0.0018655214057756511, + "preemptions": 0.0, + "prefill_token_fraction": -0.0010506631885456308, + "queue_running_mean": 2.228850923693333, + "queue_waiting_mean": -47.28131520793333, + "scheduler_steps_per_s": 7.439999999999998 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 3, + "request_hash": "2a2fabe2c4cf176aeb7e0d32fb8e7dbb1f27429a2e7a0cd18d7d186f23096f19", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.2635294117647059, + "mns": 16, + "requests_sha256": "c7fca9b2484c8ea68c6c11438b8e59db74fa60e6189dbb81d8834038156dbf68", + "result_sha256": "bb45674fa5e19ef18d8c13d08493f1a08a62fb682b5d5e8b7d8a1db2fa3b2ea6", + "trial_id": "sessions/rep3-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "d75e1711b8c0485cf1aceaa9e26ae079fa2adc5487d37f28006c01e83c5a782b", + "result_sha256": "1654b044d815beb3a5b8913fde4283b11507464a2de9f41b5ddc3cdb9f2d150b", + "trial_id": "sessions/rep3-mns64/mid/result.json" + } + } + ], + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 6, + "max": 0.7505555555555555, + "min": 0.7316666666666667, + "n": 12 + }, + "completed_fraction_of_total": { + "distinct_n": 9, + "max": 0.7472222222222221, + "min": 0.6968627450980392, + "n": 12 + } + }, + "efficacy": { + "cross_validation_possible": true, + "label_balance_sufficient": true, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "outcome_delta": { + "best_accuracy": 1.0, + "best_balanced_accuracy": 1.0, + "best_feature": "ttft_max_over_slo_max", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.19490196078431374, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.07490196078431373, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.07490196078431373, + "train_balanced_accuracy": 1.0 + } + ] + }, + "completed_over_admitted": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.02436798652037714, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.016463090812533165, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.016833187037551656, + "train_balanced_accuracy": 1.0 + } + ] + }, + "completed_pass_rate": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.27672605790645877, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.10546659304251793, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.10546659304251793, + "train_balanced_accuracy": 1.0 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.024367986520377156, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.016463090812533193, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.016833187037551697, + "train_balanced_accuracy": 1.0 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.2407730989823646, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.22234457283277126, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.2563269824471589, + "train_balanced_accuracy": 1.0 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.6666666666666666, + "balanced_accuracy": 0.6666666666666666, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": -0.000848167884689166, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.003343406070318189, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.007915214637991413, + "train_balanced_accuracy": 1.0 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5264552920028412, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5690833184198709, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5786824529156245, + "train_balanced_accuracy": 1.0 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.27171700600992804, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.12275947254744249, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.12275947254744249, + "train_balanced_accuracy": 1.0 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "positive": 3 + }, + "positive": 3, + "telemetry_delta": { + "best_accuracy": 1.0, + "best_balanced_accuracy": 1.0, + "best_feature": "queue_waiting_mean", + "features": { + "decode_batch_size.mean": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": -0.3722455467363339, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.21812996140096796, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.21812996140096796, + "train_balanced_accuracy": 0.75 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.005142163057644268, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.005156031979134446, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.005156031979134446, + "train_balanced_accuracy": 1.0 + } + ] + }, + "kv_usage_mean": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0009077895591772371, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0011186444492266587, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0017815517901913717, + "train_balanced_accuracy": 1.0 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.00013958379127598786, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.000530540760156184, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.00013958379127598786, + "train_balanced_accuracy": 1.0 + } + ] + }, + "queue_running_mean": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.9767361708133331, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 1.2061492231866664, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 1.2912157818399992, + "train_balanced_accuracy": 1.0 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -23.674163253413333, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -14.979670980066667, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -14.979670980066667, + "train_balanced_accuracy": 1.0 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 7.033333333333331, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 3.6799999999999997, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 3.6799999999999997, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "positive": 3 + }, + "telemetry_qualifying_features": [], + "transitions": { + "false->true": 3, + "true->true": 3 + } + }, + "end_fraction": 0.75, + "end_s": 225.0, + "qualifying_response_features": [ + "graph_padding_fraction", + "queue_waiting_mean" + ], + "repeat_pairs": 8, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.49210700723778444, + "action_delta": { + "distinct_n": 6, + "max": 0.7340835489721993, + "min": -1.6273974153257296, + "n": 6 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 5, + "positive": 1, + "zero": 0 + }, + "effect_to_repeat_median": 1.187549092220796, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.41438876966131266, + "repeat_abs_p95": 1.7988231433741386, + "repeat_delta": { + "distinct_n": 8, + "max": 1.2969259412104588, + "min": -2.069075483000736, + "n": 8 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 9.394001464972767, + "action_delta": { + "distinct_n": 6, + "max": 4.870249914593785, + "min": -34.29456683018134, + "n": 6 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 5, + "positive": 1, + "zero": 0 + }, + "effect_to_repeat_median": 0.5103708522227296, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 18.40622642155347, + "repeat_abs_p95": 44.616864312796594, + "repeat_delta": { + "distinct_n": 8, + "max": 20.29593930708012, + "min": -57.34701495152672, + "n": 8 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.48828147849151593, + "action_delta": { + "distinct_n": 6, + "max": 0.7235214534720207, + "min": -1.6204010215371003, + "n": 6 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 5, + "positive": 1, + "zero": 0 + }, + "effect_to_repeat_median": 1.1843674420861072, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.41227195306168873, + "repeat_abs_p95": 1.7851097930098754, + "repeat_delta": { + "distinct_n": 8, + "max": 1.2901539913069193, + "min": -2.051624455465314, + "n": 8 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.009708147868701333, + "action_delta": { + "distinct_n": 6, + "max": 0.02986827684967852, + "min": 0.003382014563382052, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 2.9605949673769345, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0032791205739644558, + "repeat_abs_p95": 0.008038878324209003, + "repeat_delta": { + "distinct_n": 8, + "max": 0.008664142823959997, + "min": -0.006877672824671444, + "n": 8 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.017857110193969867, + "action_delta": { + "distinct_n": 6, + "max": -0.008054518886335793, + "min": -0.04355833918802347, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 4.464300302534616, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.003999979612444856, + "repeat_abs_p95": 0.010649544454368921, + "repeat_delta": { + "distinct_n": 8, + "max": 0.008444852161820365, + "min": -0.011836686458048917, + "n": 8 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.6666666666666666, + "action_abs_median": 0.005156031979134446, + "action_delta": { + "distinct_n": 6, + "max": 0.009679698181700893, + "min": 0.0013844048532868167, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 5.791648307818219, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": true, + "repeat_abs_median": 0.0008902529478826008, + "repeat_abs_p95": 0.0032904967132956443, + "repeat_delta": { + "distinct_n": 8, + "max": 0.0038875531917088053, + "min": -0.000986602698250582, + "n": 8 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 0.0038856607212670036, + "action_delta": { + "distinct_n": 6, + "max": 0.002908292329360629, + "min": -0.0096678119183089, + "n": 6 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 5, + "positive": 1, + "zero": 0 + }, + "effect_to_repeat_median": 2.2991048875374145, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0016900754473315738, + "repeat_abs_p95": 0.006128076218419801, + "repeat_delta": { + "distinct_n": 8, + "max": 0.0017580081358981037, + "min": -0.007809750200217236, + "n": 8 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.019251090447259256, + "action_delta": { + "distinct_n": 6, + "max": 0.04533198050309872, + "min": 0.00626947513206999, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 7.386921150773906, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.002606104770083051, + "repeat_abs_p95": 0.016512755779962775, + "repeat_delta": { + "distinct_n": 8, + "max": 0.012755779962795644, + "min": -0.01853574275843739, + "n": 8 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 0.0011186444492266587, + "action_delta": { + "distinct_n": 6, + "max": 0.0049451651951918, + "min": -0.0002826822094093009, + "n": 6 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 2, + "positive": 4, + "zero": 0 + }, + "effect_to_repeat_median": 0.9840910262940343, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.001136728635194791, + "repeat_abs_p95": 0.0026446247991943482, + "repeat_delta": { + "distinct_n": 8, + "max": 0.00023211260883475478, + "min": -0.0029335843047816335, + "n": 8 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 6 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 6 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 8 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.00018025132142640565, + "action_delta": { + "distinct_n": 6, + "max": 0.00012710629028711562, + "min": -0.0014595808679910283, + "n": 6 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 5, + "positive": 1, + "zero": 0 + }, + "effect_to_repeat_median": 0.07514942281617999, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0023985722667133613, + "repeat_abs_p95": 0.003432972436972548, + "repeat_delta": { + "distinct_n": 8, + "max": -0.00011986074750158782, + "min": -0.0035071769437554634, + "n": 8 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.3333333333333333, + "action_abs_median": 1.371498475546666, + "action_delta": { + "distinct_n": 6, + "max": 5.284704507639999, + "min": -0.5141460273999989, + "n": 6 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 2, + "positive": 4, + "zero": 0 + }, + "effect_to_repeat_median": 5.458580798787922, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.2512555050666663, + "repeat_abs_p95": 2.2674077304893334, + "repeat_delta": { + "distinct_n": 8, + "max": 0.25758021949333276, + "min": -2.6281402471466677, + "n": 8 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 14.979670980066667, + "action_delta": { + "distinct_n": 6, + "max": -0.14998382319999998, + "min": -47.28131520793333, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 263.3579940744849, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": true, + "repeat_abs_median": 0.05687949983333334, + "repeat_abs_p95": 11.769436231303327, + "repeat_delta": { + "distinct_n": 8, + "max": 17.82422918966667, + "min": -0.5248207372, + "n": 8 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 7.033333333333331, + "action_delta": { + "distinct_n": 6, + "max": 11.493333333333325, + "min": 3.373333333333335, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 1.8158347676419953, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 3.873333333333335, + "repeat_abs_p95": 7.130000000000002, + "repeat_delta": { + "distinct_n": 8, + "max": 7.200000000000003, + "min": -3.3866666666666703, + "n": 8 + } + } + }, + "sanity": { + "invariants": { + "all_results_uncensored": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "layer1_boundary_and_internal_gaps_bounded": true, + "nonnegative_counters": true, + "per_cell_state_vectors_not_all_identical": true, + "ratios_bounded": true, + "state_vectors_not_all_identical": true + }, + "red_flags": [], + "trials": 12 + }, + "start_s": 150.0, + "trial_sanity": [ + { + "admitted_fraction": 0.7388888888888889, + "cell": "tp4_mns16", + "completed_fraction": 0.7377777777777779, + "level": "low", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.002047043, + "max_internal_gap_s": 0.302592239, + "start_gap_s": 0.003297384 + }, + "trial_id": "sessions/rep1-mns16/low/result.json" + }, + { + "admitted_fraction": 0.7384313725490196, + "cell": "tp4_mns16", + "completed_fraction": 0.7101960784313726, + "level": "mid", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.000359161, + "max_internal_gap_s": 0.188948337, + "start_gap_s": 0.002535254 + }, + "trial_id": "sessions/rep1-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.7388888888888889, + "cell": "tp4_mns64", + "completed_fraction": 0.7377777777777779, + "level": "low", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.003650633, + "max_internal_gap_s": 0.322890476, + "start_gap_s": 0.001316118 + }, + "trial_id": "sessions/rep1-mns64/low/result.json" + }, + { + "admitted_fraction": 0.7384313725490196, + "cell": "tp4_mns64", + "completed_fraction": 0.7345098039215686, + "level": "mid", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.004944353, + "max_internal_gap_s": 0.219527251, + "start_gap_s": 0.003464116 + }, + "trial_id": "sessions/rep1-mns64/mid/result.json" + }, + { + "admitted_fraction": 0.7505555555555555, + "cell": "tp4_mns64", + "completed_fraction": 0.7472222222222221, + "level": "low", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.002633243, + "max_internal_gap_s": 0.646724237, + "start_gap_s": 0.001383425 + }, + "trial_id": "sessions/rep2-mns64/low/result.json" + }, + { + "admitted_fraction": 0.7435294117647059, + "cell": "tp4_mns64", + "completed_fraction": 0.74, + "level": "mid", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.000558705, + "max_internal_gap_s": 0.183301269, + "start_gap_s": 0.161925118 + }, + "trial_id": "sessions/rep2-mns64/mid/result.json" + }, + { + "admitted_fraction": 0.7505555555555555, + "cell": "tp4_mns16", + "completed_fraction": 0.7466666666666666, + "level": "low", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.001410558, + "max_internal_gap_s": 0.583053185, + "start_gap_s": 0.155434911 + }, + "trial_id": "sessions/rep2-mns16/low/result.json" + }, + { + "admitted_fraction": 0.7435294117647059, + "cell": "tp4_mns16", + "completed_fraction": 0.7043137254901961, + "level": "mid", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.042708099, + "max_internal_gap_s": 0.179974019, + "start_gap_s": 0.02943376 + }, + "trial_id": "sessions/rep2-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.7316666666666667, + "cell": "tp4_mns16", + "completed_fraction": 0.7311111111111112, + "level": "low", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.000142513, + "max_internal_gap_s": 0.318659499, + "start_gap_s": 0.034846458 + }, + "trial_id": "sessions/rep3-mns16/low/result.json" + }, + { + "admitted_fraction": 0.7427450980392157, + "cell": "tp4_mns16", + "completed_fraction": 0.6968627450980392, + "level": "mid", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.049578949, + "max_internal_gap_s": 0.182904479, + "start_gap_s": 0.084632833 + }, + "trial_id": "sessions/rep3-mns16/mid/result.json" + }, + { + "admitted_fraction": 0.7316666666666667, + "cell": "tp4_mns64", + "completed_fraction": 0.7311111111111112, + "level": "low", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.00141287, + "max_internal_gap_s": 0.456938178, + "start_gap_s": 0.003994197 + }, + "trial_id": "sessions/rep3-mns64/low/result.json" + }, + { + "admitted_fraction": 0.7427450980392157, + "cell": "tp4_mns64", + "completed_fraction": 0.74, + "level": "mid", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.009657729, + "max_internal_gap_s": 0.179710886, + "start_gap_s": 0.001797426 + }, + "trial_id": "sessions/rep3-mns64/mid/result.json" + } + ] + }, + { + "action_pairs": 6, + "actions": [ + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0011111111111111738, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.0011111111111111113, + "tpot_max_over_slo": 0.13955613873224326, + "tpot_mean_over_slo": -0.0010090653746841205, + "ttft_max_over_slo_max": -0.162763388332678, + "ttft_mean_over_slo_max": -0.00961120104762887 + }, + "delta_state": { + "batch_size.mean": -0.36880112566985623, + "batch_tokens.mean": -6.920339794120963, + "decode_batch_size.mean": -0.3657101237877507, + "graph_full_share": 0.002401511671261014, + "graph_none_share": -0.007647132972108277, + "graph_padding_fraction": 0.0013086971950763866, + "kv_usage_end_minus_start": 0.0006854718445343799, + "kv_usage_max": 0.008271873870579816, + "kv_usage_mean": -0.00014200125936815727, + "preemptions": 0.0, + "prefill_token_fraction": 0.00022380184306991513, + "queue_running_mean": -0.2263784608933328, + "queue_waiting_mean": -0.11799772912, + "scheduler_steps_per_s": 5.7733333333333405 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 1, + "request_hash": "af50f0c3edfc6f9fb74734b2501378fc8ba6fdd3dd3fe84f2bb1d68540eee69b", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "5856cc9ff24c9a984a2abecc59ab92c1f87b02b2a9dc0a47eae7a08462169373", + "result_sha256": "4e788371d6f24cab1be2bd4aed4bfe0354b5a5c23973bac9bf69ad0aa4c1b7d9", + "trial_id": "sessions/rep1-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "a91536c628053d3cf530bf0b8ebe51628cc41e71af31b90310d3307d1badc473", + "result_sha256": "2dd1d5ae610db6d738291032ab07a3ad73c46235b30e4ff38b87ce2715204bc6", + "trial_id": "sessions/rep1-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.3949019607843137, + "completed_over_admitted": 0.03529411764705881, + "completed_pass_rate": 0.41338259441707714, + "outstanding_over_admitted": -0.03529411764705882, + "tpot_max_over_slo": 0.3362407738628878, + "tpot_mean_over_slo": 0.031421107704212525, + "ttft_max_over_slo_max": -1.0149181675127088, + "ttft_mean_over_slo_max": -0.4177688425802161 + }, + "delta_state": { + "batch_size.mean": -0.025927986090335864, + "batch_tokens.mean": 18.396156394293996, + "decode_batch_size.mean": -0.03259251609001801, + "graph_full_share": 0.02053093742241252, + "graph_none_share": -0.032184416438265945, + "graph_padding_fraction": 0.008177832545205635, + "kv_usage_end_minus_start": 0.01748278407834214, + "kv_usage_max": 0.016581905042334366, + "kv_usage_mean": 0.002461661215058867, + "preemptions": 0.0, + "prefill_token_fraction": 0.0028536948076697266, + "queue_running_mean": 1.9724076930533343, + "queue_waiting_mean": -47.152675258386665, + "scheduler_steps_per_s": 3.3066666666666578 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 1, + "request_hash": "0bb61dbc9c26875e991d0d4f984134910d37463e5063f86ee960cf4f8aafb771", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.5603921568627451, + "mns": 16, + "requests_sha256": "8dcab1a24d009309a6dd54676e762ee4e74717ed530d19b3965f30feba32f193", + "result_sha256": "ae7cbca9785fa548acf8bd82646dfe53694ffd0b58d2ef6953e3d531d4d7ec4f", + "trial_id": "sessions/rep1-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "e7bf606cb77817e67a3e64e60f035a5fb344977a7991be741a81db8b3a8d28fe", + "result_sha256": "9604c8a0fe449d3798c6de566e4c804665d66b859a3b43c716dd5ecbb60bee1c", + "trial_id": "sessions/rep1-mns64/mid/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": 0.0, + "tpot_max_over_slo": 0.17641319103142994, + "tpot_mean_over_slo": -0.004863679616933292, + "ttft_max_over_slo_max": -0.16410399084755528, + "ttft_mean_over_slo_max": -0.005948038735831104 + }, + "delta_state": { + "batch_size.mean": -0.21169295708183622, + "batch_tokens.mean": -4.472135801297441, + "decode_batch_size.mean": -0.21006339951083586, + "graph_full_share": 0.001298043937749238, + "graph_none_share": -0.004668542387939491, + "graph_padding_fraction": 0.0013031829071826536, + "kv_usage_end_minus_start": 0.0014933728423490722, + "kv_usage_max": 0.01910758793466949, + "kv_usage_mean": 0.0002283394725866271, + "preemptions": 0.0, + "prefill_token_fraction": 4.923759477581946e-05, + "queue_running_mean": 0.18394671514666605, + "queue_waiting_mean": -0.37382507898666667, + "scheduler_steps_per_s": 3.4399999999999977 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 2, + "request_hash": "adc728f189a1ce0301609cdbae7d1dc2318609b13eafb94bcb90903bb7998eb1", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "a5ce6cd64890663ef30486701236af3de6169f86b72349928b07adafcb406bcb", + "result_sha256": "289f8f99f5056b26822bab5c1feb4252042c7852b80c5399271ddb1b6db25fbb", + "trial_id": "sessions/rep2-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "1c6931ea4a0913e2460e18c59f5a07e9c21cb2650454f2dd2f2997c416f63209", + "result_sha256": "9988f2b6c9e8c45144eb7b53aa1bea62942ae8fd36aba10c9e6bf5c8cbd65a50", + "trial_id": "sessions/rep2-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.6321568627450981, + "completed_over_admitted": 0.04980392156862745, + "completed_pass_rate": 0.667771333885667, + "outstanding_over_admitted": -0.04980392156862745, + "tpot_max_over_slo": 0.485827943940801, + "tpot_mean_over_slo": 0.017498560395543927, + "ttft_max_over_slo_max": -0.9946015174985708, + "ttft_mean_over_slo_max": -0.6463312081620042 + }, + "delta_state": { + "batch_size.mean": -0.5058158071040495, + "batch_tokens.mean": -7.22996895580593, + "decode_batch_size.mean": -0.5073220598343564, + "graph_full_share": 0.026735082314620406, + "graph_none_share": -0.0367740255019158, + "graph_padding_fraction": 0.008672030283845366, + "kv_usage_end_minus_start": 0.008358614300674416, + "kv_usage_max": 0.02410580556943176, + "kv_usage_mean": 0.002518308420146869, + "preemptions": 0.0, + "prefill_token_fraction": 0.0005037242486639615, + "queue_running_mean": 2.4556977948933323, + "queue_waiting_mean": -47.22722224089333, + "scheduler_steps_per_s": 6.413333333333327 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 2, + "request_hash": "56a0616b6b54abafd37875c7cb25f8639afef2706ccc55dfbe568f45859ea382", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.31450980392156863, + "mns": 16, + "requests_sha256": "e00c0d9acfb0c22624fdde79f4e8a0300fc2aba09ad4851908d68c1f46be30d3", + "result_sha256": "d081e1ac0143b4e8a63f4c60b58905320ca6e6163fa858e78f5643221fd87c2b", + "trial_id": "sessions/rep2-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "e1543f873745dbd1d3a1461e63549747b6e1517d63e281e6cdfa974d2e2798e4", + "result_sha256": "8556783dc3ba3274269500add337c75c493e3eece4f66260449549ee86506035", + "trial_id": "sessions/rep2-mns64/mid/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": 0.0, + "completed_over_admitted": 0.0005555555555556424, + "completed_pass_rate": 0.0, + "outstanding_over_admitted": -0.0005555555555555552, + "tpot_max_over_slo": 0.18505735732578327, + "tpot_mean_over_slo": -0.007144351590753079, + "ttft_max_over_slo_max": -0.12056507783321042, + "ttft_mean_over_slo_max": -0.006789902099118429 + }, + "delta_state": { + "batch_size.mean": -0.36923651780535494, + "batch_tokens.mean": -7.266086823190449, + "decode_batch_size.mean": -0.3664963060152342, + "graph_full_share": 0.0022555226793449235, + "graph_none_share": -0.007954072283154097, + "graph_padding_fraction": 0.003323634208537993, + "kv_usage_end_minus_start": -4.647472024943422e-05, + "kv_usage_max": 0.01645626968867886, + "kv_usage_mean": 0.00016819101878902745, + "preemptions": 0.0, + "prefill_token_fraction": 3.148363682092814e-06, + "queue_running_mean": -0.07117231964000048, + "queue_waiting_mean": -0.44794735680000003, + "scheduler_steps_per_s": 5.546666666666667 + }, + "full_action_efficacy": 0, + "full_feasibility_transition": "true->true", + "group": { + "level": "low", + "offered_rate_per_gpu": 1.5, + "replicate": 3, + "request_hash": "4993c26442fc6b5a4c09b6f576fd8b3cb1be7ce83b7a771e4a57ee7600de688d", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 16, + "requests_sha256": "de93d4364969393741a920ea72831b69003cbd9f10bcfd1809848fcf4e948549", + "result_sha256": "3b32b5ff3aea07cb7b7cf252ba23852542cb8429c83123fe772b2c8093c408a6", + "trial_id": "sessions/rep3-mns16/low/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "3623a99013725d7b42845bce09678368b3dc8d82e0ab26fad549c7eab8a6b6fe", + "result_sha256": "77766d9ec36ff11893feb73626768fa593160394f2b4720ff9253d1dfd5fa950", + "trial_id": "sessions/rep3-mns64/low/result.json" + } + }, + { + "delta_outcome": { + "admitted_fraction": 0.0, + "admitted_input_tokens_mean_over_limit": 0.0, + "completed_fail_fraction_of_total": -0.6756862745098039, + "completed_over_admitted": 0.05372549019607842, + "completed_pass_rate": 0.7194154488517746, + "outstanding_over_admitted": -0.05372549019607843, + "tpot_max_over_slo": 0.37831041669495047, + "tpot_mean_over_slo": 0.007493808043833983, + "ttft_max_over_slo_max": -0.9795807876701778, + "ttft_mean_over_slo_max": -0.6540993991122069 + }, + "delta_state": { + "batch_size.mean": -1.6247957869409735, + "batch_tokens.mean": -35.238499877789366, + "decode_batch_size.mean": -1.6176751208671583, + "graph_full_share": 0.029945447264822622, + "graph_none_share": -0.04283615284473359, + "graph_padding_fraction": 0.010093615119271773, + "kv_usage_end_minus_start": -0.0001447507354654931, + "kv_usage_max": 0.03542758976690741, + "kv_usage_mean": 0.0020665348540741674, + "preemptions": 0.0, + "prefill_token_fraction": -0.0004446109784110863, + "queue_running_mean": 1.6193733474533332, + "queue_waiting_mean": -47.261676539759996, + "scheduler_steps_per_s": 11.693333333333328 + }, + "full_action_efficacy": 1, + "full_feasibility_transition": "false->true", + "group": { + "level": "mid", + "offered_rate_per_gpu": 2.125, + "replicate": 3, + "request_hash": "2a2fabe2c4cf176aeb7e0d32fb8e7dbb1f27429a2e7a0cd18d7d186f23096f19", + "tp": 4 + }, + "source": { + "cell": "tp4_mns16", + "early_stopped": false, + "full_feasible": false, + "full_pass_rate": 0.2635294117647059, + "mns": 16, + "requests_sha256": "c7fca9b2484c8ea68c6c11438b8e59db74fa60e6189dbb81d8834038156dbf68", + "result_sha256": "bb45674fa5e19ef18d8c13d08493f1a08a62fb682b5d5e8b7d8a1db2fa3b2ea6", + "trial_id": "sessions/rep3-mns16/mid/result.json" + }, + "target": { + "cell": "tp4_mns64", + "early_stopped": false, + "full_feasible": true, + "full_pass_rate": 1.0, + "mns": 64, + "requests_sha256": "d75e1711b8c0485cf1aceaa9e26ae079fa2adc5487d37f28006c01e83c5a782b", + "result_sha256": "1654b044d815beb3a5b8913fde4283b11507464a2de9f41b5ddc3cdb9f2d150b", + "trial_id": "sessions/rep3-mns64/mid/result.json" + } + } + ], + "coverage_at_end": { + "admitted_fraction_of_total": { + "distinct_n": 1, + "max": 1.0, + "min": 1.0, + "n": 12 + }, + "completed_fraction_of_total": { + "distinct_n": 10, + "max": 0.9966666666666667, + "min": 0.9392156862745098, + "n": 12 + } + }, + "efficacy": { + "cross_validation_possible": true, + "label_balance_sufficient": true, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "outcome_delta": { + "best_accuracy": 1.0, + "best_balanced_accuracy": 1.0, + "best_feature": "ttft_mean_over_slo_max", + "features": { + "admitted_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "admitted_input_tokens_mean_over_limit": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 1 + ], + "threshold": 0.0, + "train_balanced_accuracy": 0.5 + } + ] + }, + "completed_fail_fraction_of_total": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.31607843137254904, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.19745098039215686, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.19745098039215686, + "train_balanced_accuracy": 1.0 + } + ] + }, + "completed_over_admitted": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.025179738562091547, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.01820261437908499, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.01820261437908499, + "train_balanced_accuracy": 1.0 + } + ] + }, + "completed_pass_rate": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.3338856669428335, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.20669129720853857, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.20669129720853857, + "train_balanced_accuracy": 1.0 + } + ] + }, + "outstanding_over_admitted": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.025179738562091502, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.018202614379084967, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.018202614379084967, + "train_balanced_accuracy": 1.0 + } + ] + }, + "tpot_max_over_slo": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.28168388701036684, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.26064906559433554, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.2563269824471589, + "train_balanced_accuracy": 1.0 + } + ] + }, + "tpot_mean_over_slo": { + "accuracy": 0.8333333333333334, + "balanced_accuracy": 0.8333333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0013150642134503454, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.003242371334574931, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.008244747510429903, + "train_balanced_accuracy": 1.0 + } + ] + }, + "ttft_max_over_slo_max": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5718423892588665, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5711720880014279, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.5793527541730631, + "train_balanced_accuracy": 1.0 + } + ] + }, + "ttft_mean_over_slo_max": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.3265605551305613, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.2136900218139225, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -0.2136900218139225, + "train_balanced_accuracy": 1.0 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "positive": 3 + }, + "positive": 3, + "telemetry_delta": { + "best_accuracy": 1.0, + "best_balanced_accuracy": 1.0, + "best_feature": "queue_waiting_mean", + "features": { + "decode_batch_size.mean": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.4369091829247953, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.19915131993888435, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.12132795780042693, + "train_balanced_accuracy": 0.75 + } + ] + }, + "graph_padding_fraction": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.00599783224619168, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.005750733376871814, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.004743264870141011, + "train_balanced_accuracy": 1.0 + } + ] + }, + "kv_usage_mean": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0011474371633303972, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.0011173629364315974, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.001345000343822747, + "train_balanced_accuracy": 1.0 + } + ] + }, + "prefill_token_fraction": { + "accuracy": 0.5, + "balanced_accuracy": 0.5, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.00022073130736449675, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": -0.00022073130736449675, + "train_balanced_accuracy": 0.75 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 0.0003637630458669383, + "train_balanced_accuracy": 1.0 + } + ] + }, + "queue_running_mean": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.9016600312999996, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 0.7741005139066663, + "train_balanced_accuracy": 1.0 + }, + { + "direction": 1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": 1.0781772041000002, + "train_balanced_accuracy": 1.0 + } + ] + }, + "queue_waiting_mean": { + "accuracy": 1.0, + "balanced_accuracy": 1.0, + "folds": [ + { + "direction": -1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -23.837584798846667, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -23.800311307593333, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 1 + ], + "threshold": -23.763250168686668, + "train_balanced_accuracy": 1.0 + } + ] + }, + "scheduler_steps_per_s": { + "accuracy": 0.3333333333333333, + "balanced_accuracy": 0.3333333333333333, + "folds": [ + { + "direction": 1, + "held_out_replicate": 1, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 5.979999999999997, + "train_balanced_accuracy": 1.0 + }, + { + "direction": -1, + "held_out_replicate": 2, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 1, + 0 + ], + "threshold": 4.426666666666662, + "train_balanced_accuracy": 0.75 + }, + { + "direction": -1, + "held_out_replicate": 3, + "test_labels": [ + 0, + 1 + ], + "test_predictions": [ + 0, + 0 + ], + "threshold": 3.3733333333333277, + "train_balanced_accuracy": 0.75 + } + ] + } + }, + "labels": { + "distinct_n": 2, + "max": 1.0, + "min": 0.0, + "n": 6 + }, + "negative": 3, + "positive": 3 + }, + "telemetry_qualifying_features": [], + "transitions": { + "false->true": 3, + "true->true": 3 + } + }, + "end_fraction": 1.0, + "end_s": 300.0, + "qualifying_response_features": [ + "graph_padding_fraction", + "queue_waiting_mean", + "scheduler_steps_per_s" + ], + "repeat_pairs": 8, + "response_statistics": { + "batch_size.mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.3690188217376056, + "action_delta": { + "distinct_n": 6, + "max": -0.025927986090335864, + "min": -1.6247957869409735, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 0.8618355767528564, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.4281777541929288, + "repeat_abs_p95": 0.9733377249226858, + "repeat_delta": { + "distinct_n": 8, + "max": 0.7022084961695088, + "min": -1.119330386559012, + "n": 8 + } + }, + "batch_tokens.mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 7.248027889498189, + "action_delta": { + "distinct_n": 6, + "max": 18.396156394293996, + "min": -35.238499877789366, + "n": 6 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 5, + "positive": 1, + "zero": 0 + }, + "effect_to_repeat_median": 1.0330903781421452, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 7.015870095056599, + "repeat_abs_p95": 31.643382433390233, + "repeat_delta": { + "distinct_n": 8, + "max": 15.47659343283351, + "min": -40.3485765106131, + "n": 8 + } + }, + "decode_batch_size.mean": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.36610321490149245, + "action_delta": { + "distinct_n": 6, + "max": -0.03259251609001801, + "min": -1.6176751208671583, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 0.8640718152580315, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.4236953554516365, + "repeat_abs_p95": 0.965912412061184, + "repeat_delta": { + "distinct_n": 8, + "max": 0.6959888014852451, + "min": -1.1112558946789974, + "n": 8 + } + }, + "graph_full_share": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.011466224546836767, + "action_delta": { + "distinct_n": 6, + "max": 0.029945447264822622, + "min": 0.001298043937749238, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 2.7129568200282717, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.004226467764686825, + "repeat_abs_p95": 0.0058786421963618, + "repeat_delta": { + "distinct_n": 8, + "max": 0.005744029111679261, + "min": -0.005951126165037013, + "n": 8 + } + }, + "graph_none_share": { + "action_above_repeat_p95_fraction": 0.8333333333333334, + "action_abs_median": 0.02006924436071002, + "action_delta": { + "distinct_n": 6, + "max": -0.004668542387939491, + "min": -0.04283615284473359, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 6.0905462240848625, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0032951468755539302, + "repeat_abs_p95": 0.005637894418237397, + "repeat_delta": { + "distinct_n": 8, + "max": 0.005589923520491251, + "min": -0.005663724901639168, + "n": 8 + } + }, + "graph_padding_fraction": { + "action_above_repeat_p95_fraction": 0.6666666666666666, + "action_abs_median": 0.005750733376871814, + "action_delta": { + "distinct_n": 6, + "max": 0.010093615119271773, + "min": 0.0013031829071826536, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 10.921714369329514, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": true, + "repeat_abs_median": 0.000526541272038856, + "repeat_abs_p95": 0.002367923136830107, + "repeat_delta": { + "distinct_n": 8, + "max": 0.0028531471739462067, + "min": -0.0004812194431302371, + "n": 8 + } + }, + "kv_usage_end_minus_start": { + "action_above_repeat_p95_fraction": 0.16666666666666666, + "action_abs_median": 0.001089422343441726, + "action_delta": { + "distinct_n": 6, + "max": 0.01748278407834214, + "min": -0.0001447507354654931, + "n": 6 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 2, + "positive": 4, + "zero": 0 + }, + "effect_to_repeat_median": 0.1436227894695951, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.007585302774476166, + "repeat_abs_p95": 0.014011837897733656, + "repeat_delta": { + "distinct_n": 8, + "max": 0.008574911878879932, + "min": -0.015719863447740123, + "n": 8 + } + }, + "kv_usage_max": { + "action_above_repeat_p95_fraction": 0.8333333333333334, + "action_abs_median": 0.017844746488501928, + "action_delta": { + "distinct_n": 6, + "max": 0.03542758976690741, + "min": 0.008271873870579816, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 6.740328646178248, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.002647459408172903, + "repeat_abs_p95": 0.011185326764651764, + "repeat_delta": { + "distinct_n": 8, + "max": 0.012433819170465488, + "min": -0.0019690160530919654, + "n": 8 + } + }, + "kv_usage_mean": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.0011474371633303972, + "action_delta": { + "distinct_n": 6, + "max": 0.002518308420146869, + "min": -0.00014200125936815727, + "n": 6 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 1, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 1.7625178532585835, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.0006510215832475053, + "repeat_abs_p95": 0.001132085438086033, + "repeat_delta": { + "distinct_n": 8, + "max": 0.0007778503404774276, + "min": -0.0012902061862114789, + "n": 8 + } + }, + "preemptions": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0, + "action_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 6 + }, + "action_signs": { + "consistency": 0.0, + "negative": 0, + "positive": 0, + "zero": 6 + }, + "effect_to_repeat_median": 0.0, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": false, + "qualifies": false, + "repeat_abs_median": 0.0, + "repeat_abs_p95": 0.0, + "repeat_delta": { + "distinct_n": 1, + "max": 0.0, + "min": 0.0, + "n": 8 + } + }, + "prefill_token_fraction": { + "action_above_repeat_p95_fraction": 0.0, + "action_abs_median": 0.0003342064107405007, + "action_delta": { + "distinct_n": 6, + "max": 0.0028536948076697266, + "min": -0.0004446109784110863, + "n": 6 + }, + "action_signs": { + "consistency": 0.8333333333333334, + "negative": 1, + "positive": 5, + "zero": 0 + }, + "effect_to_repeat_median": 0.121317147777762, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.002754815925550158, + "repeat_abs_p95": 0.005037760791822205, + "repeat_delta": { + "distinct_n": 8, + "max": 0.00289215301640644, + "min": -0.005053892022705009, + "n": 8 + } + }, + "queue_running_mean": { + "action_above_repeat_p95_fraction": 0.5, + "action_abs_median": 0.922875904173333, + "action_delta": { + "distinct_n": 6, + "max": 2.4556977948933323, + "min": -0.2263784608933328, + "n": 6 + }, + "action_signs": { + "consistency": 0.6666666666666666, + "negative": 2, + "positive": 4, + "zero": 0 + }, + "effect_to_repeat_median": 2.147073922684241, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": false, + "repeat_abs_median": 0.4298295901333322, + "repeat_abs_p95": 0.8487974595846658, + "repeat_delta": { + "distinct_n": 7, + "max": 0.8555136968933326, + "min": -0.8363244474399991, + "n": 8 + } + }, + "queue_waiting_mean": { + "action_above_repeat_p95_fraction": 0.8333333333333334, + "action_abs_median": 23.800311307593333, + "action_delta": { + "distinct_n": 6, + "max": -0.11799772912, + "min": -47.261676539759996, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 6, + "positive": 0, + "zero": 0 + }, + "effect_to_repeat_median": 1182.9517515641658, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": true, + "repeat_abs_median": 0.020119426913332024, + "repeat_abs_p95": 0.19249742833733324, + "repeat_delta": { + "distinct_n": 8, + "max": 0.25510083812, + "min": -0.001456834226666667, + "n": 8 + } + }, + "scheduler_steps_per_s": { + "action_above_repeat_p95_fraction": 0.6666666666666666, + "action_abs_median": 5.660000000000004, + "action_delta": { + "distinct_n": 6, + "max": 11.693333333333328, + "min": 3.3066666666666578, + "n": 6 + }, + "action_signs": { + "consistency": 1.0, + "negative": 0, + "positive": 6, + "zero": 0 + }, + "effect_to_repeat_median": 2.830000000000002, + "effect_to_repeat_median_is_infinite": false, + "gate_feature": true, + "qualifies": true, + "repeat_abs_median": 2.0, + "repeat_abs_p95": 4.641333333333332, + "repeat_delta": { + "distinct_n": 8, + "max": 4.986666666666665, + "min": -4.0, + "n": 8 + } + } + }, + "sanity": { + "invariants": { + "all_results_uncensored": true, + "expected_action_pair_count": true, + "expected_repeat_pair_count": true, + "finite_deltas": true, + "layer1_boundary_and_internal_gaps_bounded": true, + "nonnegative_counters": true, + "per_cell_state_vectors_not_all_identical": true, + "ratios_bounded": true, + "state_vectors_not_all_identical": true + }, + "red_flags": [], + "trials": 12 + }, + "start_s": 225.0, + "trial_sanity": [ + { + "admitted_fraction": 1.0, + "cell": "tp4_mns16", + "completed_fraction": 0.99, + "level": "low", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.00637513, + "max_internal_gap_s": 0.526093105, + "start_gap_s": 0.001941686 + }, + "trial_id": "sessions/rep1-mns16/low/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns16", + "completed_fraction": 0.9552941176470588, + "level": "mid", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.00415122, + "max_internal_gap_s": 0.18536399, + "start_gap_s": 0.00588794 + }, + "trial_id": "sessions/rep1-mns16/mid/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns64", + "completed_fraction": 0.9911111111111112, + "level": "low", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.03110111, + "max_internal_gap_s": 0.52361076, + "start_gap_s": 0.000397498 + }, + "trial_id": "sessions/rep1-mns64/low/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns64", + "completed_fraction": 0.9905882352941177, + "level": "mid", + "replicate": 1, + "telemetry_coverage": { + "end_gap_s": 0.033263413, + "max_internal_gap_s": 0.325952494, + "start_gap_s": 0.001285913 + }, + "trial_id": "sessions/rep1-mns64/mid/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns64", + "completed_fraction": 0.9966666666666667, + "level": "low", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.002571552, + "max_internal_gap_s": 0.533068266, + "start_gap_s": 0.002834243 + }, + "trial_id": "sessions/rep2-mns64/low/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns64", + "completed_fraction": 0.9964705882352941, + "level": "mid", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.002910387, + "max_internal_gap_s": 0.26729158, + "start_gap_s": 0.006516028 + }, + "trial_id": "sessions/rep2-mns64/mid/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns16", + "completed_fraction": 0.9966666666666667, + "level": "low", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.005037876, + "max_internal_gap_s": 0.524592261, + "start_gap_s": 0.003957027 + }, + "trial_id": "sessions/rep2-mns16/low/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns16", + "completed_fraction": 0.9466666666666667, + "level": "mid", + "replicate": 2, + "telemetry_coverage": { + "end_gap_s": 0.04911377, + "max_internal_gap_s": 0.184204303, + "start_gap_s": 0.00971298 + }, + "trial_id": "sessions/rep2-mns16/mid/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns16", + "completed_fraction": 0.9961111111111111, + "level": "low", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.001459145, + "max_internal_gap_s": 0.184019771, + "start_gap_s": 0.003328883 + }, + "trial_id": "sessions/rep3-mns16/low/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns16", + "completed_fraction": 0.9392156862745098, + "level": "mid", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.024192492, + "max_internal_gap_s": 0.174342785, + "start_gap_s": 0.000436554 + }, + "trial_id": "sessions/rep3-mns16/mid/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns64", + "completed_fraction": 0.9966666666666667, + "level": "low", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.004830812, + "max_internal_gap_s": 0.197897948, + "start_gap_s": 0.002055499 + }, + "trial_id": "sessions/rep3-mns64/low/result.json" + }, + { + "admitted_fraction": 1.0, + "cell": "tp4_mns64", + "completed_fraction": 0.9929411764705882, + "level": "mid", + "replicate": 3, + "telemetry_coverage": { + "end_gap_s": 0.021026794, + "max_internal_gap_s": 0.186718691, + "start_gap_s": 0.041331534 + }, + "trial_id": "sessions/rep3-mns64/mid/result.json" + } + ] + } + ], + "sanity": { + "controller": { + "gpu_hours_total": 5.092428478929731, + "invariants": { + "all_sessions_complete": true, + "all_stream_validation_invariants_pass": true, + "completed_session_count": true, + "controller_complete": true, + "exact_session_set": true, + "no_controller_failures": true, + "under_h20_hour_cap": true + }, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/runs/pilot/controller-state.json", + "red_flags": [], + "sha256": "f928a7bc8b4f6877a1f3ec71715177711428820c6a60669b29092e7371bdd734" + }, + "cumulative_coverage": { + "invariants": { + "admitted_fraction_monotonic": true, + "completed_fraction_monotonic": true, + "same_trials_at_every_checkpoint": true + }, + "red_flags": [] + }, + "red_flags": [], + "stream_bytes": { + "distinct_n": 6, + "max": 58550249.0, + "min": 52931532.0, + "n": 6 + }, + "streams": 6 + }, + "schema": "intervention-response-phase-aware-pilot-analysis-v3", + "stable_adjacent_efficacy_features": { + "0.25->0.50": [], + "0.50->0.75": [], + "0.75->1.00": [] + }, + "stable_adjacent_features": { + "0.10->0.25": [], + "0.25->0.50": [], + "0.50->0.75": [ + "graph_padding_fraction" + ], + "0.75->1.00": [ + "graph_padding_fraction", + "queue_waiting_mean" + ] + }, + "stable_incremental_efficacy_candidates": [], + "stable_incremental_efficacy_features": [], + "status": "COMPLETE" +} diff --git a/runs/intervention-response-v2/pilot-controller-state-v3.json b/runs/intervention-response-v2/pilot-controller-state-v3.json new file mode 100644 index 0000000..8b1562c --- /dev/null +++ b/runs/intervention-response-v2/pilot-controller-state-v3.json @@ -0,0 +1,342 @@ +{ + "completed_at": 1784027453.4005148, + "completed_sessions": 6, + "failures": [], + "gpu_hours_total": 5.092428478929731, + "hard_cap_h20_hours": 6.641945318645901, + "schema": "intervention-response-phase-aware-pilot-state-v3", + "sessions": { + "rep1-mns16": { + "burnin": { + "elapsed_s": 64.194648239, + "feasible": false, + "pass_rate": 0.5235294117647059 + }, + "completed_at": 1784023620.6911042, + "gpu_hours": 0.856446467505561, + "mns": 16, + "replicate": 1, + "runs": [ + { + "early_stopped": false, + "elapsed_s": 301.068878512, + "feasible": true, + "level": "low", + "offered_req_s_per_gpu": 1.5, + "pass_rate": 1.0, + "selected_count": 1800 + }, + { + "early_stopped": false, + "elapsed_s": 313.268231477, + "feasible": false, + "level": "mid", + "offered_req_s_per_gpu": 2.125, + "pass_rate": 0.5603921568627451, + "selected_count": 2550 + } + ], + "started_at": 1784022846.257515, + "status": "complete", + "validation": { + "accounting_mode": "graceful-footer", + "cell": "tp4_mns16", + "invariants": { + "anchor_intervals_present": true, + "compile_capture_pre_ready": true, + "encoded_balanced": true, + "footer_sidecar_agrees": true, + "last_step_matches": true, + "layer1_contiguous": true, + "layer1_zero_drops": true, + "one_footer_last": true, + "one_ready_marker": true, + "sidecar_final": true, + "warmup_exact_16": true, + "warmup_long": true, + "written_matches_records": true + }, + "layer1_records": 60017, + "post_ready_capture_events": [], + "stream": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/runs/pilot/sessions/rep1-mns16/opprof/opprof-v1-dp0-pid458427-1784022917661752826.jsonl" + } + }, + "rep1-mns64": { + "burnin": { + "elapsed_s": 60.79926316, + "feasible": true, + "pass_rate": 1.0 + }, + "completed_at": 1784024382.2164423, + "gpu_hours": 0.8415956518385146, + "mns": 64, + "replicate": 1, + "runs": [ + { + "early_stopped": false, + "elapsed_s": 301.022700013, + "feasible": true, + "level": "low", + "offered_req_s_per_gpu": 1.5, + "pass_rate": 1.0, + "selected_count": 1800 + }, + { + "early_stopped": false, + "elapsed_s": 301.100993458, + "feasible": true, + "level": "mid", + "offered_req_s_per_gpu": 2.125, + "pass_rate": 1.0, + "selected_count": 2550 + } + ], + "started_at": 1784023620.9865165, + "status": "complete", + "validation": { + "accounting_mode": "graceful-footer", + "cell": "tp4_mns64", + "invariants": { + "anchor_intervals_present": true, + "compile_capture_pre_ready": true, + "encoded_balanced": true, + "footer_sidecar_agrees": true, + "last_step_matches": true, + "layer1_contiguous": true, + "layer1_zero_drops": true, + "one_footer_last": true, + "one_ready_marker": true, + "sidecar_final": true, + "warmup_exact_16": true, + "warmup_long": true, + "written_matches_records": true + }, + "layer1_records": 62456, + "post_ready_capture_events": [], + "stream": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/runs/pilot/sessions/rep1-mns64/opprof/opprof-v1-dp0-pid471212-1784023695315485142.jsonl" + } + }, + "rep2-mns16": { + "burnin": { + "elapsed_s": 63.518994307, + "feasible": false, + "pass_rate": 0.6647058823529411 + }, + "completed_at": 1784025910.710227, + "gpu_hours": 0.8481734045346578, + "mns": 16, + "replicate": 2, + "runs": [ + { + "early_stopped": false, + "elapsed_s": 315.780805484, + "feasible": false, + "level": "mid", + "offered_req_s_per_gpu": 2.125, + "pass_rate": 0.31450980392156863, + "selected_count": 2550 + }, + { + "early_stopped": false, + "elapsed_s": 300.346368314, + "feasible": true, + "level": "low", + "offered_req_s_per_gpu": 1.5, + "pass_rate": 1.0, + "selected_count": 1800 + } + ], + "started_at": 1784025143.731658, + "status": "complete", + "validation": { + "accounting_mode": "graceful-footer", + "cell": "tp4_mns16", + "invariants": { + "anchor_intervals_present": true, + "compile_capture_pre_ready": true, + "encoded_balanced": true, + "footer_sidecar_agrees": true, + "last_step_matches": true, + "layer1_contiguous": true, + "layer1_zero_drops": true, + "one_footer_last": true, + "one_ready_marker": true, + "sidecar_final": true, + "warmup_exact_16": true, + "warmup_long": true, + "written_matches_records": true + }, + "layer1_records": 58465, + "post_ready_capture_events": [], + "stream": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/runs/pilot/sessions/rep2-mns16/opprof/opprof-v1-dp0-pid496100-1784025206790752505.jsonl" + } + }, + "rep2-mns64": { + "burnin": { + "elapsed_s": 60.803771161, + "feasible": true, + "pass_rate": 1.0 + }, + "completed_at": 1784025143.4362338, + "gpu_hours": 0.8412917502721151, + "mns": 64, + "replicate": 2, + "runs": [ + { + "early_stopped": false, + "elapsed_s": 300.605515019, + "feasible": true, + "level": "mid", + "offered_req_s_per_gpu": 2.125, + "pass_rate": 1.0, + "selected_count": 2550 + }, + { + "early_stopped": false, + "elapsed_s": 300.354084826, + "feasible": true, + "level": "low", + "offered_req_s_per_gpu": 1.5, + "pass_rate": 1.0, + "selected_count": 1800 + } + ], + "started_at": 1784024382.5080929, + "status": "complete", + "validation": { + "accounting_mode": "graceful-footer", + "cell": "tp4_mns64", + "invariants": { + "anchor_intervals_present": true, + "compile_capture_pre_ready": true, + "encoded_balanced": true, + "footer_sidecar_agrees": true, + "last_step_matches": true, + "layer1_contiguous": true, + "layer1_zero_drops": true, + "one_footer_last": true, + "one_ready_marker": true, + "sidecar_final": true, + "warmup_exact_16": true, + "warmup_long": true, + "written_matches_records": true + }, + "layer1_records": 61696, + "post_ready_capture_events": [], + "stream": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/runs/pilot/sessions/rep2-mns64/opprof/opprof-v1-dp0-pid483758-1784024456752184540.jsonl" + } + }, + "rep3-mns16": { + "burnin": { + "elapsed_s": 64.324514503, + "feasible": false, + "pass_rate": 0.5588235294117647 + }, + "completed_at": 1784026691.6293187, + "gpu_hours": 0.8631463612450494, + "mns": 16, + "replicate": 3, + "runs": [ + { + "early_stopped": false, + "elapsed_s": 300.711109876, + "feasible": true, + "level": "low", + "offered_req_s_per_gpu": 1.5, + "pass_rate": 1.0, + "selected_count": 1800 + }, + { + "early_stopped": false, + "elapsed_s": 317.011840655, + "feasible": false, + "level": "mid", + "offered_req_s_per_gpu": 2.125, + "pass_rate": 0.2635294117647059, + "selected_count": 2550 + } + ], + "started_at": 1784025911.00421, + "status": "complete", + "validation": { + "accounting_mode": "graceful-footer", + "cell": "tp4_mns16", + "invariants": { + "anchor_intervals_present": true, + "compile_capture_pre_ready": true, + "encoded_balanced": true, + "footer_sidecar_agrees": true, + "last_step_matches": true, + "layer1_contiguous": true, + "layer1_zero_drops": true, + "one_footer_last": true, + "one_ready_marker": true, + "sidecar_final": true, + "warmup_exact_16": true, + "warmup_long": true, + "written_matches_records": true + }, + "layer1_records": 61438, + "post_ready_capture_events": [], + "stream": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/runs/pilot/sessions/rep3-mns16/opprof/opprof-v1-dp0-pid508913-1784025983046041167.jsonl" + } + }, + "rep3-mns64": { + "burnin": { + "elapsed_s": 60.850091109, + "feasible": true, + "pass_rate": 1.0 + }, + "completed_at": 1784027453.3790896, + "gpu_hours": 0.8417748435338338, + "mns": 64, + "replicate": 3, + "runs": [ + { + "early_stopped": false, + "elapsed_s": 300.710505069, + "feasible": true, + "level": "low", + "offered_req_s_per_gpu": 1.5, + "pass_rate": 1.0, + "selected_count": 1800 + }, + { + "early_stopped": false, + "elapsed_s": 300.869326418, + "feasible": true, + "level": "mid", + "offered_req_s_per_gpu": 2.125, + "pass_rate": 1.0, + "selected_count": 2550 + } + ], + "started_at": 1784026691.96421, + "status": "complete", + "validation": { + "accounting_mode": "graceful-footer", + "cell": "tp4_mns64", + "invariants": { + "anchor_intervals_present": true, + "compile_capture_pre_ready": true, + "encoded_balanced": true, + "footer_sidecar_agrees": true, + "last_step_matches": true, + "layer1_contiguous": true, + "layer1_zero_drops": true, + "one_footer_last": true, + "one_ready_marker": true, + "sidecar_final": true, + "warmup_exact_16": true, + "warmup_long": true, + "written_matches_records": true + }, + "layer1_records": 64776, + "post_ready_capture_events": [], + "stream": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/runs/pilot/sessions/rep3-mns64/opprof/opprof-v1-dp0-pid521797-1784026767359409666.jsonl" + } + } + }, + "started_at": 1784022845.964628, + "status": "complete" +} diff --git a/runs/intervention-response-v2/pilot-manifest-v3.json b/runs/intervention-response-v2/pilot-manifest-v3.json new file mode 100644 index 0000000..4590e51 --- /dev/null +++ b/runs/intervention-response-v2/pilot-manifest-v3.json @@ -0,0 +1,324 @@ +{ + "budget": { + "expected_h20_hours": [ + 4.8, + 6.641945318645901 + ], + "expected_wall_minutes": [ + 75, + 95 + ], + "global_hard_cap_h20_hours": 8.0, + "hard_cap_h20_hours": 6.641945318645901, + "prior_attempt_h20_hours": 1.3580546813540988, + "session_estimate_h20_hours": 1.0 + }, + "burnin": { + "anchor": 0.18919793755240089, + "arrival_order_sha256": "6c0ac4cb9a30ef501eeeacc8e6cc631c345e976db5ccf530ea5a1ec706d62a24", + "input_length_order_sha256": "7939cc20e1a00d1031d27d71508789f38decbbbb6ea59a1df18b2ec342fd2ef8", + "offered_req_s": 8.5, + "offered_req_s_per_gpu": 2.125, + "request_id_order_sha256": "84f4809acbc8acd3b1d14dfa357134a1dc0b9287341624b33f598dafeef54dc7", + "selected_count": 510, + "study": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/studies/burnin-tp4.json", + "study_sha256": "5d6c2098042909a863efd3112818fbee9bafe96f22898ac98b66846dbe1fef0f" + }, + "checkpoints": { + "fractions": [ + 0.1, + 0.25, + 0.5, + 0.75, + 1.0 + ], + "seconds": [ + 30.0, + 75.0, + 150.0, + 225.0, + 300.0 + ] + }, + "engine": { + "disable_slo_early_stop": true, + "duration_s": 300.0, + "mns_endpoints": [ + 16, + 64 + ], + "replay_time_scale": 0.5, + "safety_deadline_s": 360.0, + "tp": 4 + }, + "private": { + "merged_traces": { + "1": { + "bytes": 337429767, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep1.jsonl", + "request_id_scheme": "sha256(source_sha256:line_number:original_id)", + "rows": 9420, + "sha256": "68983266aa0e66aa589562f7c08edbd966f9ba4405e20c105adb43777d2dfbf5", + "source_sha256": [ + "b242d1d9086df3accab57b4c92445d5edd581e12f47e12cea227aa63964c6930", + "d23b549f7b69af3647308677bbf76f818a3c226a1c98f9a9f93f09ceee46be87" + ], + "sources": [ + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low1.jsonl", + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high1.jsonl" + ] + }, + "2": { + "bytes": 337509330, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep2.jsonl", + "request_id_scheme": "sha256(source_sha256:line_number:original_id)", + "rows": 9457, + "sha256": "f38e8938f6a481fc6725b71b21aa04ff7eaf79783cdfd6e41aa2f074156f00c2", + "source_sha256": [ + "4cbb0baac082bd54af562ce2f39104c5c23b4671672da365a67b1e8c146adf9f", + "bb0bcd2564a88000f435f12feb21c7c902eafc9ea5fe916adfe9d1eae47f3f9a" + ], + "sources": [ + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low2.jsonl", + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high2.jsonl" + ] + }, + "3": { + "bytes": 337450256, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep3.jsonl", + "request_id_scheme": "sha256(source_sha256:line_number:original_id)", + "rows": 9431, + "sha256": "3094084b0bb20cc02eecf465091a5c919b4e5b112f704cdc36a563d1efdcee46", + "source_sha256": [ + "1f7ececb142f9a363d2d1ca25eb7b8488b2cc319a51b55faa384f2a3d51f2142", + "6f326234791e1cff4ff866bface0d097d0d6e3844eebb1c97653d8e9c35e9397" + ], + "sources": [ + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low3.jsonl", + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high3.jsonl" + ] + } + } + }, + "repetitions": { + "1": { + "duration_s": 300.0, + "load_order": [ + "low", + "mid" + ], + "merged_trace": { + "bytes": 337429767, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep1.jsonl", + "request_id_scheme": "sha256(source_sha256:line_number:original_id)", + "rows": 9420, + "sha256": "68983266aa0e66aa589562f7c08edbd966f9ba4405e20c105adb43777d2dfbf5", + "source_sha256": [ + "b242d1d9086df3accab57b4c92445d5edd581e12f47e12cea227aa63964c6930", + "d23b549f7b69af3647308677bbf76f818a3c226a1c98f9a9f93f09ceee46be87" + ], + "sources": [ + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low1.jsonl", + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high1.jsonl" + ] + }, + "selections": { + "low": { + "anchor": 0.34815061842962747, + "arrival_order_sha256": "0b9473052a28b2eb98bb4bda59c726d6c74cd42656c00046f0559cefd38e0c42", + "input_length_order_sha256": "0172aa1fd7d994e9d95857c32293e25f6c537616b40e01034308a0d57cd94139", + "offered_req_s": 6.0, + "offered_req_s_per_gpu": 1.5, + "request_id_order_sha256": "af50f0c3edfc6f9fb74734b2501378fc8ba6fdd3dd3fe84f2bb1d68540eee69b", + "selected_count": 1800, + "target_count": 1800, + "target_req_s_per_gpu": 1.5 + }, + "mid": { + "anchor": 0.48686986110831465, + "arrival_order_sha256": "c2ad99986ce558da5901a9c5ec0a00bd69f198c981d8779235f2773a5c87f1c0", + "input_length_order_sha256": "9442bfebdc3fab5062dc1f4d688dc28c02afe3fd806c56dd8159f0ac7e6d0b94", + "offered_req_s": 8.5, + "offered_req_s_per_gpu": 2.125, + "request_id_order_sha256": "0bb61dbc9c26875e991d0d4f984134910d37463e5063f86ee960cf4f8aafb771", + "selected_count": 2550, + "target_count": 2550, + "target_req_s_per_gpu": 2.125 + } + }, + "source_roles": [ + "low1", + "high1" + ], + "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep1-tp4.json", + "study_sha256": "ecfff96e33d458eb1e3b9a6d24386f00cc6f1b19ff926e2ec6320b3f671a7ae3" + }, + "2": { + "duration_s": 300.0, + "load_order": [ + "mid", + "low" + ], + "merged_trace": { + "bytes": 337509330, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep2.jsonl", + "request_id_scheme": "sha256(source_sha256:line_number:original_id)", + "rows": 9457, + "sha256": "f38e8938f6a481fc6725b71b21aa04ff7eaf79783cdfd6e41aa2f074156f00c2", + "source_sha256": [ + "4cbb0baac082bd54af562ce2f39104c5c23b4671672da365a67b1e8c146adf9f", + "bb0bcd2564a88000f435f12feb21c7c902eafc9ea5fe916adfe9d1eae47f3f9a" + ], + "sources": [ + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low2.jsonl", + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high2.jsonl" + ] + }, + "selections": { + "low": { + "anchor": 0.3411190868318572, + "arrival_order_sha256": "f4eafbdf7420e8a97e38d46fff1aec5fdbc2a049133862a793b4635b51061b7f", + "input_length_order_sha256": "cdf82cfc546651f1859c53348eace72348572f09e6e1879ad77d51b489a998fe", + "offered_req_s": 6.0, + "offered_req_s_per_gpu": 1.5, + "request_id_order_sha256": "adc728f189a1ce0301609cdbae7d1dc2318609b13eafb94bcb90903bb7998eb1", + "selected_count": 1800, + "target_count": 1800, + "target_req_s_per_gpu": 1.5 + }, + "mid": { + "anchor": 0.4825698948735577, + "arrival_order_sha256": "b9fc12cf3f86bc8a79bee65296e65aa2b8bf2aeca46b2887094c669adcbb9a00", + "input_length_order_sha256": "d8d4bd6fc8ba852a45605b673b6b3e4f33b58f459e69f2a032d226ee175b074e", + "offered_req_s": 8.5, + "offered_req_s_per_gpu": 2.125, + "request_id_order_sha256": "56a0616b6b54abafd37875c7cb25f8639afef2706ccc55dfbe568f45859ea382", + "selected_count": 2550, + "target_count": 2550, + "target_req_s_per_gpu": 2.125 + } + }, + "source_roles": [ + "low2", + "high2" + ], + "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep2-tp4.json", + "study_sha256": "d92a576db031db24bb58f354ea725d7f7567cb76699d387117ac5a6c9317bbb9" + }, + "3": { + "duration_s": 300.0, + "load_order": [ + "low", + "mid" + ], + "merged_trace": { + "bytes": 337450256, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep3.jsonl", + "request_id_scheme": "sha256(source_sha256:line_number:original_id)", + "rows": 9431, + "sha256": "3094084b0bb20cc02eecf465091a5c919b4e5b112f704cdc36a563d1efdcee46", + "source_sha256": [ + "1f7ececb142f9a363d2d1ca25eb7b8488b2cc319a51b55faa384f2a3d51f2142", + "6f326234791e1cff4ff866bface0d097d0d6e3844eebb1c97653d8e9c35e9397" + ], + "sources": [ + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low3.jsonl", + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high3.jsonl" + ] + }, + "selections": { + "low": { + "anchor": 0.3459572828521402, + "arrival_order_sha256": "68de977e2ea7396ffe9406675a3d2c5ebecd1b9ea5afc8116c836cc53cfca188", + "input_length_order_sha256": "dffa79545e738b899bec2e9d282764a3388fe38b162982fb2d26928e17ef6016", + "offered_req_s": 6.0, + "offered_req_s_per_gpu": 1.5, + "request_id_order_sha256": "4993c26442fc6b5a4c09b6f576fd8b3cb1be7ce83b7a771e4a57ee7600de688d", + "selected_count": 1800, + "target_count": 1800, + "target_req_s_per_gpu": 1.5 + }, + "mid": { + "anchor": 0.48664343020532463, + "arrival_order_sha256": "efce7339e22d3618cb4d55e6b55bfddb2c563c18faba2a992d5829c13e3f55e9", + "input_length_order_sha256": "0792b05fff6729fbd92ab2bb4cb6d31bea7799e232ad42772936bc06efbafb54", + "offered_req_s": 8.5, + "offered_req_s_per_gpu": 2.125, + "request_id_order_sha256": "2a2fabe2c4cf176aeb7e0d32fb8e7dbb1f27429a2e7a0cd18d7d186f23096f19", + "selected_count": 2550, + "target_count": 2550, + "target_req_s_per_gpu": 2.125 + } + }, + "source_roles": [ + "low3", + "high3" + ], + "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep3-tp4.json", + "study_sha256": "fb8ffe256dace32f4ca8a8d49b662d98c3b69b94ecc8fa826e43068b238884ab" + } + }, + "sanity": { + "distinct_selected_sets": 6, + "invariants": { + "all_counts_positive": true, + "load_levels_two": true, + "selection_hashes_unique": true, + "selection_sets_disjoint_across_repetitions": true, + "six_sessions": true, + "three_repetitions": true + }, + "red_flags": [], + "selected_sets": 6 + }, + "schema": "intervention-response-phase-aware-pilot-manifest-v3", + "sessions": [ + { + "mns": 16, + "replicate": 1, + "session": "rep1-mns16" + }, + { + "mns": 64, + "replicate": 1, + "session": "rep1-mns64" + }, + { + "mns": 64, + "replicate": 2, + "session": "rep2-mns64" + }, + { + "mns": 16, + "replicate": 2, + "session": "rep2-mns16" + }, + { + "mns": 16, + "replicate": 3, + "session": "rep3-mns16" + }, + { + "mns": 64, + "replicate": 3, + "session": "rep3-mns64" + } + ], + "source": { + "base_manifest": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/runs/p1b/pilot-manifest.json", + "base_manifest_sha256": "86b249a087ab1bcd51cd8de3d1da6bf34afd41af977eb58c93229b941dbf5c8a", + "prior_attempt_failure": [ + { + "failure": "TimeoutError('client timeout: tp4_mns16 high')", + "session": "rep1-mns16" + } + ], + "prior_attempt_h20_hours": 1.3580546813540988, + "prior_attempt_state": "/home/admin/cpfs/wjh/intervention-response-v2-20260714/runs/pilot/controller-state.json", + "prior_attempt_state_sha256": "c88030ae36862abfc643a855eb3630fb18cd74a58b388514aa31ec1ab485a431", + "source_trace": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260312_1000.jsonl", + "source_trace_sha256": "875ba869775deb78086477919f03b322da14e2673c7d070e26528c4190912757", + "window_id": "chat_w20260312_1000" + }, + "status": "PASS" +} diff --git a/runs/intervention-response-v2/platform-v3.json b/runs/intervention-response-v2/platform-v3.json new file mode 100644 index 0000000..5a45de1 --- /dev/null +++ b/runs/intervention-response-v2/platform-v3.json @@ -0,0 +1,46 @@ +{ + "schema": "intervention-response-platform-v3", + "host_alias": "dash0", + "hostname": "ds-07429c65-1-6c5fd97778-9vhkr", + "os": "Ubuntu 24.04.2 LTS", + "kernel": "Linux 5.10.134-013.8.2.kangaroo.al8.x86_64 x86_64 GNU/Linux", + "cpu": { + "logical_cpus": 160, + "model": "Intel(R) Xeon(R) Processor" + }, + "gpu": { + "count": 8, + "experiment_ids": [ + 0, + 1, + 2, + 3 + ], + "memory_mib_each": 97871, + "model": "NVIDIA H20" + }, + "runtime": { + "driver": "580.95.05", + "python": "3.12.3", + "torch": "2.11.0+cu130", + "torch_cuda": "13.0", + "vllm_editable_source": "/home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0", + "vllm_git_head": "4b253fd8619764b6971a7f2e3a3aa7545f6ace05", + "vllm_git_status_clean": true, + "vllm_import_path": "/home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0/vllm/__init__.py", + "vllm_reported_version": "0.24.1.dev3+g668cfb7e2" + }, + "model": { + "bytes": 61084188040, + "path": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + }, + "aituner_commit": "7fd956355072399b1b56946ecd8007da8f9d2fb0", + "experiment": { + "completed_at": "2026-07-14T19:10:53+08:00", + "incremental_h20_hours": 5.092428478929731, + "prior_invalid_attempt_h20_hours": 1.3580546813540988, + "started_at": "2026-07-14T17:54:05+08:00", + "total_h20_hours": 6.45048316028383, + "wall_minutes": 76.79059811433156 + } +} From 823c550e53807b632f4792ce71d39dd21875655e Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 20:26:54 +0800 Subject: [PATCH 021/148] Add crossed-constraint action-aware pilot --- ...e-constraint-pilot-v0-protocol-20260714.md | 179 ++++++ runs/action-aware-v0/action_aware_client.py | 63 ++ runs/action-aware-v0/analyze_pilot.py | 584 +++++++++++++++++ runs/action-aware-v0/pilot-manifest.json | 223 +++++++ runs/action-aware-v0/pilot_controller.py | 595 ++++++++++++++++++ runs/action-aware-v0/prepare_pilot.py | 153 +++++ runs/action-aware-v0/test_pilot.py | 191 ++++++ 7 files changed, 1988 insertions(+) create mode 100644 docs/action-aware-constraint-pilot-v0-protocol-20260714.md create mode 100644 runs/action-aware-v0/action_aware_client.py create mode 100644 runs/action-aware-v0/analyze_pilot.py create mode 100644 runs/action-aware-v0/pilot-manifest.json create mode 100644 runs/action-aware-v0/pilot_controller.py create mode 100644 runs/action-aware-v0/prepare_pilot.py create mode 100644 runs/action-aware-v0/test_pilot.py diff --git a/docs/action-aware-constraint-pilot-v0-protocol-20260714.md b/docs/action-aware-constraint-pilot-v0-protocol-20260714.md new file mode 100644 index 0000000..062685d --- /dev/null +++ b/docs/action-aware-constraint-pilot-v0-protocol-20260714.md @@ -0,0 +1,179 @@ +# Action-aware constraint pilot v0 protocol + +Status: **FROZEN BEFORE NEW GPU RUNS**. + +Date: 2026-07-14 (Asia/Singapore). + +## Headline question + +Can telemetry from one complete initial-config benchmark identify which of two +competing knob families should be changed, before either target configuration +is evaluated? + +This pilot tests a narrow prerequisite, not an end-to-end tuner claim. It +uses fields already present in the per-step OpProf stream to reconstruct exact +zero-slack conditions for `max_num_seqs` (MNS) and +`max_num_batched_tokens` (MBBT). No new vLLM instrumentation is justified +unless those action-conditioned conditions predict crossed real-system +intervention responses. + +## Hypothesis + +I believe config-normalized scheduler constraints provide a stronger tuning +signal than an aggregate queue symptom because the same waiting queue can be +blocked by different admission limits. + +I will verify it by holding model, hardware, TP, request bands, arrival times, +and offered load fixed while constructing two source configurations with +different binding constraints. From each source run alone, the larger +exclusive binding fraction predicts the action family. Both candidate +actions are then measured on the same requests for the full 300-second replay. + +## Frozen platform and workload + +- Host: `dash0`, solo placement on GPUs 0-3, four NVIDIA H20 GPUs. +- Model: Qwen3-30B-A3B BF16. +- Engine: patched vLLM `0.24.1.dev3+opprof`, TP=4. +- Workload: the three disjoint `mid` bands from + `chat_w20260312_1000`, 2.125 requests/s/GPU, 300-second arrival window, + exactly 128 output tokens. +- SLO: the unchanged study TTFT/TPOT thresholds and 0.95 pass-rate target. +- Every config starts one fresh server, performs the accepted 16-request + warm-up and the existing burn-in, then runs all three disjoint measured + bands in its frozen order. +- SLO early stopping is disabled. A measured run must drain all selected + requests and finish within the 450-second client deadline. + +## Frozen configuration and action matrix + +| ID | MNS | MBBT | Role | +|---|---:|---:|---| +| `b_base` | 64 | 256 | token-budget-bound source; operational gate runs first | +| `a_base` | 16 | 8192 | MNS-bound source | +| `shared` | 64 | 8192 | MNS action from A; MBBT action from B | +| `b_mns` | 128 | 256 | competing MNS action from B | +| `a_mbbt` | 16 | 16384 | competing MBBT action from A | + +The two decisions are therefore: + +```text +Regime A: a_base -> {shared (increase MNS), a_mbbt (increase MBBT)} +Regime B: b_base -> {b_mns (increase MNS), shared (increase MBBT)} +``` + +The candidate magnitudes are intentionally large in this feasibility pilot so +that a missing crossed response is not explained by an imperceptibly small +intervention. This does not establish that these are production step sizes. + +Frozen config order is `b_base`, `a_base`, `shared`, `b_mns`, `a_mbbt`. +Frozen repetition orders are respectively `123`, `231`, `312`, `132`, and +`213`, reducing band/time alignment without reusing a server across configs. + +## Pre-action signal + +For each source run, let `waiting` include the normal and deferred waiting +queues, and let `scheduled_tokens = prefill_tokens + decode_tokens`. + +```text +mns_exclusive = waiting > 0 + and running == configured MNS + and scheduled_tokens < configured MBBT + +mbbt_exclusive = waiting > 0 + and scheduled_tokens == configured MBBT + and running < configured MNS + +both = waiting > 0 + and running == configured MNS + and scheduled_tokens == configured MBBT +``` + +Each score is the fraction of all scheduler records in the measured interval +that satisfies the condition. The predicted action is the family with the +larger exclusive fraction. This uses no target telemetry or target outcome. +KV usage and preemptions are reported as possible alternative constraints but +are not silently reassigned to either score. + +These conditions reproduce two scheduler loop boundaries, but they are still +a Level-0 proxy: they do not expose the exact request rejected at the boundary +or run a shadow schedule. The pilot explicitly tests whether that additional +engine patch is warranted. + +## Outcomes and baselines + +Primary intervention outcome: + +```text +SLO-goodput = full-run SLO pass count / 300-second arrival window +``` + +Also report pass rate, TTFT p50/p95/p99, TPOT p50/p95/p99, drain elapsed time, +KV usage, preemptions, queue area, and CUDA-graph padding. + +Required decision baselines: + +1. always choose the MNS family; +2. always choose the MBBT family; +3. queue-pressure-only, which has no candidate-specific score and therefore + must use one frozen family for both regimes; +4. the pre-action exclusive-binding prediction. + +This is a mechanism ablation. It does not compare against a trained black-box +tuner because two regimes are not a valid training surface. + +## Gates and failure meanings + +Data validity requires 15 uncensored measured runs, exact request/arrival/input +hashes across each repetition, full request accounting, one continuous OpProf +stream per config, zero dropped records, monotonic timestamps and step indices, +nonnegative counters, bounded ratios, clean GPU placement, and config values in +the result matching the server command. + +The crossed-response gate passes only if, in all three repetitions: + +- the MNS target has higher SLO-goodput than the MBBT target in Regime A; +- the MBBT target has higher SLO-goodput than the MNS target in Regime B; +- each winning target exceeds its competing target by at least 10% of the + source SLO-goodput. A source with zero goodput makes the run invalid for + this relative gate rather than changing the denominator. + +The binding gate passes only if, in both regimes: + +- the predicted family matches the measured winning family in all three + repetitions; +- the median winning-family exclusive fraction is at least 0.10; +- it is at least 5x the median competing-family exclusive fraction; +- the direction is unchanged under cumulative 25%, 50%, 75%, and 100% + checkpoints after the 25% checkpoint. + +Decision meanings: + +- `STOP_WORKLOAD_NOT_CROSSED`: candidate outcomes do not have different + winners; the experiment cannot test action selection. +- `STOP_BINDING_NOT_PREDICTIVE`: outcomes cross but source-only constraint + scores do not select them; do not implement shadow scheduling from this + hypothesis. +- `STOP_NO_NEW_INSTRUMENTATION_NEEDED`: the signal works but every required + field was already present; keep it as an analysis/tuner feature and do not + claim a new engine-instrumentation contribution. +- `OPEN_EXACT_ATTRIBUTION_ABLATION`: the signal works but unresolved/both/KV + cases are material enough that exact rejection reasons could change a + decision. Only this result authorizes a minimal vLLM attribution patch. + +Ambiguity is material only when, in either regime, the median +`both + waiting_unresolved` fraction is at least the median absolute gap +between the two exclusive fractions, or when any source run records a +preemption or median source KV maximum is at least 0.90. Otherwise all fields +needed for the observed decision were already present and the result is +`STOP_NO_NEW_INSTRUMENTATION_NEEDED`. + +No result from this development pilot is a paper-level E2E tuning claim. + +## Cost and stopping discipline + +- Hard cap: 8.0 H20-hours, including failed sessions. +- Expected: 6.0-7.2 H20-hours and 90-110 minutes wall time. +- `b_base` runs first. If its first measured band cannot drain by 450 seconds, + the controller stops before any comparative analysis; MBBT=256 is then an + operationally invalid source, not negative evidence. +- Any data red flag stops analysis before computing a tuning conclusion. diff --git a/runs/action-aware-v0/action_aware_client.py b/runs/action-aware-v0/action_aware_client.py new file mode 100644 index 0000000..58fbe27 --- /dev/null +++ b/runs/action-aware-v0/action_aware_client.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Add explicit MBBT/config provenance to the accepted Phase-6 replay client.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +PHASE6 = Path(__file__).resolve().parents[1] / "opprof-phase6" +sys.path.insert(0, str(PHASE6)) + +import opprof_phase6_client as base # noqa: E402 + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser() + result.add_argument("command", choices=("warmup", "run-anchor")) + result.add_argument("--study", required=True) + result.add_argument("--cell", required=True) + result.add_argument("--anchor", type=float, required=True) + result.add_argument("--tp", type=int, required=True) + result.add_argument("--mns", type=int, required=True) + result.add_argument("--mbbt", type=int, required=True) + result.add_argument("--base-url", required=True) + result.add_argument("--result-dir", required=True) + result.add_argument("--disable-slo-early-stop", action="store_true") + return result + + +def main() -> None: + args = parser().parse_args() + result = base.run_replay(args, warmup=args.command == "warmup") + result.update( + { + "schema": "action-aware-pilot-result-v0", + "config_id": args.cell, + "mbbt": args.mbbt, + } + ) + base.atomic_json(Path(args.result_dir) / "result.json", result) + print( + json.dumps( + { + key: result[key] + for key in ( + "config_id", + "mns", + "mbbt", + "kind", + "pass_rate", + "feasible", + ) + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/action-aware-v0/analyze_pilot.py b/runs/action-aware-v0/analyze_pilot.py new file mode 100644 index 0000000..5eafbd2 --- /dev/null +++ b/runs/action-aware-v0/analyze_pilot.py @@ -0,0 +1,584 @@ +#!/usr/bin/env python3 +"""Audit source-only constraint signals against crossed real interventions.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import statistics +import sys +from pathlib import Path +from typing import Any, Iterable, Mapping + + +HERE = Path(__file__).resolve().parent +COMMON_STATE = HERE.parent / "telemetry-residual" +sys.path.insert(0, str(COMMON_STATE)) + +from common_state import summarize_engine # noqa: E402 + + +SCHEMA = "action-aware-constraint-pilot-audit-v0" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def atomic_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + os.replace(temporary, path) + + +def numeric(values: Iterable[float]) -> dict[str, Any]: + finite = [float(value) for value in values] + if not finite: + raise ValueError("numeric summary requires values") + if any(not math.isfinite(value) for value in finite): + raise ValueError("numeric summary received non-finite values") + return { + "n": len(finite), + "min": min(finite), + "max": max(finite), + "distinct_n": len(set(finite)), + } + + +def quantile(values: Iterable[float], probability: float) -> float: + ordered = sorted(float(value) for value in values) + if not ordered: + raise ValueError("quantile requires values") + position = probability * (len(ordered) - 1) + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + weight = position - lower + return ordered[lower] * (1.0 - weight) + ordered[upper] * weight + + +def load_jsonl(path: Path) -> list[dict[str, Any]]: + records = [] + with path.open(encoding="utf-8") as source: + for line_number, line in enumerate(source, 1): + try: + records.append(json.loads(line)) + except json.JSONDecodeError as error: + raise ValueError(f"{path}:{line_number}: invalid JSON") from error + return records + + +def binding_summary( + records: list[Mapping[str, Any]], *, mns: int, mbbt: int +) -> dict[str, Any]: + if not records: + raise ValueError("binding summary requires scheduler records") + counts = { + "mns_exclusive": 0, + "mbbt_exclusive": 0, + "both": 0, + "waiting_unresolved": 0, + "waiting": 0, + } + running_utilization = [] + token_utilization = [] + kv_usage = [] + preemptions = 0 + for record in records: + waiting = int(record["queues"]["waiting"]) + int( + record["queues"]["deferred"] + ) + running = int(record["queues"]["running"]) + scheduled_tokens = int(record["prefill_tokens"]) + int( + record["decode_tokens"] + ) + if running > mns: + raise ValueError("running requests exceed configured MNS") + if scheduled_tokens > mbbt: + raise ValueError("scheduled tokens exceed configured MBBT") + mns_hit = waiting > 0 and running == mns + mbbt_hit = waiting > 0 and scheduled_tokens == mbbt + if waiting > 0: + counts["waiting"] += 1 + if mns_hit and mbbt_hit: + counts["both"] += 1 + elif mns_hit: + counts["mns_exclusive"] += 1 + elif mbbt_hit: + counts["mbbt_exclusive"] += 1 + else: + counts["waiting_unresolved"] += 1 + running_utilization.append(running / mns) + token_utilization.append(scheduled_tokens / mbbt) + kv_usage.append(float(record["kv"]["usage"])) + preemptions += int(record["preemptions"]) + count = len(records) + return { + "records": count, + **{f"{name}_count": value for name, value in counts.items()}, + **{f"{name}_fraction": value / count for name, value in counts.items()}, + "running_utilization_mean": statistics.fmean(running_utilization), + "running_utilization_max": max(running_utilization), + "token_utilization_mean": statistics.fmean(token_utilization), + "token_utilization_max": max(token_utilization), + "kv_usage_mean": statistics.fmean(kv_usage), + "kv_usage_max": max(kv_usage), + "preemptions": preemptions, + } + + +def request_summary(path: Path, expected_count: int) -> dict[str, Any]: + rows = load_jsonl(path) + if len(rows) != expected_count: + raise ValueError(f"request row count mismatch: {path}") + ttft = [float(row["ttft_ms"]) for row in rows if row["ttft_ms"] is not None] + tpot = [float(row["tpot_ms"]) for row in rows if row["tpot_ms"] is not None] + if not ttft or not tpot: + raise ValueError(f"missing request latency values: {path}") + return { + "ttft_ms": {f"p{int(p * 100)}": quantile(ttft, p) for p in (0.5, 0.95, 0.99)}, + "tpot_ms": {f"p{int(p * 100)}": quantile(tpot, p) for p in (0.5, 0.95, 0.99)}, + } + + +def load_stream(session_root: Path) -> tuple[list[dict[str, Any]], dict[str, Any]]: + streams = sorted((session_root / "opprof").glob("*.jsonl")) + sidecars = sorted((session_root / "opprof").glob("*.jsonl.footer.json")) + if len(streams) != 1 or len(sidecars) != 1: + raise ValueError(f"expected one OpProf stream and sidecar: {session_root}") + decoded = load_jsonl(streams[0]) + records = [row for row in decoded if "step_index" in row] + footers = [row for row in decoded if row.get("record_type") == "footer"] + sidecar = json.loads(sidecars[0].read_text(encoding="utf-8")) + indexes = [int(row["step_index"]) for row in records] + invariants = { + "one_footer_last": len(footers) == 1 and decoded[-1] is footers[0], + "sidecar_final": sidecar.get("final") is True, + "zero_drops": sidecar.get("dropped_records") == 0, + "written_matches_records": sidecar.get("written_records") == len(records), + "contiguous_step_indexes": indexes == list(range(len(indexes))), + "monotonic_timestamps": all( + int(right["submit_mono_ns"]) >= int(left["submit_mono_ns"]) + for left, right in zip(records, records[1:], strict=False) + ), + } + return records, { + "stream": str(streams[0]), + "stream_sha256": sha256_file(streams[0]), + "records": len(records), + "invariants": invariants, + } + + +def analyze_run( + *, + run_root: Path, + config: Mapping[str, Any], + repetition: int, + expected: Mapping[str, Any], + stream_records: list[Mapping[str, Any]], + duration_s: float, + phase_fractions: list[float], +) -> dict[str, Any]: + result_root = run_root / "sessions" / str(config["id"]) / f"rep{repetition}" + result_path = result_root / "result.json" + result = json.loads(result_path.read_text(encoding="utf-8")) + selection = result["selection"] + invariants = { + "result_schema": result.get("schema") == "action-aware-pilot-result-v0", + "config_id": result.get("config_id") == config["id"], + "tp": int(result.get("tp", -1)) == 4, + "mns": int(result.get("mns", -1)) == int(config["mns"]), + "mbbt": int(result.get("mbbt", -1)) == int(config["mbbt"]), + "uncensored": not bool(result.get("early_stopped", True)), + "slo_early_stop_disabled": result.get("slo_early_stop_disabled") is True, + "selection_count": int(selection["count"]) == int(expected["selected_count"]), + "request_accounting": int(result["observed_count"]) + == int(expected["selected_count"]), + "request_hash": selection["request_id_order_sha256"] + == expected["request_id_order_sha256"], + "arrival_hash": selection["arrival_order_sha256"] + == expected["arrival_order_sha256"], + "length_hash": selection["raw_length_order_sha256"] + == expected["input_length_order_sha256"], + } + start_ns = int(result["interval"]["start_mono_ns"]) + arrival_end_ns = start_ns + round(duration_s * 1e9) + full_records = [ + record + for record in stream_records + if start_ns <= int(record["submit_mono_ns"]) <= arrival_end_ns + ] + if not full_records: + raise ValueError(f"no telemetry records in measured window: {result_path}") + gaps = [ + (int(right["submit_mono_ns"]) - int(left["submit_mono_ns"])) / 1e9 + for left, right in zip(full_records, full_records[1:], strict=False) + ] + coverage = { + "start_gap_s": (int(full_records[0]["submit_mono_ns"]) - start_ns) / 1e9, + "end_gap_s": (arrival_end_ns - int(full_records[-1]["submit_mono_ns"])) / 1e9, + "max_internal_gap_s": max(gaps, default=0.0), + } + invariants["telemetry_coverage"] = all( + 0.0 <= value <= 1.0 for value in coverage.values() + ) + binding = binding_summary( + full_records, mns=int(config["mns"]), mbbt=int(config["mbbt"]) + ) + phases = {} + for fraction in phase_fractions: + phase_end = start_ns + round(duration_s * fraction * 1e9) + phase_records = [ + record + for record in full_records + if int(record["submit_mono_ns"]) <= phase_end + ] + phases[f"{fraction:.2f}"] = binding_summary( + phase_records, mns=int(config["mns"]), mbbt=int(config["mbbt"]) + ) + state = summarize_engine( + full_records, + start_ns=start_ns, + end_ns=arrival_end_ns, + request_count=int(result["observed_count"]), + ) + latency = request_summary( + result_root / "requests.jsonl", int(result["observed_count"]) + ) + return { + "config_id": config["id"], + "mns": int(config["mns"]), + "mbbt": int(config["mbbt"]), + "repetition": repetition, + "result_path": str(result_path), + "result_sha256": sha256_file(result_path), + "selection": { + "count": int(selection["count"]), + "request_id_order_sha256": selection["request_id_order_sha256"], + "arrival_order_sha256": selection["arrival_order_sha256"], + "raw_length_order_sha256": selection["raw_length_order_sha256"], + }, + "outcome": { + "pass_rate": float(result["pass_rate"]), + "feasible": bool(result["feasible"]), + "slo_pass_count": int(result["slo_pass_count"]), + "slo_goodput_req_s": int(result["slo_pass_count"]) / duration_s, + "elapsed_s": float(result["interval"]["elapsed_s"]), + **latency, + }, + "binding": binding, + "phases": phases, + "state": state, + "coverage": coverage, + "invariants": invariants, + } + + +def median(values: Iterable[float]) -> float: + return float(statistics.median(float(value) for value in values)) + + +def evaluate_decisions( + runs: list[Mapping[str, Any]], manifest: Mapping[str, Any] +) -> dict[str, Any]: + by_key = { + (str(run["config_id"]), int(run["repetition"])): run for run in runs + } + repetitions = sorted(int(key) for key in manifest["repetitions"]) + regime_results = {} + all_predictions = [] + crossed_pass = True + binding_pass = True + material_ambiguity = False + for regime_name, regime in manifest["regimes"].items(): + rows = [] + source_runs = [] + for repetition in repetitions: + source = by_key[(str(regime["source"]), repetition)] + mns_target = by_key[(str(regime["actions"]["mns"]), repetition)] + mbbt_target = by_key[(str(regime["actions"]["mbbt"]), repetition)] + source_runs.append(source) + source_goodput = float(source["outcome"]["slo_goodput_req_s"]) + mns_goodput = float(mns_target["outcome"]["slo_goodput_req_s"]) + mbbt_goodput = float(mbbt_target["outcome"]["slo_goodput_req_s"]) + observed = ( + "mns" + if mns_goodput > mbbt_goodput + else "mbbt" + if mbbt_goodput > mns_goodput + else "tie" + ) + mns_score = float(source["binding"]["mns_exclusive_fraction"]) + mbbt_score = float(source["binding"]["mbbt_exclusive_fraction"]) + predicted = ( + "mns" + if mns_score > mbbt_score + else "mbbt" + if mbbt_score > mns_score + else "tie" + ) + phase_predictions = {} + for phase, summary in source["phases"].items(): + left = float(summary["mns_exclusive_fraction"]) + right = float(summary["mbbt_exclusive_fraction"]) + phase_predictions[phase] = ( + "mns" if left > right else "mbbt" if right > left else "tie" + ) + margin = ( + abs(mns_goodput - mbbt_goodput) / source_goodput + if source_goodput > 0 + else None + ) + row = { + "repetition": repetition, + "source_goodput_req_s": source_goodput, + "mns_target_goodput_req_s": mns_goodput, + "mbbt_target_goodput_req_s": mbbt_goodput, + "observed_winner": observed, + "predicted_winner": predicted, + "prediction_correct": predicted == observed, + "relative_winner_margin_over_source": margin, + "mns_exclusive_fraction": mns_score, + "mbbt_exclusive_fraction": mbbt_score, + "phase_predictions": phase_predictions, + "phase_stable": all(value == predicted for value in phase_predictions.values()), + } + rows.append(row) + all_predictions.append(row) + + expected_winner = "mns" if regime_name == "A" else "mbbt" + minimum_margin = float(manifest["gates"]["minimum_relative_winner_margin"]) + regime_crossed = all( + row["observed_winner"] == expected_winner + and row["relative_winner_margin_over_source"] is not None + and row["relative_winner_margin_over_source"] >= minimum_margin + for row in rows + ) + crossed_pass &= regime_crossed + winning_key = f"{expected_winner}_exclusive_fraction" + losing_key = ( + "mbbt_exclusive_fraction" if expected_winner == "mns" else "mns_exclusive_fraction" + ) + winning_median = median(row[winning_key] for row in rows) + losing_median = median(row[losing_key] for row in rows) + ratio_pass = winning_median >= float( + manifest["gates"]["minimum_exclusive_ratio"] + ) * losing_median + regime_binding = ( + all(row["prediction_correct"] and row["phase_stable"] for row in rows) + and winning_median + >= float(manifest["gates"]["minimum_exclusive_fraction"]) + and ratio_pass + ) + binding_pass &= regime_binding + ambiguity_median = median( + float(run["binding"]["both_fraction"]) + + float(run["binding"]["waiting_unresolved_fraction"]) + for run in source_runs + ) + score_gap_median = median( + abs( + float(run["binding"]["mns_exclusive_fraction"]) + - float(run["binding"]["mbbt_exclusive_fraction"]) + ) + for run in source_runs + ) + kv_max_median = median( + float(run["binding"]["kv_usage_max"]) for run in source_runs + ) + any_preemption = any( + int(run["binding"]["preemptions"]) > 0 for run in source_runs + ) + regime_material = ( + ambiguity_median >= score_gap_median + or kv_max_median >= float(manifest["gates"]["material_kv_usage"]) + or any_preemption + ) + material_ambiguity |= regime_material + regime_results[regime_name] = { + "source": regime["source"], + "actions": regime["actions"], + "expected_winner": expected_winner, + "crossed_response_pass": regime_crossed, + "binding_pass": regime_binding, + "winning_exclusive_median": winning_median, + "losing_exclusive_median": losing_median, + "exclusive_ratio_pass": ratio_pass, + "ambiguity_median": ambiguity_median, + "exclusive_gap_median": score_gap_median, + "kv_usage_max_median": kv_max_median, + "any_preemption": any_preemption, + "material_ambiguity": regime_material, + "repetitions": rows, + } + + if not crossed_pass: + decision = "STOP_WORKLOAD_NOT_CROSSED" + elif not binding_pass: + decision = "STOP_BINDING_NOT_PREDICTIVE" + elif material_ambiguity: + decision = "OPEN_EXACT_ATTRIBUTION_ABLATION" + else: + decision = "STOP_NO_NEW_INSTRUMENTATION_NEEDED" + correct = sum(int(row["prediction_correct"]) for row in all_predictions) + return { + "decision": decision, + "crossed_response_pass": crossed_pass, + "binding_pass": binding_pass, + "material_ambiguity": material_ambiguity, + "regimes": regime_results, + "baselines": { + "always_mns_correct": sum( + int(row["observed_winner"] == "mns") for row in all_predictions + ), + "always_mbbt_correct": sum( + int(row["observed_winner"] == "mbbt") for row in all_predictions + ), + "binding_correct": correct, + "decision_count": len(all_predictions), + }, + } + + +def analyze(run_root: Path, manifest_path: Path) -> dict[str, Any]: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest.get("schema") != "action-aware-constraint-pilot-manifest-v0": + raise ValueError("unexpected manifest schema") + duration_s = float(manifest["engine"]["duration_s"]) + phase_fractions = [float(value) for value in manifest["gates"]["phase_fractions"]] + runs = [] + stream_audits = [] + for config in manifest["configs"]: + session_root = run_root / "sessions" / str(config["id"]) + stream_records, stream_audit = load_stream(session_root) + stream_audit["config_id"] = config["id"] + stream_audits.append(stream_audit) + for repetition in sorted(int(key) for key in manifest["repetitions"]): + runs.append( + analyze_run( + run_root=run_root, + config=config, + repetition=repetition, + expected=manifest["repetitions"][str(repetition)]["selection"], + stream_records=stream_records, + duration_s=duration_s, + phase_fractions=phase_fractions, + ) + ) + invariants = { + "fifteen_runs": len(runs) == 15, + "five_streams": len(stream_audits) == 5, + "all_run_invariants": all( + all(bool(value) for value in run["invariants"].values()) for run in runs + ), + "all_stream_invariants": all( + all(bool(value) for value in stream["invariants"].values()) + for stream in stream_audits + ), + "nonnegative_counters": all( + all( + float(run["binding"][key]) >= 0 + for key in ( + "mns_exclusive_count", + "mbbt_exclusive_count", + "both_count", + "waiting_unresolved_count", + "preemptions", + ) + ) + for run in runs + ), + "ratios_bounded": all( + all( + 0.0 <= float(run["binding"][key]) <= 1.0 + for key in ( + "mns_exclusive_fraction", + "mbbt_exclusive_fraction", + "both_fraction", + "waiting_unresolved_fraction", + "kv_usage_mean", + "kv_usage_max", + ) + ) + for run in runs + ), + "per_config_results_not_all_identical": len( + {float(run["outcome"]["pass_rate"]) for run in runs} + ) + > 1, + } + red_flags = [name for name, passed in invariants.items() if not passed] + decisions = ( + evaluate_decisions(runs, manifest) + if not red_flags + else { + "decision": "STOP_DATA_INVALID", + "crossed_response_pass": False, + "binding_pass": False, + "material_ambiguity": False, + "regimes": {}, + "baselines": {}, + } + ) + payload = { + "schema": SCHEMA, + "decision": decisions["decision"], + "manifest": str(manifest_path), + "manifest_sha256": sha256_file(manifest_path), + "run_root": str(run_root), + "runs": runs, + "streams": stream_audits, + "decision_audit": decisions, + "sanity": { + "runs": len(runs), + "pass_rate": numeric(run["outcome"]["pass_rate"] for run in runs), + "slo_goodput_req_s": numeric( + run["outcome"]["slo_goodput_req_s"] for run in runs + ), + "telemetry_records_per_run": numeric( + run["binding"]["records"] for run in runs + ), + "mns_values": numeric(run["mns"] for run in runs), + "mbbt_values": numeric(run["mbbt"] for run in runs), + "invariants": invariants, + "red_flags": red_flags, + }, + } + return payload + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + payload = analyze(args.run_root, args.manifest) + atomic_json(args.output, payload) + print( + json.dumps( + { + "decision": payload["decision"], + "sanity": payload["sanity"], + "decision_audit": payload["decision_audit"], + }, + indent=2, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/action-aware-v0/pilot-manifest.json b/runs/action-aware-v0/pilot-manifest.json new file mode 100644 index 0000000..170c02f --- /dev/null +++ b/runs/action-aware-v0/pilot-manifest.json @@ -0,0 +1,223 @@ +{ + "budget": { + "expected_h20_hours": [ + 6.0, + 7.2 + ], + "expected_wall_minutes": [ + 90, + 110 + ], + "hard_cap_h20_hours": 8.0, + "safety_h20_hours": 0.25, + "session_estimate_h20_hours": 1.35 + }, + "burnin": { + "anchor": 0.18919793755240089, + "arrival_order_sha256": "6c0ac4cb9a30ef501eeeacc8e6cc631c345e976db5ccf530ea5a1ec706d62a24", + "input_length_order_sha256": "7939cc20e1a00d1031d27d71508789f38decbbbb6ea59a1df18b2ec342fd2ef8", + "offered_req_s": 8.5, + "offered_req_s_per_gpu": 2.125, + "request_id_order_sha256": "84f4809acbc8acd3b1d14dfa357134a1dc0b9287341624b33f598dafeef54dc7", + "selected_count": 510, + "study": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/studies/burnin-tp4.json", + "study_sha256": "5d6c2098042909a863efd3112818fbee9bafe96f22898ac98b66846dbe1fef0f" + }, + "configs": [ + { + "id": "b_base", + "mbbt": 256, + "mns": 64, + "repetition_order": [ + 1, + 2, + 3 + ] + }, + { + "id": "a_base", + "mbbt": 8192, + "mns": 16, + "repetition_order": [ + 2, + 3, + 1 + ] + }, + { + "id": "shared", + "mbbt": 8192, + "mns": 64, + "repetition_order": [ + 3, + 1, + 2 + ] + }, + { + "id": "b_mns", + "mbbt": 256, + "mns": 128, + "repetition_order": [ + 1, + 3, + 2 + ] + }, + { + "id": "a_mbbt", + "mbbt": 16384, + "mns": 16, + "repetition_order": [ + 2, + 1, + 3 + ] + } + ], + "engine": { + "client_timeout_s": 450.0, + "disable_slo_early_stop": true, + "duration_s": 300.0, + "tp": 4 + }, + "gates": { + "material_kv_usage": 0.9, + "minimum_exclusive_fraction": 0.1, + "minimum_exclusive_ratio": 5.0, + "minimum_relative_winner_margin": 0.1, + "phase_fractions": [ + 0.25, + 0.5, + 0.75, + 1.0 + ] + }, + "regimes": { + "A": { + "actions": { + "mbbt": "a_mbbt", + "mns": "shared" + }, + "source": "a_base" + }, + "B": { + "actions": { + "mbbt": "shared", + "mns": "b_mns" + }, + "source": "b_base" + } + }, + "repetitions": { + "1": { + "merged_trace": { + "bytes": 337429767, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep1.jsonl", + "request_id_scheme": "sha256(source_sha256:line_number:original_id)", + "rows": 9420, + "sha256": "68983266aa0e66aa589562f7c08edbd966f9ba4405e20c105adb43777d2dfbf5", + "source_sha256": [ + "b242d1d9086df3accab57b4c92445d5edd581e12f47e12cea227aa63964c6930", + "d23b549f7b69af3647308677bbf76f818a3c226a1c98f9a9f93f09ceee46be87" + ], + "sources": [ + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low1.jsonl", + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high1.jsonl" + ] + }, + "selection": { + "anchor": 0.48686986110831465, + "arrival_order_sha256": "c2ad99986ce558da5901a9c5ec0a00bd69f198c981d8779235f2773a5c87f1c0", + "input_length_order_sha256": "9442bfebdc3fab5062dc1f4d688dc28c02afe3fd806c56dd8159f0ac7e6d0b94", + "offered_req_s": 8.5, + "offered_req_s_per_gpu": 2.125, + "request_id_order_sha256": "0bb61dbc9c26875e991d0d4f984134910d37463e5063f86ee960cf4f8aafb771", + "selected_count": 2550, + "target_count": 2550, + "target_req_s_per_gpu": 2.125 + }, + "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep1-tp4.json", + "study_sha256": "ecfff96e33d458eb1e3b9a6d24386f00cc6f1b19ff926e2ec6320b3f671a7ae3" + }, + "2": { + "merged_trace": { + "bytes": 337509330, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep2.jsonl", + "request_id_scheme": "sha256(source_sha256:line_number:original_id)", + "rows": 9457, + "sha256": "f38e8938f6a481fc6725b71b21aa04ff7eaf79783cdfd6e41aa2f074156f00c2", + "source_sha256": [ + "4cbb0baac082bd54af562ce2f39104c5c23b4671672da365a67b1e8c146adf9f", + "bb0bcd2564a88000f435f12feb21c7c902eafc9ea5fe916adfe9d1eae47f3f9a" + ], + "sources": [ + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low2.jsonl", + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high2.jsonl" + ] + }, + "selection": { + "anchor": 0.4825698948735577, + "arrival_order_sha256": "b9fc12cf3f86bc8a79bee65296e65aa2b8bf2aeca46b2887094c669adcbb9a00", + "input_length_order_sha256": "d8d4bd6fc8ba852a45605b673b6b3e4f33b58f459e69f2a032d226ee175b074e", + "offered_req_s": 8.5, + "offered_req_s_per_gpu": 2.125, + "request_id_order_sha256": "56a0616b6b54abafd37875c7cb25f8639afef2706ccc55dfbe568f45859ea382", + "selected_count": 2550, + "target_count": 2550, + "target_req_s_per_gpu": 2.125 + }, + "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep2-tp4.json", + "study_sha256": "d92a576db031db24bb58f354ea725d7f7567cb76699d387117ac5a6c9317bbb9" + }, + "3": { + "merged_trace": { + "bytes": 337450256, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep3.jsonl", + "request_id_scheme": "sha256(source_sha256:line_number:original_id)", + "rows": 9431, + "sha256": "3094084b0bb20cc02eecf465091a5c919b4e5b112f704cdc36a563d1efdcee46", + "source_sha256": [ + "1f7ececb142f9a363d2d1ca25eb7b8488b2cc319a51b55faa384f2a3d51f2142", + "6f326234791e1cff4ff866bface0d097d0d6e3844eebb1c97653d8e9c35e9397" + ], + "sources": [ + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low3.jsonl", + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high3.jsonl" + ] + }, + "selection": { + "anchor": 0.48664343020532463, + "arrival_order_sha256": "efce7339e22d3618cb4d55e6b55bfddb2c563c18faba2a992d5829c13e3f55e9", + "input_length_order_sha256": "0792b05fff6729fbd92ab2bb4cb6d31bea7799e232ad42772936bc06efbafb54", + "offered_req_s": 8.5, + "offered_req_s_per_gpu": 2.125, + "request_id_order_sha256": "2a2fabe2c4cf176aeb7e0d32fb8e7dbb1f27429a2e7a0cd18d7d186f23096f19", + "selected_count": 2550, + "target_count": 2550, + "target_req_s_per_gpu": 2.125 + }, + "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep3-tp4.json", + "study_sha256": "fb8ffe256dace32f4ca8a8d49b662d98c3b69b94ecc8fa826e43068b238884ab" + } + }, + "sanity": { + "invariants": { + "all_repetition_orders_are_permutations": true, + "five_unique_configs": true, + "same_load_all_repetitions": true, + "shared_endpoint_reused_by_both_regimes": true, + "three_disjoint_repetitions": true + }, + "red_flags": [] + }, + "schema": "action-aware-constraint-pilot-manifest-v0", + "source": { + "base_manifest": "/home/gahow/phd/aituner/runs/intervention-response-v2/pilot-manifest-v3.json", + "base_manifest_sha256": "273db1181dcc9d6b64439650d0642ebe553b12e6aa9adebfbe3758a7977e5611", + "source_trace": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260312_1000.jsonl", + "source_trace_sha256": "875ba869775deb78086477919f03b322da14e2673c7d070e26528c4190912757", + "window_id": "chat_w20260312_1000" + }, + "status": "PASS" +} diff --git a/runs/action-aware-v0/pilot_controller.py b/runs/action-aware-v0/pilot_controller.py new file mode 100644 index 0000000..8e8b94f --- /dev/null +++ b/runs/action-aware-v0/pilot_controller.py @@ -0,0 +1,595 @@ +#!/usr/bin/env python3 +"""Serialized controller for the crossed-constraint action-aware pilot.""" + +from __future__ import annotations + +import argparse +import json +import os +import shlex +import signal +import subprocess +import sys +import time +from pathlib import Path +from typing import Any, Mapping + + +HERE = Path(__file__).resolve().parent +PHASE6 = HERE.parent / "opprof-phase6" +sys.path.insert(0, str(PHASE6)) + +import opprof_phase6_controller as base # noqa: E402 + + +SCHEMA = "action-aware-constraint-pilot-state-v0" + + +def atomic_json(path: Path, payload: Any) -> None: + base.atomic_json(path, payload) + + +def wait_all_idle(timeout_s: float = 30.0) -> None: + deadline = time.monotonic() + timeout_s + last_error: Exception | None = None + while time.monotonic() < deadline: + try: + base.assert_all_idle() + return + except RuntimeError as error: + last_error = error + time.sleep(1.0) + raise last_error or RuntimeError("GPU idle timeout") + + +def configure(args: argparse.Namespace, manifest: Mapping[str, Any]) -> None: + base.WORKDIR = args.run_root.parent + base.RUN_ROOT = args.run_root + base.STATE = args.run_root / "controller-state.json" + base.SOURCE = args.vllm_source + base.VENV = args.venv + base.AITUNER = args.aituner_root + base.MODEL = args.model + base.CLIENT = args.client + base.GPU_LIMIT = float(manifest["budget"]["hard_cap_h20_hours"]) + base.MARKER = "action-aware-constraint-pilot-v0" + + +def validate_inputs(args: argparse.Namespace, manifest: Mapping[str, Any]) -> None: + if manifest.get("schema") != "action-aware-constraint-pilot-manifest-v0": + raise RuntimeError("unexpected action-aware manifest schema") + if manifest.get("status") != "PASS": + raise RuntimeError("action-aware manifest did not pass preflight") + red_flags = manifest.get("sanity", {}).get("red_flags", []) + if red_flags: + raise RuntimeError(f"manifest red flags: {red_flags}") + + required = { + "manifest": args.manifest, + "aituner_root": args.aituner_root, + "vllm_source": args.vllm_source, + "venv_python": args.venv / "bin/python", + "venv_vllm": args.venv / "bin/vllm", + "model": args.model, + "client": args.client, + "burnin_study": Path(manifest["burnin"]["study"]), + } + for repetition, item in manifest["repetitions"].items(): + required[f"rep{repetition}_study"] = Path(item["study"]) + required[f"rep{repetition}_trace"] = Path(item["merged_trace"]["path"]) + missing = {name: str(path) for name, path in required.items() if not path.exists()} + if missing: + raise RuntimeError(f"action-aware input paths missing: {missing}") + + +def config_map(manifest: Mapping[str, Any]) -> dict[str, dict[str, Any]]: + return {str(item["id"]): dict(item) for item in manifest["configs"]} + + +def server_command( + config: Mapping[str, Any], *, gpus: tuple[int, ...], port: int +) -> list[str]: + return [ + "taskset", + "-c", + base.cpu_mask(gpus), + str(base.VENV / "bin/vllm"), + "serve", + str(base.MODEL), + "--host", + "127.0.0.1", + "--port", + str(port), + "--served-model-name", + "qwen3-30b-a3b-community", + "--max-num-batched-tokens", + str(config["mbbt"]), + "--max-num-seqs", + str(config["mns"]), + "--tensor-parallel-size", + "4", + "--shutdown-timeout", + "120", + ] + + +def client_command( + entry: Mapping[str, Any], + config: Mapping[str, Any], + *, + study: str, + anchor: float, + output: Path, + warmup: bool, +) -> list[str]: + command = [ + "taskset", + "-c", + base.cpu_mask(entry["gpus"]), + str(base.VENV / "bin/python"), + str(base.CLIENT), + "warmup" if warmup else "run-anchor", + "--study", + study, + "--cell", + str(config["id"]), + "--anchor", + str(anchor), + "--tp", + "4", + "--mns", + str(config["mns"]), + "--mbbt", + str(config["mbbt"]), + "--base-url", + f"http://127.0.0.1:{entry['port']}", + "--result-dir", + str(output), + "--disable-slo-early-stop", + ] + return command + + +def remaining_projection( + manifest: Mapping[str, Any], *, completed_sessions: int +) -> float: + remaining = len(manifest["configs"]) - completed_sessions + return ( + remaining * float(manifest["budget"]["session_estimate_h20_hours"]) + + float(manifest["budget"]["safety_h20_hours"]) + ) + + +def dry_run_plan( + args: argparse.Namespace, manifest: Mapping[str, Any] +) -> dict[str, Any]: + sessions = [] + for index, config in enumerate(manifest["configs"]): + entry = {"gpus": (0, 1, 2, 3), "port": 9050 + index} + session_root = args.run_root / "sessions" / str(config["id"]) + first_repetition = str(config["repetition_order"][0]) + first = manifest["repetitions"][first_repetition] + commands = { + "server": server_command(config, gpus=entry["gpus"], port=entry["port"]), + "warmup": client_command( + entry, + config, + study=first["study"], + anchor=float(first["selection"]["anchor"]), + output=session_root / "warmup", + warmup=True, + ), + "burnin": client_command( + entry, + config, + study=manifest["burnin"]["study"], + anchor=float(manifest["burnin"]["anchor"]), + output=session_root / "burnin", + warmup=False, + ), + } + for repetition in config["repetition_order"]: + item = manifest["repetitions"][str(repetition)] + commands[f"rep{repetition}"] = client_command( + entry, + config, + study=item["study"], + anchor=float(item["selection"]["anchor"]), + output=session_root / f"rep{repetition}", + warmup=False, + ) + sessions.append( + { + "config": config["id"], + "mns": config["mns"], + "mbbt": config["mbbt"], + "port": entry["port"], + "repetition_order": config["repetition_order"], + "commands": { + role: shlex.join(command) for role, command in commands.items() + }, + } + ) + return { + "schema": "action-aware-constraint-pilot-dry-run-v0", + "status": "PASS", + "manifest": str(args.manifest), + "run_root": str(args.run_root), + "projected_h20_hours": remaining_projection( + manifest, completed_sessions=0 + ), + "hard_cap_h20_hours": manifest["budget"]["hard_cap_h20_hours"], + "sessions": sessions, + } + + +def load_state(path: Path, hard_cap: float) -> dict[str, Any]: + if path.exists(): + return json.loads(path.read_text(encoding="utf-8")) + return { + "schema": SCHEMA, + "status": "initialized", + "hard_cap_h20_hours": hard_cap, + "gpu_hours_total": 0.0, + "completed_sessions": 0, + "sessions": {}, + "failures": [], + "started_at": time.time(), + } + + +def append_echo(run_root: Path, line: str) -> None: + run_root.mkdir(parents=True, exist_ok=True) + with (run_root / "launch-echo.log").open("a", encoding="utf-8") as target: + target.write(line + "\n") + print(line, flush=True) + + +def start_server( + *, + args: argparse.Namespace, + config: Mapping[str, Any], + index: int, +) -> dict[str, Any]: + gpus = (0, 1, 2, 3) + session_root = args.run_root / "sessions" / str(config["id"]) + session_root.mkdir(parents=True, exist_ok=True) + port = 9050 + index + command = server_command(config, gpus=gpus, port=port) + with (session_root / "commands.log").open("a", encoding="utf-8") as log: + log.write(f"SERVER {shlex.join(command)}\n") + server_log = (session_root / "server.log").open("ab", buffering=0) + environment = os.environ.copy() + environment.update( + { + "CUDA_VISIBLE_DEVICES": "0,1,2,3", + "VLLM_OPPROF_DIR": str(session_root / "opprof"), + "OPPROF_PHASE6_MARKER": base.MARKER, + "AITUNER_ROOT": str(base.AITUNER), + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "PYTHONUNBUFFERED": "1", + } + ) + server = subprocess.Popen( + command, + cwd=base.SOURCE, + env=environment, + stdout=server_log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + base.OWNED_PGIDS.add(server.pid) + return { + "cell": str(config["id"]), + "gpus": gpus, + "port": port, + "dir": session_root, + "server": server, + "server_handle": server_log, + "spawned_at": time.time(), + "results": [], + } + + +def validate_result( + result: Mapping[str, Any], + *, + config: Mapping[str, Any], + selection: Mapping[str, Any], + role: str, + warmup: bool, +) -> None: + if result.get("schema") != "action-aware-pilot-result-v0": + raise RuntimeError(f"unexpected result schema: {role}") + if result.get("config_id") != config["id"]: + raise RuntimeError(f"config id mismatch: {role}") + if int(result["tp"]) != 4: + raise RuntimeError(f"TP mismatch: {role}") + if int(result["mns"]) != int(config["mns"]): + raise RuntimeError(f"MNS mismatch: {role}") + if int(result["mbbt"]) != int(config["mbbt"]): + raise RuntimeError(f"MBBT mismatch: {role}") + if result.get("slo_early_stop_disabled") is not True: + raise RuntimeError(f"SLO early stop was not disabled: {role}") + if warmup: + if result["kind"] != "warmup" or int(result["selection"]["count"]) != 16: + raise RuntimeError(f"invalid warmup: {role}") + return + if bool(result["early_stopped"]): + raise RuntimeError(f"uncensored run early-stopped: {role}") + if int(result["selection"]["count"]) != int(selection["selected_count"]): + raise RuntimeError(f"selection count mismatch: {role}") + if int(result["observed_count"]) != int(selection["selected_count"]): + raise RuntimeError(f"request accounting mismatch: {role}") + for result_key, selection_key in ( + ("request_id_order_sha256", "request_id_order_sha256"), + ("arrival_order_sha256", "arrival_order_sha256"), + ("raw_length_order_sha256", "input_length_order_sha256"), + ): + if result["selection"][result_key] != selection[selection_key]: + raise RuntimeError(f"selection hash mismatch {result_key}: {role}") + + +def run_client( + *, + entry: dict[str, Any], + config: Mapping[str, Any], + role: str, + study: str, + selection: Mapping[str, Any], + output: Path, + state: Mapping[str, Any], + timeout_s: float, + warmup: bool = False, +) -> dict[str, Any]: + command = client_command( + entry, + config, + study=study, + anchor=float(selection["anchor"]), + output=output, + warmup=warmup, + ) + with (entry["dir"] / "commands.log").open("a", encoding="utf-8") as log: + log.write(f"CLIENT role={role} {shlex.join(command)}\n") + handle = (output.parent / f"{output.name}.log").open("ab", buffering=0) + environment = os.environ.copy() + environment.update({"AITUNER_ROOT": str(base.AITUNER), "PYTHONUNBUFFERED": "1"}) + process = subprocess.Popen( + command, + cwd=base.WORKDIR, + env=environment, + stdout=handle, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + deadline = time.monotonic() + timeout_s + try: + while process.poll() is None: + if time.monotonic() > deadline: + raise TimeoutError(f"client timeout: {config['id']} {role}") + if entry["server"].poll() is not None: + raise RuntimeError(f"server exited during {config['id']} {role}") + base.assert_no_other_compute() + if state["gpu_hours_total"] + base.live_gpu_hours([entry]) >= base.GPU_LIMIT: + raise RuntimeError("action-aware pilot H20-hour hard cap reached") + time.sleep(1.0) + except Exception: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + process.wait(timeout=10.0) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=10.0) + raise + finally: + handle.close() + if process.returncode: + raise RuntimeError( + f"client failed: config={config['id']} role={role} rc={process.returncode}" + ) + result = json.loads((output / "result.json").read_text(encoding="utf-8")) + validate_result( + result, + config=config, + selection=selection, + role=role, + warmup=warmup, + ) + entry["results"].append( + {"anchor": float(selection["anchor"]), "dir": str(output), "kind": result["kind"]} + ) + return result + + +def execute_session( + *, + args: argparse.Namespace, + manifest: Mapping[str, Any], + config: Mapping[str, Any], + index: int, + state: dict[str, Any], + state_path: Path, +) -> None: + name = str(config["id"]) + if state["sessions"].get(name, {}).get("status") == "complete": + return + projection = remaining_projection( + manifest, completed_sessions=int(state["completed_sessions"]) + ) + if float(state["gpu_hours_total"]) + projection > base.GPU_LIMIT: + raise RuntimeError(f"projected cost exceeds cap before {name}") + echo = ( + f"ACTION_AWARE_SESSION_ECHO host=dash0 config={name} tp=4 " + f"mns={config['mns']} mbbt={config['mbbt']} gpus=0-3 " + f"workload={manifest['source']['window_id']} load_per_gpu=2.125 " + f"duration_s=300 repetitions={','.join(map(str, config['repetition_order']))} " + f"source={args.manifest} output={args.run_root / 'sessions' / name} " + f"spent_h20h={state['gpu_hours_total']:.6f} " + f"remaining_projection_h20h={projection:.3f} cap_h20h={base.GPU_LIMIT:.1f}" + ) + append_echo(args.run_root, echo) + wait_all_idle() + session_state = { + "status": "starting", + "mns": int(config["mns"]), + "mbbt": int(config["mbbt"]), + "repetition_order": list(config["repetition_order"]), + "started_at": time.time(), + "runs": [], + } + state["status"] = "running" + state["sessions"][name] = session_state + atomic_json(state_path, state) + entry = start_server(args=args, config=config, index=index) + failure: Exception | None = None + try: + base.wait_ready(entry) + first = manifest["repetitions"][str(config["repetition_order"][0])] + session_state["status"] = "warmup" + atomic_json(state_path, state) + run_client( + entry=entry, + config=config, + role="warmup", + study=first["study"], + selection=first["selection"], + output=entry["dir"] / "warmup", + state=state, + timeout_s=180.0, + warmup=True, + ) + session_state["status"] = "burnin" + atomic_json(state_path, state) + burnin = manifest["burnin"] + run_client( + entry=entry, + config=config, + role="burnin", + study=burnin["study"], + selection=burnin, + output=entry["dir"] / "burnin", + state=state, + timeout_s=float(manifest["engine"]["client_timeout_s"]), + ) + session_state["status"] = "measured" + atomic_json(state_path, state) + for repetition in config["repetition_order"]: + item = manifest["repetitions"][str(repetition)] + role = f"rep{repetition}" + result = run_client( + entry=entry, + config=config, + role=role, + study=item["study"], + selection=item["selection"], + output=entry["dir"] / role, + state=state, + timeout_s=float(manifest["engine"]["client_timeout_s"]), + ) + session_state["runs"].append( + { + "repetition": int(repetition), + "pass_rate": result["pass_rate"], + "feasible": result["feasible"], + "slo_pass_count": result["slo_pass_count"], + "elapsed_s": result["interval"]["elapsed_s"], + } + ) + atomic_json(state_path, state) + session_state["status"] = "stopping" + atomic_json(state_path, state) + except Exception as error: # noqa: BLE001 + failure = error + finally: + try: + base.stop_entry(entry) + except Exception as error: # noqa: BLE001 + failure = failure or error + time.sleep(2.0) + try: + wait_all_idle() + except Exception as error: # noqa: BLE001 + failure = failure or error + + session_hours = base.live_gpu_hours([entry]) + state["gpu_hours_total"] += session_hours + session_state["gpu_hours"] = session_hours + if failure is not None: + session_state["status"] = "failed" + session_state["failure"] = repr(failure) + state["status"] = "failed" + state["failures"].append({"session": name, "failure": repr(failure)}) + atomic_json(state_path, state) + raise failure + validation = base.validate_cell(entry) + session_state["validation"] = validation + session_state["status"] = "complete" + session_state["completed_at"] = time.time() + state["completed_sessions"] += 1 + atomic_json(state_path, state) + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser() + result.add_argument("--manifest", type=Path, required=True) + result.add_argument("--run-root", type=Path, required=True) + result.add_argument("--aituner-root", type=Path, required=True) + result.add_argument("--vllm-source", type=Path, required=True) + result.add_argument("--venv", type=Path, required=True) + result.add_argument("--model", type=Path, required=True) + result.add_argument("--client", type=Path, required=True) + result.add_argument("--dry-run", action="store_true") + return result + + +def main() -> None: + args = parser().parse_args() + manifest = json.loads(args.manifest.read_text(encoding="utf-8")) + validate_inputs(args, manifest) + configure(args, manifest) + if args.dry_run: + print(json.dumps(dry_run_plan(args, manifest), indent=2, sort_keys=True)) + return + args.run_root.mkdir(parents=True, exist_ok=True) + copied_manifest = args.run_root / "pilot-manifest.json" + if not copied_manifest.exists(): + atomic_json(copied_manifest, manifest) + state_path = args.run_root / "controller-state.json" + state = load_state(state_path, base.GPU_LIMIT) + state["status"] = "running" + atomic_json(state_path, state) + for index, config in enumerate(manifest["configs"]): + execute_session( + args=args, + manifest=manifest, + config=config, + index=index, + state=state, + state_path=state_path, + ) + state["status"] = "complete" + state["completed_at"] = time.time() + atomic_json(state_path, state) + wait_all_idle() + print( + json.dumps( + { + "status": state["status"], + "completed_sessions": state["completed_sessions"], + "gpu_hours_total": state["gpu_hours_total"], + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/action-aware-v0/prepare_pilot.py b/runs/action-aware-v0/prepare_pilot.py new file mode 100644 index 0000000..51e9fd1 --- /dev/null +++ b/runs/action-aware-v0/prepare_pilot.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Freeze the crossed-constraint action-aware development pilot.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +from typing import Any + + +SCHEMA = "action-aware-constraint-pilot-manifest-v0" +CONFIGS = ( + {"id": "b_base", "mns": 64, "mbbt": 256, "repetition_order": [1, 2, 3]}, + {"id": "a_base", "mns": 16, "mbbt": 8192, "repetition_order": [2, 3, 1]}, + {"id": "shared", "mns": 64, "mbbt": 8192, "repetition_order": [3, 1, 2]}, + {"id": "b_mns", "mns": 128, "mbbt": 256, "repetition_order": [1, 3, 2]}, + {"id": "a_mbbt", "mns": 16, "mbbt": 16384, "repetition_order": [2, 1, 3]}, +) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def atomic_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + os.replace(temporary, path) + + +def build(base_path: Path) -> dict[str, Any]: + base = json.loads(base_path.read_text(encoding="utf-8")) + if base.get("schema") != "intervention-response-phase-aware-pilot-manifest-v3": + raise ValueError("unexpected base manifest schema") + if base.get("status") != "PASS": + raise ValueError("base manifest did not pass its preflight") + if sorted(int(key) for key in base["repetitions"]) != [1, 2, 3]: + raise ValueError("base manifest must contain exactly three repetitions") + + repetitions = {} + selection_hashes = [] + for repetition in (1, 2, 3): + source = base["repetitions"][str(repetition)] + selection = dict(source["selections"]["mid"]) + selection_hashes.append(selection["request_id_order_sha256"]) + repetitions[str(repetition)] = { + "study": source["study"], + "study_sha256": source["study_sha256"], + "selection": selection, + "merged_trace": source["merged_trace"], + } + + config_ids = [str(config["id"]) for config in CONFIGS] + payload = { + "schema": SCHEMA, + "status": "PASS", + "source": { + "base_manifest": str(base_path.resolve()), + "base_manifest_sha256": sha256_file(base_path), + "window_id": base["source"]["window_id"], + "source_trace": base["source"]["source_trace"], + "source_trace_sha256": base["source"]["source_trace_sha256"], + }, + "engine": { + "tp": 4, + "duration_s": 300.0, + "disable_slo_early_stop": True, + "client_timeout_s": 450.0, + }, + "burnin": base["burnin"], + "repetitions": repetitions, + "configs": [dict(config) for config in CONFIGS], + "regimes": { + "A": { + "source": "a_base", + "actions": {"mns": "shared", "mbbt": "a_mbbt"}, + }, + "B": { + "source": "b_base", + "actions": {"mns": "b_mns", "mbbt": "shared"}, + }, + }, + "budget": { + "hard_cap_h20_hours": 8.0, + "session_estimate_h20_hours": 1.35, + "safety_h20_hours": 0.25, + "expected_h20_hours": [6.0, 7.2], + "expected_wall_minutes": [90, 110], + }, + "gates": { + "minimum_relative_winner_margin": 0.10, + "minimum_exclusive_fraction": 0.10, + "minimum_exclusive_ratio": 5.0, + "phase_fractions": [0.25, 0.50, 0.75, 1.0], + "material_kv_usage": 0.90, + }, + "sanity": { + "invariants": { + "five_unique_configs": len(config_ids) == len(set(config_ids)) == 5, + "three_disjoint_repetitions": len(set(selection_hashes)) == 3, + "same_load_all_repetitions": len( + { + float(item["selection"]["offered_req_s_per_gpu"]) + for item in repetitions.values() + } + ) + == 1, + "all_repetition_orders_are_permutations": all( + sorted(config["repetition_order"]) == [1, 2, 3] + for config in CONFIGS + ), + } + }, + } + payload["sanity"]["invariants"]["shared_endpoint_reused_by_both_regimes"] = ( + payload["regimes"]["A"]["actions"]["mns"] + == payload["regimes"]["B"]["actions"]["mbbt"] + == "shared" + ) + payload["sanity"]["red_flags"] = [ + name + for name, passed in payload["sanity"]["invariants"].items() + if not passed + ] + if payload["sanity"]["red_flags"]: + payload["status"] = "FAIL" + return payload + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-manifest", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + payload = build(args.base_manifest) + atomic_json(args.output, payload) + print(json.dumps(payload["sanity"], sort_keys=True)) + if payload["status"] != "PASS": + raise SystemExit("manifest preflight failed") + + +if __name__ == "__main__": + main() diff --git a/runs/action-aware-v0/test_pilot.py b/runs/action-aware-v0/test_pilot.py new file mode 100644 index 0000000..5b2f3af --- /dev/null +++ b/runs/action-aware-v0/test_pilot.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import copy +import importlib.util +from pathlib import Path +from types import SimpleNamespace + + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] + + +def load(name: str, filename: str): + spec = importlib.util.spec_from_file_location(name, HERE / filename) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def record(*, waiting: int, running: int, tokens: int) -> dict: + return { + "queues": {"waiting": waiting, "deferred": 0, "running": running}, + "prefill_tokens": tokens, + "decode_tokens": 0, + "kv": {"usage": 0.5}, + "preemptions": 0, + } + + +def fake_run( + config: str, + repetition: int, + *, + goodput: float, + mns_score: float = 0.0, + mbbt_score: float = 0.0, + ambiguous: float = 0.0, +) -> dict: + binding = { + "mns_exclusive_fraction": mns_score, + "mbbt_exclusive_fraction": mbbt_score, + "both_fraction": ambiguous, + "waiting_unresolved_fraction": 0.0, + "kv_usage_max": 0.5, + "preemptions": 0, + } + phases = { + phase: { + "mns_exclusive_fraction": mns_score, + "mbbt_exclusive_fraction": mbbt_score, + } + for phase in ("0.25", "0.50", "0.75", "1.00") + } + return { + "config_id": config, + "repetition": repetition, + "outcome": {"slo_goodput_req_s": goodput}, + "binding": binding, + "phases": phases, + } + + +def main() -> None: + analysis = load("action_aware_analysis", "analyze_pilot.py") + summary = analysis.binding_summary( + [ + record(waiting=1, running=16, tokens=8), + record(waiting=1, running=8, tokens=32), + record(waiting=1, running=16, tokens=32), + record(waiting=1, running=8, tokens=8), + record(waiting=0, running=8, tokens=8), + ], + mns=16, + mbbt=32, + ) + assert summary["mns_exclusive_count"] == 1 + assert summary["mbbt_exclusive_count"] == 1 + assert summary["both_count"] == 1 + assert summary["waiting_unresolved_count"] == 1 + assert summary["waiting_count"] == 4 + + manifest = { + "repetitions": {str(index): {} for index in (1, 2, 3)}, + "regimes": { + "A": { + "source": "a_base", + "actions": {"mns": "shared", "mbbt": "a_mbbt"}, + }, + "B": { + "source": "b_base", + "actions": {"mns": "b_mns", "mbbt": "shared"}, + }, + }, + "gates": { + "minimum_relative_winner_margin": 0.10, + "minimum_exclusive_fraction": 0.10, + "minimum_exclusive_ratio": 5.0, + "material_kv_usage": 0.90, + }, + } + runs = [] + for repetition in (1, 2, 3): + runs.extend( + [ + fake_run( + "a_base", + repetition, + goodput=1.0, + mns_score=0.8, + mbbt_score=0.01, + ), + fake_run( + "b_base", + repetition, + goodput=1.0, + mns_score=0.01, + mbbt_score=0.7, + ), + fake_run("shared", repetition, goodput=3.0), + fake_run("a_mbbt", repetition, goodput=1.5), + fake_run("b_mns", repetition, goodput=1.2), + ] + ) + result = analysis.evaluate_decisions(runs, manifest) + assert result["decision"] == "STOP_NO_NEW_INSTRUMENTATION_NEEDED" + assert result["baselines"] == { + "always_mns_correct": 3, + "always_mbbt_correct": 3, + "binding_correct": 6, + "decision_count": 6, + } + + ambiguous = copy.deepcopy(runs) + for run in ambiguous: + if run["config_id"] == "b_base": + run["binding"]["both_fraction"] = 0.8 + assert ( + analysis.evaluate_decisions(ambiguous, manifest)["decision"] + == "OPEN_EXACT_ATTRIBUTION_ABLATION" + ) + + wrong = copy.deepcopy(runs) + for run in wrong: + if run["config_id"] == "b_base": + run["binding"]["mns_exclusive_fraction"] = 0.8 + run["binding"]["mbbt_exclusive_fraction"] = 0.01 + for phase in run["phases"].values(): + phase["mns_exclusive_fraction"] = 0.8 + phase["mbbt_exclusive_fraction"] = 0.01 + assert ( + analysis.evaluate_decisions(wrong, manifest)["decision"] + == "STOP_BINDING_NOT_PREDICTIVE" + ) + + prepare = load("action_aware_prepare", "prepare_pilot.py") + frozen = prepare.build( + ROOT / "runs/intervention-response-v2/pilot-manifest-v3.json" + ) + assert frozen["status"] == "PASS" + assert frozen["sanity"]["red_flags"] == [] + assert [config["id"] for config in frozen["configs"]] == [ + "b_base", + "a_base", + "shared", + "b_mns", + "a_mbbt", + ] + + controller = load("action_aware_controller", "pilot_controller.py") + args = SimpleNamespace( + manifest=Path("/tmp/manifest.json"), + run_root=Path("/tmp/action-aware"), + aituner_root=Path("/tmp/aituner"), + vllm_source=Path("/tmp/vllm"), + venv=Path("/tmp/venv"), + model=Path("/tmp/model"), + client=Path("/tmp/client.py"), + ) + controller.configure(args, frozen) + plan = controller.dry_run_plan(args, frozen) + assert plan["status"] == "PASS" + assert len(plan["sessions"]) == 5 + assert plan["projected_h20_hours"] == 7.0 + assert "--max-num-batched-tokens 256" in plan["sessions"][0]["commands"]["server"] + print("action-aware constraint pilot: PASS") + + +if __name__ == "__main__": + main() From 1db737e6414620d96375690d961d0b0c385d7041 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 20:39:04 +0800 Subject: [PATCH 022/148] Revise action-aware pilot after token overload --- ...e-constraint-pilot-v1-protocol-20260714.md | 44 ++++ runs/action-aware-v0/analyze_pilot.py | 5 +- runs/action-aware-v0/pilot-manifest-v1.json | 227 ++++++++++++++++++ runs/action-aware-v0/pilot_controller.py | 20 +- runs/action-aware-v0/prepare_pilot.py | 72 ++++-- runs/action-aware-v0/test_pilot.py | 17 ++ 6 files changed, 367 insertions(+), 18 deletions(-) create mode 100644 docs/action-aware-constraint-pilot-v1-protocol-20260714.md create mode 100644 runs/action-aware-v0/pilot-manifest-v1.json diff --git a/docs/action-aware-constraint-pilot-v1-protocol-20260714.md b/docs/action-aware-constraint-pilot-v1-protocol-20260714.md new file mode 100644 index 0000000..4937903 --- /dev/null +++ b/docs/action-aware-constraint-pilot-v1-protocol-20260714.md @@ -0,0 +1,44 @@ +# Action-aware constraint pilot v1 amendment + +Status: **FROZEN AFTER V0 OPERATIONAL STOP AND BEFORE V1 GPU RUNS**. + +Date: 2026-07-14 (Asia/Singapore). + +The complete claim, workload, baselines, metrics, action matrix, analysis gates, +and data-validity requirements remain those in the v0 protocol. This amendment +changes only the token-bound source severity and adds an operational burn-in +gate. + +## Why v0 produced no comparative evidence + +The first v0 session used MNS=64 and MBBT=256. During the 510-request, +60-second burn-in, the client had run for 197 seconds and the engine still held +64 requests: 13 running and 51 waiting. The last step scheduled exactly 256 +tokens, KV usage was 0.01151, and there were zero preemptions. No measured run +or target configuration had started. + +The session was stopped and cleanly released all GPUs after consuming +0.3859868995 H20-hours. This is evidence that MBBT=256 is a real token-budget +bottleneck, but it is not an admissible source for the 2.125 requests/s/GPU +comparison because it cannot sustain the offered load. V0 contributes no +tuning label and none of its runtime data is reused by V1. + +Authoritative failure artifact: +`/home/admin/cpfs/wjh/action-aware-constraint-v0-20260714/operational-stop-v0.json`. + +## V1 changes + +- `b_base`: MNS=64, MBBT **2048** instead of 256. +- `b_mns`: MNS=128, MBBT **2048** instead of 256. +- The B-family MBBT action is therefore 2048 -> 8192. +- All five configurations and all three repetitions run fresh under a new V1 + run root. +- Before any measured run, every config's 510-request/60-second burn-in must + drain in at most **90 seconds**. A slower config is an operational failure; + the controller stops before comparative analysis. +- V1 incremental hard cap is 7.6140131005 H20-hours so that V0 plus V1 remains + within the original global 8.0 H20-hour cap. + +The V0 protocol's crossed-response and source-only binding gates are unchanged. +In particular, V1 still requires three-of-three action-family predictions in +both regimes and a different real winner in Regime A versus Regime B. diff --git a/runs/action-aware-v0/analyze_pilot.py b/runs/action-aware-v0/analyze_pilot.py index 5eafbd2..8c438ec 100644 --- a/runs/action-aware-v0/analyze_pilot.py +++ b/runs/action-aware-v0/analyze_pilot.py @@ -454,7 +454,10 @@ def evaluate_decisions( def analyze(run_root: Path, manifest_path: Path) -> dict[str, Any]: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - if manifest.get("schema") != "action-aware-constraint-pilot-manifest-v0": + if manifest.get("schema") not in { + "action-aware-constraint-pilot-manifest-v0", + "action-aware-constraint-pilot-manifest-v1", + }: raise ValueError("unexpected manifest schema") duration_s = float(manifest["engine"]["duration_s"]) phase_fractions = [float(value) for value in manifest["gates"]["phase_fractions"]] diff --git a/runs/action-aware-v0/pilot-manifest-v1.json b/runs/action-aware-v0/pilot-manifest-v1.json new file mode 100644 index 0000000..14558e7 --- /dev/null +++ b/runs/action-aware-v0/pilot-manifest-v1.json @@ -0,0 +1,227 @@ +{ + "budget": { + "expected_h20_hours": [ + 6.0, + 7.2 + ], + "expected_wall_minutes": [ + 90, + 110 + ], + "global_hard_cap_h20_hours": 8.0, + "hard_cap_h20_hours": 7.614013100465138, + "prior_attempt_artifact": "/home/admin/cpfs/wjh/action-aware-constraint-v0-20260714/operational-stop-v0.json", + "prior_attempt_h20_hours": 0.38598689953486126, + "safety_h20_hours": 0.25, + "session_estimate_h20_hours": 1.35 + }, + "burnin": { + "anchor": 0.18919793755240089, + "arrival_order_sha256": "6c0ac4cb9a30ef501eeeacc8e6cc631c345e976db5ccf530ea5a1ec706d62a24", + "input_length_order_sha256": "7939cc20e1a00d1031d27d71508789f38decbbbb6ea59a1df18b2ec342fd2ef8", + "offered_req_s": 8.5, + "offered_req_s_per_gpu": 2.125, + "request_id_order_sha256": "84f4809acbc8acd3b1d14dfa357134a1dc0b9287341624b33f598dafeef54dc7", + "selected_count": 510, + "study": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/studies/burnin-tp4.json", + "study_sha256": "5d6c2098042909a863efd3112818fbee9bafe96f22898ac98b66846dbe1fef0f" + }, + "configs": [ + { + "id": "b_base", + "mbbt": 2048, + "mns": 64, + "repetition_order": [ + 1, + 2, + 3 + ] + }, + { + "id": "a_base", + "mbbt": 8192, + "mns": 16, + "repetition_order": [ + 2, + 3, + 1 + ] + }, + { + "id": "shared", + "mbbt": 8192, + "mns": 64, + "repetition_order": [ + 3, + 1, + 2 + ] + }, + { + "id": "b_mns", + "mbbt": 2048, + "mns": 128, + "repetition_order": [ + 1, + 3, + 2 + ] + }, + { + "id": "a_mbbt", + "mbbt": 16384, + "mns": 16, + "repetition_order": [ + 2, + 1, + 3 + ] + } + ], + "engine": { + "burnin_max_elapsed_s": 90.0, + "client_timeout_s": 450.0, + "disable_slo_early_stop": true, + "duration_s": 300.0, + "tp": 4 + }, + "gates": { + "material_kv_usage": 0.9, + "minimum_exclusive_fraction": 0.1, + "minimum_exclusive_ratio": 5.0, + "minimum_relative_winner_margin": 0.1, + "phase_fractions": [ + 0.25, + 0.5, + 0.75, + 1.0 + ] + }, + "regimes": { + "A": { + "actions": { + "mbbt": "a_mbbt", + "mns": "shared" + }, + "source": "a_base" + }, + "B": { + "actions": { + "mbbt": "shared", + "mns": "b_mns" + }, + "source": "b_base" + } + }, + "repetitions": { + "1": { + "merged_trace": { + "bytes": 337429767, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep1.jsonl", + "request_id_scheme": "sha256(source_sha256:line_number:original_id)", + "rows": 9420, + "sha256": "68983266aa0e66aa589562f7c08edbd966f9ba4405e20c105adb43777d2dfbf5", + "source_sha256": [ + "b242d1d9086df3accab57b4c92445d5edd581e12f47e12cea227aa63964c6930", + "d23b549f7b69af3647308677bbf76f818a3c226a1c98f9a9f93f09ceee46be87" + ], + "sources": [ + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low1.jsonl", + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high1.jsonl" + ] + }, + "selection": { + "anchor": 0.48686986110831465, + "arrival_order_sha256": "c2ad99986ce558da5901a9c5ec0a00bd69f198c981d8779235f2773a5c87f1c0", + "input_length_order_sha256": "9442bfebdc3fab5062dc1f4d688dc28c02afe3fd806c56dd8159f0ac7e6d0b94", + "offered_req_s": 8.5, + "offered_req_s_per_gpu": 2.125, + "request_id_order_sha256": "0bb61dbc9c26875e991d0d4f984134910d37463e5063f86ee960cf4f8aafb771", + "selected_count": 2550, + "target_count": 2550, + "target_req_s_per_gpu": 2.125 + }, + "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep1-tp4.json", + "study_sha256": "ecfff96e33d458eb1e3b9a6d24386f00cc6f1b19ff926e2ec6320b3f671a7ae3" + }, + "2": { + "merged_trace": { + "bytes": 337509330, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep2.jsonl", + "request_id_scheme": "sha256(source_sha256:line_number:original_id)", + "rows": 9457, + "sha256": "f38e8938f6a481fc6725b71b21aa04ff7eaf79783cdfd6e41aa2f074156f00c2", + "source_sha256": [ + "4cbb0baac082bd54af562ce2f39104c5c23b4671672da365a67b1e8c146adf9f", + "bb0bcd2564a88000f435f12feb21c7c902eafc9ea5fe916adfe9d1eae47f3f9a" + ], + "sources": [ + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low2.jsonl", + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high2.jsonl" + ] + }, + "selection": { + "anchor": 0.4825698948735577, + "arrival_order_sha256": "b9fc12cf3f86bc8a79bee65296e65aa2b8bf2aeca46b2887094c669adcbb9a00", + "input_length_order_sha256": "d8d4bd6fc8ba852a45605b673b6b3e4f33b58f459e69f2a032d226ee175b074e", + "offered_req_s": 8.5, + "offered_req_s_per_gpu": 2.125, + "request_id_order_sha256": "56a0616b6b54abafd37875c7cb25f8639afef2706ccc55dfbe568f45859ea382", + "selected_count": 2550, + "target_count": 2550, + "target_req_s_per_gpu": 2.125 + }, + "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep2-tp4.json", + "study_sha256": "d92a576db031db24bb58f354ea725d7f7567cb76699d387117ac5a6c9317bbb9" + }, + "3": { + "merged_trace": { + "bytes": 337450256, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep3.jsonl", + "request_id_scheme": "sha256(source_sha256:line_number:original_id)", + "rows": 9431, + "sha256": "3094084b0bb20cc02eecf465091a5c919b4e5b112f704cdc36a563d1efdcee46", + "source_sha256": [ + "1f7ececb142f9a363d2d1ca25eb7b8488b2cc319a51b55faa384f2a3d51f2142", + "6f326234791e1cff4ff866bface0d097d0d6e3844eebb1c97653d8e9c35e9397" + ], + "sources": [ + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low3.jsonl", + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high3.jsonl" + ] + }, + "selection": { + "anchor": 0.48664343020532463, + "arrival_order_sha256": "efce7339e22d3618cb4d55e6b55bfddb2c563c18faba2a992d5829c13e3f55e9", + "input_length_order_sha256": "0792b05fff6729fbd92ab2bb4cb6d31bea7799e232ad42772936bc06efbafb54", + "offered_req_s": 8.5, + "offered_req_s_per_gpu": 2.125, + "request_id_order_sha256": "2a2fabe2c4cf176aeb7e0d32fb8e7dbb1f27429a2e7a0cd18d7d186f23096f19", + "selected_count": 2550, + "target_count": 2550, + "target_req_s_per_gpu": 2.125 + }, + "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep3-tp4.json", + "study_sha256": "fb8ffe256dace32f4ca8a8d49b662d98c3b69b94ecc8fa826e43068b238884ab" + } + }, + "sanity": { + "invariants": { + "all_repetition_orders_are_permutations": true, + "five_unique_configs": true, + "same_load_all_repetitions": true, + "shared_endpoint_reused_by_both_regimes": true, + "three_disjoint_repetitions": true + }, + "red_flags": [] + }, + "schema": "action-aware-constraint-pilot-manifest-v1", + "source": { + "base_manifest": "/home/gahow/phd/aituner/runs/intervention-response-v2/pilot-manifest-v3.json", + "base_manifest_sha256": "273db1181dcc9d6b64439650d0642ebe553b12e6aa9adebfbe3758a7977e5611", + "source_trace": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260312_1000.jsonl", + "source_trace_sha256": "875ba869775deb78086477919f03b322da14e2673c7d070e26528c4190912757", + "window_id": "chat_w20260312_1000" + }, + "status": "PASS" +} diff --git a/runs/action-aware-v0/pilot_controller.py b/runs/action-aware-v0/pilot_controller.py index 8e8b94f..0492116 100644 --- a/runs/action-aware-v0/pilot_controller.py +++ b/runs/action-aware-v0/pilot_controller.py @@ -56,7 +56,10 @@ def configure(args: argparse.Namespace, manifest: Mapping[str, Any]) -> None: def validate_inputs(args: argparse.Namespace, manifest: Mapping[str, Any]) -> None: - if manifest.get("schema") != "action-aware-constraint-pilot-manifest-v0": + if manifest.get("schema") not in { + "action-aware-constraint-pilot-manifest-v0", + "action-aware-constraint-pilot-manifest-v1", + }: raise RuntimeError("unexpected action-aware manifest schema") if manifest.get("status") != "PASS": raise RuntimeError("action-aware manifest did not pass preflight") @@ -455,7 +458,7 @@ def execute_session( first = manifest["repetitions"][str(config["repetition_order"][0])] session_state["status"] = "warmup" atomic_json(state_path, state) - run_client( + burnin_result = run_client( entry=entry, config=config, role="warmup", @@ -479,6 +482,19 @@ def execute_session( state=state, timeout_s=float(manifest["engine"]["client_timeout_s"]), ) + burnin_elapsed_s = float(burnin_result["interval"]["elapsed_s"]) + session_state["burnin"] = { + "elapsed_s": burnin_elapsed_s, + "pass_rate": float(burnin_result["pass_rate"]), + "feasible": bool(burnin_result["feasible"]), + } + atomic_json(state_path, state) + burnin_limit_s = float(manifest["engine"]["burnin_max_elapsed_s"]) + if burnin_elapsed_s > burnin_limit_s: + raise RuntimeError( + f"burnin throughput gate failed: {burnin_elapsed_s:.3f}s > " + f"{burnin_limit_s:.3f}s" + ) session_state["status"] = "measured" atomic_json(state_path, state) for repetition in config["repetition_order"]: diff --git a/runs/action-aware-v0/prepare_pilot.py b/runs/action-aware-v0/prepare_pilot.py index 51e9fd1..c7f6aaf 100644 --- a/runs/action-aware-v0/prepare_pilot.py +++ b/runs/action-aware-v0/prepare_pilot.py @@ -11,14 +11,28 @@ from pathlib import Path from typing import Any -SCHEMA = "action-aware-constraint-pilot-manifest-v0" -CONFIGS = ( - {"id": "b_base", "mns": 64, "mbbt": 256, "repetition_order": [1, 2, 3]}, - {"id": "a_base", "mns": 16, "mbbt": 8192, "repetition_order": [2, 3, 1]}, - {"id": "shared", "mns": 64, "mbbt": 8192, "repetition_order": [3, 1, 2]}, - {"id": "b_mns", "mns": 128, "mbbt": 256, "repetition_order": [1, 3, 2]}, - {"id": "a_mbbt", "mns": 16, "mbbt": 16384, "repetition_order": [2, 1, 3]}, -) +SCHEMA_V0 = "action-aware-constraint-pilot-manifest-v0" +SCHEMA_V1 = "action-aware-constraint-pilot-manifest-v1" + + +def configs(token_source_mbbt: int) -> tuple[dict[str, Any], ...]: + return ( + { + "id": "b_base", + "mns": 64, + "mbbt": token_source_mbbt, + "repetition_order": [1, 2, 3], + }, + {"id": "a_base", "mns": 16, "mbbt": 8192, "repetition_order": [2, 3, 1]}, + {"id": "shared", "mns": 64, "mbbt": 8192, "repetition_order": [3, 1, 2]}, + { + "id": "b_mns", + "mns": 128, + "mbbt": token_source_mbbt, + "repetition_order": [1, 3, 2], + }, + {"id": "a_mbbt", "mns": 16, "mbbt": 16384, "repetition_order": [2, 1, 3]}, + ) def sha256_file(path: Path) -> str: @@ -38,7 +52,17 @@ def atomic_json(path: Path, payload: Any) -> None: os.replace(temporary, path) -def build(base_path: Path) -> dict[str, Any]: +def build( + base_path: Path, + *, + token_source_mbbt: int = 256, + prior_attempt_h20_hours: float = 0.0, + prior_attempt_artifact: str | None = None, +) -> dict[str, Any]: + if token_source_mbbt <= 0: + raise ValueError("token source MBBT must be positive") + if prior_attempt_h20_hours < 0.0 or prior_attempt_h20_hours >= 8.0: + raise ValueError("prior attempt cost must be in [0, 8)") base = json.loads(base_path.read_text(encoding="utf-8")) if base.get("schema") != "intervention-response-phase-aware-pilot-manifest-v3": raise ValueError("unexpected base manifest schema") @@ -60,9 +84,15 @@ def build(base_path: Path) -> dict[str, Any]: "merged_trace": source["merged_trace"], } - config_ids = [str(config["id"]) for config in CONFIGS] + frozen_configs = configs(token_source_mbbt) + config_ids = [str(config["id"]) for config in frozen_configs] + schema = ( + SCHEMA_V0 + if token_source_mbbt == 256 and prior_attempt_h20_hours == 0.0 + else SCHEMA_V1 + ) payload = { - "schema": SCHEMA, + "schema": schema, "status": "PASS", "source": { "base_manifest": str(base_path.resolve()), @@ -76,10 +106,11 @@ def build(base_path: Path) -> dict[str, Any]: "duration_s": 300.0, "disable_slo_early_stop": True, "client_timeout_s": 450.0, + "burnin_max_elapsed_s": 90.0, }, "burnin": base["burnin"], "repetitions": repetitions, - "configs": [dict(config) for config in CONFIGS], + "configs": [dict(config) for config in frozen_configs], "regimes": { "A": { "source": "a_base", @@ -91,7 +122,10 @@ def build(base_path: Path) -> dict[str, Any]: }, }, "budget": { - "hard_cap_h20_hours": 8.0, + "global_hard_cap_h20_hours": 8.0, + "hard_cap_h20_hours": 8.0 - prior_attempt_h20_hours, + "prior_attempt_h20_hours": prior_attempt_h20_hours, + "prior_attempt_artifact": prior_attempt_artifact, "session_estimate_h20_hours": 1.35, "safety_h20_hours": 0.25, "expected_h20_hours": [6.0, 7.2], @@ -117,7 +151,7 @@ def build(base_path: Path) -> dict[str, Any]: == 1, "all_repetition_orders_are_permutations": all( sorted(config["repetition_order"]) == [1, 2, 3] - for config in CONFIGS + for config in frozen_configs ), } }, @@ -141,8 +175,16 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--base-manifest", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--token-source-mbbt", type=int, default=256) + parser.add_argument("--prior-attempt-h20-hours", type=float, default=0.0) + parser.add_argument("--prior-attempt-artifact") args = parser.parse_args() - payload = build(args.base_manifest) + payload = build( + args.base_manifest, + token_source_mbbt=args.token_source_mbbt, + prior_attempt_h20_hours=args.prior_attempt_h20_hours, + prior_attempt_artifact=args.prior_attempt_artifact, + ) atomic_json(args.output, payload) print(json.dumps(payload["sanity"], sort_keys=True)) if payload["status"] != "PASS": diff --git a/runs/action-aware-v0/test_pilot.py b/runs/action-aware-v0/test_pilot.py index 5b2f3af..c46af88 100644 --- a/runs/action-aware-v0/test_pilot.py +++ b/runs/action-aware-v0/test_pilot.py @@ -184,6 +184,23 @@ def main() -> None: assert len(plan["sessions"]) == 5 assert plan["projected_h20_hours"] == 7.0 assert "--max-num-batched-tokens 256" in plan["sessions"][0]["commands"]["server"] + revised = prepare.build( + ROOT / "runs/intervention-response-v2/pilot-manifest-v3.json", + token_source_mbbt=2048, + prior_attempt_h20_hours=0.38598689953486126, + prior_attempt_artifact="/tmp/operational-stop-v0.json", + ) + assert revised["schema"] == "action-aware-constraint-pilot-manifest-v1" + assert revised["configs"][0]["mbbt"] == 2048 + assert revised["configs"][3]["mbbt"] == 2048 + assert revised["budget"]["hard_cap_h20_hours"] < 8.0 + controller.configure(args, revised) + revised_plan = controller.dry_run_plan(args, revised) + assert revised_plan["projected_h20_hours"] < revised_plan["hard_cap_h20_hours"] + assert ( + "--max-num-batched-tokens 2048" + in revised_plan["sessions"][0]["commands"]["server"] + ) print("action-aware constraint pilot: PASS") From c5ab073af5e068f34b9765b6169227020338367f Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 20:47:23 +0800 Subject: [PATCH 023/148] Fix action-aware burn-in gate --- ...e-constraint-pilot-v2-protocol-20260714.md | 35 +++ runs/action-aware-v0/pilot-manifest-v2.json | 227 ++++++++++++++++++ runs/action-aware-v0/pilot_controller.py | 45 ++-- runs/action-aware-v0/prior-attempts-v2.json | 24 ++ runs/action-aware-v0/test_pilot.py | 26 ++ 5 files changed, 343 insertions(+), 14 deletions(-) create mode 100644 docs/action-aware-constraint-pilot-v2-protocol-20260714.md create mode 100644 runs/action-aware-v0/pilot-manifest-v2.json create mode 100644 runs/action-aware-v0/prior-attempts-v2.json diff --git a/docs/action-aware-constraint-pilot-v2-protocol-20260714.md b/docs/action-aware-constraint-pilot-v2-protocol-20260714.md new file mode 100644 index 0000000..d51ca29 --- /dev/null +++ b/docs/action-aware-constraint-pilot-v2-protocol-20260714.md @@ -0,0 +1,35 @@ +# Action-aware constraint pilot v2 amendment + +Status: **FROZEN AFTER V1 CONTROLLER STOP AND BEFORE V2 GPU RUNS**. + +Date: 2026-07-14 (Asia/Singapore). + +The V1 configuration matrix and every scientific gate remain unchanged. V2 +fixes one controller bug and reruns every configuration and repetition fresh. + +## V1 controller failure + +The MNS64/MBBT2048 burn-in completed all 510 requests in 61.259 seconds, below +the frozen 90-second operational limit. However, the controller accidentally +assigned the preceding 16-request warm-up result to `burnin_result`; its state +therefore recorded 4.376 seconds and evaluated the wrong object. + +The first measured replay was terminated after 75 seconds, before it produced +a result. No target configuration had started. V1 consumed +0.3184109431 H20-hours and contributes no action label or telemetry to V2. + +## V2 correction and regression gate + +- The completed burn-in `run_client()` return value is assigned to + `burnin_result`. +- A dedicated `burnin_gate()` rejects any non-anchor object, any request count + other than 510, and elapsed time above 90 seconds. +- Unit tests explicitly pass a warm-up object and require rejection, then test + accepted and over-limit burn-ins. +- All five configs and 15 measured runs use a new run root; no V0/V1 runtime + artifact is reused. + +V0 and V1 together consumed 0.7043978426 H20-hours. V2's incremental hard cap +is therefore 7.2956021574 H20-hours, preserving the original global 8.0 +H20-hour cap. The authoritative accounting file is +`runs/action-aware-v0/prior-attempts-v2.json`. diff --git a/runs/action-aware-v0/pilot-manifest-v2.json b/runs/action-aware-v0/pilot-manifest-v2.json new file mode 100644 index 0000000..47908c7 --- /dev/null +++ b/runs/action-aware-v0/pilot-manifest-v2.json @@ -0,0 +1,227 @@ +{ + "budget": { + "expected_h20_hours": [ + 6.0, + 7.2 + ], + "expected_wall_minutes": [ + 90, + 110 + ], + "global_hard_cap_h20_hours": 8.0, + "hard_cap_h20_hours": 7.295602157380846, + "prior_attempt_artifact": "/home/admin/cpfs/wjh/aituner/aituner-action-aware-20260714/runs/action-aware-v0/prior-attempts-v2.json", + "prior_attempt_h20_hours": 0.7043978426191542, + "safety_h20_hours": 0.25, + "session_estimate_h20_hours": 1.35 + }, + "burnin": { + "anchor": 0.18919793755240089, + "arrival_order_sha256": "6c0ac4cb9a30ef501eeeacc8e6cc631c345e976db5ccf530ea5a1ec706d62a24", + "input_length_order_sha256": "7939cc20e1a00d1031d27d71508789f38decbbbb6ea59a1df18b2ec342fd2ef8", + "offered_req_s": 8.5, + "offered_req_s_per_gpu": 2.125, + "request_id_order_sha256": "84f4809acbc8acd3b1d14dfa357134a1dc0b9287341624b33f598dafeef54dc7", + "selected_count": 510, + "study": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/studies/burnin-tp4.json", + "study_sha256": "5d6c2098042909a863efd3112818fbee9bafe96f22898ac98b66846dbe1fef0f" + }, + "configs": [ + { + "id": "b_base", + "mbbt": 2048, + "mns": 64, + "repetition_order": [ + 1, + 2, + 3 + ] + }, + { + "id": "a_base", + "mbbt": 8192, + "mns": 16, + "repetition_order": [ + 2, + 3, + 1 + ] + }, + { + "id": "shared", + "mbbt": 8192, + "mns": 64, + "repetition_order": [ + 3, + 1, + 2 + ] + }, + { + "id": "b_mns", + "mbbt": 2048, + "mns": 128, + "repetition_order": [ + 1, + 3, + 2 + ] + }, + { + "id": "a_mbbt", + "mbbt": 16384, + "mns": 16, + "repetition_order": [ + 2, + 1, + 3 + ] + } + ], + "engine": { + "burnin_max_elapsed_s": 90.0, + "client_timeout_s": 450.0, + "disable_slo_early_stop": true, + "duration_s": 300.0, + "tp": 4 + }, + "gates": { + "material_kv_usage": 0.9, + "minimum_exclusive_fraction": 0.1, + "minimum_exclusive_ratio": 5.0, + "minimum_relative_winner_margin": 0.1, + "phase_fractions": [ + 0.25, + 0.5, + 0.75, + 1.0 + ] + }, + "regimes": { + "A": { + "actions": { + "mbbt": "a_mbbt", + "mns": "shared" + }, + "source": "a_base" + }, + "B": { + "actions": { + "mbbt": "shared", + "mns": "b_mns" + }, + "source": "b_base" + } + }, + "repetitions": { + "1": { + "merged_trace": { + "bytes": 337429767, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep1.jsonl", + "request_id_scheme": "sha256(source_sha256:line_number:original_id)", + "rows": 9420, + "sha256": "68983266aa0e66aa589562f7c08edbd966f9ba4405e20c105adb43777d2dfbf5", + "source_sha256": [ + "b242d1d9086df3accab57b4c92445d5edd581e12f47e12cea227aa63964c6930", + "d23b549f7b69af3647308677bbf76f818a3c226a1c98f9a9f93f09ceee46be87" + ], + "sources": [ + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low1.jsonl", + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high1.jsonl" + ] + }, + "selection": { + "anchor": 0.48686986110831465, + "arrival_order_sha256": "c2ad99986ce558da5901a9c5ec0a00bd69f198c981d8779235f2773a5c87f1c0", + "input_length_order_sha256": "9442bfebdc3fab5062dc1f4d688dc28c02afe3fd806c56dd8159f0ac7e6d0b94", + "offered_req_s": 8.5, + "offered_req_s_per_gpu": 2.125, + "request_id_order_sha256": "0bb61dbc9c26875e991d0d4f984134910d37463e5063f86ee960cf4f8aafb771", + "selected_count": 2550, + "target_count": 2550, + "target_req_s_per_gpu": 2.125 + }, + "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep1-tp4.json", + "study_sha256": "ecfff96e33d458eb1e3b9a6d24386f00cc6f1b19ff926e2ec6320b3f671a7ae3" + }, + "2": { + "merged_trace": { + "bytes": 337509330, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep2.jsonl", + "request_id_scheme": "sha256(source_sha256:line_number:original_id)", + "rows": 9457, + "sha256": "f38e8938f6a481fc6725b71b21aa04ff7eaf79783cdfd6e41aa2f074156f00c2", + "source_sha256": [ + "4cbb0baac082bd54af562ce2f39104c5c23b4671672da365a67b1e8c146adf9f", + "bb0bcd2564a88000f435f12feb21c7c902eafc9ea5fe916adfe9d1eae47f3f9a" + ], + "sources": [ + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low2.jsonl", + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high2.jsonl" + ] + }, + "selection": { + "anchor": 0.4825698948735577, + "arrival_order_sha256": "b9fc12cf3f86bc8a79bee65296e65aa2b8bf2aeca46b2887094c669adcbb9a00", + "input_length_order_sha256": "d8d4bd6fc8ba852a45605b673b6b3e4f33b58f459e69f2a032d226ee175b074e", + "offered_req_s": 8.5, + "offered_req_s_per_gpu": 2.125, + "request_id_order_sha256": "56a0616b6b54abafd37875c7cb25f8639afef2706ccc55dfbe568f45859ea382", + "selected_count": 2550, + "target_count": 2550, + "target_req_s_per_gpu": 2.125 + }, + "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep2-tp4.json", + "study_sha256": "d92a576db031db24bb58f354ea725d7f7567cb76699d387117ac5a6c9317bbb9" + }, + "3": { + "merged_trace": { + "bytes": 337450256, + "path": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/traces/rep3.jsonl", + "request_id_scheme": "sha256(source_sha256:line_number:original_id)", + "rows": 9431, + "sha256": "3094084b0bb20cc02eecf465091a5c919b4e5b112f704cdc36a563d1efdcee46", + "source_sha256": [ + "1f7ececb142f9a363d2d1ca25eb7b8488b2cc319a51b55faa384f2a3d51f2142", + "6f326234791e1cff4ff866bface0d097d0d6e3844eebb1c97653d8e9c35e9397" + ], + "sources": [ + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/low3.jsonl", + "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/private/traces/high3.jsonl" + ] + }, + "selection": { + "anchor": 0.48664343020532463, + "arrival_order_sha256": "efce7339e22d3618cb4d55e6b55bfddb2c563c18faba2a992d5829c13e3f55e9", + "input_length_order_sha256": "0792b05fff6729fbd92ab2bb4cb6d31bea7799e232ad42772936bc06efbafb54", + "offered_req_s": 8.5, + "offered_req_s_per_gpu": 2.125, + "request_id_order_sha256": "2a2fabe2c4cf176aeb7e0d32fb8e7dbb1f27429a2e7a0cd18d7d186f23096f19", + "selected_count": 2550, + "target_count": 2550, + "target_req_s_per_gpu": 2.125 + }, + "study": "/home/admin/cpfs/wjh/intervention-response-v3-20260714/private/studies/rep3-tp4.json", + "study_sha256": "fb8ffe256dace32f4ca8a8d49b662d98c3b69b94ecc8fa826e43068b238884ab" + } + }, + "sanity": { + "invariants": { + "all_repetition_orders_are_permutations": true, + "five_unique_configs": true, + "same_load_all_repetitions": true, + "shared_endpoint_reused_by_both_regimes": true, + "three_disjoint_repetitions": true + }, + "red_flags": [] + }, + "schema": "action-aware-constraint-pilot-manifest-v1", + "source": { + "base_manifest": "/home/gahow/phd/aituner/runs/intervention-response-v2/pilot-manifest-v3.json", + "base_manifest_sha256": "273db1181dcc9d6b64439650d0642ebe553b12e6aa9adebfbe3758a7977e5611", + "source_trace": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260312_1000.jsonl", + "source_trace_sha256": "875ba869775deb78086477919f03b322da14e2673c7d070e26528c4190912757", + "window_id": "chat_w20260312_1000" + }, + "status": "PASS" +} diff --git a/runs/action-aware-v0/pilot_controller.py b/runs/action-aware-v0/pilot_controller.py index 0492116..14668f9 100644 --- a/runs/action-aware-v0/pilot_controller.py +++ b/runs/action-aware-v0/pilot_controller.py @@ -334,6 +334,30 @@ def validate_result( raise RuntimeError(f"selection hash mismatch {result_key}: {role}") +def burnin_gate( + result: Mapping[str, Any], + *, + expected_count: int, + maximum_elapsed_s: float, +) -> dict[str, Any]: + if result.get("kind") != "anchor": + raise RuntimeError("burnin gate received a non-anchor result") + if int(result["selection"]["count"]) != expected_count: + raise RuntimeError("burnin gate received the wrong request set") + elapsed_s = float(result["interval"]["elapsed_s"]) + summary = { + "elapsed_s": elapsed_s, + "pass_rate": float(result["pass_rate"]), + "feasible": bool(result["feasible"]), + } + if elapsed_s > maximum_elapsed_s: + raise RuntimeError( + f"burnin throughput gate failed: {elapsed_s:.3f}s > " + f"{maximum_elapsed_s:.3f}s" + ) + return summary + + def run_client( *, entry: dict[str, Any], @@ -458,7 +482,7 @@ def execute_session( first = manifest["repetitions"][str(config["repetition_order"][0])] session_state["status"] = "warmup" atomic_json(state_path, state) - burnin_result = run_client( + run_client( entry=entry, config=config, role="warmup", @@ -472,7 +496,7 @@ def execute_session( session_state["status"] = "burnin" atomic_json(state_path, state) burnin = manifest["burnin"] - run_client( + burnin_result = run_client( entry=entry, config=config, role="burnin", @@ -482,19 +506,12 @@ def execute_session( state=state, timeout_s=float(manifest["engine"]["client_timeout_s"]), ) - burnin_elapsed_s = float(burnin_result["interval"]["elapsed_s"]) - session_state["burnin"] = { - "elapsed_s": burnin_elapsed_s, - "pass_rate": float(burnin_result["pass_rate"]), - "feasible": bool(burnin_result["feasible"]), - } + session_state["burnin"] = burnin_gate( + burnin_result, + expected_count=int(burnin["selected_count"]), + maximum_elapsed_s=float(manifest["engine"]["burnin_max_elapsed_s"]), + ) atomic_json(state_path, state) - burnin_limit_s = float(manifest["engine"]["burnin_max_elapsed_s"]) - if burnin_elapsed_s > burnin_limit_s: - raise RuntimeError( - f"burnin throughput gate failed: {burnin_elapsed_s:.3f}s > " - f"{burnin_limit_s:.3f}s" - ) session_state["status"] = "measured" atomic_json(state_path, state) for repetition in config["repetition_order"]: diff --git a/runs/action-aware-v0/prior-attempts-v2.json b/runs/action-aware-v0/prior-attempts-v2.json new file mode 100644 index 0000000..3faf060 --- /dev/null +++ b/runs/action-aware-v0/prior-attempts-v2.json @@ -0,0 +1,24 @@ +{ + "global_hard_cap_h20_hours": 8.0, + "invariants": { + "all_gpus_idle_after_each_stop": true, + "no_completed_measured_runs": true, + "no_prior_runtime_data_reused": true + }, + "prior_attempt_h20_hours": 0.7043978426191542, + "schema": "action-aware-prior-attempts-v2", + "stops": [ + { + "artifact": "/home/admin/cpfs/wjh/action-aware-constraint-v0-20260714/operational-stop-v0.json", + "h20_hours": 0.38598689953486126, + "reason": "MBBT256 burn-in remained throughput-backlogged", + "stage": "burnin" + }, + { + "artifact": "/home/admin/cpfs/wjh/action-aware-constraint-v1-20260714/operational-stop-v1.json", + "h20_hours": 0.31841094308429296, + "reason": "controller passed the warmup result to the burn-in gate", + "stage": "first measured run in flight; zero measured results completed" + } + ] +} diff --git a/runs/action-aware-v0/test_pilot.py b/runs/action-aware-v0/test_pilot.py index c46af88..c2867d9 100644 --- a/runs/action-aware-v0/test_pilot.py +++ b/runs/action-aware-v0/test_pilot.py @@ -201,6 +201,32 @@ def main() -> None: "--max-num-batched-tokens 2048" in revised_plan["sessions"][0]["commands"]["server"] ) + accepted_burnin = { + "kind": "anchor", + "selection": {"count": 510}, + "interval": {"elapsed_s": 61.25}, + "pass_rate": 0.5, + "feasible": False, + } + assert controller.burnin_gate( + accepted_burnin, expected_count=510, maximum_elapsed_s=90.0 + )["elapsed_s"] == 61.25 + warmup = copy.deepcopy(accepted_burnin) + warmup["kind"] = "warmup" + try: + controller.burnin_gate(warmup, expected_count=510, maximum_elapsed_s=90.0) + except RuntimeError as error: + assert "non-anchor" in str(error) + else: + raise AssertionError("warmup incorrectly passed the burnin gate") + slow = copy.deepcopy(accepted_burnin) + slow["interval"]["elapsed_s"] = 91.0 + try: + controller.burnin_gate(slow, expected_count=510, maximum_elapsed_s=90.0) + except RuntimeError as error: + assert "throughput gate failed" in str(error) + else: + raise AssertionError("slow burnin incorrectly passed the throughput gate") print("action-aware constraint pilot: PASS") From 3facb18bcfe2875a1bf905ee9f2bcb1cb950d55f Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 22:23:43 +0800 Subject: [PATCH 024/148] Fix async telemetry coverage audit --- runs/action-aware-v0/analyze_pilot.py | 44 ++++++++++++++++++++------- runs/action-aware-v0/test_pilot.py | 19 ++++++++++++ 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/runs/action-aware-v0/analyze_pilot.py b/runs/action-aware-v0/analyze_pilot.py index 8c438ec..ba5511a 100644 --- a/runs/action-aware-v0/analyze_pilot.py +++ b/runs/action-aware-v0/analyze_pilot.py @@ -138,6 +138,37 @@ def binding_summary( } +def telemetry_coverage( + records: list[Mapping[str, Any]], *, start_ns: int, end_ns: int +) -> tuple[dict[str, float], bool]: + if not records: + raise ValueError("telemetry coverage requires records") + submit_gaps = [ + (int(right["submit_mono_ns"]) - int(left["submit_mono_ns"])) / 1e9 + for left, right in zip(records, records[1:], strict=False) + ] + uncovered_gaps = [ + max( + 0, + int(right["submit_mono_ns"]) - int(left["complete_mono_ns"]), + ) + / 1e9 + for left, right in zip(records, records[1:], strict=False) + ] + coverage = { + "start_gap_s": (int(records[0]["submit_mono_ns"]) - start_ns) / 1e9, + "end_gap_s": (end_ns - int(records[-1]["submit_mono_ns"])) / 1e9, + "max_internal_submit_gap_s": max(submit_gaps, default=0.0), + "max_uncovered_gap_s": max(uncovered_gaps, default=0.0), + } + covered = ( + 0.0 <= coverage["start_gap_s"] <= 1.0 + and 0.0 <= coverage["end_gap_s"] <= 1.0 + and 0.0 <= coverage["max_uncovered_gap_s"] <= 1.0 + ) + return coverage, covered + + def request_summary(path: Path, expected_count: int) -> dict[str, Any]: rows = load_jsonl(path) if len(rows) != expected_count: @@ -222,17 +253,8 @@ def analyze_run( ] if not full_records: raise ValueError(f"no telemetry records in measured window: {result_path}") - gaps = [ - (int(right["submit_mono_ns"]) - int(left["submit_mono_ns"])) / 1e9 - for left, right in zip(full_records, full_records[1:], strict=False) - ] - coverage = { - "start_gap_s": (int(full_records[0]["submit_mono_ns"]) - start_ns) / 1e9, - "end_gap_s": (arrival_end_ns - int(full_records[-1]["submit_mono_ns"])) / 1e9, - "max_internal_gap_s": max(gaps, default=0.0), - } - invariants["telemetry_coverage"] = all( - 0.0 <= value <= 1.0 for value in coverage.values() + coverage, invariants["telemetry_coverage"] = telemetry_coverage( + full_records, start_ns=start_ns, end_ns=arrival_end_ns ) binding = binding_summary( full_records, mns=int(config["mns"]), mbbt=int(config["mbbt"]) diff --git a/runs/action-aware-v0/test_pilot.py b/runs/action-aware-v0/test_pilot.py index c2867d9..afb1e29 100644 --- a/runs/action-aware-v0/test_pilot.py +++ b/runs/action-aware-v0/test_pilot.py @@ -81,6 +81,25 @@ def main() -> None: assert summary["waiting_unresolved_count"] == 1 assert summary["waiting_count"] == 4 + # A per-step stream may have a submit gap above one second when the + # preceding model execution itself spans that interval. Such a gap is + # covered telemetry, not a dropped-record interval. + asynchronous = [ + {"submit_mono_ns": 0, "complete_mono_ns": 1_200_000_000}, + {"submit_mono_ns": 1_100_000_000, "complete_mono_ns": 1_300_000_000}, + ] + coverage, covered = analysis.telemetry_coverage( + asynchronous, start_ns=0, end_ns=1_100_000_000 + ) + assert coverage["max_internal_submit_gap_s"] == 1.1 + assert coverage["max_uncovered_gap_s"] == 0.0 + assert covered + missing = copy.deepcopy(asynchronous) + missing[0]["complete_mono_ns"] = 0 + assert not analysis.telemetry_coverage( + missing, start_ns=0, end_ns=1_100_000_000 + )[1] + manifest = { "repetitions": {str(index): {} for index in (1, 2, 3)}, "regimes": { From 2af22dbce4a8be1b9004da03e58d9980edd95298 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 22:29:07 +0800 Subject: [PATCH 025/148] Audit action-response mechanism telemetry --- runs/action-aware-v0/analyze_pilot.py | 88 +++++++++++++++++++++++++++ runs/action-aware-v0/test_pilot.py | 39 ++++++++++++ 2 files changed, 127 insertions(+) diff --git a/runs/action-aware-v0/analyze_pilot.py b/runs/action-aware-v0/analyze_pilot.py index ba5511a..18e295d 100644 --- a/runs/action-aware-v0/analyze_pilot.py +++ b/runs/action-aware-v0/analyze_pilot.py @@ -55,6 +55,18 @@ def numeric(values: Iterable[float]) -> dict[str, Any]: } +def distribution(values: Iterable[float]) -> dict[str, Any]: + finite = [float(value) for value in values] + summary = numeric(finite) + return { + **summary, + "mean": statistics.fmean(finite), + "p50": quantile(finite, 0.50), + "p95": quantile(finite, 0.95), + "p99": quantile(finite, 0.99), + } + + def quantile(values: Iterable[float], probability: float) -> float: ordered = sorted(float(value) for value in values) if not ordered: @@ -169,6 +181,77 @@ def telemetry_coverage( return coverage, covered +def mechanism_summary(records: list[Mapping[str, Any]]) -> dict[str, Any]: + executed = [record for record in records if bool(record["model_executed"])] + if not executed: + raise ValueError("mechanism summary requires executed steps") + prefill = [record for record in executed if int(record["prefill_tokens"]) > 0] + decode_only = [ + record for record in executed if int(record["prefill_tokens"]) == 0 + ] + if not prefill or not decode_only: + raise ValueError("mechanism summary requires prefill and decode-only steps") + + def durations_ms(selected: list[Mapping[str, Any]]) -> list[float]: + values = [ + (int(record["complete_mono_ns"]) - int(record["submit_mono_ns"])) + / 1e6 + for record in selected + ] + if any(value < 0.0 for value in values): + raise ValueError("engine step duration must be non-negative") + return values + + chunk_keys = ("first", "middle", "final", "unsplit", "tokens") + chunks = { + key: sum(int(record["chunked_prefill"][key]) for record in executed) + for key in chunk_keys + } + prefill_tokens = [int(record["prefill_tokens"]) for record in prefill] + prefill_requests = sum(int(record["prefill_requests"]) for record in prefill) + prefix_queries = sum( + int(record["prefix"]["local"]["queries"]) for record in executed + ) + prefix_hits = sum( + int(record["prefix"]["local"]["hits"]) for record in executed + ) + invariants = { + "nonnegative_counts": all( + value >= 0 + for value in ( + *chunks.values(), + prefill_requests, + prefix_queries, + prefix_hits, + ) + ), + "chunk_tokens_match_prefill_tokens": chunks["tokens"] + == sum(prefill_tokens), + "prefix_hits_bounded": 0 <= prefix_hits <= prefix_queries, + } + return { + "executed_steps": len(executed), + "step_duration_ms": distribution(durations_ms(executed)), + "prefill_steps": len(prefill), + "prefill_step_duration_ms": distribution(durations_ms(prefill)), + "decode_only_steps": len(decode_only), + "decode_only_step_duration_ms": distribution(durations_ms(decode_only)), + "prefill": { + "requests": prefill_requests, + "requests_per_step": prefill_requests / len(prefill), + "tokens": sum(prefill_tokens), + "tokens_per_step": distribution(prefill_tokens), + "chunks": chunks, + }, + "prefix": { + "queries": prefix_queries, + "hits": prefix_hits, + "hit_rate": prefix_hits / prefix_queries if prefix_queries else 0.0, + }, + "sanity": {"invariants": invariants}, + } + + def request_summary(path: Path, expected_count: int) -> dict[str, Any]: rows = load_jsonl(path) if len(rows) != expected_count: @@ -259,6 +342,10 @@ def analyze_run( binding = binding_summary( full_records, mns=int(config["mns"]), mbbt=int(config["mbbt"]) ) + mechanism = mechanism_summary(full_records) + invariants["mechanism_summary"] = all( + mechanism["sanity"]["invariants"].values() + ) phases = {} for fraction in phase_fractions: phase_end = start_ns + round(duration_s * fraction * 1e9) @@ -301,6 +388,7 @@ def analyze_run( **latency, }, "binding": binding, + "mechanism": mechanism, "phases": phases, "state": state, "coverage": coverage, diff --git a/runs/action-aware-v0/test_pilot.py b/runs/action-aware-v0/test_pilot.py index afb1e29..3b25456 100644 --- a/runs/action-aware-v0/test_pilot.py +++ b/runs/action-aware-v0/test_pilot.py @@ -100,6 +100,45 @@ def main() -> None: missing, start_ns=0, end_ns=1_100_000_000 )[1] + mechanism = analysis.mechanism_summary( + [ + { + "model_executed": True, + "submit_mono_ns": 0, + "complete_mono_ns": 2_000_000, + "prefill_tokens": 8, + "prefill_requests": 2, + "chunked_prefill": { + "first": 1, + "middle": 0, + "final": 0, + "unsplit": 1, + "tokens": 8, + }, + "prefix": {"local": {"queries": 10, "hits": 2}}, + }, + { + "model_executed": True, + "submit_mono_ns": 2_000_000, + "complete_mono_ns": 3_000_000, + "prefill_tokens": 0, + "prefill_requests": 0, + "chunked_prefill": { + "first": 0, + "middle": 0, + "final": 0, + "unsplit": 0, + "tokens": 0, + }, + "prefix": {"local": {"queries": 0, "hits": 0}}, + }, + ] + ) + assert mechanism["prefill"]["requests_per_step"] == 2.0 + assert mechanism["prefill"]["chunks"]["first"] == 1 + assert mechanism["prefix"]["hit_rate"] == 0.2 + assert all(mechanism["sanity"]["invariants"].values()) + manifest = { "repetitions": {str(index): {} for index in (1, 2, 3)}, "regimes": { From 8c930ba3a1f6e7b6481ff0a310e45a30c877988e Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Tue, 14 Jul 2026 22:34:26 +0800 Subject: [PATCH 026/148] Report action-aware constraint pilot results --- ...re-constraint-pilot-v2-results-20260714.md | 260 ++++++++++++++++++ 1 file changed, 260 insertions(+) create mode 100644 docs/action-aware-constraint-pilot-v2-results-20260714.md diff --git a/docs/action-aware-constraint-pilot-v2-results-20260714.md b/docs/action-aware-constraint-pilot-v2-results-20260714.md new file mode 100644 index 0000000..b0f1ed0 --- /dev/null +++ b/docs/action-aware-constraint-pilot-v2-results-20260714.md @@ -0,0 +1,260 @@ +# Action-aware constraint pilot v2 results + +Date: 2026-07-14 (Asia/Singapore). + +Decision: **`STOP_WORKLOAD_NOT_CROSSED`**. + +The pilot produced one valid positive regime and one invalid-for-effect-size +regime. It supports continuing a narrower action-response investigation, but +it does not justify an end-to-end telemetry-guided tuner claim or new engine +instrumentation yet. + +## Question tested + +Given only a completed source run, can existing engine telemetry distinguish +which of two one-knob interventions will improve SLO-goodput more? + +The frozen score counted scheduler steps with backlog where either MNS or MBBT +was exclusively at its configured limit. It made two pre-intervention +predictions on the same workload and offered load: + +- Regime A: source `(MNS=16, MBBT=8192)` predicts increasing MNS to 64 over + increasing MBBT to 16384. +- Regime B: source `(MNS=64, MBBT=2048)` predicts increasing MBBT to 8192 over + increasing MNS to 128. + +The primary outcome was 300-second SLO-goodput. A predicted action had to beat +the alternative on every paired request band by at least 10% of that band's +source goodput. Telemetry direction also had to remain stable at 25%, 50%, +75%, and 100% of the replay. + +## Setup + +- Host: `dash0`, GPU 0-3 used exclusively; four NVIDIA H20 GPUs; TP=4. GPU + 4-7 remained idle to avoid co-location effects. +- Model: Qwen3-30B-A3B BF16 at + `/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B`. +- Runtime: patched vLLM `0.24.1.dev3+g668cfb7e2`, source commit `4b253fd`, with + OpProf Layer-1 telemetry. +- Workload: `chat_w20260312_1000`, 2.125 requests/s/GPU, 300-second arrival + window, 128 output tokens, three disjoint request bands. +- Five fresh-server configurations, three measured runs each, counter-rotated + repetition order, 16-request warm-up, and a 510-request/60-second burn-in. + SLO early stopping was disabled. +- Exact request-id, arrival-order, and input-length hashes matched for every + paired comparison. + +The authoritative run root is +`/home/admin/cpfs/wjh/action-aware-constraint-v2-20260714`. The final audit is +`pilot-audit-final.json`, SHA256 +`7ebe080fcc4970bef423bc587253d157e75aed1ea8b410bd37770c17708135ab`. + +## End-to-end result + +### Regime A: strong MNS constraint + +| Rep | Source 16/8192 | MBBT action 16/16384 | MNS action 64/8192 | MNS-only source steps | MBBT-only source steps | `(MNS action - MBBT action) / source` | +|---:|---:|---:|---:|---:|---:|---:| +| 1 | 4.710 | 7.687 | 8.500 | 79.85% | 0.038% | 17.27% | +| 2 | 2.803 | 4.150 | 8.500 | 94.61% | 0.005% | 155.17% | +| 3 | 2.227 | 3.943 | 8.500 | 79.87% | 0.005% | 204.64% | + +Units are SLO-goodput requests/s except the percentage columns. The source +prediction was stable at every phase checkpoint and correct in all three +paired bands. The predicted MNS action cleared the frozen 10% material-margin +gate in all three bands. + +This is a real but limited positive result. The source was an extreme case: +MNS was full on nearly every scheduler step that retained backlog, TTFT p50 was +1.64-5.61 seconds, KV usage remained below 2.45%, and no preemption occurred. +An expert or a simple rule could identify this case without a learned tuner. + +### Regime B: MBBT direction at an outcome ceiling + +| Rep | Source 64/2048 | MNS action 128/2048 | MBBT action 64/8192 | MBBT-only source steps | `(MBBT action - MNS action) / source` | +|---:|---:|---:|---:|---:| +| 1 | 8.423 | 8.420 | 8.500 | 10.90% | 0.950% | +| 2 | 8.447 | 8.500 | 8.500 | 12.03% | 0% | +| 3 | 8.500 | 8.497 | 8.500 | 8.79% | 0.039% | + +The telemetry direction was stable in all phase checkpoints. The predicted +action won twice and tied once, while the wrong MNS action left the MBBT-only +state intact. However, the source already delivered 99.10-100% of the offered +8.5 requests/s. Even a perfect action could not reach the preregistered 10% +margin. Regime B therefore does not test material weak-signal value; it is a +workload-selection failure, not evidence that telemetry does or does not help +near a decision boundary. + +The missing preflight condition is mathematical. For an effect threshold +`delta` and offered goodput ceiling `G`, the source must satisfy +`source <= G / (1 + delta)`. Here `G=8.5` and `delta=0.10`, so any source above +7.727 requests/s cannot possibly pass before either target is measured. + +## Why the exclusive-limit rule is incomplete + +The alternative MBBT action improved Regime A by 48.0%, 63.2%, and 77.1% over +the source even though MBBT was almost never the exclusive backlog constraint. +This rules out the binary interpretation "a knob that is not exclusively at +its cap cannot help." + +Existing richer telemetry provides a plausible mechanism: + +| Rep | Split-prefill requests, source -> MBBT action | Prefill steps, source -> MBBT action | Prefill requests/step, source -> MBBT action | Prefix-hit rate, source -> MBBT action | +|---:|---:|---:|---:|---:| +| 1 | 41 -> 2 | 2324 -> 2022 | 1.071 -> 1.249 | 13.851% -> 13.747% | +| 2 | 7 -> 0 | 2410 -> 2291 | 1.012 -> 1.076 | 13.078% -> 12.988% | +| 3 | 12 -> 1 | 2377 -> 2270 | 1.018 -> 1.096 | 13.613% -> 13.604% | + +Increasing MBBT allows more prefill work to be packed into one iteration and +nearly eliminates split prefills. Under MNS=16, this can reduce the number of +iterations for which long prompts occupy scarce running slots. Prefix-cache +hit rates differ by at most 0.104 percentage points, and exact workload hashes +match, so neither explains the gain. Step-duration p99 also remains similar; +one 1.127-second decode-step outlier appears in `a_mbbt/rep2`, but the same +action direction occurs in all three bands. + +This is a mechanism-consistent explanation, not a completed causal +decomposition. MBBT simultaneously changes total per-iteration token budget, +per-request chunk size, and multi-request packing. Instrumentation observes +their joint response but cannot separate those effects without another +intervention. + +## Instrumentation decision + +Do **not** add a new engine patch for this mechanism yet. The existing OpProf +stream already records submit/complete timestamps, prefill/decode composition, +chunked-prefill categories, prefix hits, queues, KV usage, and CUDA graph mode. +Those fields are sufficient to identify the interaction missed by the initial +exclusive-limit rule. + +The next narrow mechanism ablation is available in the current vLLM runtime: + +1. `(MNS=16, MBBT=8192, long-prefill-threshold=0)` is the current source. +2. `(MNS=16, MBBT=16384, long-prefill-threshold=8192)` keeps individual long + chunks at 8192 while increasing total packing budget. +3. `(MNS=16, MBBT=16384, long-prefill-threshold=0)` is the current MBBT action. + +The runtime exposes `--long-prefill-token-threshold`; with a threshold of 8192, +the second arm separates total packing headroom from the larger per-request +chunk allowed by the third arm. A formal test must rerun all three arms fresh +with counter-rotated order rather than reuse today's endpoints. + +## Correct tuning-research route + +The pilot does not support turning the frozen equality checks into a larger +rule tree. The supported route is **intervention-calibrated, action-conditioned +system identification**: + +```text +source event sequence + normalized config delta + -> predicted distribution of Delta SLO-goodput and evaluation cost + -> uncertainty-aware next-config selection +``` + +The policy input should retain continuous distributions and phase evolution: +queue/running residency, MNS and token slack, prefill/decode composition, +partial-prefill occupancy, step time, KV state, and graph behavior. Human +bottleneck labels and hand-authored `if queue then increase MNS` mappings are +not policy inputs. Mechanism summaries remain audit and interpretation tools; +the action response is learned from paired real interventions. + +The harness has a narrower, non-heuristic role: + +- define legal configurations and exact paired workloads; +- reject source points without outcome headroom before a full sweep; +- randomize/counter-rotate execution order and preserve failures/cost; +- validate stream coverage, hashes, request accounting, and censoring; +- expose target outcomes only after a source-only prediction is frozen; +- evaluate fixed-budget regret and H20-hours, not explanation quality alone. + +The next tuning experiment should use non-extreme, non-ceiling source points +and a local two-dimensional MNS/MBBT neighborhood. A short run may screen load +only; every inferential telemetry and outcome result remains a 300-second run. +At least one held-out workload must be reserved before choosing features or +thresholds. + +Primary evaluation is H20-hours/trials to reach 95% of the real local oracle +and cost-normalized regret AUC. Required baselines are random search, +config/outcome-only sequential search, the current rule heuristic, and the +same action-response model with telemetry removed. Action-ranking accuracy is +supporting evidence only. + +## What this pilot establishes and does not establish + +Established: + +- Long-window engine state can make a correct, phase-stable action-family + prediction in an extreme MNS-constrained regime. +- A naive `queue > 0 -> increase MNS` rule would choose an ineffective action + in Regime B; action-conditioned state distinguishes the mechanism direction, + although the measured effect is immaterial at the selected load. +- Binary exclusive-cap attribution misses a substantial MNS/MBBT interaction; + existing chunk/step telemetry reveals a plausible explanation. +- Source outcome headroom must be an explicit experiment admission gate. + +Not established: + +- telemetry improves an end-to-end tuner over an outcome-only baseline; +- weak or mixed constraints can be ranked with material gain; +- the response transfers across workloads, models, TP, or hardware; +- new engine instrumentation is necessary; +- the chunking/packing breakdown is causal rather than mechanism-consistent. + +## Change and verification + +Reproduction: + +```bash +python3 runs/action-aware-v0/test_pilot.py + +python3 runs/action-aware-v0/analyze_pilot.py \ + --run-root /home/admin/cpfs/wjh/action-aware-constraint-v2-20260714/runs/pilot \ + --manifest runs/action-aware-v0/pilot-manifest-v2.json \ + --output /home/admin/cpfs/wjh/action-aware-constraint-v2-20260714/pilot-audit-final.json +``` + +The fresh GPU run used AITuner commit `c5ab073`; asynchronous coverage was +corrected in `3facb18`; reproducible mechanism summaries were added in +`2af22db`. The raw run is unchanged across those analyzer-only commits. + +Change: added a crossed real-intervention controller and audit, fixed the +burn-in result gate, corrected asynchronous per-step coverage accounting, and +added reproducible step/chunk/prefix mechanism summaries. + +Expected effect: distinguish descriptive telemetry from source-only action +predictions that survive paired real interventions. + +Verification: local and remote action-aware test suites pass; all five sessions +completed; all stream/footer and request-accounting invariants pass; the final +analyzer was run twice and produced byte-identical output. + +Result: Regime A passes; Regime B is invalid for the frozen effect-size test; +the global decision is `STOP_WORKLOAD_NOT_CROSSED`. + +Remaining risk: one model, one TP, one trace family, three bands, two action +families, and deliberately constructed endpoints are development evidence +only. The strong positive regime is too obvious to support a paper claim. + +## Data sanity + +- Measured runs: n=15; elapsed 300.610-317.350 seconds; 15 distinct. Pass + rate min/max 0.2620/1.0 with 11 distinct values; SLO-goodput min/max + 2.2267/8.5 requests/s with 11 distinct values. +- Telemetry intervals: n=15; records min/max 13,621/23,711; 15 distinct. + Start gaps min/max 0.0412/0.1227 seconds; end gaps 0.00053/0.0649; uncovered + internal gaps 0/0.3231. One submit gap reached 1.1190 seconds but was fully + covered by a 1.1269-second recorded execution; contiguous indices and zero + drops were preserved. +- Sessions: n=5; cost min/max 1.1691/1.2730 H20-hours; 5 distinct. V2 cost + was 6.0862 H20-hours; V0/V1/V2 total was 6.7906, below the 8.0 cap. +- Regime-A split-prefill observations: n=6; min/max 0/41 requests; 6 distinct. + Prefix-hit rates: n=6; min/max 0.12988/0.13851; 6 distinct. +- Checked invariants: non-negative counters and durations; ratios in `[0,1]`; + exact request, arrival, and length hashes; 2550/2550 request accounting per + measured run; uncensored outcomes; outcomes across configurations not all + identical; + five complete streams; monotonic timestamps; contiguous step indices; zero + drops; footer/sidecar agreement; chunk-token accounting; bounded prefix hits; + no OOM, controller error, or residual GPU allocation. No unresolved red + flag remains. The three identical shared goodputs are reported as the + offered-load ceiling, not treated as independent performance variation. From 0d16838097c4b9467d8df58ce449fbbebee2f78b Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 01:41:25 +0800 Subject: [PATCH 027/148] Audit tuning cost and core challenges --- ...ing-core-challenges-cost-audit-20260715.md | 391 ++++++++++ runs/tuning-cost/analyze.py | 507 ++++++++++++ runs/tuning-cost/manifest.json | 42 + runs/tuning-cost/metrics.json | 736 ++++++++++++++++++ runs/tuning-cost/test_analysis.py | 57 ++ 5 files changed, 1733 insertions(+) create mode 100644 docs/tuning-core-challenges-cost-audit-20260715.md create mode 100644 runs/tuning-cost/analyze.py create mode 100644 runs/tuning-cost/manifest.json create mode 100644 runs/tuning-cost/metrics.json create mode 100644 runs/tuning-cost/test_analysis.py diff --git a/docs/tuning-core-challenges-cost-audit-20260715.md b/docs/tuning-core-challenges-cost-audit-20260715.md new file mode 100644 index 0000000..200bdae --- /dev/null +++ b/docs/tuning-core-challenges-cost-audit-20260715.md @@ -0,0 +1,391 @@ +# AITuner tuning：核心挑战、统一成本口径与研究路线 + +日期：2026-07-15（Asia/Singapore） + +状态：**问题定义与历史成本审计完成；新的 tuner 贡献尚未建立。** + +## 结论先行 + +我们不应该把 tuning 定义成“根据当前 telemetry 判断哪个 cap 满了，再调对应 knob”。这个定义同时遗漏了 knob interaction、反事实识别、实验成本和跨任务失配。更准确的问题是： + +> 给定模型、engine version、hardware、workload、SLO 和一个声明好的合法配置空间，tuner 如何用最少的真实 GPU 成本，依次选择可能包含多个 knob 的 intervention，找到 SLO-goodput regret 不超过 `epsilon` 的配置？ + +AITuner 可以形成的系统贡献应当是： + +> **一个 intervention-calibrated、action-conditioned、cost-aware 的 tuner：它从真实 engine trajectory 和已测 intervention 中学习联合 config action 的反事实收益分布，并以 cost-to-oracle 而非规则命中率作为目标。Harness 只负责实验语义、合法性、配对、记账和可复现性，不负责用人工 bottleneck rule 决定 action。** + +现有结果支持这个问题值得做，但不支持宣称它已经解决： + +- 在真实 `TP x MNS` surface 上，one-knob-at-a-time 会停在比 oracle 低 **25.6%** 的 coordinate-wise local optimum。 +- 在 action-aware pilot 中，增加 MBBT 在“几乎从未独占打满 MBBT cap”的情况下仍把 source goodput 提高 **48.0%--77.1%**；因此 `cap -> knob` 不是完整模型。 +- 同一 dash0 任务上，当前 guided harness 到 5% empirical regret 只比纯 LLM 少 **5.85%** H20-hours；到 2% regret 则少 **61.09%**。这说明必须比较完整 cost--regret curve，不能只比较最终最好值。 +- Frontier 的 decision-bearing throughput top-1 在 12-cell surface 上有 **30.46%** real regret。Simulator 本身的边际 GPU cost 是 0，但通过 real-final 恢复 oracle 需要 tie-expanded 4 个真实 cell，即 **0.7828 reconstructed H20-hours**。 + +## 1. Tuning 问题和成功标准 + +固定 task context： + +```text +T = {model, engine build, hardware, workload/trace, SLO, legal config space C} +``` + +每个完整配置 `c in C` 的目标为： + +```text +f_T(c) = max request_rate_per_gpu + subject to request SLO pass rate >= target +``` + +有限空间 oracle 为： + +```text +f*_T = max_{c in C} f_T(c) +regret(c) = 1 - f_T(c) / f*_T +``` + +顺序 tuner 在第 `t` 步基于历史 `D_t` 选择一个完整 config intervention： + +```text +a_t = c_t -> c_{t+1} +``` + +成功不是“最后找到一个不错的值”，而是同时满足： + +1. `regret(best_t) <= epsilon`； +2. 达到该点之前的 all-in H20-hours 最小； +3. launch、correctness、SLO 和失败率约束不退化； +4. 结论在 held-out task 上成立，而不是在用于设计规则的 task 上成立。 + +### 1.1 GPU cost 的统一定义 + +未来实验的 task-marginal cost 应定义为： + +```text +C_task = sum_j allocated_GPU_count_j + * (GPU_idle_or_release_time_j - allocation_start_time_j) +``` + +它包括 method 实际触发的 startup、warm-up、prefix/full replay、confirmation、failure、cleanup；如果 LLM 思考期间 GPU 仍被占用，也计入。Simulator/模型的一次性 onboarding 成本单独报告： + +```text +C_e2e(N tasks) = C_profile_or_training / N + C_task +``` + +另外报告 CPU-hours、LLM API latency/cost，但不把它们伪装成 GPU-hours。构建 benchmark oracle 的 exhaustive annotation cost 是公共评测成本，单独报告，不计入任何方法；同时可给一个将其等量加回所有方法的 conservative view。 + +历史记录没有 allocation start/release timestamp。本次只能从每个 `engine.log` 的首末时间戳重建： + +```text +C_engine_lower_bound = parallel_size * engine_log_span / 3600 +``` + +因此下面所有历史 H20-hour 数字都是 **engine-lifetime lower bound**，不是 all-in cost。尤其 simulator 的一次性 H20 operator profiling 成本没有记录，不能称为完全免费。 + +### 1.2 两种 oracle 必须分开 + +- **Exact finite-surface oracle**：声明好的 12-cell `TP x MNS` 空间全部真实测量，oracle 是 `TP2/MNS32 = 3.2833 req/s/GPU`。 +- **Broader empirical reference**：dash0 两个 sequential run 中观察到的最好值 `3.35 req/s/GPU`。它包含 surface 外的 MBBT/chunk/GMU action，但只是 best observed，不是全局 oracle。 + +不能把 empirical best 写成 global oracle，也不能让每个方法使用不同的 oracle 定义。 + +## 2. 现有方案的 cost-to-oracle 审计 + +可复算输入和完整结果在： + +- `runs/tuning-cost/manifest.json` +- `runs/tuning-cost/analyze.py` +- `runs/tuning-cost/metrics.json` + +### 2.1 严格同任务对照：纯 LLM vs 当前 guided harness + +两组均为 dash0、Qwen3-30B-A3B、community-vLLM 0.20.0、8xH20 可见、`chat_w20260311_1000`、input 0--8k、output 128、replay scale 0.1、TTFT 2/4/6s、TPOT 50ms、pass rate 0.95。除 tuner method 和服务端口外，固定 task spec 相同。 + +Reference 是两组中 best observed `3.35 req/s/GPU`： + +| Method | 到 <=5% regret | 到 <=2% regret | 到 <=1% regret | 完整 run 成本 | 最终 best | +|---|---:|---:|---:|---:|---:| +| Pure LLM, no harness | 0.2847 H20h，trial 2，regret 2.736% | 1.1458，trial 6，regret 1.493% | 1.3719，trial 7，regret 0% | 2.2825 | 3.35 | +| Guided harness v2 | 0.2681 H20h，trial 2，regret 2.736% | 0.4458，trial 3，regret 1.990% | 未达到 | 0.6231 | 3.30，regret 1.493% | + +直接结论： + +- 5% endpoint：guided 比 pure LLM 少 **5.85%**，不是 material contribution。 +- 2% endpoint：guided 比 pure LLM 少 **61.09%**，有明显 headroom signal，但只有一个 task，不能外推。 +- Pure LLM 在 trial 7 已找到 best observed，之后又花了 `2.2825 - 1.3719 = 0.9106 H20h` 而没有改进，说明 trustworthy stopping 本身就是成本来源。 +- Pure LLM 的 trial 3 使用当前 binary 不支持的 `--expert-parallel-size` 并在 launch 前失败。当前 harness 的 legality/version contract 有实际价值，但它仍不是性能 action-ranking 贡献。 + +### 2.2 Simulator：零边际 GPU cost 不等于零 tuning cost + +Frontier fidelity suite 在 CPU 上执行 184 个 simulation，耗时 **2.055 CPU-hours**，simulation 本身为 0 marginal H20-hours。其对应的 exact dash1 12-cell real surface annotation lower bound 为 **3.5953 H20-hours**。 + +Decision-bearing `frozen-calibrated/throughput-proxy`： + +| Policy | Real cells evaluated | Real-final H20h lower bound | Selected real regret | +|---|---:|---:|---:| +| Simulator-only top-1 | 0 | 0 | **30.46%**，选 TP1/MNS64 | +| Throughput top-1 + real final | 1 | 0.1353 | **30.46%** | +| Throughput top-2 + real final | 2 | 0.2672 | **30.46%** | +| Throughput nominal top-3 + real final | tie-expanded 4 | 0.7828 | 0%，找到 TP2/MNS32 | + +Post-hoc `SLO-gated` reading 把 `{TP2/MNS32, TP2/MNS64}` 放在 top tie bucket；测两个 cell 需 **0.5156 H20h** 并能找到 oracle。但它不是 preregistered decision-bearing policy，而且 anchor verdict 中有 21 个 false-feasible、7 个 false-infeasible，只能作为诊断上界，不能反写成 prospective simulator 结果。 + +Pure LLM/harness 数据来自 dash0，simulator exact surface 来自 dash1。模型、engine、trace、GPU type 匹配，但 host 和 campaign 不同。因此两块内部可以直接比较，跨块只能做 development-level 指示；paper 结论必须在同 host、同 task execution protocol 下重跑。 + +### 2.3 我们要达到的成本目标 + +在当前 reconstructed lower-bound 口径下，一个有意义的单任务 development bar 是： + +| Endpoint | 当前最强同任务 baseline | 20% reduction bar | 兼顾 post-hoc sim+real 的 30% bar | 暂定目标 | +|---|---:|---:|---:|---:| +| <=5% empirical regret | guided 0.2681 | 0.2144 | 0.3609 | **<=0.2144 H20h** | +| <=2% empirical regret | guided 0.4458 | 0.3567 | 0.3609 | **<=0.3567 H20h** | + +这两个数字不是 paper result，只用于检查 proposed method 是否有足够 headroom： + +- 5% endpoint 已经由 baseline + TP2 两个完整 trial 达到。任何必须先跑 source 再跑 target 的 telemetry tuner 都不能靠减少 trial count 获得 20% 优势；它必须能够 one-shot warm-start、跳过 baseline，或安全地缩短其中一次测量。 +- 2% endpoint 有更合理的结构性空间：从一个 source 直接选择 joint `TP2 + MBBT/chunk` target，可能跳过当前中间 trial；如果仍按当前三次完整 trial 顺序执行，就不会达到 bar。 + +Paper-facing gate 不使用这些跨 campaign 绝对数，而使用 prospective same-host all-in cost：在每个 held-out task 上 regret <=5%，相对最强 safe outcome-only/current harness 至少省 20%，相对 frozen simulator+real 至少省 30%，并报告 task-level paired confidence interval。 + +## 3. 四个最核心的 tuning challenge + +### Challenge 1：响应面是联合、条件化且 regime-dependent 的 + +#### 问题本质 + +一般情况下： + +```text +f(c) != base + sum_k effect_k(c_k) +``` + +一个 knob 的 effect 是当前完整 context 的函数： + +```text +Delta_x(c, workload, engine state) +``` + +它可能随 topology、另一个 runtime knob、load、SLO 或 engine version 改变大小甚至改变符号。因此不能先分别求每个 knob 的最优值再 merge，也不能固定一个低质量 context 去判断另一个 knob。 + +#### 已有真实证据 + +在 C1 12-cell real surface： + +- `MNS 8 -> 32` 在 TP1/TP2/TP4 下分别提升约 **8.7% / 44.3% / 90.3%**。 +- 从同一 `TP1/MNS8` 起点，先 tune MNS 再 TP 会停在 `TP4/MNS16 = 2.4417`；该点沿任一单维都没有 strictly improving move，但 joint/global surface oracle `TP2/MNS32 = 3.2833` 高 **34.5% relative to the local point**，即 local point 对 oracle 有 **25.6% regret**。 +- C3 中 `MBT 256 -> 384` 的 effect 根据 topology/MNS 从 0 到约 -9.2%；`MNS 64 -> 128` 从 0 到约 +10.1%。 +- Action-aware Regime A 中 MBBT 几乎从不作为 exclusive cap，但 MBBT action 仍把 source goodput 提高 48.0%--77.1%。它通过 chunk size、prefill packing 和 scarce MNS slot residency 的联合变化获得收益。 + +这直接否定两类通用策略：OAT/coordinate greedy，以及 `which cap is full -> tune that knob`。 + +#### Tuner 必须具备的能力 + +- Action 的基本单位是完整 `config delta`，允许 sparse joint action，而不是孤立 knob/value。 +- 对 topology/runtime family 使用 crossed anchors 或信息增益设计，主动测 interaction；不是默认所有 interaction 都强。 +- 能从数据判断 task 是 topology-dominant、runtime-interaction-dominant 还是 flat/noisy，并据此分配实验，而不是把固定 search order 写进规则。 + +### Challenge 2：当前状态是 observational signal，tuning 需要 counterfactual identification + +#### 问题本质 + +一次 telemetry trace 只能告诉我们： + +```text +P(engine trajectory | current config, workload) +``` + +Tuning 真正需要的是： + +```text +P(Delta SLO-goodput, failure, cost + | source trajectory, proposed full-config action) +``` + +Queue、KV、padding、split prefill 等状态既可能是原因，也可能是 workload/config 的结果。看见某种状态，不等于知道哪个 action 能修复它。一个 action 也可能同时改变多条机制；例如 MBBT 同时改变总 token budget、per-request chunk 和 multi-request packing，现有 telemetry 的解释是 mechanism-consistent，不是已完成的 causal decomposition。 + +#### 已有真实证据 + +- 5/10 秒 telemetry 确实太短；300 秒 phase-aware experiment 中，MNS action 的 queue/padding 机制直到 replay 75%--100% 才稳定出现。 +- 但 external TTFT outcome 在 25% 已完美区分该 action 是否修复 SLO。Telemetry 解释了 why，却没有比 outcome 更早或更可靠地指导 tuning。 +- 3.125 req/s/GPU 的 source 无法在 timeout 内 drain；另一组 source 已达 offered ceiling 的 99.1%--100%，数学上不可能通过 10% improvement gate。没有 exposure/headroom 和 censoring control，模型学到的不是 action response。 +- Same-config repeats 与 matched intervention 的波动不可忽略；只比较两个未经配对的 run 会混入 arrival/order/warm-state noise。 + +#### Tuner 必须具备的能力 + +- 训练样本必须是 exact-workload paired intervention：`(source trajectory, action) -> target delta`，保留失败和 censoring。 +- 使用 phase-binned continuous trajectory，而不是人工 bottleneck label 或 threshold rule。 +- 输出 response distribution 和 uncertainty；证据不足时 abstain，而不是强行给 diagnosis。 +- Telemetry 的价值必须通过同 cutoff、同 model capacity 的 outcome-only ablation 证明。若不能降低 end-to-end H20-hours，instrumentation 只保留为 debugging/解释工具。 + +### Challenge 3：这是异构成本下的 sequential experimental design，不是静态 ranking + +#### 问题本质 + +每个 trial 的成本不同：TP4 是 TP1 的四倍 GPU multiplier，startup/warm-up 可能主导短 probe，失败也有成本；同时 tuner 不知道 oracle，只能在 exploitation、information gain 和 cost 之间权衡。选对 top-1 的 accuracy 不能代表 tuning 效果。 + +必须回答三个连续问题： + +1. 下一次测哪个联合 action？ +2. 测多久，何时 continuation/confirmation？ +3. 什么证据允许停止，并声称 best 已在 `epsilon` 内？ + +#### 已有真实证据 + +- Pure LLM 达到 best observed 后仍浪费 0.9106 reconstructed H20h。 +- Simulator top-1 虽然 0 marginal GPUh，却因 rank error 损失 30.46%；real-final 的 k 增大又迅速增加 H20h。 +- 5% endpoint 上两个方法都只需两个 trial，selection-count headroom 很小；2% endpoint 才暴露 action quality 和 stopping 的巨大差异。 +- Prefix 不是天然便宜：如果 startup、warm-up 和稳定状态形成占主要成本，缩短 replay window 未必带来等比例 H20h reduction。 + +#### Tuner 必须具备的能力 + +- Acquisition 直接优化 expected regret reduction / predicted H20 cost，并把 failure probability 纳入约束。 +- 在 run 前做与 tuning policy 分离的 workload admissibility check：避免 outcome ceiling、无法 drain、无请求或 measurement cap。 +- 使用 uncertainty-aware continuation 和 stop；stop criterion 针对声明的 candidate set 中“仍存在 >epsilon improvement 的概率”，而不是连续几次没提升。 +- 主结果报告 H20-hours-to-5%/2%/1%、fixed-budget regret 和 cost-normalized regret AUC，不 metric shopping。 + +### Challenge 4：任何 mechanism model 都有 fidelity 和 transfer boundary + +#### 问题本质 + +Simulator、learned surrogate、LLM prior 都是近似。Workload、SLO、model、hardware、engine version 改变后，operator cost、scheduler state transition、合法 flag 和 response surface 都可能变化。模型在 calibration task 上解释得好，不表示能在 held-out task 上排序正确。 + +#### 已有真实证据 + +- Frontier throughput reading 在完全匹配的 12-cell task 上仍把 real oracle 排错，top-1 regret 30.46%。这说明预测绝对 throughput 还不够，局部 rank fidelity 才是 tuning 关键。 +- Post-hoc SLO reading 的 top bucket 正确，但有大量 anchor feasibility error，也没有 prospective policy status。 +- Pure LLM 提出了当前 community-vLLM binary 不支持的 flag；engine/API version knowledge 本身会漂移。 +- 已有 cross-version experiment 中 vLLM 0.20 的强配置在 0.24 上出现大幅退化，说明 response prior 不能无条件迁移。 + +#### Tuner 必须具备的能力 + +- Simulator 只能作为 prior mean 或 candidate prior；真实 outcome 是 authoritative update。 +- 学习 simulator residual：把 `sim prediction + source state + action` 映射到 real response，而不是用 telemetry 重新实现另一个无校准 simulator。 +- 对 task-level OOD 显式提高 uncertainty/abstain；train/test 按完整 task 分割，不能按 request、anchor 或同一 surface cell 随机分割。 +- 分开报告 cold-start profile/training cost 与 per-task marginal cost，并在 N=1/10/100 等 amortization horizon 下展示。 + +## 4. 对应的系统设计 + +### 4.1 Harness：从 rule-based tuner 收缩成 experimental control plane + +Harness 保留以下确定性职责： + +- engine-version-aware config schema、合法性和资源约束； +- 完整 config/action canonicalization，禁止隐式 merge 和重复试验； +- exact trace/request/arrival/length hash，配对、随机化和 counter-rotation； +- engine trajectory、external outcome、failure/censoring 的统一时间轴； +- all-in GPU cost ledger、oracle annotation 分账、budget enforcement； +- data sanity、coverage、SLO/correctness 和 stop-proof audit。 + +Harness **不**包含 `queue > N -> increase MNS`、`cap full -> tune knob` 或人工 diagnosis-to-action mapping。这里的规则是实验语义和安全 invariant，不是性能决策 heuristic。 + +### 4.2 Action-conditioned response model + +每条学习记录为： + +```text +x = {source full config, + workload/SLO context, + source external outcome, + phase-binned engine trajectory} +a = normalized full-config delta +y = {Delta SLO-goodput, target feasibility/failure, measured H20 cost} +``` + +学习： + +```text +p_theta(y | x, a, optional simulator prediction) +``` + +第一版应使用适合小数据且有 uncertainty 的 action-conditioned Gaussian-process/bootstrapped surrogate；kernel/feature ablation包括： + +1. config + external outcome； +2. 同样输入 + telemetry trajectory； +3. simulator + config + outcome； +4. 同样输入 + telemetry residual features。 + +Telemetry 保留 continuous phase distributions：queue/running residency、MNS/token slack、prefill/decode composition、partial/split prefill、step duration、KV、graph/padding。模型学习它们与 action 的 interaction；不先压成 bottleneck label。 + +### 4.3 Cost-aware policy + +在合法的 single/joint candidate set 上选择： + +```text +a* = argmax_a + expected constrained improvement(a) + / expected all-in H20 cost(a) +``` + +探索项来自 posterior uncertainty/information gain；launch/SLO failure 有显式 penalty。Simulator 可提供 prior mean，但 simulator 与 real discrepancy 会被 posterior residual 更新。一次 target measurement 后更新 response model，并重新计算下一步 action 或停止概率。 + +LLM 在这个 tuning core 中不是 telemetry classifier。它最多作为可移除的 candidate/prior source，提出 schema 内的 sparse joint actions 或检索 engine mechanism；每个 proposal 都由同一个 response model、cost acquisition 和 real validator 评分。只有 `with LLM` 相对 `same tuner without LLM` 在 held-out tasks 上继续降低 cost-to-oracle，才能讨论 LLM 必要性。 + +### 4.4 Stop 条件 + +对一个预先声明的有限 candidate set，满足以下条件才 stop： + +```text +P(exists c: f(c) > best_observed / (1 - epsilon) | D_t) < alpha +``` + +并且 best config 通过独立 confirmation、SLO/correctness gate，remaining candidate 的 cost-aware value of information 低于阈值。停止原因、posterior coverage 和未测区域必须写入 audit。 + +## 5. 下一阶段如何证明，而不是再次构造 heuristic + +### R0：已有数据 retrospective premise check + +- 用 C1/C3 response surfaces 检查 joint model 是否能避免 OAT trap。 +- 用 action-aware paired records 比较 outcome-only 与 +telemetry 的 action-delta calibration。 +- 用 SimFid surface 比较 direct model 与 simulator-residual model 的 rank/regret。 +- 所有 feature、kernel、candidate encoding 在 held-out task 结果之前冻结。 + +R0 只能筛选 model family，不能作为 paper result，因为现有 tasks 已参与路线设计。 + +### R1：prospective same-host cost-to-oracle pilot + +- dash0 8xH20，固定 engine build/model；serialized placement，禁止共置干扰。 +- 至少一个未参与 feature/threshold 选择的新 trace window；选择非 ceiling、可 drain 的 offered load。 +- 声明一个可穷举的小 surface，至少包含 topology/runtime crossed actions，而不是只有一个 MNS ladder。 +- Oracle annotation 与 tuner online actions 分开记账；method 只能看到当时可用的数据。 +- 运行 random/search、OAT、纯 LLM、当前 guided harness、frozen simulator+real、outcome-only response、+telemetry response、sim-residual +telemetry。 +- 比较完整 H20 cost-to-regret curve，而不是 action classification accuracy。 + +Pilot opening gate： + +1. telemetry model 相对相同 response model 去掉 telemetry，确实改变至少一个正确的 prospective action ranking； +2. 最终 regret <=5%，无 false-safe accept； +3. all-in H20-hours 相对 strongest safe outcome-only 至少下降 20%； +4. 如果使用 simulator，需相对 frozen simulator+real 至少下降 30%； +5. instrumentation overhead <=1%，所有成本和失败均计入。 + +若 1--5 任一失败，就不能把 telemetry/harness 写成 tuning contribution；保留其 debugging/measurement 价值即可。 + +### R2：task-held-out replication + +至少 3 个 workload window x 2 个 SLO regime，按完整 task 做 leave-one-task-out 或固定 train/test split。报告每个 task 的 regret、安全和成本，以及 task-level paired bootstrap CI。只有 R2 通过，才能把单 task 的 61.09% lower-bound saving 升级为项目贡献。 + +## 6. 当前能说与不能说的贡献 + +当前能说： + +- 我们有真实反例证明 OAT 和 cap-to-knob mapping 不是通用 tuning strategy。 +- Harness 的 legality、exact replay、failure/cost accounting 有必要的实验基础设施价值。 +- 当前 guided sequence 在一个严格同任务比较中显著减少了达到 2% empirical regret 的 reconstructed engine cost。 +- Simulator 的边际计算便宜，但 rank error 会转化成显著 real regret 或更多 real-final 成本。 + +当前不能说： + +- telemetry 已经对 end-to-end tuning 提供独立增益；现有 direct pilot 对此为 negative。 +- 当前 harness 的 heuristic action ranking 是系统贡献；5% endpoint 只省 5.85%。 +- LLM 是必要组件；尚无同 policy 的 with/without LLM held-out ablation。 +- simulator 总 tuning cost 是 0；profile GPU cost 未审计，real verification 不能忽略。 +- 3.35 是 global oracle，或 dash0 与 dash1 数字是完全 controlled comparison。 + +## Data sanity + +- Dash0 sequential numeric scores：n=9，min/max `1.1042/3.35`，distinct=7；两组 config outcome 不全相同。 +- Exact surface scores：n=12，min/max `1.2833/3.2833`，distinct=8；12 cells 完整且与 simulator metrics 中的 real scores 一致。 +- Reconstructed trial/cell attempts 包括 4 个无 engine timestamp 的失败：n=32，min/max `0/0.49778 H20h`，distinct=26；所有可重建成本均非负。 +- Sequential regret observations：n=16，min/max `0/0.34328`，distinct=6，全部在 `[0,1]`。 +- Checked invariants：dash0 fixed task contexts 相同（除 method/port）；trial counts 与 manifest 相符；engine log timestamps monotonic；surface cell 唯一且 MBT=8192；simulator 无失败且 predictions 不全相同；scores/results 不全相同；cost 非负；regret bounded。 +- Measurement limitation：primary 12-cell campaign 的 4 个 TP4 pre-ready failure 没有 engine timestamp，随后由 companion campaign 完整重跑；其失败成本在 engine-lifetime reconstruction 中为 0。因此 `3.5953 H20h` 是 completed annotation lower bound，不能作为 all-in annotation cost。这个缺口已显式保留，没有在其上建立 total-cost claim。 diff --git a/runs/tuning-cost/analyze.py b/runs/tuning-cost/analyze.py new file mode 100644 index 0000000..be3003a --- /dev/null +++ b/runs/tuning-cost/analyze.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 +"""Reconstruct tuning cost and cost-to-oracle curves from existing runs. + +The historical engine logs do not contain controller setup/cleanup timestamps. +Consequently the reported GPU cost is an engine-lifetime lower bound: +parallel_size * (last engine timestamp - first engine timestamp). It must not +be presented as all-in method cost. New experiments should record allocation +start/end directly. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import re +import subprocess +from datetime import datetime +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit + + +SCHEMA = "aituner-tuning-cost-analysis-v1" +TIMESTAMP = re.compile(r"\b(\d{2}-\d{2} \d{2}:\d{2}:\d{2})(?:\.\d+)?\b") + + +def numeric_summary(values: list[float]) -> dict[str, Any]: + values = [float(value) for value in values] + return { + "n": len(values), + "min": min(values) if values else None, + "max": max(values) if values else None, + "distinct_n": len(set(values)), + } + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +class Reader: + def __init__(self, repo_root: Path): + self.repo_root = repo_root + + def _split(self, locator: str) -> tuple[str | None, str]: + if locator.startswith("ssh://"): + parsed = urlsplit(locator) + if not parsed.hostname or not parsed.path.startswith("/"): + raise ValueError(f"invalid SSH locator: {locator}") + return parsed.hostname, parsed.path + path = Path(locator) + if not path.is_absolute(): + path = self.repo_root / path + return None, str(path) + + def read_text(self, locator: str) -> str: + host, path = self._split(locator) + if host is None: + return Path(path).read_text(encoding="utf-8", errors="replace") + completed = subprocess.run( + ["ssh", host, "cat", "--", path], + check=True, + capture_output=True, + text=True, + ) + return completed.stdout + +def join_locator(root: str, *parts: str) -> str: + return "/".join([root.rstrip("/"), *(part.strip("/") for part in parts)]) + + +def timestamp_span(log_text: str, year: int) -> tuple[float | None, bool, int]: + parsed = [ + datetime.strptime(f"{year}-{match}", "%Y-%m-%d %H:%M:%S") + for match in TIMESTAMP.findall(log_text) + ] + if not parsed: + return None, True, 0 + monotonic = all(right >= left for left, right in zip(parsed, parsed[1:])) + return (max(parsed) - min(parsed)).total_seconds(), monotonic, len(parsed) + + +def load_campaign( + reader: Reader, + root: str, + year: int, + missing_duration_s: dict[str, float] | None = None, + missing_reason: dict[str, str] | None = None, +) -> dict[str, Any]: + missing_duration_s = missing_duration_s or {} + missing_reason = missing_reason or {} + state_text = reader.read_text(join_locator(root, "state.json")) + state = json.loads(state_text) + trials = [] + for trial in state["trials"]: + trial_id = str(trial["trial_id"]) + log_text = reader.read_text(join_locator(root, "trials", trial_id, "engine.log")) + duration_s, monotonic, timestamp_n = timestamp_span(log_text, year) + duration_source = "engine_log_span" + duration_note = "" + if duration_s is None: + duration_s = float(missing_duration_s.get(trial_id, 0.0)) + duration_source = ( + "conservative_fallback" if trial_id in missing_duration_s else "no_engine_timestamp" + ) + duration_note = missing_reason.get(trial_id, "") + parallel_size = int(trial["parallel_size"]) + score = trial.get("best_request_rate_per_gpu") + trials.append( + { + "trial_id": trial_id, + "status": trial["status"], + "failure_stage": trial.get("failure_stage", ""), + "parallel_size": parallel_size, + "score_req_s_per_gpu": None if score is None else float(score), + "config_patch": trial["config_patch"], + "duration_s": duration_s, + "duration_source": duration_source, + "duration_note": duration_note, + "engine_timestamp_n": timestamp_n, + "engine_timestamps_monotonic": monotonic, + "engine_h20_hours_lower_bound": duration_s * parallel_size / 3600.0, + "engine_log_sha256": sha256_text(log_text), + } + ) + return { + "root": root, + "state_sha256": sha256_text(state_text), + "trials": trials, + } + + +def fixed_task_context(spec: dict[str, Any]) -> dict[str, Any]: + flags = dict(spec["engine"]["base_flags"]) + flags.pop("port", None) + return { + "model": spec["model"], + "hardware": spec["hardware"], + "engine_version": spec["engine"]["engine_version"], + "launch_args": spec["engine"]["launch_args"], + "base_flags_without_port": flags, + "search": spec["search"], + "slo": spec["slo"], + "trace": spec["trace"], + } + + +def regret(score: float, reference: float) -> float: + if reference <= 0: + raise ValueError("reference score must be positive") + return max(0.0, 1.0 - float(score) / float(reference)) + + +def sequential_curve( + trials: list[dict[str, Any]], reference: float, thresholds: list[float] +) -> dict[str, Any]: + cumulative_cost = 0.0 + best_score: float | None = None + points = [] + for trial in trials: + cumulative_cost += float(trial["engine_h20_hours_lower_bound"]) + score = trial["score_req_s_per_gpu"] + if score is not None: + best_score = score if best_score is None else max(best_score, score) + points.append( + { + "trial_id": trial["trial_id"], + "cumulative_engine_h20_hours_lower_bound": cumulative_cost, + "best_score_req_s_per_gpu": best_score, + "regret": None if best_score is None else regret(best_score, reference), + } + ) + hits = {} + for threshold in thresholds: + hit = next( + ( + point + for point in points + if point["regret"] is not None + and float(point["regret"]) <= float(threshold) + 1e-12 + ), + None, + ) + hits[f"regret_le_{threshold:g}"] = hit + return { + "reference_score_req_s_per_gpu": reference, + "points": points, + "cost_to_threshold": hits, + "total_engine_h20_hours_lower_bound": cumulative_cost, + } + + +def surface_cell(trial: dict[str, Any]) -> str: + flags = trial["config_patch"]["flag_patch"] + return f"tp{int(flags['tensor-parallel-size'])}_mns{int(flags['max-num-seqs'])}" + + +def tie_expanded_candidates(scores: dict[str, float], nominal_k: int) -> list[str]: + ordered = sorted(scores, key=lambda cell: (-float(scores[cell]), cell)) + if nominal_k <= 0 or nominal_k > len(ordered): + raise ValueError("nominal k outside score surface") + cutoff = float(scores[ordered[nominal_k - 1]]) + tolerance = max(1e-12, abs(cutoff) * 1e-12) + return [cell for cell in ordered if float(scores[cell]) >= cutoff - tolerance] + + +def real_final_policy( + candidates: list[str], real_scores: dict[str, float], cell_costs: dict[str, float] +) -> dict[str, Any]: + oracle = max(real_scores.values()) + selected = max(candidates, key=lambda cell: (real_scores[cell], cell)) + return { + "candidate_cells": candidates, + "real_evaluations": len(candidates), + "selected_cell": selected, + "selected_real_score_req_s_per_gpu": real_scores[selected], + "real_regret": regret(real_scores[selected], oracle), + "engine_h20_hours_lower_bound": sum(cell_costs[cell] for cell in candidates), + } + + +def percentage_saving(new: float, old: float) -> float: + if old <= 0: + raise ValueError("baseline cost must be positive") + return 1.0 - new / old + + +def build_analysis(manifest_path: Path) -> dict[str, Any]: + repo_root = manifest_path.resolve().parents[2] + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + reader = Reader(repo_root) + year = int(manifest["year"]) + + sequential = {} + task_contexts = {} + for name, run in manifest["sequential_runs"].items(): + sequential[name] = load_campaign( + reader, + run["root"], + year, + run.get("missing_log_duration_s"), + run.get("missing_log_reason"), + ) + spec_text = reader.read_text(join_locator(run["root"], "study_spec.snapshot.json")) + sequential[name]["study_spec_sha256"] = sha256_text(spec_text) + task_contexts[name] = fixed_task_context(json.loads(spec_text)) + + all_sequential_scores = [ + trial["score_req_s_per_gpu"] + for campaign in sequential.values() + for trial in campaign["trials"] + if trial["score_req_s_per_gpu"] is not None + ] + empirical_reference = max(all_sequential_scores) + thresholds = [float(value) for value in manifest["threshold_regrets"]] + for campaign in sequential.values(): + campaign["curve"] = sequential_curve( + campaign["trials"], empirical_reference, thresholds + ) + + surface_manifest = manifest["real_surface"] + primary = load_campaign(reader, surface_manifest["primary_root"], year) + companion = load_campaign(reader, surface_manifest["tp4_companion_root"], year) + completed_surface_trials = [ + trial + for trial in primary["trials"] + companion["trials"] + if trial["status"] == "completed" and trial["score_req_s_per_gpu"] is not None + ] + cells: dict[str, dict[str, Any]] = {} + for trial in completed_surface_trials: + flags = trial["config_patch"]["flag_patch"] + if int(flags["max-num-batched-tokens"]) != int( + surface_manifest["fixed_max_num_batched_tokens"] + ): + raise ValueError("surface MBT invariant failed") + cell = surface_cell(trial) + if cell in cells: + raise ValueError(f"duplicate completed surface cell: {cell}") + cells[cell] = trial + real_scores = {cell: float(trial["score_req_s_per_gpu"]) for cell, trial in cells.items()} + cell_costs = { + cell: float(trial["engine_h20_hours_lower_bound"]) for cell, trial in cells.items() + } + surface_oracle_score = max(real_scores.values()) + surface_oracle_cells = [ + cell for cell, score in real_scores.items() if math.isclose(score, surface_oracle_score) + ] + + simulator_text = reader.read_text(manifest["simulator_metrics"]) + simulator = json.loads(simulator_text) + simulator_real_scores = { + cell: float(score) for cell, score in simulator["real_scores"].items() + } + surface_matches_simulator = set(real_scores) == set(simulator_real_scores) and all( + math.isclose(real_scores[cell], simulator_real_scores[cell], abs_tol=1e-12) + for cell in real_scores + ) + throughput = simulator["analyses"]["frozen-calibrated/throughput-proxy"] + throughput_scores = { + cell: float(score) for cell, score in throughput["simulated_scores"].items() + } + throughput_real_final = { + f"nominal_k_{k}": real_final_policy( + tie_expanded_candidates(throughput_scores, k), real_scores, cell_costs + ) + for k in (1, 2, 3) + } + throughput_top1 = tie_expanded_candidates(throughput_scores, 1) + simulator_only_cell = throughput_top1[0] + slo = simulator["analyses"]["frozen-calibrated/SLO-gated"] + slo_top_bucket = list(slo["metrics"]["top1"]["candidate_cells"]) + slo_diagnostic = real_final_policy(slo_top_bucket, real_scores, cell_costs) + + pure_hits = sequential["pure_llm"]["curve"]["cost_to_threshold"] + guided_hits = sequential["guided_harness"]["curve"]["cost_to_threshold"] + direct_comparison = {} + for threshold in (0.05, 0.02): + key = f"regret_le_{threshold:g}" + pure_hit = pure_hits[key] + guided_hit = guided_hits[key] + if pure_hit is None or guided_hit is None: + continue + pure_cost = float(pure_hit["cumulative_engine_h20_hours_lower_bound"]) + guided_cost = float(guided_hit["cumulative_engine_h20_hours_lower_bound"]) + direct_comparison[key] = { + "pure_llm_h20_hours_lower_bound": pure_cost, + "guided_harness_h20_hours_lower_bound": guided_cost, + "guided_saving_vs_pure_llm": percentage_saving(guided_cost, pure_cost), + } + + five_cost = direct_comparison["regret_le_0.05"][ + "guided_harness_h20_hours_lower_bound" + ] + two_cost = direct_comparison["regret_le_0.02"][ + "guided_harness_h20_hours_lower_bound" + ] + sim_real_cost = float(slo_diagnostic["engine_h20_hours_lower_bound"]) + target_bars = { + "five_percent_regret": { + "twenty_percent_below_current_guided": 0.8 * five_cost, + "thirty_percent_below_posthoc_sim_slo_real_final": 0.7 * sim_real_cost, + "development_target_h20_hours_lower_bound": min( + 0.8 * five_cost, 0.7 * sim_real_cost + ), + }, + "two_percent_regret": { + "twenty_percent_below_current_guided": 0.8 * two_cost, + "thirty_percent_below_posthoc_sim_slo_real_final": 0.7 * sim_real_cost, + "development_target_h20_hours_lower_bound": min( + 0.8 * two_cost, 0.7 * sim_real_cost + ), + }, + } + + all_trial_costs = [ + float(trial["engine_h20_hours_lower_bound"]) + for campaign in sequential.values() + for trial in campaign["trials"] + ] + [ + float(trial["engine_h20_hours_lower_bound"]) + for campaign in (primary, companion) + for trial in campaign["trials"] + ] + all_regrets = [ + float(point["regret"]) + for campaign in sequential.values() + for point in campaign["curve"]["points"] + if point["regret"] is not None + ] + monotonic_logs = all( + trial["engine_timestamps_monotonic"] + for campaign in [*sequential.values(), primary, companion] + for trial in campaign["trials"] + ) + invariants = { + "dash0_task_contexts_equal_except_method_and_port": len( + {json.dumps(value, sort_keys=True) for value in task_contexts.values()} + ) + == 1, + "surface_has_expected_cell_count": len(cells) + == int(surface_manifest["expected_cells"]), + "surface_matches_simulator_real_scores": surface_matches_simulator, + "all_costs_non_negative": all(value >= 0 for value in all_trial_costs), + "all_regrets_in_0_1": all(0 <= value <= 1 for value in all_regrets), + "surface_scores_not_all_identical": len(set(real_scores.values())) > 1, + "sequential_scores_not_all_identical": len(set(all_sequential_scores)) > 1, + "engine_log_timestamps_monotonic": monotonic_logs, + "sequential_trial_counts_match_manifest": all( + len(sequential[name]["trials"]) == int(run["expected_trials"]) + for name, run in manifest["sequential_runs"].items() + ), + "simulator_suite_has_no_failed_runs": int(simulator["execution"]["failed_runs"]) + == 0, + "simulator_scores_not_all_identical": len(set(throughput_scores.values())) > 1, + } + failed_invariants = [name for name, passed in invariants.items() if not passed] + if failed_invariants: + raise RuntimeError(f"data sanity invariant failed: {failed_invariants}") + + failed_primary_attempts = [ + trial for trial in primary["trials"] if trial["status"] != "completed" + ] + return { + "schema": SCHEMA, + "cost_definition": { + "reported_metric": "engine H20-hours lower bound", + "formula": "parallel_size * (last_engine_log_timestamp - first_engine_log_timestamp) / 3600", + "included": ["engine startup after first timestamp", "warm-up/probes until last timestamp"], + "not_reconstructable": [ + "GPU allocation before first engine timestamp", + "controller/LLM latency", + "cleanup after last engine timestamp", + "one-time simulator operator profiling GPU-hours", + ], + "future_all_in_metric": "allocation_start_to_GPU_idle * allocated_GPU_count, including failures", + }, + "comparison_scope": { + "pure_llm_vs_guided_harness": "direct: same dash0 fixed task context", + "simulator_vs_surface": "direct: simulator predictions and exact dash1 12-cell real surface", + "dash0_methods_vs_dash1_simulator": "indicative only: matched model/engine/workload/GPU type, different host and campaign", + }, + "empirical_reference": { + "score_req_s_per_gpu": empirical_reference, + "meaning": "best observed across the two dash0 sequential runs; not a global oracle", + }, + "sequential_runs": sequential, + "direct_dash0_comparison": direct_comparison, + "real_surface": { + "cells": { + cell: { + "score_req_s_per_gpu": real_scores[cell], + "engine_h20_hours_lower_bound": cell_costs[cell], + } + for cell in sorted(cells) + }, + "oracle_score_req_s_per_gpu": surface_oracle_score, + "oracle_cells": surface_oracle_cells, + "completed_annotation_engine_h20_hours_lower_bound": sum(cell_costs.values()), + "failed_primary_attempt_n": len(failed_primary_attempts), + "failed_primary_attempt_engine_h20_hours_lower_bound": sum( + float(trial["engine_h20_hours_lower_bound"]) + for trial in failed_primary_attempts + ), + "primary_state_sha256": primary["state_sha256"], + "tp4_companion_state_sha256": companion["state_sha256"], + }, + "simulator": { + "marginal_gpu_hours_without_real_verification": 0.0, + "observed_fidelity_suite_cpu_hours": float( + simulator["execution"]["suite_elapsed_seconds"] + ) + / 3600.0, + "observed_fidelity_suite_runs": int(simulator["execution"]["attempted_runs"]), + "one_time_profile_gpu_hours": None, + "one_time_profile_cost_status": "not recorded; total cold-start cost is unknown", + "decision_bearing_throughput_proxy_sim_only_top1": { + "selected_cell": simulator_only_cell, + "selected_real_score_req_s_per_gpu": real_scores[simulator_only_cell], + "real_regret": regret(real_scores[simulator_only_cell], surface_oracle_score), + "gpu_hours": 0.0, + }, + "decision_bearing_throughput_proxy_plus_real_final": throughput_real_final, + "posthoc_slo_gated_plus_real_final": { + **slo_diagnostic, + "status": "diagnostic/post-hoc, not a preregistered prospective policy", + "false_feasible": int(slo["false_feasibility"]["overall"]["false_feasible"]), + "false_infeasible": int(slo["false_feasibility"]["overall"]["false_infeasible"]), + }, + "metrics_sha256": sha256_text(simulator_text), + }, + "provisional_development_targets": { + "status": "lower-bound, single-task targets; require same-host prospective validation", + **target_bars, + }, + "data_sanity": { + "invariants": invariants, + "sequential_score_summary": numeric_summary(all_sequential_scores), + "surface_score_summary": numeric_summary(list(real_scores.values())), + "simulator_throughput_score_summary": numeric_summary( + list(throughput_scores.values()) + ), + "trial_cost_summary": numeric_summary(all_trial_costs), + "regret_summary": numeric_summary(all_regrets), + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--manifest", type=Path, default=Path(__file__).with_name("manifest.json")) + parser.add_argument("--output", type=Path, default=Path(__file__).with_name("metrics.json")) + args = parser.parse_args() + analysis = build_analysis(args.manifest) + args.output.write_text(json.dumps(analysis, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({ + "status": "ok", + "output": str(args.output), + "empirical_reference": analysis["empirical_reference"], + "surface_oracle": analysis["real_surface"]["oracle_score_req_s_per_gpu"], + "sanity": analysis["data_sanity"], + }, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/runs/tuning-cost/manifest.json b/runs/tuning-cost/manifest.json new file mode 100644 index 0000000..fabfb53 --- /dev/null +++ b/runs/tuning-cost/manifest.json @@ -0,0 +1,42 @@ +{ + "schema": "aituner-tuning-cost-v1", + "year": 2026, + "task": { + "model": "Qwen/Qwen3-30B-A3B", + "engine": "community-vLLM 0.20.0", + "gpu": "NVIDIA H20", + "trace_window": "chat_w20260311_1000", + "input_tokens": [0, 8192], + "output_tokens": 128, + "replay_time_scale": 0.1, + "target_pass_rate": 0.95, + "ttft_ms": [2000, 4000, 6000], + "tpot_ms": 50 + }, + "sequential_runs": { + "pure_llm": { + "root": "ssh://dash0/home/admin/cpfs/wjh/aituner/aituner/.aituner-community-vllm020/dash0-qwen30b-a3b-community-vllm020-chat-0-8k-out128-scale01-high1-noharness", + "expected_trials": 12, + "missing_log_duration_s": { + "trial-0003": 13.0 + }, + "missing_log_reason": { + "trial-0003": "conservative trial_spec-to-result mtime envelope for a pre-ready CLI failure" + } + }, + "guided_harness": { + "root": "ssh://dash0/home/admin/cpfs/wjh/aituner/aituner/.aituner-community-vllm020/dash0-qwen30b-a3b-community-vllm020-chat-0-8k-out128-scale01-high1-harness-guided-v2", + "expected_trials": 4, + "missing_log_duration_s": {}, + "missing_log_reason": {} + } + }, + "real_surface": { + "primary_root": "recovered-stores/aituner-interaction-runs-dash1-20260710/interaction-mixed-qwen30b-tp-mns-surface-high1-dash1-d8899c5-20260701T095858Z/store/interaction-mixed-qwen30b-tp-mns-surface-high1-dash1-d8899c5-20260701T095858Z", + "tp4_companion_root": "recovered-stores/aituner-interaction-runs-dash1-20260710/interaction-mixed-qwen30b-tp4-mns-nocap-qps20-dash1-d8899c5-20260701T161900Z/store/interaction-mixed-qwen30b-tp4-mns-nocap-qps20-dash1-d8899c5-20260701T161900Z", + "expected_cells": 12, + "fixed_max_num_batched_tokens": 8192 + }, + "simulator_metrics": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/metrics.json", + "threshold_regrets": [0.05, 0.02, 0.01, 0.0] +} diff --git a/runs/tuning-cost/metrics.json b/runs/tuning-cost/metrics.json new file mode 100644 index 0000000..e246bea --- /dev/null +++ b/runs/tuning-cost/metrics.json @@ -0,0 +1,736 @@ +{ + "comparison_scope": { + "dash0_methods_vs_dash1_simulator": "indicative only: matched model/engine/workload/GPU type, different host and campaign", + "pure_llm_vs_guided_harness": "direct: same dash0 fixed task context", + "simulator_vs_surface": "direct: simulator predictions and exact dash1 12-cell real surface" + }, + "cost_definition": { + "formula": "parallel_size * (last_engine_log_timestamp - first_engine_log_timestamp) / 3600", + "future_all_in_metric": "allocation_start_to_GPU_idle * allocated_GPU_count, including failures", + "included": [ + "engine startup after first timestamp", + "warm-up/probes until last timestamp" + ], + "not_reconstructable": [ + "GPU allocation before first engine timestamp", + "controller/LLM latency", + "cleanup after last engine timestamp", + "one-time simulator operator profiling GPU-hours" + ], + "reported_metric": "engine H20-hours lower bound" + }, + "data_sanity": { + "invariants": { + "all_costs_non_negative": true, + "all_regrets_in_0_1": true, + "dash0_task_contexts_equal_except_method_and_port": true, + "engine_log_timestamps_monotonic": true, + "sequential_scores_not_all_identical": true, + "sequential_trial_counts_match_manifest": true, + "simulator_scores_not_all_identical": true, + "simulator_suite_has_no_failed_runs": true, + "surface_has_expected_cell_count": true, + "surface_matches_simulator_real_scores": true, + "surface_scores_not_all_identical": true + }, + "regret_summary": { + "distinct_n": 6, + "max": 0.34328358208955223, + "min": 0.0, + "n": 16 + }, + "sequential_score_summary": { + "distinct_n": 7, + "max": 3.35, + "min": 1.1041666666666667, + "n": 9 + }, + "simulator_throughput_score_summary": { + "distinct_n": 10, + "max": 4.356763578770651, + "min": 1.5449814460277083, + "n": 12 + }, + "surface_score_summary": { + "distinct_n": 8, + "max": 3.283333333333333, + "min": 1.2833333333333334, + "n": 12 + }, + "trial_cost_summary": { + "distinct_n": 26, + "max": 0.49777777777777776, + "min": 0.0, + "n": 32 + } + }, + "direct_dash0_comparison": { + "regret_le_0.02": { + "guided_harness_h20_hours_lower_bound": 0.4458333333333333, + "guided_saving_vs_pure_llm": 0.6109090909090908, + "pure_llm_h20_hours_lower_bound": 1.1458333333333333 + }, + "regret_le_0.05": { + "guided_harness_h20_hours_lower_bound": 0.26805555555555555, + "guided_saving_vs_pure_llm": 0.0585365853658536, + "pure_llm_h20_hours_lower_bound": 0.2847222222222222 + } + }, + "empirical_reference": { + "meaning": "best observed across the two dash0 sequential runs; not a global oracle", + "score_req_s_per_gpu": 3.35 + }, + "provisional_development_targets": { + "five_percent_regret": { + "development_target_h20_hours_lower_bound": 0.21444444444444444, + "thirty_percent_below_posthoc_sim_slo_real_final": 0.36088888888888887, + "twenty_percent_below_current_guided": 0.21444444444444444 + }, + "status": "lower-bound, single-task targets; require same-host prospective validation", + "two_percent_regret": { + "development_target_h20_hours_lower_bound": 0.3566666666666667, + "thirty_percent_below_posthoc_sim_slo_real_final": 0.36088888888888887, + "twenty_percent_below_current_guided": 0.3566666666666667 + } + }, + "real_surface": { + "cells": { + "tp1_mns16": { + "engine_h20_hours_lower_bound": 0.14083333333333334, + "score_req_s_per_gpu": 2.35 + }, + "tp1_mns32": { + "engine_h20_hours_lower_bound": 0.13194444444444445, + "score_req_s_per_gpu": 2.283333333333333 + }, + "tp1_mns64": { + "engine_h20_hours_lower_bound": 0.13527777777777777, + "score_req_s_per_gpu": 2.283333333333333 + }, + "tp1_mns8": { + "engine_h20_hours_lower_bound": 0.13333333333333333, + "score_req_s_per_gpu": 2.1 + }, + "tp2_mns16": { + "engine_h20_hours_lower_bound": 0.29944444444444446, + "score_req_s_per_gpu": 2.275 + }, + "tp2_mns32": { + "engine_h20_hours_lower_bound": 0.2816666666666667, + "score_req_s_per_gpu": 3.283333333333333 + }, + "tp2_mns64": { + "engine_h20_hours_lower_bound": 0.2338888888888889, + "score_req_s_per_gpu": 3.2583333333333333 + }, + "tp2_mns8": { + "engine_h20_hours_lower_bound": 0.2777777777777778, + "score_req_s_per_gpu": 2.275 + }, + "tp4_mns16": { + "engine_h20_hours_lower_bound": 0.4866666666666667, + "score_req_s_per_gpu": 2.441666666666667 + }, + "tp4_mns32": { + "engine_h20_hours_lower_bound": 0.49777777777777776, + "score_req_s_per_gpu": 2.441666666666667 + }, + "tp4_mns64": { + "engine_h20_hours_lower_bound": 0.49666666666666665, + "score_req_s_per_gpu": 2.441666666666667 + }, + "tp4_mns8": { + "engine_h20_hours_lower_bound": 0.48, + "score_req_s_per_gpu": 1.2833333333333334 + } + }, + "completed_annotation_engine_h20_hours_lower_bound": 3.5952777777777776, + "failed_primary_attempt_engine_h20_hours_lower_bound": 0.0, + "failed_primary_attempt_n": 4, + "oracle_cells": [ + "tp2_mns32" + ], + "oracle_score_req_s_per_gpu": 3.283333333333333, + "primary_state_sha256": "790bf6df9045e22c46a8abdc022390bf842d884ef13ade4b627936722b9c3366", + "tp4_companion_state_sha256": "1fc4f584a69a90fb0ff1d64f6a07a8383c368f4e306a0b7a3c19d2304042fd73" + }, + "schema": "aituner-tuning-cost-analysis-v1", + "sequential_runs": { + "guided_harness": { + "curve": { + "cost_to_threshold": { + "regret_le_0": null, + "regret_le_0.01": null, + "regret_le_0.02": { + "best_score_req_s_per_gpu": 3.283333333333333, + "cumulative_engine_h20_hours_lower_bound": 0.4458333333333333, + "regret": 0.01990049751243783, + "trial_id": "trial-0003" + }, + "regret_le_0.05": { + "best_score_req_s_per_gpu": 3.2583333333333333, + "cumulative_engine_h20_hours_lower_bound": 0.26805555555555555, + "regret": 0.027363184079602032, + "trial_id": "trial-0002" + } + }, + "points": [ + { + "best_score_req_s_per_gpu": 2.3833333333333333, + "cumulative_engine_h20_hours_lower_bound": 0.09305555555555556, + "regret": 0.2885572139303483, + "trial_id": "trial-0001" + }, + { + "best_score_req_s_per_gpu": 3.2583333333333333, + "cumulative_engine_h20_hours_lower_bound": 0.26805555555555555, + "regret": 0.027363184079602032, + "trial_id": "trial-0002" + }, + { + "best_score_req_s_per_gpu": 3.283333333333333, + "cumulative_engine_h20_hours_lower_bound": 0.4458333333333333, + "regret": 0.01990049751243783, + "trial_id": "trial-0003" + }, + { + "best_score_req_s_per_gpu": 3.3, + "cumulative_engine_h20_hours_lower_bound": 0.6230555555555555, + "regret": 0.014925373134328401, + "trial_id": "trial-0004" + } + ], + "reference_score_req_s_per_gpu": 3.35, + "total_engine_h20_hours_lower_bound": 0.6230555555555555 + }, + "root": "ssh://dash0/home/admin/cpfs/wjh/aituner/aituner/.aituner-community-vllm020/dash0-qwen30b-a3b-community-vllm020-chat-0-8k-out128-scale01-high1-harness-guided-v2", + "state_sha256": "76e439d1f20e0f9af54785005258cbe39f205ae107fc2398955e360e263c7718", + "study_spec_sha256": "03241089052f1a07dbbb2ba6100c6737635877b683d275813b3c59f2769fc2d7", + "trials": [ + { + "config_patch": { + "env_patch": {}, + "flag_patch": {} + }, + "duration_note": "", + "duration_s": 335.0, + "duration_source": "engine_log_span", + "engine_h20_hours_lower_bound": 0.09305555555555556, + "engine_log_sha256": "6d8b908085c03d4796e78679eed13737581fc7d2e18947fb72d7031a9f557c9d", + "engine_timestamp_n": 108, + "engine_timestamps_monotonic": true, + "failure_stage": "", + "parallel_size": 1, + "score_req_s_per_gpu": 2.3833333333333333, + "status": "completed", + "trial_id": "trial-0001" + }, + { + "config_patch": { + "env_patch": {}, + "flag_patch": { + "tensor-parallel-size": 2 + } + }, + "duration_note": "", + "duration_s": 315.0, + "duration_source": "engine_log_span", + "engine_h20_hours_lower_bound": 0.175, + "engine_log_sha256": "b09caadd930495c27effea57e4164e52f58c6461cbbc0f4e873a309d45ec3443", + "engine_timestamp_n": 134, + "engine_timestamps_monotonic": true, + "failure_stage": "", + "parallel_size": 2, + "score_req_s_per_gpu": 3.2583333333333333, + "status": "completed", + "trial_id": "trial-0002" + }, + { + "config_patch": { + "env_patch": {}, + "flag_patch": { + "enable-chunked-prefill": true, + "gpu-memory-utilization": 0.95, + "max-num-batched-tokens": 16384, + "tensor-parallel-size": 2 + } + }, + "duration_note": "", + "duration_s": 320.0, + "duration_source": "engine_log_span", + "engine_h20_hours_lower_bound": 0.17777777777777778, + "engine_log_sha256": "8f6f2637cff8566d39b8e6c17de71ed37a01f2aa6a4460ff36ecc9728d033c3a", + "engine_timestamp_n": 134, + "engine_timestamps_monotonic": true, + "failure_stage": "", + "parallel_size": 2, + "score_req_s_per_gpu": 3.283333333333333, + "status": "completed", + "trial_id": "trial-0003" + }, + { + "config_patch": { + "env_patch": {}, + "flag_patch": { + "enable-chunked-prefill": true, + "gpu-memory-utilization": 0.95, + "max-num-batched-tokens": 24576, + "tensor-parallel-size": 2 + } + }, + "duration_note": "", + "duration_s": 319.0, + "duration_source": "engine_log_span", + "engine_h20_hours_lower_bound": 0.17722222222222223, + "engine_log_sha256": "29b8fc588b5422ceae60456188d6373070c179610243258fa213aeead9f07a00", + "engine_timestamp_n": 137, + "engine_timestamps_monotonic": true, + "failure_stage": "", + "parallel_size": 2, + "score_req_s_per_gpu": 3.3, + "status": "completed", + "trial_id": "trial-0004" + } + ] + }, + "pure_llm": { + "curve": { + "cost_to_threshold": { + "regret_le_0": { + "best_score_req_s_per_gpu": 3.35, + "cumulative_engine_h20_hours_lower_bound": 1.3719444444444444, + "regret": 0.0, + "trial_id": "trial-0007" + }, + "regret_le_0.01": { + "best_score_req_s_per_gpu": 3.35, + "cumulative_engine_h20_hours_lower_bound": 1.3719444444444444, + "regret": 0.0, + "trial_id": "trial-0007" + }, + "regret_le_0.02": { + "best_score_req_s_per_gpu": 3.3, + "cumulative_engine_h20_hours_lower_bound": 1.1458333333333333, + "regret": 0.014925373134328401, + "trial_id": "trial-0006" + }, + "regret_le_0.05": { + "best_score_req_s_per_gpu": 3.2583333333333333, + "cumulative_engine_h20_hours_lower_bound": 0.2847222222222222, + "regret": 0.027363184079602032, + "trial_id": "trial-0002" + } + }, + "points": [ + { + "best_score_req_s_per_gpu": 2.2, + "cumulative_engine_h20_hours_lower_bound": 0.10861111111111112, + "regret": 0.34328358208955223, + "trial_id": "trial-0001" + }, + { + "best_score_req_s_per_gpu": 3.2583333333333333, + "cumulative_engine_h20_hours_lower_bound": 0.2847222222222222, + "regret": 0.027363184079602032, + "trial_id": "trial-0002" + }, + { + "best_score_req_s_per_gpu": 3.2583333333333333, + "cumulative_engine_h20_hours_lower_bound": 0.29194444444444445, + "regret": 0.027363184079602032, + "trial_id": "trial-0003" + }, + { + "best_score_req_s_per_gpu": 3.2583333333333333, + "cumulative_engine_h20_hours_lower_bound": 0.4280555555555555, + "regret": 0.027363184079602032, + "trial_id": "trial-0004" + }, + { + "best_score_req_s_per_gpu": 3.2583333333333333, + "cumulative_engine_h20_hours_lower_bound": 0.9247222222222222, + "regret": 0.027363184079602032, + "trial_id": "trial-0005" + }, + { + "best_score_req_s_per_gpu": 3.3, + "cumulative_engine_h20_hours_lower_bound": 1.1458333333333333, + "regret": 0.014925373134328401, + "trial_id": "trial-0006" + }, + { + "best_score_req_s_per_gpu": 3.35, + "cumulative_engine_h20_hours_lower_bound": 1.3719444444444444, + "regret": 0.0, + "trial_id": "trial-0007" + }, + { + "best_score_req_s_per_gpu": 3.35, + "cumulative_engine_h20_hours_lower_bound": 1.561388888888889, + "regret": 0.0, + "trial_id": "trial-0008" + }, + { + "best_score_req_s_per_gpu": 3.35, + "cumulative_engine_h20_hours_lower_bound": 1.7497222222222222, + "regret": 0.0, + "trial_id": "trial-0009" + }, + { + "best_score_req_s_per_gpu": 3.35, + "cumulative_engine_h20_hours_lower_bound": 1.9047222222222222, + "regret": 0.0, + "trial_id": "trial-0010" + }, + { + "best_score_req_s_per_gpu": 3.35, + "cumulative_engine_h20_hours_lower_bound": 2.0930555555555554, + "regret": 0.0, + "trial_id": "trial-0011" + }, + { + "best_score_req_s_per_gpu": 3.35, + "cumulative_engine_h20_hours_lower_bound": 2.2824999999999998, + "regret": 0.0, + "trial_id": "trial-0012" + } + ], + "reference_score_req_s_per_gpu": 3.35, + "total_engine_h20_hours_lower_bound": 2.2824999999999998 + }, + "root": "ssh://dash0/home/admin/cpfs/wjh/aituner/aituner/.aituner-community-vllm020/dash0-qwen30b-a3b-community-vllm020-chat-0-8k-out128-scale01-high1-noharness", + "state_sha256": "342cde3546299cd312ebc078b1c9589773b841d23c4ed4632370883be598fd4a", + "study_spec_sha256": "3238dc89477108405c09b25eee434c3c10c0f5be245d2bfa3b90509a372c088e", + "trials": [ + { + "config_patch": { + "env_patch": {}, + "flag_patch": {} + }, + "duration_note": "", + "duration_s": 391.0, + "duration_source": "engine_log_span", + "engine_h20_hours_lower_bound": 0.10861111111111112, + "engine_log_sha256": "c5caf9e9f6faa73a4a39adb71a7a6c04bcf623d5c0c91b231116bb596c8a11be", + "engine_timestamp_n": 113, + "engine_timestamps_monotonic": true, + "failure_stage": "", + "parallel_size": 1, + "score_req_s_per_gpu": 2.2, + "status": "completed", + "trial_id": "trial-0001" + }, + { + "config_patch": { + "env_patch": {}, + "flag_patch": { + "tensor-parallel-size": 2 + } + }, + "duration_note": "", + "duration_s": 317.0, + "duration_source": "engine_log_span", + "engine_h20_hours_lower_bound": 0.1761111111111111, + "engine_log_sha256": "239ff7349d7b9a1b24334fbe6e2a45a967bfa2d16380e98da4f81520d422cd0d", + "engine_timestamp_n": 134, + "engine_timestamps_monotonic": true, + "failure_stage": "", + "parallel_size": 2, + "score_req_s_per_gpu": 3.2583333333333333, + "status": "completed", + "trial_id": "trial-0002" + }, + { + "config_patch": { + "env_patch": {}, + "flag_patch": { + "enable-expert-parallel": true, + "expert-parallel-size": 2, + "tensor-parallel-size": 2 + } + }, + "duration_note": "conservative trial_spec-to-result mtime envelope for a pre-ready CLI failure", + "duration_s": 13.0, + "duration_source": "conservative_fallback", + "engine_h20_hours_lower_bound": 0.007222222222222222, + "engine_log_sha256": "4707735f33c8bf9030054eccd36b66985a3879050d80a853f93f166282414d8f", + "engine_timestamp_n": 0, + "engine_timestamps_monotonic": true, + "failure_stage": "engine_launch", + "parallel_size": 2, + "score_req_s_per_gpu": null, + "status": "failed", + "trial_id": "trial-0003" + }, + { + "config_patch": { + "env_patch": {}, + "flag_patch": { + "data-parallel-size": 2, + "tensor-parallel-size": 1 + } + }, + "duration_note": "", + "duration_s": 245.0, + "duration_source": "engine_log_span", + "engine_h20_hours_lower_bound": 0.1361111111111111, + "engine_log_sha256": "215ed31c6d227e4a6771ecb637c2b4e3e38e766d73a8882a923d1fa8b97a23da", + "engine_timestamp_n": 170, + "engine_timestamps_monotonic": true, + "failure_stage": "", + "parallel_size": 2, + "score_req_s_per_gpu": null, + "status": "completed", + "trial_id": "trial-0004" + }, + { + "config_patch": { + "env_patch": {}, + "flag_patch": { + "data-parallel-size": 2, + "tensor-parallel-size": 2 + } + }, + "duration_note": "", + "duration_s": 447.0, + "duration_source": "engine_log_span", + "engine_h20_hours_lower_bound": 0.49666666666666665, + "engine_log_sha256": "6057b3477ccca14e0ebd467ac96b777662df67b57827c851d58caf3a77b95ca4", + "engine_timestamp_n": 201, + "engine_timestamps_monotonic": true, + "failure_stage": "", + "parallel_size": 4, + "score_req_s_per_gpu": 1.1041666666666667, + "status": "completed", + "trial_id": "trial-0005" + }, + { + "config_patch": { + "env_patch": {}, + "flag_patch": { + "enable-chunked-prefill": true, + "gpu-memory-utilization": 0.95, + "max-num-batched-tokens": 16384, + "tensor-parallel-size": 2 + } + }, + "duration_note": "", + "duration_s": 398.0, + "duration_source": "engine_log_span", + "engine_h20_hours_lower_bound": 0.22111111111111112, + "engine_log_sha256": "c5c336f74d419645639b469eea9f92575780164637fdda078fa5c84c96915f1f", + "engine_timestamp_n": 138, + "engine_timestamps_monotonic": true, + "failure_stage": "", + "parallel_size": 2, + "score_req_s_per_gpu": 3.3, + "status": "completed", + "trial_id": "trial-0006" + }, + { + "config_patch": { + "env_patch": {}, + "flag_patch": { + "enable-chunked-prefill": true, + "gpu-memory-utilization": 0.95, + "max-num-batched-tokens": 24576, + "tensor-parallel-size": 2 + } + }, + "duration_note": "", + "duration_s": 407.0, + "duration_source": "engine_log_span", + "engine_h20_hours_lower_bound": 0.22611111111111112, + "engine_log_sha256": "5e2829479ecd4771e7c781d40209f74e4c2843bbca15522974e435e06daa12a2", + "engine_timestamp_n": 142, + "engine_timestamps_monotonic": true, + "failure_stage": "", + "parallel_size": 2, + "score_req_s_per_gpu": 3.35, + "status": "completed", + "trial_id": "trial-0007" + }, + { + "config_patch": { + "env_patch": {}, + "flag_patch": { + "enable-chunked-prefill": true, + "gpu-memory-utilization": 0.95, + "max-num-batched-tokens": 32768, + "tensor-parallel-size": 2 + } + }, + "duration_note": "", + "duration_s": 341.0, + "duration_source": "engine_log_span", + "engine_h20_hours_lower_bound": 0.18944444444444444, + "engine_log_sha256": "fde628968fba10fccfe5b09675aa57cf725f507be83c10dd3c0b6c109b5d72eb", + "engine_timestamp_n": 136, + "engine_timestamps_monotonic": true, + "failure_stage": "", + "parallel_size": 2, + "score_req_s_per_gpu": null, + "status": "completed", + "trial_id": "trial-0008" + }, + { + "config_patch": { + "env_patch": {}, + "flag_patch": { + "enable-chunked-prefill": true, + "gpu-memory-utilization": 0.95, + "max-num-batched-tokens": 28672, + "tensor-parallel-size": 2 + } + }, + "duration_note": "", + "duration_s": 339.0, + "duration_source": "engine_log_span", + "engine_h20_hours_lower_bound": 0.18833333333333332, + "engine_log_sha256": "fc60765578d57af08e0310983ff42e2a200739a90f11c45f9d4b1f6b7b789cc8", + "engine_timestamp_n": 136, + "engine_timestamps_monotonic": true, + "failure_stage": "", + "parallel_size": 2, + "score_req_s_per_gpu": null, + "status": "completed", + "trial_id": "trial-0009" + }, + { + "config_patch": { + "env_patch": {}, + "flag_patch": { + "enable-chunked-prefill": true, + "enable-prefix-caching": true, + "gpu-memory-utilization": 0.95, + "max-num-batched-tokens": 24576, + "tensor-parallel-size": 2 + } + }, + "duration_note": "", + "duration_s": 279.0, + "duration_source": "engine_log_span", + "engine_h20_hours_lower_bound": 0.155, + "engine_log_sha256": "22c473b39662df6c8c4c64a616346652ab889d63256efab18257b1f4445ce13b", + "engine_timestamp_n": 133, + "engine_timestamps_monotonic": true, + "failure_stage": "", + "parallel_size": 2, + "score_req_s_per_gpu": null, + "status": "completed", + "trial_id": "trial-0010" + }, + { + "config_patch": { + "env_patch": {}, + "flag_patch": { + "enable-chunked-prefill": true, + "gpu-memory-utilization": 0.95, + "max-num-batched-tokens": 25600, + "tensor-parallel-size": 2 + } + }, + "duration_note": "", + "duration_s": 339.0, + "duration_source": "engine_log_span", + "engine_h20_hours_lower_bound": 0.18833333333333332, + "engine_log_sha256": "19b19de4a0a11a5a112b3e8ef04c3ca4d90d6c6c9097b5180c024d8570c9a608", + "engine_timestamp_n": 136, + "engine_timestamps_monotonic": true, + "failure_stage": "", + "parallel_size": 2, + "score_req_s_per_gpu": null, + "status": "completed", + "trial_id": "trial-0011" + }, + { + "config_patch": { + "env_patch": {}, + "flag_patch": { + "enable-chunked-prefill": true, + "gpu-memory-utilization": 0.95, + "max-num-batched-tokens": 25088, + "tensor-parallel-size": 2 + } + }, + "duration_note": "", + "duration_s": 341.0, + "duration_source": "engine_log_span", + "engine_h20_hours_lower_bound": 0.18944444444444444, + "engine_log_sha256": "40f7307d6677f2bb55c0132d42fc82fce78ffa6a558d5c69b01ec60d1e40b3ae", + "engine_timestamp_n": 136, + "engine_timestamps_monotonic": true, + "failure_stage": "", + "parallel_size": 2, + "score_req_s_per_gpu": null, + "status": "completed", + "trial_id": "trial-0012" + } + ] + } + }, + "simulator": { + "decision_bearing_throughput_proxy_plus_real_final": { + "nominal_k_1": { + "candidate_cells": [ + "tp1_mns64" + ], + "engine_h20_hours_lower_bound": 0.13527777777777777, + "real_evaluations": 1, + "real_regret": 0.30456852791878175, + "selected_cell": "tp1_mns64", + "selected_real_score_req_s_per_gpu": 2.283333333333333 + }, + "nominal_k_2": { + "candidate_cells": [ + "tp1_mns64", + "tp1_mns32" + ], + "engine_h20_hours_lower_bound": 0.26722222222222225, + "real_evaluations": 2, + "real_regret": 0.30456852791878175, + "selected_cell": "tp1_mns64", + "selected_real_score_req_s_per_gpu": 2.283333333333333 + }, + "nominal_k_3": { + "candidate_cells": [ + "tp1_mns64", + "tp1_mns32", + "tp2_mns32", + "tp2_mns64" + ], + "engine_h20_hours_lower_bound": 0.7827777777777778, + "real_evaluations": 4, + "real_regret": 0.0, + "selected_cell": "tp2_mns32", + "selected_real_score_req_s_per_gpu": 3.283333333333333 + } + }, + "decision_bearing_throughput_proxy_sim_only_top1": { + "gpu_hours": 0.0, + "real_regret": 0.30456852791878175, + "selected_cell": "tp1_mns64", + "selected_real_score_req_s_per_gpu": 2.283333333333333 + }, + "marginal_gpu_hours_without_real_verification": 0.0, + "metrics_sha256": "55edb37d5692e979ab6f6dc6c65913a9db0aa0a836c350e4c05d9c38eee78206", + "observed_fidelity_suite_cpu_hours": 2.055026211017717, + "observed_fidelity_suite_runs": 184, + "one_time_profile_cost_status": "not recorded; total cold-start cost is unknown", + "one_time_profile_gpu_hours": null, + "posthoc_slo_gated_plus_real_final": { + "candidate_cells": [ + "tp2_mns32", + "tp2_mns64" + ], + "engine_h20_hours_lower_bound": 0.5155555555555555, + "false_feasible": 21, + "false_infeasible": 7, + "real_evaluations": 2, + "real_regret": 0.0, + "selected_cell": "tp2_mns32", + "selected_real_score_req_s_per_gpu": 3.283333333333333, + "status": "diagnostic/post-hoc, not a preregistered prospective policy" + } + } +} diff --git a/runs/tuning-cost/test_analysis.py b/runs/tuning-cost/test_analysis.py new file mode 100644 index 0000000..ca698c7 --- /dev/null +++ b/runs/tuning-cost/test_analysis.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import importlib.util +import math +import sys +from pathlib import Path + + +HERE = Path(__file__).resolve().parent + + +def load_analysis(): + spec = importlib.util.spec_from_file_location("tuning_cost", HERE / "analyze.py") + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def main() -> None: + analysis = load_analysis() + duration, monotonic, count = analysis.timestamp_span( + "INFO 07-01 10:00:00 x\nINFO 07-01 10:05:30 y\n", 2026 + ) + assert duration == 330.0 + assert monotonic + assert count == 2 + + trials = [ + {"trial_id": "t1", "engine_h20_hours_lower_bound": 0.1, "score_req_s_per_gpu": 8.0}, + {"trial_id": "t2", "engine_h20_hours_lower_bound": 0.2, "score_req_s_per_gpu": None}, + {"trial_id": "t3", "engine_h20_hours_lower_bound": 0.3, "score_req_s_per_gpu": 9.6}, + ] + curve = analysis.sequential_curve(trials, 10.0, [0.05, 0.04]) + assert curve["cost_to_threshold"]["regret_le_0.05"]["trial_id"] == "t3" + assert curve["cost_to_threshold"]["regret_le_0.04"]["trial_id"] == "t3" + assert math.isclose(curve["total_engine_h20_hours_lower_bound"], 0.6) + + candidates = analysis.tie_expanded_candidates( + {"a": 3.0, "b": 2.0, "c": 2.0, "d": 1.0}, 2 + ) + assert candidates == ["a", "b", "c"] + policy = analysis.real_final_policy( + candidates, + {"a": 1.0, "b": 4.0, "c": 2.0, "d": 3.0}, + {"a": 0.1, "b": 0.2, "c": 0.3, "d": 0.4}, + ) + assert policy["selected_cell"] == "b" + assert policy["real_regret"] == 0.0 + assert math.isclose(policy["engine_h20_hours_lower_bound"], 0.6) + print("tuning cost analysis: PASS") + + +if __name__ == "__main__": + main() From d229f2a85eac9dc9a262c530149358151a00d4d3 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 01:42:37 +0800 Subject: [PATCH 028/148] Add action-conditioned intervention feasibility model --- .../extract_training.py | 324 ++++++++++++++++++ runs/active-intervention-v0/model.py | 287 ++++++++++++++++ runs/active-intervention-v0/test_model.py | 91 +++++ runs/active-intervention-v0/test_pipeline.py | 196 +++++++++++ runs/active-intervention-v0/train_policy.py | 318 +++++++++++++++++ 5 files changed, 1216 insertions(+) create mode 100644 runs/active-intervention-v0/extract_training.py create mode 100644 runs/active-intervention-v0/model.py create mode 100644 runs/active-intervention-v0/test_model.py create mode 100644 runs/active-intervention-v0/test_pipeline.py create mode 100644 runs/active-intervention-v0/train_policy.py diff --git a/runs/active-intervention-v0/extract_training.py b/runs/active-intervention-v0/extract_training.py new file mode 100644 index 0000000..8f7c8bf --- /dev/null +++ b/runs/active-intervention-v0/extract_training.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 +"""Extract paired source/action examples from the accepted action-aware run.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import sys +from pathlib import Path +from statistics import fmean +from typing import Any, Mapping + + +PHASES = ("0.25", "0.50", "0.75", "1.00") +HERE = Path(__file__).resolve().parent +COMMON_STATE = HERE.parent / "telemetry-residual" +sys.path.insert(0, str(COMMON_STATE)) + +from common_state import summarize_engine # noqa: E402 + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def atomic_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + os.replace(temporary, path) + + +def load_jsonl(path: Path) -> list[dict[str, Any]]: + records = [] + with path.open(encoding="utf-8") as source: + for line_number, line in enumerate(source, 1): + if not line.strip(): + continue + try: + records.append(json.loads(line)) + except json.JSONDecodeError as error: + raise ValueError(f"{path}:{line_number}: invalid JSON") from error + if not records: + raise ValueError(f"{path}: no request records") + return records + + +def prefix_outcome( + requests: list[Mapping[str, Any]], *, cutoff_s: float, offered_total: float +) -> dict[str, float]: + admitted = [request for request in requests if float(request["arrival_s"]) <= cutoff_s] + completed = [ + request + for request in requests + if request.get("completed_elapsed_s") is not None + and float(request["completed_elapsed_s"]) <= cutoff_s + ] + if not admitted: + raise ValueError("prefix has no admitted requests") + admitted_ids = {str(request["request_id"]) for request in admitted} + if any(str(request["request_id"]) not in admitted_ids for request in completed): + raise ValueError("prefix completion precedes admission") + passed = sum(bool(request["slo_pass"]) for request in completed) + ttft = [float(request["ttft_ms"]) for request in completed] + tpot = [float(request["tpot_ms"]) for request in completed] + total = len(requests) + return { + "normalized_slo_goodput": passed / cutoff_s / offered_total, + "admitted_fraction": len(admitted) / total, + "completed_over_admitted": len(completed) / len(admitted), + "completed_pass_rate": passed / max(1, len(completed)), + "completed_fail_fraction_of_total": (len(completed) - passed) / total, + "outstanding_over_admitted": (len(admitted) - len(completed)) / len(admitted), + "ttft_max_over_slo_max": max(ttft, default=0.0) / 6000.0, + "ttft_mean_over_slo_max": fmean(ttft) / 6000.0 if ttft else 0.0, + "tpot_max_over_slo": max(tpot, default=0.0) / 50.0, + "tpot_mean_over_slo": fmean(tpot) / 50.0 if tpot else 0.0, + "admitted_input_tokens_mean_over_limit": fmean( + float(request["raw_input_tokens"]) for request in admitted + ) + / 8192.0, + } + + +def telemetry_record(state: Mapping[str, Any]) -> dict[str, float]: + common = state["common"] + engine = state["engine_only"] + executed_steps = int(state["sanity"]["executed_steps"]) + if executed_steps <= 0: + raise ValueError("telemetry phase contains no executed engine steps") + return { + "scheduler_steps_per_s": float(common["scheduler_steps_per_s"]), + "batch_size_mean": float(common["batch_size"]["mean"]), + "batch_size_cv": float(common["batch_size"]["cv"]), + "batch_tokens_mean": float(common["batch_tokens"]["mean"]), + "batch_tokens_cv": float(common["batch_tokens"]["cv"]), + "decode_batch_size_mean": float(common["decode_batch_size"]["mean"]), + "decode_batch_size_cv": float(common["decode_batch_size"]["cv"]), + "prefill_token_fraction": float(common["prefill_token_fraction"]), + "queue_waiting_mean": float(common["queue_waiting_mean"]), + "queue_running_mean": float(common["queue_running_mean"]), + "preemptions_per_step": float(common["preemptions"]) / executed_steps, + "kv_usage_mean": float(engine["kv_usage_mean"]), + "kv_usage_max": float(engine["kv_usage_max"]), + "kv_usage_end_minus_start": float(engine["kv_usage_end_minus_start"]), + "graph_none_share": float(engine["graph_none_share"]), + "graph_full_share": float(engine["graph_full_share"]), + "graph_padding_fraction": float(engine["graph_padding_fraction"]), + } + + +def load_stream(path: Path, *, expected_sha256: str) -> list[dict[str, Any]]: + if sha256_file(path) != expected_sha256: + raise ValueError(f"engine stream hash mismatch: {path}") + decoded = load_jsonl(path) + records = [row for row in decoded if "step_index" in row] + if not records: + raise ValueError(f"engine stream has no Layer-1 records: {path}") + return records + + +def build_dataset( + *, audit_path: Path, manifest_path: Path, run_root: Path +) -> dict[str, Any]: + audit = json.loads(audit_path.read_text(encoding="utf-8")) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if audit.get("schema") != "action-aware-constraint-pilot-audit-v0": + raise ValueError("unexpected action-aware audit schema") + if audit["sanity"]["red_flags"]: + raise ValueError(f"action-aware audit red flags: {audit['sanity']['red_flags']}") + configs = {str(item["id"]): item for item in manifest["configs"]} + runs = { + (str(run["config_id"]), int(run["repetition"])): run + for run in audit["runs"] + } + source_ids = {str(regime["source"]) for regime in manifest["regimes"].values()} + stream_entries = { + str(item["config_id"]): item + for item in audit["streams"] + if str(item["config_id"]) in source_ids + } + if set(stream_entries) != source_ids: + raise ValueError("audit is missing a source config engine stream") + streams = { + config_id: load_stream( + Path(item["stream"]), expected_sha256=str(item["stream_sha256"]) + ) + for config_id, item in stream_entries.items() + } + examples = [] + request_hashes = [] + for regime_name, regime in sorted(manifest["regimes"].items()): + source_id = str(regime["source"]) + for repetition in sorted(int(value) for value in manifest["repetitions"]): + source_run = runs[(source_id, repetition)] + source_config = configs[source_id] + request_path = run_root / "sessions" / source_id / f"rep{repetition}" / "requests.jsonl" + requests = load_jsonl(request_path) + request_hashes.append(sha256_file(request_path)) + offered_rate_per_gpu = float( + manifest["repetitions"][str(repetition)]["selection"][ + "offered_req_s_per_gpu" + ] + ) + offered_total = offered_rate_per_gpu * int(manifest["engine"]["tp"]) + source_goodput = float(source_run["outcome"]["slo_goodput_req_s"]) + source_normalized = min(1.0, source_goodput / offered_total) + decision_id = f"{regime_name}-rep{repetition}" + for phase in PHASES: + cutoff_s = float(manifest["engine"]["duration_s"]) * float(phase) + outcome = prefix_outcome( + requests, cutoff_s=cutoff_s, offered_total=offered_total + ) + admitted_count = sum( + float(request["arrival_s"]) <= cutoff_s for request in requests + ) + start_ns = int(source_run["state"]["interval"]["start_ns"]) + phase_state = summarize_engine( + streams[source_id], + start_ns=start_ns, + end_ns=start_ns + round(cutoff_s * 1e9), + request_count=admitted_count, + ) + if not all(phase_state["sanity"]["invariants"].values()): + raise ValueError( + f"engine state invariant failed: {decision_id} phase {phase}" + ) + telemetry = telemetry_record(phase_state) + actions = {"noop": source_id, **regime["actions"]} + for action_name, target_id in sorted(actions.items()): + target_run = runs[(str(target_id), repetition)] + target_config = configs[str(target_id)] + target_goodput = float(target_run["outcome"]["slo_goodput_req_s"]) + normalized = target_goodput / offered_total + if not 0.0 <= normalized <= 1.0 + 1e-12: + raise ValueError("target normalized goodput is outside [0, 1]") + examples.append( + { + "phase": phase, + "cutoff_s": cutoff_s, + "decision_id": decision_id, + "regime": regime_name, + "repetition": repetition, + "source": { + "config_id": source_id, + "mns": int(source_config["mns"]), + "mbbt": int(source_config["mbbt"]), + "offered_rate_per_gpu": offered_rate_per_gpu, + "outcome": outcome, + "telemetry": telemetry, + }, + "action": { + "id": action_name, + "target_config_id": str(target_id), + "target_mns": int(target_config["mns"]), + "target_mbbt": int(target_config["mbbt"]), + }, + "target_slo_goodput_req_s": target_goodput, + "target_normalized_goodput": min(1.0, normalized), + "source_normalized_goodput": source_normalized, + "target_delta_normalized_goodput": min(1.0, normalized) + - source_normalized, + } + ) + invariants = { + "expected_examples": len(examples) == len(PHASES) * 2 * 3 * 3, + "four_phases": sorted({example["phase"] for example in examples}) + == sorted(PHASES), + "six_decisions": len({example["decision_id"] for example in examples}) == 6, + "three_actions_per_decision_phase": all( + sum( + item["decision_id"] == decision + and item["phase"] == phase + for item in examples + ) + == 3 + for decision in {item["decision_id"] for item in examples} + for phase in PHASES + ), + "targets_not_all_identical": len( + {example["target_normalized_goodput"] for example in examples} + ) + > 1, + "bounded_prefix_ratios": all( + 0.0 <= float(value) <= 1.0 + for example in examples + for key, value in example["source"]["outcome"].items() + if key + in { + "admitted_fraction", + "completed_over_admitted", + "completed_pass_rate", + "completed_fail_fraction_of_total", + "outstanding_over_admitted", + } + ), + "direct_telemetry_without_binding_labels": all( + not any(token in key for token in ("exclusive", "unresolved", "both")) + for example in examples + for key in example["source"]["telemetry"] + ), + "treatment_effects_bounded": all( + -1.0 <= float(example["target_delta_normalized_goodput"]) <= 1.0 + for example in examples + ), + } + red_flags = [name for name, passed in invariants.items() if not passed] + if red_flags: + raise RuntimeError(f"training dataset sanity failed: {red_flags}") + return { + "schema": "active-intervention-training-v0", + "status": "VALID", + "provenance": { + "audit": str(audit_path), + "audit_sha256": sha256_file(audit_path), + "manifest": str(manifest_path), + "manifest_sha256": sha256_file(manifest_path), + "run_root": str(run_root), + "source_request_sha256": sorted(set(request_hashes)), + "source_stream_sha256": sorted( + str(item["stream_sha256"]) for item in stream_entries.values() + ), + }, + "examples": examples, + "sanity": {"invariants": invariants, "red_flags": red_flags}, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--audit", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + dataset = build_dataset( + audit_path=args.audit, + manifest_path=args.manifest, + run_root=args.run_root, + ) + atomic_json(args.output, dataset) + print( + json.dumps( + { + "status": dataset["status"], + "examples": len(dataset["examples"]), + "sanity": dataset["sanity"], + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/active-intervention-v0/model.py b/runs/active-intervention-v0/model.py new file mode 100644 index 0000000..fe815e6 --- /dev/null +++ b/runs/active-intervention-v0/model.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +"""Small-data action-response model for the active intervention pilot. + +The model predicts the paired normalized SLO-goodput treatment effect from a +source measurement and a full MNS/MBBT action. Telemetry features are direct, +continuous engine measurements; there is no diagnosis-to-action rule here. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Any, Iterable, Mapping, Sequence + +import numpy as np + + +PREFIX_FEATURES = ( + "normalized_slo_goodput", + "admitted_fraction", + "completed_over_admitted", + "completed_pass_rate", + "completed_fail_fraction_of_total", + "outstanding_over_admitted", + "ttft_max_over_slo_max", + "ttft_mean_over_slo_max", + "tpot_max_over_slo", + "tpot_mean_over_slo", + "admitted_input_tokens_mean_over_limit", +) + +TELEMETRY_FEATURES = ( + "scheduler_steps_per_s", + "batch_size_mean", + "batch_size_cv", + "batch_tokens_mean", + "batch_tokens_cv", + "decode_batch_size_mean", + "decode_batch_size_cv", + "prefill_token_fraction", + "queue_waiting_mean", + "queue_running_mean", + "preemptions_per_step", + "kv_usage_mean", + "kv_usage_max", + "kv_usage_end_minus_start", + "graph_none_share", + "graph_full_share", + "graph_padding_fraction", +) + + +def _finite(value: Any, name: str) -> float: + result = float(value) + if not math.isfinite(result): + raise ValueError(f"{name} must be finite") + return result + + +def feature_vector( + example: Mapping[str, Any], *, include_telemetry: bool +) -> tuple[list[str], np.ndarray]: + source = example["source"] + action = example["action"] + source_log_mns = math.log2(_finite(source["mns"], "source MNS")) + source_log_mbbt = math.log2(_finite(source["mbbt"], "source MBBT")) + target_log_mns = math.log2(_finite(action["target_mns"], "target MNS")) + target_log_mbbt = math.log2(_finite(action["target_mbbt"], "target MBBT")) + delta_mns = target_log_mns - source_log_mns + delta_mbbt = target_log_mbbt - source_log_mbbt + names = [ + "source_log2_mns", + "source_log2_mbbt", + "target_log2_mns", + "target_log2_mbbt", + "delta_log2_mns", + "delta_log2_mbbt", + "delta_product", + "offered_rate_per_gpu", + ] + values = [ + source_log_mns, + source_log_mbbt, + target_log_mns, + target_log_mbbt, + delta_mns, + delta_mbbt, + delta_mns * delta_mbbt, + _finite(source["offered_rate_per_gpu"], "offered rate"), + ] + for name in PREFIX_FEATURES: + names.append(f"outcome.{name}") + values.append(_finite(source["outcome"][name], name)) + if include_telemetry: + for name in TELEMETRY_FEATURES: + value = _finite(source["telemetry"][name], name) + names.extend( + ( + f"telemetry.{name}", + f"telemetry.{name}*delta_mns", + f"telemetry.{name}*delta_mbbt", + ) + ) + values.extend((value, value * delta_mns, value * delta_mbbt)) + vector = np.asarray(values, dtype=np.float64) + if not np.all(np.isfinite(vector)): + raise ValueError("feature vector contains a non-finite value") + return names, vector + + +@dataclass(frozen=True) +class RidgeModel: + feature_names: tuple[str, ...] + mean: np.ndarray + scale: np.ndarray + weights: np.ndarray + intercept: float + regularization: float + + def predict(self, values: np.ndarray) -> float: + if values.shape != self.mean.shape: + raise ValueError("ridge prediction feature shape mismatch") + normalized = (values - self.mean) / self.scale + return float(self.intercept + normalized @ self.weights) + + def to_json(self) -> dict[str, Any]: + return { + "feature_names": list(self.feature_names), + "mean": self.mean.tolist(), + "scale": self.scale.tolist(), + "weights": self.weights.tolist(), + "intercept": self.intercept, + "regularization": self.regularization, + } + + @classmethod + def from_json(cls, payload: Mapping[str, Any]) -> "RidgeModel": + return cls( + feature_names=tuple(str(value) for value in payload["feature_names"]), + mean=np.asarray(payload["mean"], dtype=np.float64), + scale=np.asarray(payload["scale"], dtype=np.float64), + weights=np.asarray(payload["weights"], dtype=np.float64), + intercept=float(payload["intercept"]), + regularization=float(payload["regularization"]), + ) + + +def fit_ridge( + examples: Sequence[Mapping[str, Any]], + *, + include_telemetry: bool, + regularization: float, +) -> RidgeModel: + if not examples: + raise ValueError("ridge fit requires examples") + if regularization <= 0: + raise ValueError("ridge regularization must be positive") + encoded = [ + feature_vector(example, include_telemetry=include_telemetry) + for example in examples + ] + names = encoded[0][0] + if any(item[0] != names for item in encoded): + raise ValueError("feature names changed across examples") + x = np.stack([item[1] for item in encoded]) + y = np.asarray( + [ + _finite(example["target_delta_normalized_goodput"], "target effect") + for example in examples + ], + dtype=np.float64, + ) + mean = x.mean(axis=0) + scale = x.std(axis=0) + scale[scale < 1e-12] = 1.0 + normalized = (x - mean) / scale + intercept = float(y.mean()) + centered = y - intercept + system = normalized.T @ normalized + regularization * np.eye(x.shape[1]) + weights = np.linalg.solve(system, normalized.T @ centered) + return RidgeModel( + feature_names=tuple(names), + mean=mean, + scale=scale, + weights=weights, + intercept=intercept, + regularization=regularization, + ) + + +def fit_jackknife_ensemble( + examples: Sequence[Mapping[str, Any]], + *, + include_telemetry: bool, + regularization: float, + group_key: str = "decision_id", +) -> list[RidgeModel]: + groups = sorted({str(example[group_key]) for example in examples}) + if len(groups) < 3: + raise ValueError("jackknife ensemble requires at least three groups") + models = [] + for held_out in groups: + training = [ + example for example in examples if str(example[group_key]) != held_out + ] + models.append( + fit_ridge( + training, + include_telemetry=include_telemetry, + regularization=regularization, + ) + ) + return models + + +def ensemble_predict( + models: Sequence[RidgeModel], + example: Mapping[str, Any], + *, + include_telemetry: bool, +) -> dict[str, float]: + if not models: + raise ValueError("ensemble prediction requires models") + source = example["source"] + action = example["action"] + if ( + int(action["target_mns"]) == int(source["mns"]) + and int(action["target_mbbt"]) == int(source["mbbt"]) + ): + return {"mean": 0.0, "std": 0.0, "min": 0.0, "max": 0.0, "distinct_n": 1} + names, values = feature_vector(example, include_telemetry=include_telemetry) + if any(model.feature_names != tuple(names) for model in models): + raise ValueError("ensemble feature schema mismatch") + raw = np.asarray([model.predict(values) for model in models], dtype=np.float64) + clipped = np.clip(raw, -1.0, 1.0) + return { + "mean": float(clipped.mean()), + "std": float(clipped.std(ddof=0)), + "min": float(clipped.min()), + "max": float(clipped.max()), + "distinct_n": len(set(float(value) for value in clipped)), + } + + +def select_action( + models: Sequence[RidgeModel], + candidates: Sequence[Mapping[str, Any]], + *, + include_telemetry: bool, + confidence_z: float = 1.0, + minimum_margin: float = 0.02, +) -> dict[str, Any]: + if len(candidates) < 2: + raise ValueError("action selection requires at least two candidates") + rows = [] + for example in candidates: + prediction = ensemble_predict( + models, example, include_telemetry=include_telemetry + ) + rows.append( + { + "action_id": str(example["action"]["id"]), + "prediction": prediction, + "lower": prediction["mean"] - confidence_z * prediction["std"], + "upper": prediction["mean"] + confidence_z * prediction["std"], + } + ) + rows.sort(key=lambda row: (-row["prediction"]["mean"], row["action_id"])) + best, second = rows[:2] + margin = float(best["prediction"]["mean"] - second["prediction"]["mean"]) + confident = bool( + margin >= minimum_margin and best["lower"] > second["upper"] + ) + return { + "selected_action": best["action_id"], + "confident": confident, + "predicted_margin": margin, + "candidates": rows, + } + + +def models_to_json(models: Iterable[RidgeModel]) -> list[dict[str, Any]]: + return [model.to_json() for model in models] + + +def models_from_json(payload: Iterable[Mapping[str, Any]]) -> list[RidgeModel]: + return [RidgeModel.from_json(item) for item in payload] diff --git a/runs/active-intervention-v0/test_model.py b/runs/active-intervention-v0/test_model.py new file mode 100644 index 0000000..c49401a --- /dev/null +++ b/runs/active-intervention-v0/test_model.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + + +HERE = Path(__file__).resolve().parent + + +def load_model(): + spec = importlib.util.spec_from_file_location( + "active_intervention_model", HERE / "model.py" + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def example(model, decision: str, action: str, pressure: float, target: float): + outcome = { + name: 0.5 for name in model.PREFIX_FEATURES + } + telemetry = {name: 0.0 for name in model.TELEMETRY_FEATURES} + telemetry["queue_waiting_mean"] = pressure + telemetry["batch_size_mean"] = pressure + return { + "decision_id": decision, + "source": { + "mns": 16, + "mbbt": 8192, + "offered_rate_per_gpu": 2.0, + "outcome": outcome, + "telemetry": telemetry, + }, + "action": { + "id": action, + "target_mns": 64 if action == "mns" else 16, + "target_mbbt": 8192 if action == "mns" else 16384, + }, + "target_normalized_goodput": target, + "target_delta_normalized_goodput": target - 0.5, + } + + +def main() -> None: + model = load_model() + examples = [] + for index, pressure in enumerate((0.2, 0.5, 0.8), 1): + examples.extend( + ( + example(model, f"d{index}", "mns", pressure, 0.5 + pressure / 2), + example(model, f"d{index}", "mbbt", pressure, 0.6 - pressure / 4), + ) + ) + fitted = model.fit_ridge( + examples, include_telemetry=True, regularization=1.0 + ) + encoded = fitted.to_json() + restored = model.RidgeModel.from_json(encoded) + names, values = model.feature_vector(examples[-2], include_telemetry=True) + assert tuple(names) == restored.feature_names + assert abs(fitted.predict(values) - restored.predict(values)) < 1e-12 + ensemble = model.fit_jackknife_ensemble( + examples, include_telemetry=True, regularization=1.0 + ) + decision = model.select_action( + ensemble, examples[-2:], include_telemetry=True, minimum_margin=0.0 + ) + assert decision["selected_action"] == "mns" + assert all(-1.0 <= row["prediction"]["mean"] <= 1.0 for row in decision["candidates"]) + noop = example(model, "noop", "noop", 0.8, 0.5) + noop["action"]["target_mbbt"] = 8192 + prediction = model.ensemble_predict( + ensemble, noop, include_telemetry=True + ) + assert prediction == { + "mean": 0.0, + "std": 0.0, + "min": 0.0, + "max": 0.0, + "distinct_n": 1, + } + print("active intervention model: PASS") + + +if __name__ == "__main__": + main() diff --git a/runs/active-intervention-v0/test_pipeline.py b/runs/active-intervention-v0/test_pipeline.py new file mode 100644 index 0000000..ee7805c --- /dev/null +++ b/runs/active-intervention-v0/test_pipeline.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import importlib.util +import json +import sys +import tempfile +from pathlib import Path + + +HERE = Path(__file__).resolve().parent + + +def load(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def write_json(path: Path, payload) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload) + "\n", encoding="utf-8") + + +def write_jsonl(path: Path, rows) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8" + ) + + +def engine_record(index: int, timestamp_ns: int) -> dict: + alternate = index % 2 + return { + "step_index": index, + "submit_mono_ns": timestamp_ns, + "model_executed": True, + "scheduled_requests": 1 + alternate, + "decode_batch_size": alternate, + "prefill_tokens": 8 + alternate, + "decode_tokens": alternate, + "preemptions": 0, + "queues": {"waiting": alternate, "running": 1 + alternate}, + "kv": {"usage": 0.1 + 0.01 * alternate}, + "cudagraph": { + "runtime_mode": "FULL" if alternate else "NONE", + "bucket_tokens": 16, + "padding_tokens": alternate, + }, + "dropped_records_before": 0, + } + + +def main() -> None: + extractor = load("active_intervention_extract_test", HERE / "extract_training.py") + trainer = load("active_intervention_train_test", HERE / "train_policy.py") + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + run_root = root / "runs" + configs = [ + {"id": "a_base", "mns": 16, "mbbt": 8192}, + {"id": "a_mns", "mns": 64, "mbbt": 8192}, + {"id": "a_mbbt", "mns": 16, "mbbt": 16384}, + {"id": "b_base", "mns": 64, "mbbt": 2048}, + {"id": "b_mns", "mns": 128, "mbbt": 2048}, + {"id": "b_mbbt", "mns": 64, "mbbt": 8192}, + ] + manifest = { + "engine": {"duration_s": 300.0, "tp": 4}, + "configs": configs, + "repetitions": { + str(rep): {"selection": {"offered_req_s_per_gpu": 0.01}} + for rep in (1, 2, 3) + }, + "regimes": { + "A": { + "source": "a_base", + "actions": {"mns": "a_mns", "mbbt": "a_mbbt"}, + }, + "B": { + "source": "b_base", + "actions": {"mns": "b_mns", "mbbt": "b_mbbt"}, + }, + }, + } + manifest_path = root / "manifest.json" + write_json(manifest_path, manifest) + + streams = [] + source_starts: dict[tuple[str, int], int] = {} + for source_index, source_id in enumerate(("a_base", "b_base")): + rows = [] + index = 0 + for repetition in (1, 2, 3): + start_ns = int((source_index * 2000 + repetition * 400) * 1e9) + source_starts[(source_id, repetition)] = start_ns + for second in (1, 30, 76, 105, 151, 180, 226, 255): + rows.append(engine_record(index, start_ns + int(second * 1e9))) + index += 1 + stream_path = root / f"{source_id}-stream.jsonl" + write_jsonl(stream_path, rows) + streams.append( + { + "config_id": source_id, + "stream": str(stream_path), + "stream_sha256": extractor.sha256_file(stream_path), + } + ) + + request_rows = [ + { + "request_id": f"r{index}", + "arrival_s": arrival, + "completed_elapsed_s": arrival + 10, + "slo_pass": index != 3, + "ttft_ms": 1000 + index * 100, + "tpot_ms": 20 + index, + "raw_input_tokens": 1000 + index * 100, + } + for index, arrival in enumerate((5.0, 80.0, 155.0, 230.0), 1) + ] + for source_id in ("a_base", "b_base"): + for repetition in (1, 2, 3): + write_jsonl( + run_root + / "sessions" + / source_id + / f"rep{repetition}" + / "requests.jsonl", + request_rows, + ) + + goodput = { + "a_base": 0.020, + "a_mns": 0.036, + "a_mbbt": 0.028, + "b_base": 0.032, + "b_mns": 0.030, + "b_mbbt": 0.038, + } + runs = [] + for config in configs: + for repetition in (1, 2, 3): + item = { + "config_id": config["id"], + "repetition": repetition, + "outcome": { + "slo_goodput_req_s": goodput[config["id"]] + + repetition * 0.0001 + }, + } + if config["id"] in ("a_base", "b_base"): + start_ns = source_starts[(config["id"], repetition)] + item["state"] = { + "interval": { + "start_ns": start_ns, + "end_ns": start_ns + int(300 * 1e9), + } + } + runs.append(item) + audit = { + "schema": "action-aware-constraint-pilot-audit-v0", + "sanity": {"red_flags": []}, + "streams": streams, + "runs": runs, + } + audit_path = root / "audit.json" + write_json(audit_path, audit) + + dataset = extractor.build_dataset( + audit_path=audit_path, manifest_path=manifest_path, run_root=run_root + ) + assert dataset["status"] == "VALID" + assert len(dataset["examples"]) == 72 + assert not dataset["sanity"]["red_flags"] + assert all( + "exclusive" not in feature + for example in dataset["examples"] + for feature in example["source"]["telemetry"] + ) + dataset_path = root / "dataset.json" + write_json(dataset_path, dataset) + policy = trainer.build_policy(dataset_path) + assert policy["status"] in { + "RETROSPECTIVE_INCREMENTAL_SIGNAL", + "NO_RETROSPECTIVE_INCREMENTAL_SIGNAL", + } + assert not policy["sanity"]["red_flags"] + print("active intervention pipeline: PASS") + + +if __name__ == "__main__": + main() diff --git a/runs/active-intervention-v0/train_policy.py b/runs/active-intervention-v0/train_policy.py new file mode 100644 index 0000000..8bbae02 --- /dev/null +++ b/runs/active-intervention-v0/train_policy.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""Train and audit outcome-only versus telemetry action-response policies.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import math +import os +import sys +from collections import defaultdict +from pathlib import Path +from typing import Any, Mapping, Sequence + + +HERE = Path(__file__).resolve().parent + + +def _load_model(): + spec = importlib.util.spec_from_file_location( + "active_intervention_model", HERE / "model.py" + ) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +MODEL = _load_model() +REGULARIZATION = 10.0 +MINIMUM_MARGIN = 0.02 +CONFIDENCE_Z = 1.0 + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def atomic_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + os.replace(temporary, path) + + +def grouped( + examples: Sequence[Mapping[str, Any]], key: str +) -> dict[str, list[Mapping[str, Any]]]: + result: dict[str, list[Mapping[str, Any]]] = defaultdict(list) + for example in examples: + result[str(example[key])].append(example) + return dict(result) + + +def evaluate_grouped_cv( + examples: Sequence[Mapping[str, Any]], + *, + include_telemetry: bool, + holdout_key: str, +) -> dict[str, Any]: + holdouts = grouped(examples, holdout_key) + decision_rows = [] + for held_out, test_examples in sorted(holdouts.items()): + training = [example for example in examples if str(example[holdout_key]) != held_out] + if len({str(example["decision_id"]) for example in training}) < 2: + continue + model = MODEL.fit_ridge( + training, + include_telemetry=include_telemetry, + regularization=REGULARIZATION, + ) + for decision_id, candidates in sorted(grouped(test_examples, "decision_id").items()): + predictions = [] + for candidate in candidates: + source = candidate["source"] + action = candidate["action"] + noop = ( + int(action["target_mns"]) == int(source["mns"]) + and int(action["target_mbbt"]) == int(source["mbbt"]) + ) + if noop: + prediction = 0.0 + else: + names, vector = MODEL.feature_vector( + candidate, include_telemetry=include_telemetry + ) + if tuple(names) != model.feature_names: + raise ValueError("cross-validation feature schema mismatch") + prediction = max(-1.0, min(1.0, model.predict(vector))) + predictions.append( + { + "action_id": str(candidate["action"]["id"]), + "prediction": prediction, + "real": float(candidate["target_normalized_goodput"]), + } + ) + predictions.sort(key=lambda row: (-row["prediction"], row["action_id"])) + selected = predictions[0] + oracle = max(row["real"] for row in predictions) + regret = 1.0 - selected["real"] / oracle if oracle > 0 else 0.0 + best_actions = { + row["action_id"] for row in predictions if math.isclose(row["real"], oracle) + } + decision_rows.append( + { + "holdout": held_out, + "decision_id": decision_id, + "selected_action": selected["action_id"], + "best_actions": sorted(best_actions), + "correct": selected["action_id"] in best_actions, + "selected_real": selected["real"], + "oracle_real": oracle, + "regret": regret, + "predictions": predictions, + } + ) + if not decision_rows: + return {"status": "INSUFFICIENT_GROUPS", "decisions": []} + regrets = [float(row["regret"]) for row in decision_rows] + return { + "status": "VALID", + "holdout_key": holdout_key, + "decision_n": len(decision_rows), + "correct_n": sum(bool(row["correct"]) for row in decision_rows), + "accuracy": sum(bool(row["correct"]) for row in decision_rows) / len(decision_rows), + "mean_regret": sum(regrets) / len(regrets), + "max_regret": max(regrets), + "decisions": decision_rows, + } + + +def paired_delta(outcome: Mapping[str, Any], telemetry: Mapping[str, Any]) -> dict[str, Any]: + if outcome.get("status") != "VALID" or telemetry.get("status") != "VALID": + return {"status": "INSUFFICIENT_GROUPS"} + outcome_by_id = {row["decision_id"]: row for row in outcome["decisions"]} + telemetry_by_id = {row["decision_id"]: row for row in telemetry["decisions"]} + common = sorted(set(outcome_by_id) & set(telemetry_by_id)) + rows = [] + for decision_id in common: + before = outcome_by_id[decision_id] + after = telemetry_by_id[decision_id] + rows.append( + { + "decision_id": decision_id, + "outcome_action": before["selected_action"], + "telemetry_action": after["selected_action"], + "action_changed": before["selected_action"] != after["selected_action"], + "regret_delta": float(after["regret"]) - float(before["regret"]), + "telemetry_corrected": (not before["correct"]) and bool(after["correct"]), + "telemetry_harmed": bool(before["correct"]) and (not after["correct"]), + } + ) + return { + "status": "VALID", + "decision_n": len(rows), + "action_changed_n": sum(row["action_changed"] for row in rows), + "corrected_n": sum(row["telemetry_corrected"] for row in rows), + "harmed_n": sum(row["telemetry_harmed"] for row in rows), + "mean_regret_delta": ( + sum(float(row["regret_delta"]) for row in rows) / len(rows) if rows else 0.0 + ), + "rows": rows, + } + + +def build_policy(dataset_path: Path) -> dict[str, Any]: + dataset = json.loads(dataset_path.read_text(encoding="utf-8")) + if dataset.get("status") != "VALID" or dataset["sanity"]["red_flags"]: + raise ValueError("training dataset is not valid") + examples = dataset["examples"] + phases = sorted({str(example["phase"]) for example in examples}, key=float) + phase_results = {} + incremental_candidates = [] + for phase in phases: + selected = [example for example in examples if str(example["phase"]) == phase] + outcome_cv = evaluate_grouped_cv( + selected, include_telemetry=False, holdout_key="repetition" + ) + telemetry_cv = evaluate_grouped_cv( + selected, include_telemetry=True, holdout_key="repetition" + ) + outcome_regime = evaluate_grouped_cv( + selected, include_telemetry=False, holdout_key="regime" + ) + telemetry_regime = evaluate_grouped_cv( + selected, include_telemetry=True, holdout_key="regime" + ) + delta = paired_delta(outcome_cv, telemetry_cv) + outcome_models = MODEL.fit_jackknife_ensemble( + selected, + include_telemetry=False, + regularization=REGULARIZATION, + ) + telemetry_models = MODEL.fit_jackknife_ensemble( + selected, + include_telemetry=True, + regularization=REGULARIZATION, + ) + incremental = bool( + delta.get("status") == "VALID" + and int(delta["corrected_n"]) >= 1 + and int(delta["harmed_n"]) == 0 + and float(delta["mean_regret_delta"]) < -1e-12 + and float(telemetry_cv["max_regret"]) <= 0.05 + ) + if incremental: + incremental_candidates.append(phase) + phase_results[phase] = { + "cutoff_s": float(selected[0]["cutoff_s"]), + "outcome_only": { + "leave_repetition_out": outcome_cv, + "leave_regime_out": outcome_regime, + "models": MODEL.models_to_json(outcome_models), + }, + "telemetry": { + "leave_repetition_out": telemetry_cv, + "leave_regime_out": telemetry_regime, + "models": MODEL.models_to_json(telemetry_models), + }, + "paired_incremental": delta, + "incremental_gate": incremental, + } + selected_phase = incremental_candidates[0] if incremental_candidates else phases[-1] + status = ( + "RETROSPECTIVE_INCREMENTAL_SIGNAL" + if incremental_candidates + else "NO_RETROSPECTIVE_INCREMENTAL_SIGNAL" + ) + target_values = [float(example["target_normalized_goodput"]) for example in examples] + effect_values = [ + float(example["target_delta_normalized_goodput"]) for example in examples + ] + invariants = { + "four_phases": len(phases) == 4, + "targets_bounded": all(0.0 <= value <= 1.0 for value in target_values), + "targets_not_all_identical": len(set(target_values)) > 1, + "effects_bounded": all(-1.0 <= value <= 1.0 for value in effect_values), + "effects_not_all_identical": len(set(effect_values)) > 1, + "models_present_every_phase": all( + phase_results[phase][mode]["models"] + for phase in phases + for mode in ("outcome_only", "telemetry") + ), + } + red_flags = [name for name, passed in invariants.items() if not passed] + if red_flags: + raise RuntimeError(f"policy sanity failed: {red_flags}") + return { + "schema": "active-intervention-policy-v0", + "status": status, + "training": { + "dataset": str(dataset_path), + "dataset_sha256": sha256_file(dataset_path), + "examples": len(examples), + "decisions": len({example["decision_id"] for example in examples}), + "regularization": REGULARIZATION, + "confidence_z": CONFIDENCE_Z, + "minimum_margin": MINIMUM_MARGIN, + }, + "measurement_policy": { + "selected_phase": selected_phase, + "selected_cutoff_s": phase_results[selected_phase]["cutoff_s"], + "selection_reason": ( + "earliest phase passing the frozen incremental gate" + if incremental_candidates + else "no incremental phase; retain full measurement for exploratory held-out test" + ), + }, + "phases": phase_results, + "sanity": { + "invariants": invariants, + "red_flags": red_flags, + "target_normalized_goodput": { + "n": len(target_values), + "min": min(target_values), + "max": max(target_values), + "distinct_n": len(set(target_values)), + }, + "target_delta_normalized_goodput": { + "n": len(effect_values), + "min": min(effect_values), + "max": max(effect_values), + "distinct_n": len(set(effect_values)), + }, + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--dataset", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + policy = build_policy(args.dataset) + atomic_json(args.output, policy) + print( + json.dumps( + { + "status": policy["status"], + "measurement_policy": policy["measurement_policy"], + "sanity": policy["sanity"], + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() From e5fd463f05e3351ad931ed2773289b46056d157f Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 02:03:38 +0800 Subject: [PATCH 029/148] Add prospective active intervention experiment --- ...ctive-intervention-v0-protocol-20260715.md | 129 +++++ runs/action-aware-v0/pilot_controller.py | 14 +- .../analyze_prospective.py | 325 +++++++++++++ .../prepare_prospective.py | 362 ++++++++++++++ .../prospective_controller.py | 198 ++++++++ .../prospective_decision.py | 441 ++++++++++++++++++ runs/active-intervention-v0/test_pipeline.py | 7 +- .../test_prospective.py | 190 ++++++++ runs/active-intervention-v0/train_policy.py | 225 ++++++++- 9 files changed, 1875 insertions(+), 16 deletions(-) create mode 100644 docs/active-intervention-v0-protocol-20260715.md create mode 100644 runs/active-intervention-v0/analyze_prospective.py create mode 100644 runs/active-intervention-v0/prepare_prospective.py create mode 100644 runs/active-intervention-v0/prospective_controller.py create mode 100644 runs/active-intervention-v0/prospective_decision.py create mode 100644 runs/active-intervention-v0/test_prospective.py diff --git a/docs/active-intervention-v0-protocol-20260715.md b/docs/active-intervention-v0-protocol-20260715.md new file mode 100644 index 0000000..cf6a5e4 --- /dev/null +++ b/docs/active-intervention-v0-protocol-20260715.md @@ -0,0 +1,129 @@ +# Active intervention + measurement v0 protocol + +Date: 2026-07-15 (Asia/Singapore) + +Status: **FROZEN BEFORE THE `chat_w20260313_1000` GPU RUN**. + +## Research question + +This experiment asks whether a tuner conditioned on direct engine-state +trajectories can choose both a measurement horizon and a coupled configuration +intervention with lower real-GPU cost than the same tuner using only external +prefix outcomes. + +The contribution is not the controller, legality checks, telemetry collection, +or the ridge model. The route remains open only if engine state changes an +actual decision and reduces cost-to-near-oracle on unseen workloads. + +## Development result that motivates, but does not pass, the route + +The frozen trace-12 dataset contains 72 examples: six source decisions, four +measurement checkpoints, and `noop/MNS/MBBT` actions. Features are direct +continuous Layer-1 state summaries; cap-exclusive and bottleneck labels are +excluded. Leave-one-repetition-out sequential replay uses the same model, +candidate set, confidence rule, and checkpoint set for both modes. + +The external-outcome policy and telemetry policy both put all six decisions +within 2% regret. Outcome-only selected a mean 262.5-second source measurement +and cost 3.750 replay H20-hours across the six replayed decisions; telemetry +selected 275 seconds and cost 3.833 H20-hours. Telemetry therefore increased +the replay lower-bound cost by 2.22%, with no regret reduction. This is a +negative result. It does not settle the question because the dataset has only +two source regimes, one source is at the offered ceiling, and there is no joint +MNS+MBBT action. + +Sanity: n=6 decisions; regret min=0, max=0.009412, distinct=3; source cutoff +min=150s, max=300s, distinct=3 across the two policies; all costs are +non-negative, regrets are in `[0,1]`, target results are not all identical, and +the six decisions are complete exact-workload pairs. + +## Frozen prospective setup + +- Host: `dash0`, 8 NVIDIA H20 GPUs available; each TP4 server runs alone on + GPUs 0-3. Co-location is prohibited for SLO verdicts. +- Engine: patched vLLM `0.24.1.dev3+opprof` from + `/home/admin/cpfs/wjh/vllm-opprof-phase3` in + `/tmp/wjh/venvs/vllm-0.20.0-cu129`. +- Model: `/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B`, BF16. +- Workload: unseen `chat_w20260313_1000`; input 0-8192; output exactly 128; + replay scale 0.5; 300-second arrival window. +- Three disjoint repetitions: source rows are assigned by a deterministic + SHA-256 modulo-3 partition before input filtering. Each repetition selects + approximately 3300 requests, or 2.75 requests/s/GPU at TP4. +- SLO: at least 95% pass; stepped TTFT 2/4/6 seconds; TPOT at most 50 ms. +- Checkpoints: 75, 150, 225, and 300 seconds. +- Full 2x2 surface: + - source: `MNS=32, MBBT=4096`; + - MNS action: `64,4096`; + - MBBT action: `32,8192`; + - joint action: `64,8192`; + - `noop` retains the source. +- Four config sessions are serialized. Each session uses a fresh server, + warm-up, burn-in, and counter-rotated repetition order. +- Expected campaign cost: 4.6-5.5 H20-hours; hard cap: 6.0 H20-hours; + expected wall time: 75-100 minutes. + +The source is executed first. The frozen telemetry policy selects the next +real config session; all remaining cells are then measured only to construct +the exact finite-surface oracle. Oracle annotation after the selected action +is reported separately from tuner cost. + +## Frozen policies + +Both policies fit the paired treatment effect + +```text +target normalized SLO-goodput - source normalized SLO-goodput +``` + +from source config, full config delta, offered load, and external prefix +outcomes. The telemetry policy additionally receives fixed direct Layer-1 +summaries and their interactions with `delta_log2(MNS)` and +`delta_log2(MBBT)`. It does not receive a bottleneck label or a +diagnosis-to-knob rule. + +At each checkpoint, jackknife models produce an effect distribution for +`noop`, MNS, MBBT, and joint actions. Measurement stops at the earliest second +consecutive checkpoint with the same confident best action; otherwise it uses +the full 300 seconds. Confidence requires a predicted margin of at least 0.02 +and the best lower bound to exceed the second-best upper bound. If the final +choice is not confident, the next run is the positive-UCB action, explicitly +marked as a diagnostic intervention. The exact same rule is used for the +outcome-only baseline. + +## Hypotheses and gates + +### H1: action value + +Engine state must change the selected intervention or its ranking and reduce +real action regret. Prediction error or bottleneck-label accuracy is not a +success metric. + +### H2: measurement value + +Engine state must select a shorter stable source measurement without increasing +action regret. A shorter reconstructed prefix is only a trigger; it is not an +actual GPU-cost claim until an early-terminated confirmation run measures +startup, warm-up, drain, and cleanup. + +### H3: end-to-end cost + +Primary development metric is H20-hours to first reach a configuration within +2% of the exact median-goodput oracle. The outcome-only and telemetry policies +use the same measured config costs and differ only in source information. + +- At least 10% prospective replay cost reduction, telemetry regret at most 2%, + and no outcome-only-to-telemetry harm triggers an actual early-stop + confirmation. +- At least 20% measured all-in H20-hour reduction is required for a contribution + claim. This one task can only establish development feasibility; a paper + claim additionally requires task-held-out replication. +- Source median normalized goodput at or above 0.98 stops the surface before + target runs because the workload has no material improvement headroom. +- Any hash mismatch, missing/censored result, telemetry drop, non-monotonic + phase, negative cost, ratio outside `[0,1]`, or all-identical config outcomes + is a red flag and stops analysis. + +If the 10% trigger fails, this route is closed for the current engine-state +representation. The experimental control plane is not retained as a fallback +research contribution. diff --git a/runs/action-aware-v0/pilot_controller.py b/runs/action-aware-v0/pilot_controller.py index 14668f9..7ecddb3 100644 --- a/runs/action-aware-v0/pilot_controller.py +++ b/runs/action-aware-v0/pilot_controller.py @@ -453,11 +453,21 @@ def execute_session( ) if float(state["gpu_hours_total"]) + projection > base.GPU_LIMIT: raise RuntimeError(f"projected cost exceeds cap before {name}") + load_values = { + float(item["selection"]["offered_req_s_per_gpu"]) + for item in manifest["repetitions"].values() + } + load_text = ( + f"{next(iter(load_values)):.6g}" + if len(load_values) == 1 + else ",".join(f"{value:.6g}" for value in sorted(load_values)) + ) echo = ( f"ACTION_AWARE_SESSION_ECHO host=dash0 config={name} tp=4 " f"mns={config['mns']} mbbt={config['mbbt']} gpus=0-3 " - f"workload={manifest['source']['window_id']} load_per_gpu=2.125 " - f"duration_s=300 repetitions={','.join(map(str, config['repetition_order']))} " + f"workload={manifest['source']['window_id']} load_per_gpu={load_text} " + f"duration_s={manifest['engine']['duration_s']} " + f"repetitions={','.join(map(str, config['repetition_order']))} " f"source={args.manifest} output={args.run_root / 'sessions' / name} " f"spent_h20h={state['gpu_hours_total']:.6f} " f"remaining_projection_h20h={projection:.3f} cap_h20h={base.GPU_LIMIT:.1f}" diff --git a/runs/active-intervention-v0/analyze_prospective.py b/runs/active-intervention-v0/analyze_prospective.py new file mode 100644 index 0000000..3442769 --- /dev/null +++ b/runs/active-intervention-v0/analyze_prospective.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 +"""Audit held-out action/measurement choices against the exact 2x2 surface.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import statistics +from pathlib import Path +from typing import Any, Mapping + + +SCHEMA = "active-intervention-prospective-audit-v0" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def atomic_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + os.replace(temporary, path) + + +def numeric(values: list[float]) -> dict[str, Any]: + finite = [float(value) for value in values] + if not finite or any(not math.isfinite(value) for value in finite): + raise ValueError("numeric summary requires finite values") + return { + "n": len(finite), + "min": min(finite), + "max": max(finite), + "distinct_n": len(set(finite)), + } + + +def load_surface( + manifest: Mapping[str, Any], run_root: Path +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + rows = [] + aggregate = {} + duration_s = float(manifest["engine"]["duration_s"]) + tp = int(manifest["engine"]["tp"]) + for config in manifest["configs"]: + config_id = str(config["id"]) + values = [] + for repetition in sorted(int(key) for key in manifest["repetitions"]): + expected = manifest["repetitions"][str(repetition)]["selection"] + result_path = ( + run_root / "sessions" / config_id / f"rep{repetition}" / "result.json" + ) + result = json.loads(result_path.read_text(encoding="utf-8")) + if result["selection"]["request_id_order_sha256"] != expected[ + "request_id_order_sha256" + ]: + raise ValueError(f"request hash mismatch: {config_id} rep{repetition}") + offered_total = float(expected["offered_req_s_per_gpu"]) * tp + normalized = float(result["slo_pass_count"]) / duration_s / offered_total + values.append(normalized) + rows.append( + { + "config_id": config_id, + "mns": int(config["mns"]), + "mbbt": int(config["mbbt"]), + "repetition": repetition, + "normalized_slo_goodput": normalized, + "slo_goodput_req_s": float(result["slo_pass_count"]) / duration_s, + "pass_rate": float(result["pass_rate"]), + "elapsed_s": float(result["interval"]["elapsed_s"]), + "result": str(result_path), + "result_sha256": sha256_file(result_path), + } + ) + aggregate[config_id] = { + "normalized_slo_goodput_values": values, + "median_normalized_slo_goodput": float(statistics.median(values)), + "sanity": numeric(values), + } + return aggregate, rows + + +def source_cost_estimate( + *, + source_session: Mapping[str, Any], + source_rows: list[Mapping[str, Any]], + cutoff_s: float, + tp: int, +) -> dict[str, float]: + actual_h20_hours = float(source_session["gpu_hours"]) + measured_replay_h20_hours = ( + tp * sum(float(row["elapsed_s"]) for row in source_rows) / 3600.0 + ) + fixed_h20_hours = max(0.0, actual_h20_hours - measured_replay_h20_hours) + prefix_replay_h20_hours = tp * len(source_rows) * cutoff_s / 3600.0 + return { + "actual_full_session_h20_hours": actual_h20_hours, + "fixed_startup_warmup_burnin_cleanup_h20_hours": fixed_h20_hours, + "prefix_replay_h20_hours_lower_bound": prefix_replay_h20_hours, + "counterfactual_all_in_h20_hours_lower_bound": fixed_h20_hours + + prefix_replay_h20_hours, + } + + +def replay_policy( + *, + mode: str, + manifest: Mapping[str, Any], + decision: Mapping[str, Any], + surface: Mapping[str, Any], + session_costs: Mapping[str, float], + source_cost: Mapping[str, float], +) -> dict[str, Any]: + acceptable_regret = float(manifest["gates"]["acceptable_regret"]) + source_id = str(manifest["source_config_id"]) + oracle = max( + float(item["median_normalized_slo_goodput"]) for item in surface.values() + ) + cumulative = float(source_cost["counterfactual_all_in_h20_hours_lower_bound"]) + source_score = float(surface[source_id]["median_normalized_slo_goodput"]) + source_regret = 1.0 - source_score / oracle if oracle > 0 else 0.0 + points = [ + { + "action_id": "noop", + "config_id": source_id, + "score": source_score, + "regret": source_regret, + "cumulative_h20_hours_lower_bound": cumulative, + } + ] + hit = points[0] if source_regret <= acceptable_regret + 1e-12 else None + seen = {source_id} + for action_id in decision["decisions"][mode]["intervention_order"]: + config_id = str(manifest["actions"][action_id]) + if config_id in seen: + continue + seen.add(config_id) + cumulative += float(session_costs[config_id]) + score = float(surface[config_id]["median_normalized_slo_goodput"]) + regret = 1.0 - score / oracle if oracle > 0 else 0.0 + point = { + "action_id": action_id, + "config_id": config_id, + "score": score, + "regret": regret, + "cumulative_h20_hours_lower_bound": cumulative, + } + points.append(point) + if hit is None and regret <= acceptable_regret + 1e-12: + hit = point + return { + "mode": mode, + "measurement_cutoff_s": float( + decision["decisions"][mode]["selected_cutoff_s"] + ), + "selected_action": decision["decisions"][mode]["selected_action"], + "decision_kind": decision["decisions"][mode]["decision_kind"], + "intervention_order": decision["decisions"][mode]["intervention_order"], + "source_cost": dict(source_cost), + "oracle_normalized_slo_goodput": oracle, + "cost_to_acceptable": hit, + "reached_acceptable": hit is not None, + "points": points, + } + + +def build_audit( + *, manifest_path: Path, decision_path: Path, run_root: Path +) -> dict[str, Any]: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + decision = json.loads(decision_path.read_text(encoding="utf-8")) + state_path = run_root / "controller-state.json" + state = json.loads(state_path.read_text(encoding="utf-8")) + if manifest.get("schema") != "active-intervention-prospective-manifest-v0": + raise ValueError("unexpected prospective manifest schema") + if decision.get("schema") != "active-intervention-prospective-decision-v0": + raise ValueError("unexpected prospective decision schema") + if decision["manifest_sha256"] != sha256_file(manifest_path): + raise ValueError("decision does not match prospective manifest") + surface, rows = load_surface(manifest, run_root) + source_id = str(manifest["source_config_id"]) + sessions = state["sessions"] + session_costs = { + config_id: float(sessions[config_id]["gpu_hours"]) + for config_id in surface + } + source_rows = [row for row in rows if row["config_id"] == source_id] + policies = {} + for mode in ("outcome_only", "telemetry"): + cost = source_cost_estimate( + source_session=sessions[source_id], + source_rows=source_rows, + cutoff_s=float(decision["decisions"][mode]["selected_cutoff_s"]), + tp=int(manifest["engine"]["tp"]), + ) + policies[mode] = replay_policy( + mode=mode, + manifest=manifest, + decision=decision, + surface=surface, + session_costs=session_costs, + source_cost=cost, + ) + outcome_hit = policies["outcome_only"]["cost_to_acceptable"] + telemetry_hit = policies["telemetry"]["cost_to_acceptable"] + if outcome_hit is None or telemetry_hit is None: + reduction = None + else: + outcome_cost = float(outcome_hit["cumulative_h20_hours_lower_bound"]) + telemetry_cost = float(telemetry_hit["cumulative_h20_hours_lower_bound"]) + reduction = 1.0 - telemetry_cost / outcome_cost if outcome_cost > 0 else 0.0 + confirmation_trigger = bool( + reduction is not None + and reduction + >= float(manifest["gates"]["confirmation_trigger_gpu_cost_reduction"]) + and policies["telemetry"]["reached_acceptable"] + ) + contribution_gate = bool( + reduction is not None + and reduction >= float(manifest["gates"]["contribution_gpu_cost_reduction"]) + and policies["telemetry"]["reached_acceptable"] + ) + status = ( + "TRIGGER_ACTUAL_EARLY_STOP_CONFIRMATION" + if confirmation_trigger + else "STOP_NO_PROSPECTIVE_GPU_COST_SIGNAL" + ) + normalized_values = [float(row["normalized_slo_goodput"]) for row in rows] + costs = list(session_costs.values()) + invariants = { + "controller_complete": state.get("status") == "complete", + "four_sessions_complete": len(sessions) == 4 + and all(item.get("status") == "complete" for item in sessions.values()), + "twelve_surface_outcomes": len(rows) == 12, + "nonnegative_goodput": all(value >= 0.0 for value in normalized_values), + "normalized_goodput_bounded": all(value <= 1.0 + 1e-12 for value in normalized_values), + "surface_not_all_identical": len(set(normalized_values)) > 1, + "nonnegative_session_costs": all(value >= 0.0 for value in costs), + "policy_replay_reaches_oracle_surface": all( + policy["reached_acceptable"] for policy in policies.values() + ), + } + red_flags = [name for name, passed in invariants.items() if not passed] + if red_flags: + status = "STOP_SANITY" + return { + "schema": SCHEMA, + "status": status, + "claim_boundary": ( + "Prospective exact-surface replay. Prefix source costs reconstruct the " + "measured fixed overhead plus selected replay seconds; actual early-stop " + "confirmation is required before claiming GPU-cost reduction." + ), + "manifest": str(manifest_path), + "manifest_sha256": sha256_file(manifest_path), + "decision": str(decision_path), + "decision_sha256": sha256_file(decision_path), + "controller_state": str(state_path), + "controller_state_sha256": sha256_file(state_path), + "surface": surface, + "rows": rows, + "session_costs_h20_hours": session_costs, + "annotation_campaign_h20_hours": float(state["gpu_hours_total"]), + "policies": policies, + "comparison": { + "telemetry_gpu_cost_reduction_fraction": reduction, + "confirmation_trigger": confirmation_trigger, + "contribution_gate": contribution_gate, + "confirmation_trigger_threshold": manifest["gates"][ + "confirmation_trigger_gpu_cost_reduction" + ], + "contribution_threshold": manifest["gates"][ + "contribution_gpu_cost_reduction" + ], + "action_changed": policies["outcome_only"]["selected_action"] + != policies["telemetry"]["selected_action"], + "measurement_changed": policies["outcome_only"]["measurement_cutoff_s"] + != policies["telemetry"]["measurement_cutoff_s"], + }, + "sanity": { + "invariants": invariants, + "red_flags": red_flags, + "normalized_slo_goodput": numeric(normalized_values), + "session_h20_hours": numeric(costs), + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--decision", type=Path, required=True) + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + audit = build_audit( + manifest_path=args.manifest, + decision_path=args.decision, + run_root=args.run_root, + ) + atomic_json(args.output, audit) + print( + json.dumps( + { + "status": audit["status"], + "comparison": audit["comparison"], + "sanity": audit["sanity"], + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/active-intervention-v0/prepare_prospective.py b/runs/active-intervention-v0/prepare_prospective.py new file mode 100644 index 0000000..0268537 --- /dev/null +++ b/runs/active-intervention-v0/prepare_prospective.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python3 +"""Freeze the unseen-trace 2x2 active intervention development surface.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import sys +from pathlib import Path +from typing import Any + + +AITUNER_ROOT = Path(os.environ.get("AITUNER_ROOT", Path(__file__).resolve().parents[2])) +sys.path.insert(0, str(AITUNER_ROOT / "src")) + +from aituner.spec import load_study_spec # noqa: E402 +from aituner.trace import load_trace_requests, select_requests_for_threshold # noqa: E402 + + +SCHEMA = "active-intervention-prospective-manifest-v0" +TP = 4 +REPETITIONS = (1, 2, 3) +DURATION_S = 300.0 +REPLAY_TIME_SCALE = 0.5 +OFFERED_RATE_PER_GPU = 2.75 +TARGET_COUNT = round(OFFERED_RATE_PER_GPU * DURATION_S * TP) +WINDOW_ID = "chat_w20260313_1000" + + +def atomic_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + os.replace(temporary, path) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def order_hash(values: list[str]) -> str: + return hashlib.sha256("\n".join(values).encode()).hexdigest() + + +def configs() -> list[dict[str, Any]]: + return [ + { + "id": "source_mns32_mbbt4096", + "mns": 32, + "mbbt": 4096, + "repetition_order": [1, 2, 3], + }, + { + "id": "mns64_mbbt4096", + "mns": 64, + "mbbt": 4096, + "repetition_order": [2, 3, 1], + }, + { + "id": "mns32_mbbt8192", + "mns": 32, + "mbbt": 8192, + "repetition_order": [3, 1, 2], + }, + { + "id": "joint_mns64_mbbt8192", + "mns": 64, + "mbbt": 8192, + "repetition_order": [1, 3, 2], + }, + ] + + +def partition_trace(source: Path, output_root: Path) -> dict[str, Any]: + source_sha = sha256_file(source) + output_root.mkdir(parents=True, exist_ok=True) + paths = {rep: output_root / f"rep{rep}.jsonl" for rep in REPETITIONS} + temporary = {rep: path.with_suffix(".jsonl.tmp") for rep, path in paths.items()} + handles = {rep: temporary[rep].open("w", encoding="utf-8") for rep in REPETITIONS} + counts = {rep: 0 for rep in REPETITIONS} + id_digests = {rep: hashlib.sha256() for rep in REPETITIONS} + total = 0 + try: + with source.open(encoding="utf-8") as input_file: + for line_number, line in enumerate(input_file, start=1): + if not line.strip(): + continue + row = json.loads(line) + original_id = str(row.get("request_id") or row.get("id") or line_number) + digest = hashlib.sha256( + f"{source_sha}:{line_number}:{original_id}".encode() + ).hexdigest() + repetition = int(digest[:16], 16) % len(REPETITIONS) + 1 + row["request_id"] = f"active-r{repetition}-{digest}" + handles[repetition].write(json.dumps(row, ensure_ascii=False) + "\n") + counts[repetition] += 1 + total += 1 + id_digests[repetition].update(row["request_id"].encode() + b"\n") + finally: + for handle in handles.values(): + handle.close() + for repetition in REPETITIONS: + os.replace(temporary[repetition], paths[repetition]) + partitions = { + str(rep): { + "path": str(paths[rep]), + "rows": counts[rep], + "bytes": paths[rep].stat().st_size, + "sha256": sha256_file(paths[rep]), + "request_id_order_sha256": id_digests[rep].hexdigest(), + } + for rep in REPETITIONS + } + return { + "source": str(source), + "source_sha256": source_sha, + "source_rows": total, + "partition_rule": "sha256(source_sha:line_number:original_id) modulo 3", + "partitions": partitions, + } + + +def materialize_study( + base_study: Path, + target: Path, + *, + repetition: int, + trace_path: Path, + windows_path: Path, +) -> None: + payload = json.loads(base_study.read_text(encoding="utf-8")) + payload["study_id"] = f"active-intervention-trace13-rep{repetition}" + payload["hardware"]["host_candidates"] = ["dash0"] + payload["engine"]["engine_version"] = "0.24.1.dev3+opprof" + trace = payload["trace"] + trace.update( + { + "windows_path": str(windows_path), + "window_id": WINDOW_ID, + "trace_file_override": str(trace_path), + "completion_tokens_override": 128, + "replay_time_scale": REPLAY_TIME_SCALE, + "early_stop_max_lag_s": None, + "early_stop_max_elapsed_s": 360.0, + "restart_engine_after_early_stop": False, + "adaptive_stop": {"enabled": False}, + } + ) + atomic_json(target, payload) + + +def attainable_anchor(requests: list[Any], target_count: int) -> tuple[float, list[Any]]: + ordered = sorted(float(request.sampling_u) for request in requests) + if target_count <= 0 or target_count > len(ordered): + raise ValueError( + f"target count {target_count} is outside available range 1..{len(ordered)}" + ) + candidates = [] + for index in sorted({target_count - 1, min(target_count, len(ordered) - 1)}): + anchor = ordered[index] + selected = select_requests_for_threshold(requests, threshold=anchor) + candidates.append((abs(len(selected) - target_count), len(selected), anchor, selected)) + _error, _count, anchor, selected = min( + candidates, key=lambda item: (item[0], item[1], item[2]) + ) + return anchor, selected + + +def selection_record(selected: list[Any]) -> dict[str, Any]: + return { + "anchor": max(float(request.sampling_u) for request in selected), + "selected_count": len(selected), + "target_count": TARGET_COUNT, + "offered_req_s": len(selected) / DURATION_S, + "offered_req_s_per_gpu": len(selected) / DURATION_S / TP, + "request_id_order_sha256": order_hash([request.row_id for request in selected]), + "arrival_order_sha256": order_hash( + [f"{request.arrival_s:.12f}" for request in selected] + ), + "input_length_order_sha256": order_hash( + [str(request.prompt_tokens_hint) for request in selected] + ), + } + + +def build( + *, + base_study: Path, + base_action_manifest: Path, + source_trace: Path, + windows_path: Path, + private_root: Path, + policy_path: Path, +) -> dict[str, Any]: + base_manifest = json.loads(base_action_manifest.read_text(encoding="utf-8")) + if base_manifest.get("status") != "PASS": + raise ValueError("base action-aware manifest did not pass") + policy = json.loads(policy_path.read_text(encoding="utf-8")) + if policy.get("schema") != "active-intervention-policy-v0": + raise ValueError("unexpected frozen policy schema") + if policy.get("sanity", {}).get("red_flags"): + raise ValueError("frozen policy contains red flags") + + partition = partition_trace(source_trace, private_root / "traces") + repetitions = {} + selected_sets: list[set[str]] = [] + for repetition in REPETITIONS: + trace_path = Path(partition["partitions"][str(repetition)]["path"]) + study_path = private_root / "studies" / f"rep{repetition}-tp4.json" + materialize_study( + base_study, + study_path, + repetition=repetition, + trace_path=trace_path, + windows_path=windows_path, + ) + study = load_study_spec(study_path) + window, requests = load_trace_requests(study, study_spec_path=study_path) + duration_s = float(window.window_end - window.window_start) + if not math.isclose(duration_s, DURATION_S, abs_tol=1e-9): + raise ValueError(f"rep{repetition}: duration {duration_s} != {DURATION_S}") + _anchor, selected = attainable_anchor(requests, TARGET_COUNT) + record = selection_record(selected) + selected_sets.append({request.row_id for request in selected}) + repetitions[str(repetition)] = { + "study": str(study_path), + "study_sha256": sha256_file(study_path), + "trace": partition["partitions"][str(repetition)], + "available_filtered_requests": len(requests), + "selection": record, + } + + frozen_configs = configs() + config_ids = {str(config["id"]) for config in frozen_configs} + invariants = { + "three_nonempty_trace_partitions": all( + int(item["rows"]) > 0 for item in partition["partitions"].values() + ), + "partition_rows_conserved": sum( + int(item["rows"]) for item in partition["partitions"].values() + ) + == int(partition["source_rows"]), + "selected_sets_disjoint": all( + not selected_sets[left] & selected_sets[right] + for left in range(len(selected_sets)) + for right in range(left + 1, len(selected_sets)) + ), + "target_count_attained": all( + abs(int(item["selection"]["selected_count"]) - TARGET_COUNT) <= 1 + for item in repetitions.values() + ), + "four_unique_configs": len(config_ids) == 4, + "two_by_two_surface": { + (int(config["mns"]), int(config["mbbt"])) + for config in frozen_configs + } + == {(32, 4096), (64, 4096), (32, 8192), (64, 8192)}, + "repetition_orders_are_permutations": all( + sorted(config["repetition_order"]) == list(REPETITIONS) + for config in frozen_configs + ), + } + red_flags = [name for name, passed in invariants.items() if not passed] + return { + "schema": SCHEMA, + "status": "PASS" if not red_flags else "STOP", + "source": { + "window_id": WINDOW_ID, + "source_trace": str(source_trace), + "source_trace_sha256": partition["source_sha256"], + "windows_path": str(windows_path), + "base_study": str(base_study), + "base_study_sha256": sha256_file(base_study), + "base_action_manifest": str(base_action_manifest), + "base_action_manifest_sha256": sha256_file(base_action_manifest), + }, + "policy": { + "path": str(policy_path), + "sha256": sha256_file(policy_path), + "status": policy["status"], + "training": policy["training"], + "measurement_policy": policy["measurement_policy"], + "launch_reason": ( + "bounded unseen-trace joint-action test after a negative narrow " + "retrospective replay" + ), + }, + "engine": { + "tp": TP, + "duration_s": DURATION_S, + "client_timeout_s": 450.0, + "burnin_max_elapsed_s": 90.0, + "disable_slo_early_stop": True, + }, + "burnin": base_manifest["burnin"], + "private": {"trace_partition": partition}, + "repetitions": repetitions, + "configs": frozen_configs, + "source_config_id": "source_mns32_mbbt4096", + "actions": { + "noop": "source_mns32_mbbt4096", + "mns": "mns64_mbbt4096", + "mbbt": "mns32_mbbt8192", + "joint": "joint_mns64_mbbt8192", + }, + "checkpoints": { + "fractions": [0.25, 0.50, 0.75, 1.0], + "seconds": [75.0, 150.0, 225.0, 300.0], + }, + "gates": { + "acceptable_regret": 0.02, + "source_ceiling_normalized_goodput": 0.98, + "confirmation_trigger_gpu_cost_reduction": 0.10, + "contribution_gpu_cost_reduction": 0.20, + "maximum_task_regret": 0.05, + }, + "budget": { + "hard_cap_h20_hours": 6.0, + "session_estimate_h20_hours": 1.3, + "safety_h20_hours": 0.3, + "expected_h20_hours": [4.6, 5.5], + "expected_wall_minutes": [75, 100], + }, + "sanity": {"invariants": invariants, "red_flags": red_flags}, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-study", type=Path, required=True) + parser.add_argument("--base-action-manifest", type=Path, required=True) + parser.add_argument("--source-trace", type=Path, required=True) + parser.add_argument("--windows-path", type=Path, required=True) + parser.add_argument("--private-root", type=Path, required=True) + parser.add_argument("--policy", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + payload = build( + base_study=args.base_study, + base_action_manifest=args.base_action_manifest, + source_trace=args.source_trace, + windows_path=args.windows_path, + private_root=args.private_root, + policy_path=args.policy, + ) + atomic_json(args.output, payload) + print(json.dumps({"status": payload["status"], "sanity": payload["sanity"]})) + if payload["status"] != "PASS": + raise SystemExit("prospective manifest preflight failed") + + +if __name__ == "__main__": + main() diff --git a/runs/active-intervention-v0/prospective_controller.py b/runs/active-intervention-v0/prospective_controller.py new file mode 100644 index 0000000..086ea93 --- /dev/null +++ b/runs/active-intervention-v0/prospective_controller.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +"""Run source first, select the next intervention, then annotate the 2x2 surface.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any, Mapping + + +HERE = Path(__file__).resolve().parent +ACTION_DIR = HERE.parent / "action-aware-v0" +sys.path.insert(0, str(ACTION_DIR)) +sys.path.insert(0, str(HERE)) + +import pilot_controller as action_controller # noqa: E402 +import prospective_decision # noqa: E402 + + +SCHEMA = "active-intervention-prospective-state-v0" + + +def validate_inputs(args: argparse.Namespace, manifest: Mapping[str, Any]) -> None: + if manifest.get("schema") != "active-intervention-prospective-manifest-v0": + raise RuntimeError("unexpected active intervention manifest schema") + if manifest.get("status") != "PASS" or manifest["sanity"]["red_flags"]: + raise RuntimeError("active intervention manifest did not pass preflight") + required = { + "manifest": args.manifest, + "policy": args.policy, + "aituner_root": args.aituner_root, + "vllm_source": args.vllm_source, + "venv_python": args.venv / "bin/python", + "venv_vllm": args.venv / "bin/vllm", + "model": args.model, + "client": args.client, + "burnin_study": Path(manifest["burnin"]["study"]), + } + for repetition, item in manifest["repetitions"].items(): + required[f"rep{repetition}_study"] = Path(item["study"]) + required[f"rep{repetition}_trace"] = Path(item["trace"]["path"]) + missing = {name: str(path) for name, path in required.items() if not path.exists()} + if missing: + raise RuntimeError(f"active intervention input paths missing: {missing}") + if prospective_decision.sha256_file(args.policy) != manifest["policy"]["sha256"]: + raise RuntimeError("active intervention policy hash mismatch") + + +def dry_run(args: argparse.Namespace, manifest: Mapping[str, Any]) -> dict[str, Any]: + plan = action_controller.dry_run_plan(args, manifest) + return { + "schema": "active-intervention-prospective-dry-run-v0", + "status": "PASS", + "manifest": str(args.manifest), + "policy": str(args.policy), + "source_first": manifest["source_config_id"], + "post_source_order": "selected by telemetry policy; all remaining cells then annotated", + "candidate_actions": manifest["actions"], + "projected_h20_hours": plan["projected_h20_hours"], + "hard_cap_h20_hours": plan["hard_cap_h20_hours"], + "sessions": plan["sessions"], + } + + +def load_or_build_decision( + *, args: argparse.Namespace, run_root: Path +) -> dict[str, Any]: + path = run_root / "active-decision.json" + if path.exists(): + decision = json.loads(path.read_text(encoding="utf-8")) + if decision.get("manifest_sha256") != prospective_decision.sha256_file( + args.manifest + ): + raise RuntimeError("existing active decision has a different manifest") + if decision.get("policy_sha256") != prospective_decision.sha256_file(args.policy): + raise RuntimeError("existing active decision has a different policy") + return decision + decision = prospective_decision.build_decision( + manifest_path=args.manifest, + policy_path=args.policy, + run_root=run_root, + ) + prospective_decision.atomic_json(path, decision) + return decision + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser() + result.add_argument("--manifest", type=Path, required=True) + result.add_argument("--policy", type=Path, required=True) + result.add_argument("--run-root", type=Path, required=True) + result.add_argument("--aituner-root", type=Path, required=True) + result.add_argument("--vllm-source", type=Path, required=True) + result.add_argument("--venv", type=Path, required=True) + result.add_argument("--model", type=Path, required=True) + result.add_argument("--client", type=Path, required=True) + result.add_argument("--dry-run", action="store_true") + return result + + +def main() -> None: + args = parser().parse_args() + manifest = json.loads(args.manifest.read_text(encoding="utf-8")) + validate_inputs(args, manifest) + action_controller.configure(args, manifest) + action_controller.base.MARKER = "active-intervention-prospective-v0" + if args.dry_run: + print(json.dumps(dry_run(args, manifest), indent=2, sort_keys=True)) + return + + args.run_root.mkdir(parents=True, exist_ok=True) + copied_manifest = args.run_root / "prospective-manifest.json" + if not copied_manifest.exists(): + action_controller.atomic_json(copied_manifest, manifest) + state_path = args.run_root / "controller-state.json" + state = action_controller.load_state( + state_path, float(manifest["budget"]["hard_cap_h20_hours"]) + ) + state["schema"] = SCHEMA + state["status"] = "running" + action_controller.atomic_json(state_path, state) + + configs = {str(item["id"]): dict(item) for item in manifest["configs"]} + config_indexes = { + str(item["id"]): index for index, item in enumerate(manifest["configs"]) + } + source_id = str(manifest["source_config_id"]) + action_controller.execute_session( + args=args, + manifest=manifest, + config=configs[source_id], + index=config_indexes[source_id], + state=state, + state_path=state_path, + ) + decision = load_or_build_decision(args=args, run_root=args.run_root) + state["active_decision"] = { + "path": str(args.run_root / "active-decision.json"), + "status": decision["status"], + "outcome_only": { + key: decision["decisions"]["outcome_only"][key] + for key in ("selected_cutoff_s", "decision_kind", "selected_action") + }, + "telemetry": { + key: decision["decisions"]["telemetry"][key] + for key in ("selected_cutoff_s", "decision_kind", "selected_action") + }, + } + action_controller.atomic_json(state_path, state) + if decision["status"] != "SELECTED": + state["status"] = decision["status"].lower() + state["completed_at"] = action_controller.time.time() + action_controller.atomic_json(state_path, state) + action_controller.wait_all_idle() + print(json.dumps({"status": state["status"], "decision": decision["status"]})) + return + + action_order = decision["decisions"]["telemetry"]["intervention_order"] + execution_order = [source_id] + for action_id in action_order: + target_id = str(manifest["actions"][action_id]) + if target_id not in execution_order: + execution_order.append(target_id) + for config_id in configs: + if config_id not in execution_order: + execution_order.append(config_id) + state["execution_order"] = execution_order + action_controller.atomic_json(state_path, state) + for config_id in execution_order[1:]: + action_controller.execute_session( + args=args, + manifest=manifest, + config=configs[config_id], + index=config_indexes[config_id], + state=state, + state_path=state_path, + ) + state["status"] = "complete" + state["completed_at"] = action_controller.time.time() + action_controller.atomic_json(state_path, state) + action_controller.wait_all_idle() + print( + json.dumps( + { + "status": state["status"], + "completed_sessions": state["completed_sessions"], + "gpu_hours_total": state["gpu_hours_total"], + "execution_order": execution_order, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/active-intervention-v0/prospective_decision.py b/runs/active-intervention-v0/prospective_decision.py new file mode 100644 index 0000000..7b489d7 --- /dev/null +++ b/runs/active-intervention-v0/prospective_decision.py @@ -0,0 +1,441 @@ +#!/usr/bin/env python3 +"""Choose measurement horizon and next intervention from a completed source run.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import math +import os +import statistics +import sys +from pathlib import Path +from typing import Any, Mapping, Sequence + +import numpy as np + + +HERE = Path(__file__).resolve().parent +COMMON_STATE = HERE.parent / "telemetry-residual" +sys.path.insert(0, str(COMMON_STATE)) + +from common_state import summarize_engine # noqa: E402 + + +SCHEMA = "active-intervention-prospective-decision-v0" + + +def load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +MODEL = load_module("active_intervention_prospective_model", HERE / "model.py") +EXTRACT = load_module( + "active_intervention_prospective_extract", HERE / "extract_training.py" +) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def atomic_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + os.replace(temporary, path) + + +def numeric(values: Sequence[float]) -> dict[str, Any]: + finite = [float(value) for value in values] + if not finite or any(not math.isfinite(value) for value in finite): + raise ValueError("numeric summary requires finite values") + return { + "n": len(finite), + "min": min(finite), + "max": max(finite), + "distinct_n": len(set(finite)), + } + + +def load_engine_records(source_root: Path) -> tuple[list[dict[str, Any]], Path]: + streams = sorted((source_root / "opprof").glob("*.jsonl")) + if len(streams) != 1: + raise ValueError(f"expected one source engine stream, found {len(streams)}") + records = [ + row for row in EXTRACT.load_jsonl(streams[0]) if "step_index" in row + ] + if not records: + raise ValueError("source engine stream has no Layer-1 records") + return records, streams[0] + + +def candidate_example( + *, + source_config: Mapping[str, Any], + target_config: Mapping[str, Any], + action_id: str, + offered_rate_per_gpu: float, + outcome: Mapping[str, float], + telemetry: Mapping[str, float], +) -> dict[str, Any]: + return { + "source": { + "mns": int(source_config["mns"]), + "mbbt": int(source_config["mbbt"]), + "offered_rate_per_gpu": float(offered_rate_per_gpu), + "outcome": dict(outcome), + "telemetry": dict(telemetry), + }, + "action": { + "id": action_id, + "target_mns": int(target_config["mns"]), + "target_mbbt": int(target_config["mbbt"]), + }, + } + + +def aggregate_checkpoint( + *, + models: Sequence[Any], + examples_by_action: Mapping[str, Sequence[Mapping[str, Any]]], + include_telemetry: bool, + confidence_z: float, + minimum_margin: float, +) -> dict[str, Any]: + rows = [] + for action_id, examples in sorted(examples_by_action.items()): + raw = [] + for example in examples: + source = example["source"] + action = example["action"] + noop = ( + int(source["mns"]) == int(action["target_mns"]) + and int(source["mbbt"]) == int(action["target_mbbt"]) + ) + if noop: + raw.extend(0.0 for _model in models) + continue + names, values = MODEL.feature_vector( + example, include_telemetry=include_telemetry + ) + if any(model.feature_names != tuple(names) for model in models): + raise ValueError("prospective feature schema does not match frozen model") + raw.extend(model.predict(values) for model in models) + clipped = np.clip(np.asarray(raw, dtype=np.float64), -1.0, 1.0) + prediction = { + "mean": float(clipped.mean()), + "std": float(clipped.std(ddof=0)), + "min": float(clipped.min()), + "max": float(clipped.max()), + "distinct_n": len(set(float(value) for value in clipped)), + "sample_n": int(clipped.size), + } + rows.append( + { + "action_id": action_id, + "prediction": prediction, + "lower": prediction["mean"] - confidence_z * prediction["std"], + "upper": prediction["mean"] + confidence_z * prediction["std"], + } + ) + rows.sort(key=lambda row: (-row["prediction"]["mean"], row["action_id"])) + best, second = rows[:2] + margin = float(best["prediction"]["mean"] - second["prediction"]["mean"]) + confident = bool( + margin >= minimum_margin and best["lower"] > second["upper"] + ) + return { + "selected_action": best["action_id"], + "confident": confident, + "predicted_margin": margin, + "candidates": rows, + } + + +def apply_measurement_and_acquisition(checkpoints: list[dict[str, Any]]) -> dict[str, Any]: + selected = checkpoints[-1] + stop_reason = "full_measurement_fallback" + for previous, current in zip(checkpoints, checkpoints[1:], strict=False): + if ( + previous["confident"] + and current["confident"] + and previous["selected_action"] == current["selected_action"] + ): + selected = current + stop_reason = "two_consecutive_confident_checkpoints" + break + candidates = selected["candidates"] + mean_best = candidates[0] + non_noop = [row for row in candidates if row["action_id"] != "noop"] + if selected["confident"]: + chosen = mean_best + decision_kind = "exploit" + else: + positive_ucb = [row for row in non_noop if float(row["upper"]) > 0.0] + if positive_ucb: + chosen = max( + positive_ucb, + key=lambda row: (float(row["upper"]), row["action_id"]), + ) + decision_kind = "diagnostic_ucb" + else: + chosen = next(row for row in candidates if row["action_id"] == "noop") + decision_kind = "abstain_no_positive_ucb" + remaining = [row for row in candidates if row["action_id"] != chosen["action_id"]] + remaining.sort(key=lambda row: (-float(row["upper"]), row["action_id"])) + order = [chosen["action_id"], *(row["action_id"] for row in remaining)] + return { + "selected_phase": selected["phase"], + "selected_cutoff_s": selected["cutoff_s"], + "measurement_stop_reason": stop_reason, + "decision_kind": decision_kind, + "selected_action": chosen["action_id"], + "intervention_order": order, + "selected_checkpoint": selected, + "checkpoints": checkpoints, + } + + +def build_decision( + *, manifest_path: Path, policy_path: Path, run_root: Path +) -> dict[str, Any]: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + policy = json.loads(policy_path.read_text(encoding="utf-8")) + if manifest.get("schema") != "active-intervention-prospective-manifest-v0": + raise ValueError("unexpected prospective manifest schema") + if policy.get("schema") != "active-intervention-policy-v0": + raise ValueError("unexpected frozen policy schema") + if sha256_file(policy_path) != manifest["policy"]["sha256"]: + raise ValueError("frozen policy hash changed after manifest preparation") + configs = {str(item["id"]): item for item in manifest["configs"]} + source_id = str(manifest["source_config_id"]) + source_config = configs[source_id] + source_root = run_root / "sessions" / source_id + engine_records, stream_path = load_engine_records(source_root) + phases = [f"{fraction:.2f}" for fraction in manifest["checkpoints"]["fractions"]] + confidence_z = float(policy["measurement_policy"]["confidence_z"]) + minimum_margin = float(policy["measurement_policy"]["minimum_margin"]) + + examples: dict[str, dict[str, dict[str, Mapping[str, Any]]]] = {} + source_measurements: dict[str, dict[str, Any]] = {} + source_normalized = [] + telemetry_values = [] + for repetition in sorted(int(key) for key in manifest["repetitions"]): + item = manifest["repetitions"][str(repetition)] + result_root = source_root / f"rep{repetition}" + result = json.loads((result_root / "result.json").read_text(encoding="utf-8")) + if result["selection"]["request_id_order_sha256"] != item["selection"][ + "request_id_order_sha256" + ]: + raise ValueError(f"source request hash mismatch: rep{repetition}") + requests = EXTRACT.load_jsonl(result_root / "requests.jsonl") + offered_rate = float(item["selection"]["offered_req_s_per_gpu"]) + offered_total = offered_rate * int(manifest["engine"]["tp"]) + source_normalized.append( + float(result["slo_pass_count"]) + / float(manifest["engine"]["duration_s"]) + / offered_total + ) + start_ns = int(result["interval"]["start_mono_ns"]) + examples[str(repetition)] = {} + source_measurements[str(repetition)] = { + "result": str(result_root / "result.json"), + "result_sha256": sha256_file(result_root / "result.json"), + "request_sha256": sha256_file(result_root / "requests.jsonl"), + "phases": {}, + } + for phase, cutoff_s in zip( + phases, manifest["checkpoints"]["seconds"], strict=True + ): + outcome = EXTRACT.prefix_outcome( + requests, cutoff_s=float(cutoff_s), offered_total=offered_total + ) + admitted_count = sum( + float(request["arrival_s"]) <= float(cutoff_s) + for request in requests + ) + state = summarize_engine( + engine_records, + start_ns=start_ns, + end_ns=start_ns + round(float(cutoff_s) * 1e9), + request_count=admitted_count, + ) + if not all(state["sanity"]["invariants"].values()): + raise ValueError( + f"source engine state invariant failed: rep{repetition} {phase}" + ) + telemetry = EXTRACT.telemetry_record(state) + telemetry_values.extend(float(value) for value in telemetry.values()) + source_measurements[str(repetition)]["phases"][phase] = { + "cutoff_s": float(cutoff_s), + "outcome": outcome, + "telemetry": telemetry, + "engine_sanity": state["sanity"], + } + examples[str(repetition)][phase] = { + action_id: candidate_example( + source_config=source_config, + target_config=configs[str(target_id)], + action_id=action_id, + offered_rate_per_gpu=offered_rate, + outcome=outcome, + telemetry=telemetry, + ) + for action_id, target_id in manifest["actions"].items() + } + + decisions = {} + for mode, include_telemetry in (("outcome_only", False), ("telemetry", True)): + checkpoints = [] + for phase, cutoff_s in zip( + phases, manifest["checkpoints"]["seconds"], strict=True + ): + models = MODEL.models_from_json(policy["phases"][phase][mode]["models"]) + examples_by_action = { + action_id: [ + examples[str(repetition)][phase][action_id] + for repetition in sorted(int(key) for key in manifest["repetitions"]) + ] + for action_id in manifest["actions"] + } + checkpoint = aggregate_checkpoint( + models=models, + examples_by_action=examples_by_action, + include_telemetry=include_telemetry, + confidence_z=confidence_z, + minimum_margin=minimum_margin, + ) + checkpoints.append( + {"phase": phase, "cutoff_s": float(cutoff_s), **checkpoint} + ) + decisions[mode] = apply_measurement_and_acquisition(checkpoints) + + ceiling = float(manifest["gates"]["source_ceiling_normalized_goodput"]) + source_median = float(statistics.median(source_normalized)) + status = "STOP_SOURCE_CEILING" if source_median >= ceiling else "SELECTED" + phase_admission_monotonic = all( + all( + left <= right + 1e-12 + for left, right in zip(values, values[1:], strict=False) + ) + for repetition in source_measurements.values() + for values in ( + [ + float(repetition["phases"][phase]["outcome"]["admitted_fraction"]) + for phase in phases + ], + ) + ) + telemetry_ratio_keys = { + "prefill_token_fraction", + "kv_usage_mean", + "kv_usage_max", + "graph_none_share", + "graph_full_share", + "graph_padding_fraction", + } + telemetry_records = [ + measurement["telemetry"] + for repetition in source_measurements.values() + for measurement in repetition["phases"].values() + ] + invariants = { + "three_source_repetitions": len(source_normalized) == 3, + "source_goodput_nonnegative": all(value >= 0.0 for value in source_normalized), + "source_goodput_bounded": all( + value <= 1.0 + 1e-12 for value in source_normalized + ), + "four_actions": set(manifest["actions"]) == {"noop", "mns", "mbbt", "joint"}, + "four_checkpoints": len(phases) == 4, + "finite_telemetry": all(math.isfinite(value) for value in telemetry_values), + "nonnegative_telemetry": all( + float(value) >= 0.0 + for record in telemetry_records + for key, value in record.items() + if key != "kv_usage_end_minus_start" + ), + "telemetry_ratios_bounded": all( + 0.0 <= float(record[key]) <= 1.0 + 1e-12 + for record in telemetry_records + for key in telemetry_ratio_keys + ), + "telemetry_not_all_identical": len(set(telemetry_values)) > 1, + "phase_admission_monotonic": phase_admission_monotonic, + "orders_are_permutations": all( + set(decisions[mode]["intervention_order"]) == set(manifest["actions"]) + for mode in decisions + ), + } + red_flags = [name for name, passed in invariants.items() if not passed] + if red_flags: + status = "STOP_SANITY" + return { + "schema": SCHEMA, + "status": status, + "manifest": str(manifest_path), + "manifest_sha256": sha256_file(manifest_path), + "policy": str(policy_path), + "policy_sha256": sha256_file(policy_path), + "source_stream": str(stream_path), + "source_stream_sha256": sha256_file(stream_path), + "source_measurements": source_measurements, + "source_normalized_goodput": { + "values": source_normalized, + "median": source_median, + **numeric(source_normalized), + }, + "decisions": decisions, + "sanity": { + "invariants": invariants, + "red_flags": red_flags, + "telemetry_values": numeric(telemetry_values), + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--policy", type=Path, required=True) + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + decision = build_decision( + manifest_path=args.manifest, policy_path=args.policy, run_root=args.run_root + ) + atomic_json(args.output, decision) + print( + json.dumps( + { + "status": decision["status"], + "source_normalized_goodput": decision["source_normalized_goodput"], + "outcome_only": { + key: decision["decisions"]["outcome_only"][key] + for key in ("selected_cutoff_s", "decision_kind", "selected_action") + }, + "telemetry": { + key: decision["decisions"]["telemetry"][key] + for key in ("selected_cutoff_s", "decision_kind", "selected_action") + }, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/active-intervention-v0/test_pipeline.py b/runs/active-intervention-v0/test_pipeline.py index ee7805c..a6597eb 100644 --- a/runs/active-intervention-v0/test_pipeline.py +++ b/runs/active-intervention-v0/test_pipeline.py @@ -185,9 +185,12 @@ def main() -> None: write_json(dataset_path, dataset) policy = trainer.build_policy(dataset_path) assert policy["status"] in { - "RETROSPECTIVE_INCREMENTAL_SIGNAL", - "NO_RETROSPECTIVE_INCREMENTAL_SIGNAL", + "RETROSPECTIVE_GPU_COST_SIGNAL", + "NO_RETROSPECTIVE_GPU_COST_SIGNAL", } + assert policy["training"]["acceptable_regret"] == 0.02 + assert policy["sequential_replay"]["outcome_only"]["decision_n"] == 6 + assert policy["sequential_replay"]["telemetry"]["decision_n"] == 6 assert not policy["sanity"]["red_flags"] print("active intervention pipeline: PASS") diff --git a/runs/active-intervention-v0/test_prospective.py b/runs/active-intervention-v0/test_prospective.py new file mode 100644 index 0000000..7c95331 --- /dev/null +++ b/runs/active-intervention-v0/test_prospective.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import importlib.util +import json +import sys +import tempfile +from pathlib import Path + + +HERE = Path(__file__).resolve().parent + + +def load(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def write_json(path: Path, payload) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload) + "\n", encoding="utf-8") + + +def main() -> None: + prepare = load("active_intervention_prepare_test", HERE / "prepare_prospective.py") + decision_module = load( + "active_intervention_decision_test", HERE / "prospective_decision.py" + ) + analyzer = load("active_intervention_audit_test", HERE / "analyze_prospective.py") + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / "source.jsonl" + source.write_text( + "".join( + json.dumps( + { + "request_id": f"request-{index}", + "timestamp": float(index), + "sampling_u": index / 100.0, + } + ) + + "\n" + for index in range(60) + ), + encoding="utf-8", + ) + partition = prepare.partition_trace(source, root / "partitions") + assert sum(item["rows"] for item in partition["partitions"].values()) == 60 + ids = [] + for item in partition["partitions"].values(): + assert item["rows"] > 0 + ids.extend( + json.loads(line)["request_id"] + for line in Path(item["path"]).read_text(encoding="utf-8").splitlines() + ) + assert len(ids) == len(set(ids)) == 60 + + checkpoints = [ + { + "phase": "0.25", + "cutoff_s": 75.0, + "selected_action": "joint", + "confident": True, + "candidates": [ + {"action_id": "joint", "upper": 0.5, "prediction": {"mean": 0.4}}, + {"action_id": "mns", "upper": 0.2, "prediction": {"mean": 0.1}}, + {"action_id": "mbbt", "upper": 0.1, "prediction": {"mean": 0.05}}, + {"action_id": "noop", "upper": 0.0, "prediction": {"mean": 0.0}}, + ], + }, + { + "phase": "0.50", + "cutoff_s": 150.0, + "selected_action": "joint", + "confident": True, + "candidates": [ + {"action_id": "joint", "upper": 0.45, "prediction": {"mean": 0.4}}, + {"action_id": "mns", "upper": 0.2, "prediction": {"mean": 0.1}}, + {"action_id": "mbbt", "upper": 0.1, "prediction": {"mean": 0.05}}, + {"action_id": "noop", "upper": 0.0, "prediction": {"mean": 0.0}}, + ], + }, + ] + selected = decision_module.apply_measurement_and_acquisition(checkpoints) + assert selected["selected_cutoff_s"] == 150.0 + assert selected["selected_action"] == "joint" + + configs = prepare.configs() + repetitions = { + str(rep): { + "selection": { + "offered_req_s_per_gpu": 0.25, + "request_id_order_sha256": f"hash-{rep}", + } + } + for rep in (1, 2, 3) + } + manifest = { + "schema": "active-intervention-prospective-manifest-v0", + "engine": {"duration_s": 300.0, "tp": 4}, + "repetitions": repetitions, + "configs": configs, + "source_config_id": "source_mns32_mbbt4096", + "actions": { + "noop": "source_mns32_mbbt4096", + "mns": "mns64_mbbt4096", + "mbbt": "mns32_mbbt8192", + "joint": "joint_mns64_mbbt8192", + }, + "gates": { + "acceptable_regret": 0.02, + "confirmation_trigger_gpu_cost_reduction": 0.10, + "contribution_gpu_cost_reduction": 0.20, + }, + } + manifest_path = root / "manifest.json" + write_json(manifest_path, manifest) + run_root = root / "run" + scores = { + "source_mns32_mbbt4096": 0.5, + "mns64_mbbt4096": 0.8, + "mns32_mbbt8192": 0.7, + "joint_mns64_mbbt8192": 1.0, + } + sessions = {} + for config in configs: + config_id = config["id"] + sessions[config_id] = {"status": "complete", "gpu_hours": 1.2} + for repetition in (1, 2, 3): + result = { + "selection": { + "request_id_order_sha256": f"hash-{repetition}" + }, + "slo_pass_count": round(scores[config_id] * 300), + "pass_rate": scores[config_id], + "interval": {"elapsed_s": 300.0}, + } + write_json( + run_root + / "sessions" + / config_id + / f"rep{repetition}" + / "result.json", + result, + ) + state = { + "status": "complete", + "gpu_hours_total": 4.8, + "sessions": sessions, + } + write_json(run_root / "controller-state.json", state) + mode_base = { + "selected_cutoff_s": 300.0, + "selected_action": "mns", + "decision_kind": "exploit", + "intervention_order": ["mns", "mbbt", "joint", "noop"], + } + mode_telemetry = { + "selected_cutoff_s": 150.0, + "selected_action": "joint", + "decision_kind": "exploit", + "intervention_order": ["joint", "mns", "mbbt", "noop"], + } + decision = { + "schema": "active-intervention-prospective-decision-v0", + "manifest_sha256": analyzer.sha256_file(manifest_path), + "decisions": { + "outcome_only": mode_base, + "telemetry": mode_telemetry, + }, + } + decision_path = root / "decision.json" + write_json(decision_path, decision) + audit = analyzer.build_audit( + manifest_path=manifest_path, + decision_path=decision_path, + run_root=run_root, + ) + assert audit["status"] == "TRIGGER_ACTUAL_EARLY_STOP_CONFIRMATION" + assert audit["comparison"]["telemetry_gpu_cost_reduction_fraction"] > 0.10 + assert not audit["sanity"]["red_flags"] + print("active intervention prospective pipeline: PASS") + + +if __name__ == "__main__": + main() diff --git a/runs/active-intervention-v0/train_policy.py b/runs/active-intervention-v0/train_policy.py index 8bbae02..bd8359c 100644 --- a/runs/active-intervention-v0/train_policy.py +++ b/runs/active-intervention-v0/train_policy.py @@ -33,6 +33,7 @@ MODEL = _load_model() REGULARIZATION = 10.0 MINIMUM_MARGIN = 0.02 CONFIDENCE_Z = 1.0 +ACCEPTABLE_REGRET = 0.02 def sha256_file(path: Path) -> str: @@ -110,13 +111,20 @@ def evaluate_grouped_cv( best_actions = { row["action_id"] for row in predictions if math.isclose(row["real"], oracle) } + acceptable_actions = { + row["action_id"] + for row in predictions + if oracle <= 0 + or 1.0 - float(row["real"]) / oracle <= ACCEPTABLE_REGRET + 1e-12 + } decision_rows.append( { "holdout": held_out, "decision_id": decision_id, "selected_action": selected["action_id"], "best_actions": sorted(best_actions), - "correct": selected["action_id"] in best_actions, + "acceptable_actions": sorted(acceptable_actions), + "correct": regret <= ACCEPTABLE_REGRET + 1e-12, "selected_real": selected["real"], "oracle_real": oracle, "regret": regret, @@ -129,6 +137,7 @@ def evaluate_grouped_cv( return { "status": "VALID", "holdout_key": holdout_key, + "acceptable_regret": ACCEPTABLE_REGRET, "decision_n": len(decision_rows), "correct_n": sum(bool(row["correct"]) for row in decision_rows), "accuracy": sum(bool(row["correct"]) for row in decision_rows) / len(decision_rows), @@ -172,6 +181,173 @@ def paired_delta(outcome: Mapping[str, Any], telemetry: Mapping[str, Any]) -> di } +def evaluate_sequential_measurement_cv( + examples: Sequence[Mapping[str, Any]], + *, + include_telemetry: bool, + holdout_key: str, +) -> dict[str, Any]: + """Replay a two-consecutive-confident-checkpoint measurement policy.""" + + phases = sorted({str(example["phase"]) for example in examples}, key=float) + holdouts = grouped(examples, holdout_key) + rows = [] + full_duration_s = max(float(example["cutoff_s"]) for example in examples) + for held_out, test_examples in sorted(holdouts.items()): + training = [ + example for example in examples if str(example[holdout_key]) != held_out + ] + if len({str(example["decision_id"]) for example in training}) < 3: + continue + phase_models = {} + for phase in phases: + phase_training = [ + example for example in training if str(example["phase"]) == phase + ] + phase_models[phase] = MODEL.fit_jackknife_ensemble( + phase_training, + include_telemetry=include_telemetry, + regularization=REGULARIZATION, + ) + for decision_id, decision_examples in sorted( + grouped(test_examples, "decision_id").items() + ): + checkpoints = [] + by_phase = grouped(decision_examples, "phase") + for phase in phases: + candidates = by_phase[phase] + decision = MODEL.select_action( + phase_models[phase], + candidates, + include_telemetry=include_telemetry, + confidence_z=CONFIDENCE_Z, + minimum_margin=MINIMUM_MARGIN, + ) + checkpoints.append( + { + "phase": phase, + "cutoff_s": float(candidates[0]["cutoff_s"]), + **decision, + } + ) + selected_checkpoint = checkpoints[-1] + stop_reason = "full_measurement_fallback" + for previous, current in zip(checkpoints, checkpoints[1:], strict=False): + if ( + previous["confident"] + and current["confident"] + and previous["selected_action"] == current["selected_action"] + ): + selected_checkpoint = current + stop_reason = "two_consecutive_confident_checkpoints" + break + candidates = by_phase[str(selected_checkpoint["phase"])] + real_by_action = { + str(candidate["action"]["id"]): float( + candidate["target_normalized_goodput"] + ) + for candidate in candidates + } + target_by_action = { + str(candidate["action"]["id"]): str( + candidate["action"]["target_config_id"] + ) + for candidate in candidates + } + selected_action = str(selected_checkpoint["selected_action"]) + oracle = max(real_by_action.values()) + selected_real = real_by_action[selected_action] + regret = 1.0 - selected_real / oracle if oracle > 0 else 0.0 + source_tp = 4 + target_s = 0.0 if selected_action == "noop" else full_duration_s + replay_gpu_seconds = source_tp * ( + float(selected_checkpoint["cutoff_s"]) + target_s + ) + rows.append( + { + "holdout": held_out, + "decision_id": decision_id, + "selected_phase": str(selected_checkpoint["phase"]), + "selected_cutoff_s": float(selected_checkpoint["cutoff_s"]), + "stop_reason": stop_reason, + "selected_action": selected_action, + "selected_target_config_id": target_by_action[selected_action], + "selected_real": selected_real, + "oracle_real": oracle, + "regret": regret, + "acceptable": regret <= ACCEPTABLE_REGRET + 1e-12, + "replay_gpu_seconds_lower_bound": replay_gpu_seconds, + "checkpoints": checkpoints, + } + ) + if not rows: + return {"status": "INSUFFICIENT_GROUPS", "decisions": []} + regrets = [float(row["regret"]) for row in rows] + cutoffs = [float(row["selected_cutoff_s"]) for row in rows] + costs = [float(row["replay_gpu_seconds_lower_bound"]) for row in rows] + return { + "status": "VALID", + "holdout_key": holdout_key, + "measurement_rule": "earliest two consecutive confident checkpoints; otherwise full", + "acceptable_regret": ACCEPTABLE_REGRET, + "decision_n": len(rows), + "acceptable_n": sum(bool(row["acceptable"]) for row in rows), + "mean_regret": sum(regrets) / len(regrets), + "max_regret": max(regrets), + "mean_cutoff_s": sum(cutoffs) / len(cutoffs), + "total_replay_gpu_seconds_lower_bound": sum(costs), + "total_replay_h20_hours_lower_bound": sum(costs) / 3600.0, + "decisions": rows, + } + + +def paired_sequential_delta( + outcome: Mapping[str, Any], telemetry: Mapping[str, Any] +) -> dict[str, Any]: + if outcome.get("status") != "VALID" or telemetry.get("status") != "VALID": + return {"status": "INSUFFICIENT_GROUPS"} + before_by_id = {row["decision_id"]: row for row in outcome["decisions"]} + after_by_id = {row["decision_id"]: row for row in telemetry["decisions"]} + rows = [] + for decision_id in sorted(set(before_by_id) & set(after_by_id)): + before = before_by_id[decision_id] + after = after_by_id[decision_id] + rows.append( + { + "decision_id": decision_id, + "outcome_action": before["selected_action"], + "telemetry_action": after["selected_action"], + "outcome_cutoff_s": before["selected_cutoff_s"], + "telemetry_cutoff_s": after["selected_cutoff_s"], + "outcome_regret": before["regret"], + "telemetry_regret": after["regret"], + "regret_delta": float(after["regret"]) - float(before["regret"]), + "gpu_seconds_delta": float( + after["replay_gpu_seconds_lower_bound"] + ) + - float(before["replay_gpu_seconds_lower_bound"]), + "telemetry_corrected": (not before["acceptable"]) + and bool(after["acceptable"]), + "telemetry_harmed": bool(before["acceptable"]) + and (not after["acceptable"]), + } + ) + outcome_cost = float(outcome["total_replay_gpu_seconds_lower_bound"]) + telemetry_cost = float(telemetry["total_replay_gpu_seconds_lower_bound"]) + return { + "status": "VALID", + "decision_n": len(rows), + "corrected_n": sum(row["telemetry_corrected"] for row in rows), + "harmed_n": sum(row["telemetry_harmed"] for row in rows), + "outcome_replay_gpu_seconds_lower_bound": outcome_cost, + "telemetry_replay_gpu_seconds_lower_bound": telemetry_cost, + "gpu_cost_reduction_fraction": ( + 1.0 - telemetry_cost / outcome_cost if outcome_cost > 0 else 0.0 + ), + "rows": rows, + } + + def build_policy(dataset_path: Path) -> dict[str, Any]: dataset = json.loads(dataset_path.read_text(encoding="utf-8")) if dataset.get("status") != "VALID" or dataset["sanity"]["red_flags"]: @@ -211,6 +387,10 @@ def build_policy(dataset_path: Path) -> dict[str, Any]: and int(delta["harmed_n"]) == 0 and float(delta["mean_regret_delta"]) < -1e-12 and float(telemetry_cv["max_regret"]) <= 0.05 + and telemetry_regime.get("status") == "VALID" + and float(telemetry_regime["mean_regret"]) + <= float(outcome_regime["mean_regret"]) + 1e-12 + and float(telemetry_regime["max_regret"]) <= 0.05 ) if incremental: incremental_candidates.append(phase) @@ -229,11 +409,27 @@ def build_policy(dataset_path: Path) -> dict[str, Any]: "paired_incremental": delta, "incremental_gate": incremental, } - selected_phase = incremental_candidates[0] if incremental_candidates else phases[-1] + outcome_sequential = evaluate_sequential_measurement_cv( + examples, include_telemetry=False, holdout_key="repetition" + ) + telemetry_sequential = evaluate_sequential_measurement_cv( + examples, include_telemetry=True, holdout_key="repetition" + ) + sequential_delta = paired_sequential_delta( + outcome_sequential, telemetry_sequential + ) + retrospective_cost_gate = bool( + sequential_delta.get("status") == "VALID" + and int(sequential_delta["harmed_n"]) == 0 + and int(telemetry_sequential["acceptable_n"]) + >= int(outcome_sequential["acceptable_n"]) + and float(telemetry_sequential["max_regret"]) <= 0.05 + and float(sequential_delta["gpu_cost_reduction_fraction"]) >= 0.10 + ) status = ( - "RETROSPECTIVE_INCREMENTAL_SIGNAL" - if incremental_candidates - else "NO_RETROSPECTIVE_INCREMENTAL_SIGNAL" + "RETROSPECTIVE_GPU_COST_SIGNAL" + if retrospective_cost_gate + else "NO_RETROSPECTIVE_GPU_COST_SIGNAL" ) target_values = [float(example["target_normalized_goodput"]) for example in examples] effect_values = [ @@ -265,15 +461,20 @@ def build_policy(dataset_path: Path) -> dict[str, Any]: "regularization": REGULARIZATION, "confidence_z": CONFIDENCE_Z, "minimum_margin": MINIMUM_MARGIN, + "acceptable_regret": ACCEPTABLE_REGRET, }, "measurement_policy": { - "selected_phase": selected_phase, - "selected_cutoff_s": phase_results[selected_phase]["cutoff_s"], - "selection_reason": ( - "earliest phase passing the frozen incremental gate" - if incremental_candidates - else "no incremental phase; retain full measurement for exploratory held-out test" - ), + "rule": "earliest two consecutive confident checkpoints; otherwise full", + "checkpoints": [phase_results[phase]["cutoff_s"] for phase in phases], + "confidence_z": CONFIDENCE_Z, + "minimum_margin": MINIMUM_MARGIN, + }, + "sequential_replay": { + "outcome_only": outcome_sequential, + "telemetry": telemetry_sequential, + "paired_delta": sequential_delta, + "retrospective_gpu_cost_gate": retrospective_cost_gate, + "minimum_cost_reduction_fraction": 0.10, }, "phases": phase_results, "sanity": { From 39b767e384fc49da53b99ad06a3e2ca1b6ac37d6 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 02:07:27 +0800 Subject: [PATCH 030/148] Correct active intervention engine provenance --- docs/active-intervention-v0-protocol-20260715.md | 13 ++++++++++--- runs/active-intervention-v0/prepare_prospective.py | 3 ++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/active-intervention-v0-protocol-20260715.md b/docs/active-intervention-v0-protocol-20260715.md index cf6a5e4..6d68231 100644 --- a/docs/active-intervention-v0-protocol-20260715.md +++ b/docs/active-intervention-v0-protocol-20260715.md @@ -41,9 +41,10 @@ the six decisions are complete exact-workload pairs. - Host: `dash0`, 8 NVIDIA H20 GPUs available; each TP4 server runs alone on GPUs 0-3. Co-location is prohibited for SLO verdicts. -- Engine: patched vLLM `0.24.1.dev3+opprof` from - `/home/admin/cpfs/wjh/vllm-opprof-phase3` in - `/tmp/wjh/venvs/vllm-0.20.0-cu129`. +- Engine: patched vLLM `0.24.1.dev3+g668cfb7e2` from clean source commit + `4b253fd8619764b6971a7f2e3a3aa7545f6ace05` at + `/home/admin/cpfs/wjh/opprof-phase2-dash0-20260711/vllm-v0.24.0`, using + `/tmp/wjh-opprof-phase2-dash0-20260711/.venv`. - Model: `/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B`, BF16. - Workload: unseen `chat_w20260313_1000`; input 0-8192; output exactly 128; replay scale 0.5; 300-second arrival window. @@ -127,3 +128,9 @@ use the same measured config costs and differ only in source information. If the 10% trigger fails, this route is closed for the current engine-state representation. The experimental control plane is not retained as a fallback research contribution. + +Pre-run provenance amendment: the first controller dry-run on 2026-07-15 +rejected two stale engine paths before starting a server. The paths and exact +runtime version above were recovered from the accepted trace-12 campaign and +corrected before any trace-13 GPU work. No scientific treatment or gate was +changed. diff --git a/runs/active-intervention-v0/prepare_prospective.py b/runs/active-intervention-v0/prepare_prospective.py index 0268537..7750acb 100644 --- a/runs/active-intervention-v0/prepare_prospective.py +++ b/runs/active-intervention-v0/prepare_prospective.py @@ -28,6 +28,7 @@ REPLAY_TIME_SCALE = 0.5 OFFERED_RATE_PER_GPU = 2.75 TARGET_COUNT = round(OFFERED_RATE_PER_GPU * DURATION_S * TP) WINDOW_ID = "chat_w20260313_1000" +ENGINE_VERSION = "0.24.1.dev3+g668cfb7e2" def atomic_json(path: Path, payload: Any) -> None: @@ -140,7 +141,7 @@ def materialize_study( payload = json.loads(base_study.read_text(encoding="utf-8")) payload["study_id"] = f"active-intervention-trace13-rep{repetition}" payload["hardware"]["host_candidates"] = ["dash0"] - payload["engine"]["engine_version"] = "0.24.1.dev3+opprof" + payload["engine"]["engine_version"] = ENGINE_VERSION trace = payload["trace"] trace.update( { From 9c8570f36b1753d1b77c7cf1b0d16ecc360cad33 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 03:34:49 +0800 Subject: [PATCH 031/148] Report held-out active intervention result --- ...active-intervention-v0-results-20260715.md | 106 ++++++++++++ .../trace13-results.json | 151 ++++++++++++++++++ 2 files changed, 257 insertions(+) create mode 100644 docs/active-intervention-v0-results-20260715.md create mode 100644 runs/active-intervention-v0/trace13-results.json diff --git a/docs/active-intervention-v0-results-20260715.md b/docs/active-intervention-v0-results-20260715.md new file mode 100644 index 0000000..d4fc98c --- /dev/null +++ b/docs/active-intervention-v0-results-20260715.md @@ -0,0 +1,106 @@ +# Active intervention v0: held-out trace-13 result + +Date: 2026-07-15 (Asia/Singapore) + +Decision: **close the passive-telemetry treatment-effect route**. The held-out +campaign produced no telemetry-induced action change, measurement reduction, or +GPU-cost reduction. It did show that the engine state contained the correct +action-specific mechanism; the current feature model failed to use it. + +## Headline result + +The outcome-only and telemetry policies both measured the source for 300 +seconds, selected `joint=(MNS64,MBBT8192)`, and produced the same complete +acquisition order. Both reached the exact finite-surface oracle after the +first intervention at a reconstructed all-in lower-bound cost of 2.4284 +H20-hours. Telemetry GPU-cost reduction was therefore exactly 0%, below the +10% confirmation trigger and 20% contribution gate. No actual early-stop +confirmation was launched. + +The complete annotation campaign cost 5.0379 H20-hours, below the 6.0 H20-hour +hard cap. It ran 12 uncensored real-GPU outcomes: four configs, three disjoint +request partitions, and a fresh server per config. + +## Exact response surface + +Median normalized SLO-goodput was: + +| Config | Rep values | Median | +|---|---|---:| +| `MNS32,MBBT4096` source | 0.40091 / 0.39788 / 0.42061 | 0.40091 | +| `MNS64,MBBT4096` | 1.00000 / 0.99970 / 1.00000 | 1.00000 | +| `MNS32,MBBT8192` | 0.44394 / 0.41515 / 0.42606 | 0.42606 | +| `MNS64,MBBT8192` joint | 1.00000 / 1.00000 / 1.00000 | 1.00000 | + +Increasing MNS alone was sufficient and joint was redundant. Increasing MBBT +alone improved the median by only 0.02515, versus 0.59909 for MNS. This is a +strong non-additive action response, not a setting where independently tuning +the knobs and merging their improvements is valid. + +## What the telemetry actually said + +Across 41,086 source scheduler records, 93.12% of steps had waiting work, +85.36% were MNS-exclusive binding, 1.11% were MBBT-exclusive, mean running-slot +utilization was 97.39%, mean token-budget utilization was 15.69%, mean KV usage +was 2.75%, and there were no preemptions. + +The intervention transition agreed with that state: + +| Config | Waiting | MNS-exclusive | MBBT-exclusive | Median goodput | +|---|---:|---:|---:|---:| +| source | 93.12% | 85.36% | 1.11% | 0.40091 | +| MNS only | 5.38% | 0% | 5.38% | 1.00000 | +| MBBT only | 91.19% | 91.09% | 0.04% | 0.42606 | +| joint | 0.89% | 0% | 0.89% | 1.00000 | + +Thus this experiment does **not** support the claim that engine telemetry lacks +tuning information. It rejects the narrower claim that adding passive state +summaries to the current small-data ridge policy converts that information into +lower tuning cost. + +## Why the learned policy failed + +At 300 seconds, the telemetry model predicted joint, MNS, and MBBT effects of +0.35190, 0.26118, and 0.09686. The actual median effects were 0.59909, +0.59909, and 0.02515. Telemetry therefore made the nonexistent joint-over-MNS +gap larger: 0.09072 predicted versus 0 actual; the outcome-only model predicted +0.03188. + +The failure has three concrete causes: + +1. The six training decisions contain no joint intervention. The + `delta_product` feature has no support, so joint ranking is extrapolation. +2. Passive raw summaries do not represent the counterfactual scheduler work + unlocked by each action. Capacity-normalized MNS pressure was visible, but + the model was not structurally required to map it to MNS marginal value. +3. The policy maximizes predicted effect. It does not identify the smallest + epsilon-optimal intervention or price unsupported action complexity. + +## Research implication + +Do not retain the harness or the passive telemetry model as a contribution. +The next defensible route is engine-native, action-conditional counterfactual +instrumentation: at a real scheduling state, shadow-replay the exact scheduler +decision under an MNS relaxation, MBBT relaxation, and their joint relaxation, +then expose the incremental queued work admitted by each action. Real paired +interventions calibrate how those one-step shadow effects map to E2E SLO +goodput. This is distinct from a hand-written cap-to-knob rule and from a +full-system simulator: it reuses the exact live queue, scheduler, and cache +state while simulating only the local decision boundary. + +That route should be evaluated against outcome-only search, the present passive +telemetry model, a cap-hit expert rule, and a full simulator. The paper-level +gate remains at least 20% measured H20-hour reduction to a 2%-oracle config on +task-held-out workloads with at most 2% regret. + +## Sanity + +Surface outcomes: n=12, min=0.39788, max=1.0, distinct=8. Session costs: n=4, +min=1.1702, max=1.3566 H20-hours, distinct=4. Scheduler-record counts: n=4, +min=37,001, max=41,348, distinct=4. All counters and costs were non-negative; +all ratios were in `[0,1]`; request hashes matched; all 12 runs were uncensored; +the controller and four sessions completed; and config outcomes were not all +identical. No red flags were found. + +Machine-readable summary: `runs/active-intervention-v0/trace13-results.json`. +Raw immutable root: `/home/admin/cpfs/wjh/active-intervention-prospective-20260715`. diff --git a/runs/active-intervention-v0/trace13-results.json b/runs/active-intervention-v0/trace13-results.json new file mode 100644 index 0000000..b5c3e01 --- /dev/null +++ b/runs/active-intervention-v0/trace13-results.json @@ -0,0 +1,151 @@ +{ + "schema": "active-intervention-trace13-result-summary-v0", + "status": "STOP_NO_PROSPECTIVE_GPU_COST_SIGNAL", + "provenance": { + "aituner_commit": "39b767e384fc49da53b99ad06a3e2ca1b6ac37d6", + "vllm_commit": "4b253fd8619764b6971a7f2e3a3aa7545f6ace05", + "manifest_sha256": "3bd25ae0ca040729a6351635f14447b3c789d4d86fb0fe4d65940735ad225a78", + "policy_sha256": "4f096d3a5f8c38771e956dfd576dd6cd5d5691286ab258028dbddbe07b20078c", + "decision_sha256": "86b7089151c480e18b5ae6ed65c4e4a3e11159dd0c16d5851312d4d9196d5ca2", + "controller_state_sha256": "d10d3cbe5ce20eacc1392f52e79d16ae23b5e15301123d5e85e48d53ae676cda", + "audit_sha256": "bd95b45e5d4cb93b5ad2f722b7dc96553d64eb8a5d5aae3a82f29c2d015fe3f6", + "remote_root": "/home/admin/cpfs/wjh/active-intervention-prospective-20260715" + }, + "cost": { + "annotation_campaign_h20_hours": 5.0379046784506905, + "hard_cap_h20_hours": 6.0, + "outcome_only_cost_to_acceptable_h20_hours_lower_bound": 2.4284364508172893, + "telemetry_cost_to_acceptable_h20_hours_lower_bound": 2.4284364508172893, + "telemetry_gpu_cost_reduction_fraction": 0.0, + "session_h20_hours": { + "source_mns32_mbbt4096": 1.3566088432735868, + "mns64_mbbt4096": 1.256969277858734, + "mns32_mbbt8192": 1.254111782974667, + "joint_mns64_mbbt8192": 1.1702147743437026 + } + }, + "policy_comparison": { + "outcome_only": { + "measurement_cutoff_s": 300.0, + "selected_action": "joint", + "intervention_order": ["joint", "mns", "mbbt", "noop"] + }, + "telemetry": { + "measurement_cutoff_s": 300.0, + "selected_action": "joint", + "intervention_order": ["joint", "mns", "mbbt", "noop"] + }, + "action_changed": false, + "measurement_changed": false, + "confirmation_trigger": false, + "contribution_gate": false + }, + "surface": { + "source_mns32_mbbt4096": { + "normalized_slo_goodput": [ + 0.40090909090909094, + 0.3978787878787879, + 0.42060606060606065 + ], + "median": 0.40090909090909094 + }, + "mns64_mbbt4096": { + "normalized_slo_goodput": [1.0, 0.9996969696969698, 1.0], + "median": 1.0 + }, + "mns32_mbbt8192": { + "normalized_slo_goodput": [ + 0.44393939393939397, + 0.41515151515151516, + 0.4260606060606061 + ], + "median": 0.42606060606060603 + }, + "joint_mns64_mbbt8192": { + "normalized_slo_goodput": [1.0, 1.0, 1.0], + "median": 1.0 + } + }, + "selected_checkpoint_prediction": { + "actual_median_effect": { + "noop": 0.0, + "mns": 0.5990909090909091, + "mbbt": 0.02515151515151509, + "joint": 0.5990909090909091 + }, + "outcome_only_predicted_effect": { + "noop": 0.0, + "mns": 0.2886250281729182, + "mbbt": 0.17933598309437812, + "joint": 0.3205015384324615 + }, + "telemetry_predicted_effect": { + "noop": 0.0, + "mns": 0.26117798146236215, + "mbbt": 0.09686132563074483, + "joint": 0.35190199346536294 + }, + "actual_joint_minus_mns": 0.0, + "outcome_only_joint_minus_mns": 0.0318765102595433, + "telemetry_joint_minus_mns": 0.09072401200300079 + }, + "engine_mechanism": { + "source_mns32_mbbt4096": { + "scheduler_records": 41086, + "waiting_fraction": 0.9312174463320839, + "mns_exclusive_fraction": 0.8536484447256973, + "mbbt_exclusive_fraction": 0.01114734946210388, + "both_fraction": 0.06642165214428272, + "running_utilization_mean": 0.9738878510928297, + "token_utilization_mean": 0.15694342925509905, + "kv_usage_mean": 0.027507593814715858, + "preemptions": 0 + }, + "mns64_mbbt4096": { + "scheduler_records": 37001, + "waiting_fraction": 0.053809356503878275, + "mns_exclusive_fraction": 0.0, + "mbbt_exclusive_fraction": 0.053809356503878275, + "running_utilization_mean": 0.5410364753655307, + "token_utilization_mean": 0.17425695631536986, + "kv_usage_mean": 0.030549112146415616, + "preemptions": 0 + }, + "mns32_mbbt8192": { + "scheduler_records": 41348, + "waiting_fraction": 0.9119425365192996, + "mns_exclusive_fraction": 0.9108542130211861, + "mbbt_exclusive_fraction": 0.0003627744993711909, + "running_utilization_mean": 0.9652567838831383, + "token_utilization_mean": 0.0779606414231039, + "kv_usage_mean": 0.027355900366122385, + "preemptions": 0 + }, + "joint_mns64_mbbt8192": { + "scheduler_records": 40416, + "waiting_fraction": 0.0088826207442597, + "mns_exclusive_fraction": 0.0, + "mbbt_exclusive_fraction": 0.0088826207442597, + "running_utilization_mean": 0.49403392220902614, + "token_utilization_mean": 0.07978070546782215, + "kv_usage_mean": 0.028003770774946098, + "preemptions": 0 + } + }, + "sanity": { + "surface_outcomes": {"n": 12, "min": 0.3978787878787879, "max": 1.0, "distinct_n": 8}, + "session_h20_hours": {"n": 4, "min": 1.1702147743437026, "max": 1.3566088432735868, "distinct_n": 4}, + "scheduler_records": {"n": 4, "min": 37001, "max": 41348, "distinct_n": 4}, + "invariants": { + "controller_complete": true, + "four_sessions_complete": true, + "twelve_surface_outcomes": true, + "ratios_bounded": true, + "nonnegative_counts_and_costs": true, + "surface_not_all_identical": true, + "request_hashes_match": true, + "no_censored_runs": true + }, + "red_flags": [] + } +} From 684a2de413cfb3212c7e89cf8db8c46162102b5d Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 18:34:13 +0800 Subject: [PATCH 032/148] Align Frontier FP8 profiling with vLLM runtime --- .../frontier_fp8_profile_contract.patch | 89 +++++++ .../frontier_moe_serving_path.patch | 216 +++++++++++++++++ .../best_effort/moe_path_parity.py | 220 ++++++++++++++++++ 3 files changed, 525 insertions(+) create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/frontier_fp8_profile_contract.patch create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_serving_path.patch create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/moe_path_parity.py diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_fp8_profile_contract.patch b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_fp8_profile_contract.patch new file mode 100644 index 0000000..012fd79 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_fp8_profile_contract.patch @@ -0,0 +1,89 @@ +diff --git a/frontier/config/quantization_manager.py b/frontier/config/quantization_manager.py +--- a/frontier/config/quantization_manager.py ++++ b/frontier/config/quantization_manager.py +@@ -356,9 +356,17 @@ class QuantizationManager: + target_precision = self.get_precision(op_name) + precision_match = target_precision == profiling_precision + quant_match = profiling_quant_signature == expected_quant_signature ++ # A mixed-precision profiling CSV records the model's output dtype at ++ # file level. Quantized operators in that CSV were nevertheless ++ # measured with the exact quantization scheme identified by the ++ # quant signature. Treating those samples as BF16 and scaling them ++ # again would double-apply the FP8 speedup. ++ exact_fp8_profile = quant_match and target_precision == PrecisionType.FP8 + with self._lock: +- self._operation_profiling_precision[op_name] = profiling_precision +- if precision_match: ++ self._operation_profiling_precision[op_name] = ( ++ target_precision if exact_fp8_profile else profiling_precision ++ ) ++ if precision_match or exact_fp8_profile: + self._operation_data_sources[op_name] = "profiling" + self._operation_approximation_factors.pop(op_name, None) + self._operation_speedup_factors.pop(op_name, None) +diff --git a/tests/unit/test_quantization_profile_contract.py b/tests/unit/test_quantization_profile_contract.py +new file mode 100644 +--- /dev/null ++++ b/tests/unit/test_quantization_profile_contract.py +@@ -0,0 +1,61 @@ ++from unittest.mock import MagicMock ++ ++import pytest ++ ++from frontier.config.model_config import QuantizationConfig ++from frontier.config.precision_type import PrecisionType ++from frontier.config.quantization_manager import QuantizationManager ++ ++ ++def test_exact_fp8_quant_signature_does_not_rescale_mixed_profile() -> None: ++ manager = QuantizationManager() ++ manager.load_config() ++ ++ quant_config = QuantizationConfig( ++ quant_method="fp8", ++ activation_scheme="dynamic", ++ is_checkpoint_fp8_serialized=True, ++ weight_block_size=(128, 128), ++ ) ++ quant_signature = quant_config.get_quant_signature() ++ model_config = MagicMock() ++ model_config.get_default_precision.return_value = PrecisionType.BF16 ++ model_config.get_name.return_value = "Qwen3-235B-A22B-FP8" ++ model_config.torch_dtype = "bfloat16" ++ model_config.quantization_config = quant_config ++ model_config.get_quant_signature.return_value = quant_signature ++ ++ manager.configure_from_model_config(model_config) ++ manager.register_profiling_metadata( ++ operation_names=["attn_pre_proj"], ++ profiling_precision=PrecisionType.BF16, ++ profiling_quant_signature=quant_signature, ++ expected_quant_signature=quant_signature, ++ file_path="linear_op.csv", ++ ) ++ ++ metadata = { ++ item["operation"]: item ++ for item in manager.get_operation_precision_metadata() ++ } ++ assert metadata["attn_pre_proj"]["data_source"] == "profiling" ++ assert metadata["attn_pre_proj"]["approximation_factor"] is None ++ assert manager.has_precision_mismatch("attn_pre_proj") is False ++ assert manager.adjust_compute_time("attn_pre_proj", 1.25) == pytest.approx(1.25) ++ ++ ++def test_mismatched_fp8_quant_signature_still_uses_approximation() -> None: ++ manager = QuantizationManager() ++ manager.load_config() ++ manager._operation_precisions["attn_pre_proj"] = PrecisionType.FP8 ++ ++ manager.register_profiling_metadata( ++ operation_names=["attn_pre_proj"], ++ profiling_precision=PrecisionType.BF16, ++ profiling_quant_signature="none", ++ expected_quant_signature="method=fp8", ++ file_path="linear_op.csv", ++ ) ++ ++ assert manager.has_precision_mismatch("attn_pre_proj") is True ++ assert manager.adjust_compute_time("attn_pre_proj", 1.0) == pytest.approx(0.5) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_serving_path.patch b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_serving_path.patch new file mode 100644 index 0000000..6702a90 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_serving_path.patch @@ -0,0 +1,216 @@ +diff --git a/frontier/profiling/moe/moe_vllm_kernel.py b/frontier/profiling/moe/moe_vllm_kernel.py +--- a/frontier/profiling/moe/moe_vllm_kernel.py ++++ b/frontier/profiling/moe/moe_vllm_kernel.py +@@ -34,6 +34,7 @@ try: + import vllm + VLLM_VERSION = vllm.__version__ + ++ from vllm import _custom_ops as ops + # Import vLLM 0.10.x functions + from vllm.model_executor.layers.fused_moe.fused_moe import ( + fused_moe_kernel, +@@ -232,7 +233,7 @@ def _invoke_kernel( + """ + # Determine compute_type - for FP8, we accumulate in FP16/BF16 + if use_fp8: +- compute_type = tl.float16 # FP8 accumulates in FP16 ++ compute_type = tl.bfloat16 + else: + dtype = A.dtype + if dtype == torch.bfloat16: +@@ -275,7 +276,9 @@ def _run_fused_moe_iteration( + w1: torch.Tensor, + w2: torch.Tensor, + intermediate_cache1: torch.Tensor, + intermediate_cache2: torch.Tensor, ++ intermediate_cache3: torch.Tensor, ++ output: torch.Tensor, + topk_weights: torch.Tensor, + sorted_token_ids: torch.Tensor, + expert_ids: torch.Tensor, +@@ -292,8 +295,17 @@ def _run_fused_moe_iteration( + per_channel_quant: bool = False, + block_shape: Optional[List[int]] = None, + ) -> None: ++ first_input = A ++ first_A_scale = A_scale ++ if use_fp8: ++ group_size = block_dims[1] if block_dims else 128 ++ first_input, first_A_scale = quantize_activations_to_fp8( ++ A, ++ group_size=group_size, ++ ) ++ + _invoke_kernel( +- A=A.contiguous(), ++ A=first_input.contiguous(), + B=w1.contiguous(), + C=intermediate_cache1.contiguous(), + topk_weights=topk_weights.contiguous(), +@@ -305,15 +316,18 @@ def _run_fused_moe_iteration( + mul_routed_weight=False, + top_k=top_k, + config=config, +- A_scale=A_scale, ++ A_scale=first_A_scale, + B_scale=w1_scale, + use_fp8=use_fp8, + per_channel_quant=per_channel_quant, + block_shape=block_shape, + ) + +- intermediate_cache1_flat = intermediate_cache1.view(-1, intermediate_cache1.shape[-1]) +- intermediate_cache2_input = intermediate_cache1_flat[:, :expert_hidden_dim_per_partition].contiguous() ++ torch.ops._C.silu_and_mul( ++ intermediate_cache2, ++ intermediate_cache1.view(-1, intermediate_cache1.shape[-1]), ++ ) ++ second_input = intermediate_cache2 + + intermediate_A_scale = None + if use_fp8: +@@ -321,13 +334,13 @@ def _run_fused_moe_iteration( + group_size = block_dims[1] if block_dims else 128 +- intermediate_cache2_input, intermediate_A_scale = quantize_activations_to_fp8( +- intermediate_cache2_input, ++ second_input, intermediate_A_scale = quantize_activations_to_fp8( ++ intermediate_cache2, + group_size=group_size, + ) + + _invoke_kernel( +- A=intermediate_cache2_input, ++ A=second_input, + B=w2.contiguous(), +- C=intermediate_cache2.contiguous(), ++ C=intermediate_cache3.contiguous(), + topk_weights=topk_weights.contiguous(), + sorted_token_ids=sorted_token_ids.contiguous(), + expert_ids=expert_ids.contiguous(), +@@ -335,4 +350,6 @@ def _run_fused_moe_iteration( + ) ++ ++ ops.moe_sum(intermediate_cache3, output) + + + def _collect_cuda_event_stats(step_fn, active_steps: int) -> Dict: +@@ -493,6 +508,5 @@ def profile_fused_moe_kernel( + w1_scale = None + w2_scale = None +- A_scale = None + + block_dims = _validate_block_shape(block_shape) + if use_fp8: +@@ -509,10 +521,8 @@ def profile_fused_moe_kernel( + per_channel=per_channel_quant, + block_shape=block_shape, + ) +- group_size = block_dims[1] if block_dims else 128 +- A, A_scale = quantize_activations_to_fp8(A, group_size=group_size) + +- config_dtype = get_config_dtype_str(base_dtype) ++ config_dtype = get_config_dtype_str(base_dtype, use_fp8_w8a8=use_fp8) + config = try_get_optimal_moe_config( + w1_shape=w1.shape, + w2_shape=w2.shape, +@@ -535,13 +544,25 @@ def profile_fused_moe_kernel( + device=device, + dtype=output_dtype, + ) + intermediate_cache2 = torch.empty( +- num_tokens, +- top_k, +- hidden_dim, ++ num_tokens * top_k, ++ expert_hidden_dim_per_partition, ++ device=device, ++ dtype=output_dtype, ++ ) ++ intermediate_cache3 = torch.empty( ++ num_tokens, ++ top_k, ++ hidden_dim, + device=device, + dtype=output_dtype, + ) ++ output = torch.empty( ++ num_tokens, ++ hidden_dim, ++ device=device, ++ dtype=output_dtype, ++ ) + + def _step() -> None: + _run_fused_moe_iteration( +@@ -552,6 +571,8 @@ def profile_fused_moe_kernel( + w2=w2, + intermediate_cache1=intermediate_cache1, + intermediate_cache2=intermediate_cache2, ++ intermediate_cache3=intermediate_cache3, ++ output=output, + topk_weights=topk_weights, + sorted_token_ids=sorted_token_ids, + expert_ids=expert_ids, +@@ -562,6 +583,6 @@ def profile_fused_moe_kernel( + expert_hidden_dim_per_partition=expert_hidden_dim_per_partition, + block_dims=block_dims, +- A_scale=A_scale, ++ A_scale=None, + w1_scale=w1_scale, + w2_scale=w2_scale, + use_fp8=use_fp8, +diff --git a/frontier/profiling/moe/moe_impl.py b/frontier/profiling/moe/moe_impl.py +--- a/frontier/profiling/moe/moe_impl.py ++++ b/frontier/profiling/moe/moe_impl.py +@@ -245,10 +245,12 @@ class MoETokenShuffler(nn.Module): + def __init__( + self, + num_experts: int, + router_topk: int, + hidden_dim: int, + expert_hidden_dim: int, + dtype: torch.dtype, + use_gated: bool, + num_local_experts: Optional[int] = None, ++ use_fp8: bool = False, ++ block_shape: Optional[list[int]] = None, + ): +@@ -264,6 +266,8 @@ class MoETokenShuffler(nn.Module): + self.router_topk = router_topk + self.hidden_dim = hidden_dim + self.expert_hidden_dim = expert_hidden_dim + self.dtype = dtype + self.use_gated = use_gated ++ self.use_fp8 = use_fp8 ++ self.block_shape = block_shape + self._block_size_cache = {} +@@ -325,9 +329,12 @@ class MoETokenShuffler(nn.Module): +- config_dtype = get_config_dtype_str(dtype=self.dtype) ++ config_dtype = get_config_dtype_str( ++ dtype=self.dtype, ++ use_fp8_w8a8=self.use_fp8, ++ ) + config = try_get_optimal_moe_config( + w1_shape=w1_shape, + w2_shape=w2_shape, + top_k=self.router_topk, + dtype=config_dtype, + M=num_tokens, +- block_shape=None, ++ block_shape=self.block_shape, + ) +diff --git a/frontier/profiling/moe/moe_wrapper.py b/frontier/profiling/moe/moe_wrapper.py +--- a/frontier/profiling/moe/moe_wrapper.py ++++ b/frontier/profiling/moe/moe_wrapper.py +@@ -149,9 +149,11 @@ class MoEWrapper: + self.shuffler = MoETokenShuffler( + num_experts=self.num_experts, + num_local_experts=self.num_experts_per_device, + router_topk=self.router_topk, + hidden_dim=self.hidden_dim, + expert_hidden_dim=self.expert_hidden_dim, + dtype=self._dtype, + use_gated=self.use_gated, ++ use_fp8=self.use_fp8, ++ block_shape=self.block_shape, + ).to(dtype=self._dtype).cuda().eval() diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/moe_path_parity.py b/runs/frontier-multicase-sufficiency-v0/best_effort/moe_path_parity.py new file mode 100644 index 0000000..e90654f --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/moe_path_parity.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""Compare patched Frontier MoE decomposition with vLLM's serving path.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import torch + +from frontier.profiling.moe.moe_vllm_kernel import ( + profile_fused_moe_kernel, + quantize_weights_to_fp8, +) +from vllm.model_executor.layers.fused_moe.fused_moe import ( + fused_experts, + get_config_dtype_str, + moe_align_block_size, + try_get_optimal_moe_config, +) + + +def _measure(step, warmup: int, active: int) -> dict[str, float]: + for _ in range(warmup): + step() + torch.cuda.synchronize() + + samples = [] + for _ in range(active): + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + step() + end.record() + torch.cuda.synchronize() + samples.append(start.elapsed_time(end)) + values = torch.tensor(samples) + return { + "min": float(values.min()), + "median": float(values.median()), + "mean": float(values.mean()), + "max": float(values.max()), + "std": float(values.std()), + } + + +def _routing(num_tokens: int, top_k: int, num_experts: int, seed: int): + generator = torch.Generator(device="cuda") + generator.manual_seed(seed) + topk_ids = torch.randint( + num_experts, + (num_tokens, top_k), + generator=generator, + device="cuda", + dtype=torch.int64, + ) + topk_weights = torch.rand( + (num_tokens, top_k), + generator=generator, + device="cuda", + dtype=torch.float32, + ) + topk_weights /= topk_weights.sum(dim=-1, keepdim=True) + return topk_weights, topk_ids + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--tokens", nargs="+", type=int, default=[16, 256, 1024]) + parser.add_argument("--tp", type=int, default=4) + parser.add_argument("--ep", type=int, default=1) + parser.add_argument("--warmup", type=int, default=2) + parser.add_argument("--active", type=int, default=20) + parser.add_argument("--seed", type=int, default=20260715) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + hidden_dim = 4096 + expert_hidden_dim = 1536 + global_num_experts = 128 + top_k = 8 + block_shape = [128, 128] + if global_num_experts % args.ep: + raise ValueError("EP must divide 128 experts") + if expert_hidden_dim % args.tp: + raise ValueError("TP must divide the expert intermediate dimension") + + local_num_experts = global_num_experts // args.ep + local_intermediate = expert_hidden_dim // args.tp + device = torch.device("cuda") + + torch.manual_seed(args.seed) + w1_bf16 = torch.randn( + local_num_experts, + 2 * local_intermediate, + hidden_dim, + dtype=torch.bfloat16, + device=device, + ) + w2_bf16 = torch.randn( + local_num_experts, + hidden_dim, + local_intermediate, + dtype=torch.bfloat16, + device=device, + ) + w1, w1_scale = quantize_weights_to_fp8(w1_bf16, block_shape=block_shape) + w2, w2_scale = quantize_weights_to_fp8(w2_bf16, block_shape=block_shape) + del w1_bf16, w2_bf16 + torch.cuda.empty_cache() + + rows = [] + for index, num_tokens in enumerate(args.tokens): + topk_weights, topk_ids = _routing( + num_tokens, + top_k, + global_num_experts, + args.seed + index, + ) + hidden_states = torch.randn( + num_tokens, + hidden_dim, + dtype=torch.bfloat16, + device=device, + ) + + frontier_grouped = profile_fused_moe_kernel( + num_tokens=num_tokens, + num_experts=local_num_experts, + hidden_dim=hidden_dim, + expert_hidden_dim=expert_hidden_dim, + top_k=top_k, + topk_weights=topk_weights, + topk_ids=topk_ids, + tensor_parallel_size=args.tp, + dtype=torch.bfloat16, + warmup_steps=args.warmup, + active_steps=args.active, + use_fp8=True, + per_channel_quant=False, + block_shape=block_shape, + global_num_experts=global_num_experts, + ) + + config = try_get_optimal_moe_config( + w1_shape=w1.shape, + w2_shape=w2.shape, + top_k=top_k, + dtype=get_config_dtype_str( + torch.bfloat16, + use_fp8_w8a8=True, + ), + M=num_tokens, + block_shape=block_shape, + ) + + def align_step() -> None: + moe_align_block_size( + topk_ids, + config["BLOCK_SIZE_M"], + global_num_experts, + ) + + alignment = _measure(align_step, args.warmup, args.active) + + def serving_step() -> None: + fused_experts( + hidden_states=hidden_states, + w1=w1, + w2=w2, + topk_weights=topk_weights, + topk_ids=topk_ids, + inplace=False, + use_fp8_w8a8=True, + per_channel_quant=False, + global_num_experts=global_num_experts, + w1_scale=w1_scale, + w2_scale=w2_scale, + block_shape=block_shape, + ) + + serving = _measure(serving_step, args.warmup, args.active) + decomposed_ms = frontier_grouped["median"] + alignment["median"] + rows.append( + { + "num_tokens": num_tokens, + "block_size_m": config["BLOCK_SIZE_M"], + "frontier_grouped_ms": frontier_grouped, + "frontier_alignment_ms": alignment, + "frontier_decomposed_median_ms": decomposed_ms, + "vllm_fused_experts_ms": serving, + "decomposed_over_serving": decomposed_ms / serving["median"], + } + ) + + result = { + "contract": "Frontier grouped_gemm + shuffling alignment vs vLLM fused_experts", + "model_shape": { + "hidden_dim": hidden_dim, + "expert_hidden_dim": expert_hidden_dim, + "global_num_experts": global_num_experts, + "local_num_experts": local_num_experts, + "top_k": top_k, + "tp": args.tp, + "ep": args.ep, + "dtype": "block_fp8_w8a8_bf16_output", + "block_shape": block_shape, + }, + "warmup": args.warmup, + "active": args.active, + "rows": rows, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + main() From d9df3003ddc47e30ba57f092f4cf2f6b0425eddd Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 18:37:26 +0800 Subject: [PATCH 033/148] Match in-place vLLM MoE serving path --- .../best_effort/moe_path_parity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/moe_path_parity.py b/runs/frontier-multicase-sufficiency-v0/best_effort/moe_path_parity.py index e90654f..b8c1a66 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/moe_path_parity.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/moe_path_parity.py @@ -171,7 +171,7 @@ def main() -> None: w2=w2, topk_weights=topk_weights, topk_ids=topk_ids, - inplace=False, + inplace=True, use_fp8_w8a8=True, per_channel_quant=False, global_num_experts=global_num_experts, From c9a5f0a0c3e0c31d2ab5770af6ae2b527585be45 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 18:41:23 +0800 Subject: [PATCH 034/148] Profile exact vLLM MoE serving entrypoint --- .../frontier_moe_serving_entrypoint.patch | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_serving_entrypoint.patch diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_serving_entrypoint.patch b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_serving_entrypoint.patch new file mode 100644 index 0000000..164af22 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_serving_entrypoint.patch @@ -0,0 +1,127 @@ +diff --git a/frontier/profiling/moe/moe_vllm_kernel.py b/frontier/profiling/moe/moe_vllm_kernel.py +--- a/frontier/profiling/moe/moe_vllm_kernel.py ++++ b/frontier/profiling/moe/moe_vllm_kernel.py +@@ -33,6 +33,7 @@ try: + from vllm import _custom_ops as ops + # Import vLLM 0.10.x functions + from vllm.model_executor.layers.fused_moe.fused_moe import ( ++ fused_experts, + fused_moe_kernel, + invoke_fused_moe_kernel, + moe_align_block_size, +@@ -531,82 +532,58 @@ def profile_fused_moe_kernel( + block_shape=block_shape, + ) + +- sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size( +- topk_ids, +- config["BLOCK_SIZE_M"], +- align_num_experts, +- expert_map=expert_map, +- ) +- +- output_dtype = base_dtype +- intermediate_cache1 = torch.empty( +- num_tokens, +- top_k, +- w1.shape[1], +- device=device, +- dtype=output_dtype, +- ) +- intermediate_cache2 = torch.empty( +- num_tokens * top_k, +- expert_hidden_dim_per_partition, +- device=device, +- dtype=output_dtype, +- ) +- intermediate_cache3 = torch.empty( +- num_tokens, +- top_k, +- hidden_dim, +- device=device, +- dtype=output_dtype, +- ) +- output = torch.empty( +- num_tokens, +- hidden_dim, +- device=device, +- dtype=output_dtype, +- ) +- + def _step() -> None: +- _run_fused_moe_iteration( +- A=A, ++ fused_experts( ++ hidden_states=A, + w1=w1, + w2=w2, +- intermediate_cache1=intermediate_cache1, +- intermediate_cache2=intermediate_cache2, +- intermediate_cache3=intermediate_cache3, +- output=output, + topk_weights=topk_weights, +- sorted_token_ids=sorted_token_ids, +- expert_ids=expert_ids, +- num_tokens_post_padded=num_tokens_post_padded, +- top_k=top_k, +- config=config, +- expert_hidden_dim_per_partition=expert_hidden_dim_per_partition, +- block_dims=block_dims, +- A_scale=None, ++ topk_ids=topk_ids, ++ inplace=True, ++ global_num_experts=align_num_experts, ++ expert_map=expert_map, ++ use_fp8_w8a8=use_fp8, ++ per_channel_quant=per_channel_quant, + w1_scale=w1_scale, + w2_scale=w2_scale, +- use_fp8=use_fp8, +- per_channel_quant=per_channel_quant, + block_shape=block_shape, + ) + ++ def _alignment_step() -> None: ++ moe_align_block_size( ++ topk_ids, ++ config["BLOCK_SIZE_M"], ++ align_num_experts, ++ expert_map=expert_map, ++ ) ++ + for _ in range(warmup_steps): + _step() + torch.cuda.synchronize() + + if profile_method == "record_function": +- return _collect_record_function_stats( +- step_fn=_step, +- active_steps=active_steps, +- output_dir=output_dir, +- operation_name="moe_grouped_gemm", ++ raise ValueError( ++ "Serving-entrypoint MoE profiling requires cuda_event so the local " ++ "alignment component can be subtracted without double counting." + ) + +- return _collect_cuda_event_stats( ++ serving_stats = _collect_cuda_event_stats( + step_fn=_step, + active_steps=active_steps, + ) ++ alignment_stats = _collect_cuda_event_stats( ++ step_fn=_alignment_step, ++ active_steps=active_steps, ++ ) ++ return { ++ "min": max(0.0, serving_stats["min"] - alignment_stats["max"]), ++ "max": max(0.0, serving_stats["max"] - alignment_stats["min"]), ++ "mean": max(0.0, serving_stats["mean"] - alignment_stats["mean"]), ++ "median": max(0.0, serving_stats["median"] - alignment_stats["median"]), ++ "std": ( ++ serving_stats["std"] ** 2 + alignment_stats["std"] ** 2 ++ ) ** 0.5, ++ } + + + def generate_expert_weights( From e6e6fef41a0b7d8b6bb67501114f0fbca8d500b1 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 18:51:52 +0800 Subject: [PATCH 035/148] Skip dense FFN surrogate for MoE profiles --- .../frontier_moe_linear_plan.patch | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_linear_plan.patch diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_linear_plan.patch b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_linear_plan.patch new file mode 100644 index 0000000..1910739 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_linear_plan.patch @@ -0,0 +1,72 @@ +diff --git a/frontier/profiling/linear_op/profiling_plan.py b/frontier/profiling/linear_op/profiling_plan.py +--- a/frontier/profiling/linear_op/profiling_plan.py ++++ b/frontier/profiling/linear_op/profiling_plan.py +@@ -112,5 +112,11 @@ def build_profiling_plan( + + ffn_sharded_enabled = tp_size in ffn_tp_set ++ if is_moe and not _supports_share_expert(model_config): ++ # Routed experts are profiled by the MoE profiler. Keeping the ++ # dense FFN surrogate enabled executes an unprofiled fake weight ++ # path and can reject otherwise-valid attention TP layouts (for ++ # example Qwen3-235B TP8 with 128x128 block FP8 weights). ++ ffn_sharded_enabled = False + + padded_n_embd = model_config.embedding_dim + padded_n_expanded_embd = model_config.mlp_hidden_dim +diff --git a/tests/unit/test_moe_linear_profiling_plan.py b/tests/unit/test_moe_linear_profiling_plan.py +new file mode 100644 +--- /dev/null ++++ b/tests/unit/test_moe_linear_profiling_plan.py +@@ -0,0 +1,52 @@ ++from types import SimpleNamespace ++ ++from frontier.profiling.linear_op.profiling_plan import build_profiling_plan ++ ++ ++def test_routed_moe_plan_does_not_execute_dense_ffn_surrogate() -> None: ++ model_config = SimpleNamespace( ++ no_tensor_parallel=False, ++ embedding_dim=4096, ++ mlp_hidden_dim=1536, ++ num_q_heads=64, ++ num_kv_heads=4, ++ model_type="qwen3_moe", ++ post_attn_norm=True, ++ is_moe=True, ++ is_step2_mini=False, ++ ) ++ ++ plan = build_profiling_plan( ++ model_config=model_config, ++ tp_size=8, ++ attn_tp=[8], ++ ffn_tp=[8], ++ disable_replicated=False, ++ is_moe=True, ++ ) ++ ++ assert plan["attn_sharded_enabled"] is True ++ assert plan["ffn_sharded_enabled"] is False ++ assert plan["ffn_enabled"] is True ++ assert plan["padded_n_expanded_embd"] == 1536 ++ assert "attn_pre_proj" in plan["enabled_ops"] ++ assert "post_attention_layernorm" in plan["enabled_ops"] ++ assert "mlp_up_proj" not in plan["enabled_ops"] ++ assert "mlp_down_proj" not in plan["enabled_ops"] ++ ++ ++def test_dense_plan_still_executes_ffn_surrogate() -> None: ++ model_config = SimpleNamespace( ++ no_tensor_parallel=False, ++ embedding_dim=4096, ++ mlp_hidden_dim=1536, ++ num_q_heads=64, ++ num_kv_heads=4, ++ model_type="qwen2", ++ post_attn_norm=True, ++ is_moe=False, ++ is_step2_mini=False, ++ ) ++ plan = build_profiling_plan(model_config, 8, [8], [8], is_moe=False) ++ assert plan["ffn_sharded_enabled"] is True ++ assert "mlp_up_proj" in plan["enabled_ops"] From 97e66ae2769d43575fdacb300974b6b5eccc89da Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 19:04:40 +0800 Subject: [PATCH 036/148] Add validated Frontier profile assembly --- .../best_effort/assemble_profiles.py | 323 ++++++++++++++++++ 1 file changed, 323 insertions(+) create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/assemble_profiles.py diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/assemble_profiles.py b/runs/frontier-multicase-sufficiency-v0/best_effort/assemble_profiles.py new file mode 100644 index 0000000..a9e4053 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/assemble_profiles.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +"""Assemble and validate the Qwen3-235B Frontier best-effort profiles.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path + +import pandas as pd + + +MODEL = "Qwen3-235B-A22B-FP8" +HARDWARE = "h20" +TOKENS = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384] +QUANT_SIGNATURE = "method=fp8|act=dynamic|serialized=True|block=128x128" +MEASUREMENT_TYPE = "CUDA_EVENT" +PROFILING_PRECISION = "BF16" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--linear-tp4", type=Path, required=True) + parser.add_argument("--linear-tp8", type=Path, required=True) + parser.add_argument("--attention-standard", type=Path, required=True) + parser.add_argument("--attention-mixed", type=Path, required=True) + parser.add_argument("--moe-tp4-ep1", type=Path, required=True) + parser.add_argument("--moe-tp1-ep8", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument( + "--frontier-commit", + default="d9cfeb6d8791fbf2f295dd9744c56a666171776e", + ) + return parser.parse_args() + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def read_csv(path: Path, label: str) -> pd.DataFrame: + if not path.is_file(): + raise FileNotFoundError(f"{label}: missing input CSV: {path}") + data = pd.read_csv(path) + if data.empty: + raise ValueError(f"{label}: input CSV is empty: {path}") + return data + + +def require_columns(data: pd.DataFrame, columns: list[str], label: str) -> None: + missing = [column for column in columns if column not in data.columns] + if missing: + raise ValueError(f"{label}: missing columns: {missing}") + + +def require_exact_values( + data: pd.DataFrame, column: str, expected: set[object], label: str +) -> None: + actual = set(data[column].dropna().unique().tolist()) + if actual != expected: + raise ValueError( + f"{label}: {column} values mismatch: expected={sorted(expected)!r}, " + f"actual={sorted(actual)!r}" + ) + + +def validate_common(data: pd.DataFrame, label: str) -> None: + columns = ["quant_signature", "profiling_precision", "measurement_type"] + require_columns(data, columns, label) + require_exact_values(data, "quant_signature", {QUANT_SIGNATURE}, label) + require_exact_values(data, "profiling_precision", {PROFILING_PRECISION}, label) + require_exact_values(data, "measurement_type", {MEASUREMENT_TYPE}, label) + + +def validate_time_columns( + data: pd.DataFrame, columns: list[str], label: str +) -> None: + require_columns(data, columns, label) + if data[columns].isna().any(axis=None): + counts = data[columns].isna().sum() + raise ValueError(f"{label}: NaN timing values: {counts[counts > 0].to_dict()}") + if (data[columns] < 0).any(axis=None): + raise ValueError(f"{label}: negative timing value") + + +def validate_token_grid( + data: pd.DataFrame, + group_columns: list[str], + expected_rows_per_point: int, + label: str, +) -> None: + for group, rows in data.groupby(group_columns, dropna=False): + counts = rows["num_tokens"].value_counts().to_dict() + expected = {token: expected_rows_per_point for token in TOKENS} + if counts != expected: + raise ValueError( + f"{label}: token grid mismatch for group={group}: " + f"expected={expected}, actual={counts}" + ) + + +def assemble_linear(tp4: pd.DataFrame, tp8: pd.DataFrame) -> pd.DataFrame: + label = "linear" + for source_label, data, expected_tp in ( + ("linear-tp4", tp4, {1, 4}), + ("linear-tp8", tp8, {1, 8}), + ): + validate_common(data, source_label) + require_columns(data, ["num_tensor_parallel_workers", "num_tokens"], source_label) + require_exact_values(data, "num_tensor_parallel_workers", expected_tp, source_label) + validate_token_grid(data, ["num_tensor_parallel_workers"], 1, source_label) + + # The TP=1 rows are replicated operators. Keep the rows from the TP4 run and + # add only the TP=8 sharded operators from the second run. + combined = pd.concat( + [tp4, tp8[tp8["num_tensor_parallel_workers"] == 8]], ignore_index=True + ) + require_exact_values(combined, "num_tensor_parallel_workers", {1, 4, 8}, label) + validate_token_grid(combined, ["num_tensor_parallel_workers"], 1, label) + + replicated = combined[combined["num_tensor_parallel_workers"] == 1] + sharded = combined[combined["num_tensor_parallel_workers"] > 1] + validate_time_columns( + replicated, + [ + "time_stats.emb.median", + "time_stats.input_layernorm.median", + "time_stats.post_attention_layernorm.median", + ], + "linear replicated operators", + ) + validate_time_columns( + sharded, + [ + "time_stats.attn_pre_proj.median", + "time_stats.attn_rope.median", + "time_stats.attn_post_proj.median", + ], + "linear sharded attention operators", + ) + return combined.sort_values( + ["num_tensor_parallel_workers", "num_tokens"], kind="stable" + ).reset_index(drop=True) + + +def assemble_attention(standard: pd.DataFrame, mixed: pd.DataFrame) -> pd.DataFrame: + timing_columns = [ + "time_stats.attn_input_reshape.median", + "time_stats.attn_kv_cache_save.median", + "time_stats.attn_prefill.median", + "time_stats.attn_decode.median", + "time_stats.attn_output_reshape.median", + ] + for label, data, expected_mixed, expected_rows in ( + ("attention-standard", standard, {False}, 390), + ("attention-mixed", mixed, {True}, 336), + ): + validate_common(data, label) + require_columns( + data, + ["num_tensor_parallel_workers", "is_prefill", "is_mixed_batch"], + label, + ) + require_exact_values(data, "num_tensor_parallel_workers", {4, 8}, label) + require_exact_values(data, "is_prefill", {True}, label) + require_exact_values(data, "is_mixed_batch", expected_mixed, label) + validate_time_columns(data, timing_columns, label) + if len(data) != expected_rows: + raise ValueError( + f"{label}: row count mismatch: expected={expected_rows}, actual={len(data)}" + ) + + combined = pd.concat([standard, mixed], ignore_index=True) + if len(combined) != 726: + raise ValueError(f"attention: expected 726 rows, got {len(combined)}") + sort_columns = [ + "num_tensor_parallel_workers", + "is_mixed_batch", + "total_tokens", + "total_prefill_tokens", + "kv_cache_size", + "batch_size", + ] + return combined.sort_values(sort_columns, kind="stable").reset_index(drop=True) + + +def assemble_moe(tp4_ep1: pd.DataFrame, tp1_ep8: pd.DataFrame) -> pd.DataFrame: + timing_columns = [ + "time_stats.moe_gating_linear.median", + "time_stats.moe_gating_routing_topk.median", + "time_stats.moe_shuffling.median", + "time_stats.moe_grouped_gemm.median", + ] + cases = ( + ("moe-tp4-ep1", tp4_ep1, 4, 1, 128), + ("moe-tp1-ep8", tp1_ep8, 1, 8, 16), + ) + for label, data, expected_tp, expected_ep, expected_local_experts in cases: + validate_common(data, label) + require_columns( + data, + [ + "num_tensor_parallel_workers", + "expert_parallel_size", + "num_experts_per_device", + "num_tokens", + "load_distribution", + "seed", + "routing_runtime_path", + "gating_runtime_context", + ], + label, + ) + require_exact_values(data, "num_tensor_parallel_workers", {expected_tp}, label) + require_exact_values(data, "expert_parallel_size", {expected_ep}, label) + require_exact_values(data, "num_experts_per_device", {expected_local_experts}, label) + require_exact_values( + data, + "load_distribution", + {"uniform", "skewed", "extremely_skewed"}, + label, + ) + require_exact_values(data, "seed", {0, 1}, label) + require_exact_values(data, "routing_runtime_path", {"standard_fused_topk"}, label) + require_exact_values(data, "gating_runtime_context", {"prefill_hot"}, label) + validate_token_grid( + data, + ["num_tensor_parallel_workers", "expert_parallel_size"], + 6, + label, + ) + validate_time_columns(data, timing_columns, label) + if len(data) != 90: + raise ValueError(f"{label}: expected 90 rows, got {len(data)}") + + combined = pd.concat([tp4_ep1, tp1_ep8], ignore_index=True) + return combined.sort_values( + [ + "num_tensor_parallel_workers", + "expert_parallel_size", + "num_tokens", + "load_distribution", + "seed", + ], + kind="stable", + ).reset_index(drop=True) + + +def main() -> None: + args = parse_args() + input_paths = { + "linear_tp4": args.linear_tp4, + "linear_tp8": args.linear_tp8, + "attention_standard": args.attention_standard, + "attention_mixed": args.attention_mixed, + "moe_tp4_ep1": args.moe_tp4_ep1, + "moe_tp1_ep8": args.moe_tp1_ep8, + } + inputs = {name: read_csv(path, name) for name, path in input_paths.items()} + + outputs = { + "linear_op.csv": assemble_linear(inputs["linear_tp4"], inputs["linear_tp8"]), + "attention.csv": assemble_attention( + inputs["attention_standard"], inputs["attention_mixed"] + ), + "moe.csv": assemble_moe(inputs["moe_tp4_ep1"], inputs["moe_tp1_ep8"]), + } + + output_dir = args.output_root / "compute" / HARDWARE / MODEL + output_dir.mkdir(parents=True, exist_ok=True) + output_paths: dict[str, Path] = {} + for name, data in outputs.items(): + path = output_dir / name + data.to_csv(path, index=False) + output_paths[name] = path + + manifest = { + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "frontier_commit": args.frontier_commit, + "model": MODEL, + "hardware": HARDWARE, + "contract": { + "quant_signature": QUANT_SIGNATURE, + "profiling_precision": PROFILING_PRECISION, + "profiling_precision_semantics": "BF16 kernel output/accumulation; weights and activations are block FP8 W8A8", + "measurement_type": MEASUREMENT_TYPE, + "linear_tensor_parallel_sizes": [1, 4, 8], + "attention_tensor_parallel_sizes": [4, 8], + "moe_layouts": [ + {"tensor_parallel_size": 4, "expert_parallel_size": 1}, + {"tensor_parallel_size": 1, "expert_parallel_size": 8}, + ], + }, + "inputs": { + name: { + "path": str(path.resolve()), + "sha256": sha256(path), + "rows": len(inputs[name]), + } + for name, path in input_paths.items() + }, + "outputs": { + name: { + "path": str(path.resolve()), + "sha256": sha256(path), + "rows": len(outputs[name]), + } + for name, path in output_paths.items() + }, + } + manifest_path = args.output_root / "profile_manifest.json" + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + print(json.dumps(manifest, indent=2)) + + +if __name__ == "__main__": + main() From c71f37911079ea71b59f2fac445a794d10c07e0f Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 19:10:52 +0800 Subject: [PATCH 037/148] Add Qwen235B NCCL profile runner --- .../best_effort/profile_allreduce.py | 256 ++++++++++++++++++ 1 file changed, 256 insertions(+) create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/profile_allreduce.py diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/profile_allreduce.py b/runs/frontier-multicase-sufficiency-v0/best_effort/profile_allreduce.py new file mode 100644 index 0000000..484a2e7 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/profile_allreduce.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +"""Profile the NCCL all-reduce paths consumed by the Qwen235B experiment.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import os +import statistics +import sys +from datetime import datetime, timezone +from pathlib import Path + +import torch +import torch.distributed as dist + + +DEFAULT_TOKENS = [ + 1, + 2, + 3, + 4, + 6, + 8, + 12, + 16, + 24, + 32, + 48, + 63, + 64, + 65, + 96, + 127, + 128, + 129, + 192, + 255, + 256, + 257, + 384, + 511, + 512, + 513, + 768, + 1023, + 1024, + 1025, + 1536, + 2047, + 2048, + 2049, + 3072, + 4095, + 4096, + 4097, + 6144, + 8191, + 8192, + 8193, + 12288, + 16383, + 16384, +] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--hidden-size", type=int, default=4096) + parser.add_argument("--world-sizes", type=int, nargs="+", default=[4, 8]) + parser.add_argument("--tokens", type=int, nargs="+", default=DEFAULT_TOKENS) + parser.add_argument("--warmup-iterations", type=int, default=5) + parser.add_argument("--measured-iterations", type=int, default=30) + return parser.parse_args() + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def profile_one_size( + *, + group: dist.ProcessGroup, + group_size: int, + num_tokens: int, + hidden_size: int, + warmup_iterations: int, + measured_iterations: int, +) -> list[float]: + tensor = torch.ones( + (num_tokens, hidden_size), dtype=torch.bfloat16, device="cuda" + ) + for _ in range(warmup_iterations): + dist.all_reduce(tensor, group=group) + dist.barrier(group=group) + torch.cuda.synchronize() + + local_samples: list[float] = [] + for _ in range(measured_iterations): + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + dist.all_reduce(tensor, group=group) + end.record() + end.synchronize() + local_samples.append(float(start.elapsed_time(end))) + + samples_by_rank: list[list[float] | None] = [None] * group_size + dist.all_gather_object(samples_by_rank, local_samples, group=group) + del tensor + + # Collective completion is limited by the slowest rank. Preserve that + # critical-path statistic instead of reporting only rank 0. + return [ + max(float(rank_samples[index]) for rank_samples in samples_by_rank if rank_samples) + for index in range(measured_iterations) + ] + + +def profile_group( + *, + group: dist.ProcessGroup, + group_ranks: list[int], + args: argparse.Namespace, +) -> list[dict[str, object]]: + global_rank = dist.get_rank() + if global_rank not in group_ranks: + return [] + + group_size = len(group_ranks) + group_rank = dist.get_rank(group=group) + rows: list[dict[str, object]] = [] + trial_orders = (("forward", args.tokens), ("reverse", list(reversed(args.tokens)))) + for trial_id, (order_name, tokens) in enumerate(trial_orders): + for num_tokens in tokens: + samples = profile_one_size( + group=group, + group_size=group_size, + num_tokens=num_tokens, + hidden_size=args.hidden_size, + warmup_iterations=args.warmup_iterations, + measured_iterations=args.measured_iterations, + ) + if group_rank != 0: + continue + ordered = sorted(samples) + p95_index = int(0.95 * (len(ordered) - 1)) + rows.append( + { + "time_stats.all_reduce.min": min(samples), + "time_stats.all_reduce.max": max(samples), + "time_stats.all_reduce.mean": statistics.fmean(samples), + "time_stats.all_reduce.median": statistics.median(samples), + "time_stats.all_reduce.std": statistics.stdev(samples), + "time_stats.all_reduce.p95": ordered[p95_index], + "rank": 0, + "num_workers": group_size, + "size": num_tokens * args.hidden_size * 2, + "collective": "all_reduce", + "devices_per_node": group_size, + "max_devices_per_node": dist.get_world_size(), + "profiling_precision": "BF16", + "measurement_type": "CUDA_EVENT", + "backend": "nccl", + "num_tokens": num_tokens, + "hidden_size": args.hidden_size, + "trial_id": trial_id, + "order": order_name, + "warmup_iterations": args.warmup_iterations, + "measured_iterations": args.measured_iterations, + } + ) + return rows + + +def main() -> None: + args = parse_args() + if args.hidden_size <= 0: + raise ValueError("hidden-size must be positive") + if args.warmup_iterations <= 0 or args.measured_iterations < 2: + raise ValueError("warmup must be positive and measured iterations must be >= 2") + if sorted(set(args.tokens)) != sorted(args.tokens) or min(args.tokens) <= 0: + raise ValueError("tokens must be unique positive integers") + + local_rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(local_rank) + dist.init_process_group(backend="nccl") + global_rank = dist.get_rank() + global_world_size = dist.get_world_size() + if max(args.world_sizes) > global_world_size: + raise ValueError( + f"requested world size {max(args.world_sizes)} exceeds torchrun world size " + f"{global_world_size}" + ) + + groups: dict[int, tuple[dist.ProcessGroup, list[int]]] = {} + for world_size in sorted(set(args.world_sizes)): + ranks = list(range(world_size)) + group = dist.group.WORLD if world_size == global_world_size else dist.new_group(ranks) + groups[world_size] = (group, ranks) + + all_rows: list[dict[str, object]] = [] + for world_size in args.world_sizes: + group, ranks = groups[world_size] + all_rows.extend(profile_group(group=group, group_ranks=ranks, args=args)) + dist.barrier() + + if global_rank == 0: + expected_rows = len(set(args.world_sizes)) * len(args.tokens) * 2 + if len(all_rows) != expected_rows: + raise RuntimeError( + f"profile row count mismatch: expected={expected_rows}, actual={len(all_rows)}" + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("w", newline="", encoding="utf-8") as output_file: + writer = csv.DictWriter(output_file, fieldnames=list(all_rows[0])) + writer.writeheader() + writer.writerows(all_rows) + + manifest = { + "generated_at_utc": datetime.now(timezone.utc).isoformat(), + "command": [sys.executable, *sys.argv], + "output": str(args.output.resolve()), + "output_sha256": sha256(args.output), + "rows": len(all_rows), + "world_sizes": args.world_sizes, + "tokens": args.tokens, + "hidden_size": args.hidden_size, + "dtype": "torch.bfloat16", + "backend": dist.get_backend(), + "torch_version": torch.__version__, + "cuda_version": torch.version.cuda, + "nccl_version": list(torch.cuda.nccl.version()), + "gpu": torch.cuda.get_device_name(0), + "warmup_iterations": args.warmup_iterations, + "measured_iterations": args.measured_iterations, + "sample_semantics": "per-iteration maximum CUDA-event time across participating ranks", + } + manifest_path = args.output.with_suffix(args.output.suffix + ".manifest.json") + manifest_path.write_text( + json.dumps(manifest, indent=2) + "\n", encoding="utf-8" + ) + print(json.dumps(manifest, indent=2)) + + dist.destroy_process_group() + + +if __name__ == "__main__": + main() From a54f69352de209922ed01794d995d3cfd4e72d83 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 19:28:39 +0800 Subject: [PATCH 038/148] Add Frontier MoE gating context closure --- .../best_effort/assemble_profiles.py | 71 ++++++++++++++++--- 1 file changed, 63 insertions(+), 8 deletions(-) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/assemble_profiles.py b/runs/frontier-multicase-sufficiency-v0/best_effort/assemble_profiles.py index a9e4053..a060526 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/assemble_profiles.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/assemble_profiles.py @@ -28,6 +28,8 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--attention-mixed", type=Path, required=True) parser.add_argument("--moe-tp4-ep1", type=Path, required=True) parser.add_argument("--moe-tp1-ep8", type=Path, required=True) + parser.add_argument("--moe-standalone-tp4-ep1", type=Path, required=True) + parser.add_argument("--moe-standalone-tp1-ep8", type=Path, required=True) parser.add_argument("--output-root", type=Path, required=True) parser.add_argument( "--frontier-commit", @@ -190,7 +192,12 @@ def assemble_attention(standard: pd.DataFrame, mixed: pd.DataFrame) -> pd.DataFr return combined.sort_values(sort_columns, kind="stable").reset_index(drop=True) -def assemble_moe(tp4_ep1: pd.DataFrame, tp1_ep8: pd.DataFrame) -> pd.DataFrame: +def assemble_moe( + tp4_ep1: pd.DataFrame, + tp1_ep8: pd.DataFrame, + standalone_tp4_ep1: pd.DataFrame, + standalone_tp1_ep8: pd.DataFrame, +) -> pd.DataFrame: timing_columns = [ "time_stats.moe_gating_linear.median", "time_stats.moe_gating_routing_topk.median", @@ -198,10 +205,33 @@ def assemble_moe(tp4_ep1: pd.DataFrame, tp1_ep8: pd.DataFrame) -> pd.DataFrame: "time_stats.moe_grouped_gemm.median", ] cases = ( - ("moe-tp4-ep1", tp4_ep1, 4, 1, 128), - ("moe-tp1-ep8", tp1_ep8, 1, 8, 16), + ("moe-tp4-ep1", tp4_ep1, 4, 1, 128, "prefill_hot"), + ("moe-tp1-ep8", tp1_ep8, 1, 8, 16, "prefill_hot"), + ( + "moe-standalone-tp4-ep1", + standalone_tp4_ep1, + 4, + 1, + 128, + "standalone_legacy", + ), + ( + "moe-standalone-tp1-ep8", + standalone_tp1_ep8, + 1, + 8, + 16, + "standalone_legacy", + ), ) - for label, data, expected_tp, expected_ep, expected_local_experts in cases: + for ( + label, + data, + expected_tp, + expected_ep, + expected_local_experts, + expected_context, + ) in cases: validate_common(data, label) require_columns( data, @@ -228,22 +258,36 @@ def assemble_moe(tp4_ep1: pd.DataFrame, tp1_ep8: pd.DataFrame) -> pd.DataFrame: ) require_exact_values(data, "seed", {0, 1}, label) require_exact_values(data, "routing_runtime_path", {"standard_fused_topk"}, label) - require_exact_values(data, "gating_runtime_context", {"prefill_hot"}, label) + require_exact_values( + data, "gating_runtime_context", {expected_context}, label + ) validate_token_grid( data, ["num_tensor_parallel_workers", "expert_parallel_size"], 6, label, ) - validate_time_columns(data, timing_columns, label) + validate_time_columns( + data, + timing_columns if expected_context == "prefill_hot" else timing_columns[:2], + label, + ) if len(data) != 90: raise ValueError(f"{label}: expected 90 rows, got {len(data)}") - combined = pd.concat([tp4_ep1, tp1_ep8], ignore_index=True) + # Standalone rows are needed only for Frontier's pure-decode gating models. + # Do not duplicate the shuffling/grouped-GEMM observations merely because + # the profiler collected them while measuring a second gating context. + standalone = pd.concat( + [standalone_tp4_ep1, standalone_tp1_ep8], ignore_index=True + ) + standalone[timing_columns[2:]] = float("nan") + combined = pd.concat([tp4_ep1, tp1_ep8, standalone], ignore_index=True) return combined.sort_values( [ "num_tensor_parallel_workers", "expert_parallel_size", + "gating_runtime_context", "num_tokens", "load_distribution", "seed", @@ -261,6 +305,8 @@ def main() -> None: "attention_mixed": args.attention_mixed, "moe_tp4_ep1": args.moe_tp4_ep1, "moe_tp1_ep8": args.moe_tp1_ep8, + "moe_standalone_tp4_ep1": args.moe_standalone_tp4_ep1, + "moe_standalone_tp1_ep8": args.moe_standalone_tp1_ep8, } inputs = {name: read_csv(path, name) for name, path in input_paths.items()} @@ -269,7 +315,12 @@ def main() -> None: "attention.csv": assemble_attention( inputs["attention_standard"], inputs["attention_mixed"] ), - "moe.csv": assemble_moe(inputs["moe_tp4_ep1"], inputs["moe_tp1_ep8"]), + "moe.csv": assemble_moe( + inputs["moe_tp4_ep1"], + inputs["moe_tp1_ep8"], + inputs["moe_standalone_tp4_ep1"], + inputs["moe_standalone_tp1_ep8"], + ), } output_dir = args.output_root / "compute" / HARDWARE / MODEL @@ -296,6 +347,10 @@ def main() -> None: {"tensor_parallel_size": 4, "expert_parallel_size": 1}, {"tensor_parallel_size": 1, "expert_parallel_size": 8}, ], + "moe_gating_runtime_contexts": [ + "prefill_hot", + "standalone_legacy", + ], }, "inputs": { name: { From a6f101b09e5ba1089ee982fadef303d6b0b8b344 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 19:39:30 +0800 Subject: [PATCH 039/148] Add reproducible Frontier prefill grid freeze --- .../best_effort/frontier_prefill_grid.py | 703 ++++++++++++++++++ .../best_effort/test_frontier_prefill_grid.py | 66 ++ 2 files changed, 769 insertions(+) create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/frontier_prefill_grid.py create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/test_frontier_prefill_grid.py diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_prefill_grid.py b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_prefill_grid.py new file mode 100644 index 0000000..ce6bec5 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_prefill_grid.py @@ -0,0 +1,703 @@ +#!/usr/bin/env python3 +"""Freeze a profile-only Frontier ranking for the Qwen235B prefill grid.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import os +import subprocess +import sys +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Iterable + + +SCHEMA = "frontier-qwen235b-prefill-grid-v1" +MODEL = "Qwen3-235B-A22B-FP8" +TRACE_SHA256 = "f878e9af18f94dcfaced94a8e1e6b20a2f7d97d64aa862448025660dbbd965b2" +PROFILE_RELATIVE = Path("compute/h20") / MODEL +NETWORK_RELATIVE = Path("network/h20_nccl/all_reduce.csv") +SEARCH_LOW = 0.0 +SEARCH_HIGH = 0.125 +SEARCH_PROBES = 6 +WINDOW_DURATION_S = 600.0 +MAX_INPUT_TOKENS = 32768 +MAX_MODEL_TOKENS = 40960 +BLOCK_SIZE_TOKENS = 16 +MOE_ROUTING_SEED = 42 + + +@dataclass(frozen=True) +class GridConfig: + tp: int + mns: int + mbt: int + moe_tp: int + moe_ep: int + num_gpu_blocks: int + + @property + def name(self) -> str: + return f"tp{self.tp}_mns{self.mns}_mbt{self.mbt}" + + @property + def gpu_count(self) -> int: + return self.tp + + +GRID = tuple( + GridConfig( + tp=tp, + mns=mns, + mbt=mbt, + moe_tp=4 if tp == 4 else 1, + moe_ep=1 if tp == 4 else 8, + # Measured by community vLLM on the same checkpoint/runtime. The real + # grid will use --num-gpu-blocks-override with these same values. + num_gpu_blocks=26101 if tp == 4 else 62351, + ) + for tp in (4, 8) + for mns in (64, 128) + for mbt in (8192, 16384) +) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def order_hash(values: Iterable[object]) -> str: + payload = "\n".join(str(value) for value in values).encode() + return hashlib.sha256(payload).hexdigest() + + +def write_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + os.replace(temporary, path) + + +def sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def frontier_source_fingerprint(source: Path) -> dict[str, Any]: + commit = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=source, + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout.strip() + status = subprocess.run( + ["git", "status", "--porcelain=v1"], + cwd=source, + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout + diff = subprocess.run( + ["git", "diff", "--binary", "HEAD"], + cwd=source, + check=True, + stdout=subprocess.PIPE, + ).stdout + model_config = source / "data" / "config" / "models" / f"{MODEL}.json" + if not model_config.is_file(): + raise FileNotFoundError(f"Frontier model config is missing: {model_config}") + return { + "commit": commit, + "status_porcelain": status.splitlines(), + "tracked_diff_sha256": sha256_bytes(diff), + "model_config": { + "path": str(model_config.resolve()), + "sha256": sha256(model_config), + }, + } + + +def anchor_key(anchor: float) -> str: + return f"{anchor:.12f}" + + +def anchor_filename(anchor: float) -> str: + return f"u_{anchor_key(anchor).replace('.', 'p')}.csv" + + +def binary_search_lattice() -> list[float]: + intervals = [(SEARCH_LOW, SEARCH_HIGH)] + anchors: list[float] = [] + for _ in range(SEARCH_PROBES): + next_intervals = [] + for low, high in intervals: + midpoint = (low + high) / 2.0 + anchors.append(midpoint) + next_intervals.extend(((low, midpoint), (midpoint, high))) + intervals = next_intervals + return sorted(set(anchors)) + + +def ttft_slo_ms(input_tokens: int) -> float: + return 1000.0 if input_tokens <= 8191 else 2000.0 + + +def load_trace_rows(path: Path) -> list[dict[str, Any]]: + rows = [] + with path.open(encoding="utf-8") as source: + for source_index, line in enumerate(source): + if not line.strip(): + continue + raw = json.loads(line) + input_tokens = int(raw["input_length"]) + if not 0 <= input_tokens <= MAX_INPUT_TOKENS: + continue + timestamp = float(raw["timestamp"]) + sampling_u = float(raw["sampling_u"]) + rows.append( + { + "arrived_at": timestamp, + "num_prefill_tokens": input_tokens, + "num_decode_tokens": 1, + "source_row_index": source_index, + "source_request_id": str( + raw.get("request_id") or raw.get("id") or source_index + ), + "sampling_u": sampling_u, + "slo_ttft_ms": ttft_slo_ms(input_tokens), + } + ) + # This reproduces AITuner's stable arrival-only sort. Tied timestamps retain + # source order; sampling_u must not become a secondary key. + rows.sort(key=lambda row: row["arrived_at"]) + return rows + + +def prepare_traces(trace: Path, output_root: Path, expected_sha256: str) -> Path: + actual_sha256 = sha256(trace) + if expected_sha256 and actual_sha256 != expected_sha256: + raise ValueError( + f"trace SHA256 mismatch: expected={expected_sha256}, actual={actual_sha256}" + ) + rows = load_trace_rows(trace) + if not rows: + raise ValueError("no trace rows remain after input-length filtering") + if any( + rows[index]["arrived_at"] > rows[index + 1]["arrived_at"] + for index in range(len(rows) - 1) + ): + raise ValueError("materialized trace is not monotonic by arrival") + + trace_dir = output_root / "traces" + trace_dir.mkdir(parents=True, exist_ok=True) + fields = [ + "arrived_at", + "num_prefill_tokens", + "num_decode_tokens", + "source_row_index", + "source_request_id", + "sampling_u", + "slo_ttft_ms", + ] + anchors: dict[str, Any] = {} + for anchor in binary_search_lattice(): + selected = [row for row in rows if row["sampling_u"] <= anchor] + target = trace_dir / anchor_filename(anchor) + with target.open("w", encoding="utf-8", newline="") as output: + writer = csv.DictWriter(output, fieldnames=fields) + writer.writeheader() + writer.writerows(selected) + key = anchor_key(anchor) + anchors[key] = { + "anchor": anchor, + "path": str(target.resolve()), + "sha256": sha256(target), + "request_count": len(selected), + "request_rate": len(selected) / WINDOW_DURATION_S, + "source_row_order_sha256": order_hash( + row["source_row_index"] for row in selected + ), + "arrival_order_sha256": order_hash( + f"{row['arrived_at']:.12f}" for row in selected + ), + "input_length_order_sha256": order_hash( + row["num_prefill_tokens"] for row in selected + ), + } + + manifest = { + "schema": SCHEMA, + "source": { + "path": str(trace.resolve()), + "sha256": actual_sha256, + "filtered_request_count": len(rows), + }, + "selection_contract": { + "input_tokens": [0, MAX_INPUT_TOKENS], + "completion_tokens_override": 1, + "stable_sort_key": "arrived_at_only", + "selection": "sampling_u <= anchor", + "window_duration_s": WINDOW_DURATION_S, + "search_low": SEARCH_LOW, + "search_high": SEARCH_HIGH, + "search_probes": SEARCH_PROBES, + "lattice_anchor_count": len(anchors), + "ttft_slo_ms": {"input_le_8191": 1000, "otherwise": 2000}, + "target_pass_rate": 0.95, + }, + "anchors": anchors, + } + manifest_path = output_root / "trace_manifest.json" + write_json(manifest_path, manifest) + return manifest_path + + +def resolve_profile_paths(profile_root: Path) -> dict[str, Path]: + paths = { + "linear": profile_root / PROFILE_RELATIVE / "linear_op.csv", + "attention": profile_root / PROFILE_RELATIVE / "attention.csv", + "moe": profile_root / PROFILE_RELATIVE / "moe.csv", + "all_reduce": profile_root / NETWORK_RELATIVE, + "manifest": profile_root / "profile_manifest.json", + } + missing = [f"{name}={path}" for name, path in paths.items() if not path.is_file()] + if missing: + raise FileNotFoundError("missing profile inputs: " + ", ".join(missing)) + return paths + + +def build_command( + *, + python: Path, + frontier_source: Path, + profile_root: Path, + profile_paths: dict[str, Path], + trace: Path, + config: GridConfig, + probe_dir: Path, + run_id: str, + cache_root: Path, +) -> list[str]: + return [ + str(python), + "-m", + "frontier.main", + "--simulation_mode", + "offline", + "--sys_arch", + "co-location", + "--cluster_config_num_replicas", + "1", + "--replica_config_model_name", + MODEL, + "--replica_config_attn_tensor_parallel_size", + str(config.tp), + "--replica_config_attn_data_parallel_size", + "1", + "--replica_config_moe_tensor_parallel_size", + str(config.moe_tp), + "--replica_config_moe_expert_parallel_size", + str(config.moe_ep), + "--replica_config_total_expert_num", + "128", + "--replica_config_router_topk", + "8", + "--replica_config_moe_routing_mode", + "simulation", + "--replica_config_moe_routing_seed", + str(MOE_ROUTING_SEED), + "--replica_config_num_pipeline_stages", + "1", + "--replica_config_device", + "h20", + "--replica_config_network_device", + "h20_dgx", + "--cc_backend_config_type", + "vidur", + "--vidur_cc_backend_config_profiling_data_dir", + str(profile_root), + "--vidur_cc_backend_config_cache_dir", + str(cache_root / "collectives"), + "--vidur_cc_backend_config_all_reduce_input_file", + str(profile_paths["all_reduce"]), + "--replica_scheduler_config_type", + "vllm_v1", + "--decode_cuda_graph_mode", + "none", + "--vllm_v1_scheduler_config_batch_size_cap", + str(config.mns), + "--vllm_v1_scheduler_config_block_size", + str(BLOCK_SIZE_TOKENS), + "--vllm_v1_scheduler_config_num_blocks", + str(config.num_gpu_blocks), + "--vllm_v1_scheduler_config_num_blocks_mode", + "explicit", + "--vllm_v1_scheduler_config_max_tokens_in_batch", + str(config.mbt), + "--vllm_v1_scheduler_config_enable_chunked_prefill", + "--no-vllm_v1_scheduler_config_enable_prefix_caching", + "--request_generator_config_type", + "trace_replay", + "--trace_request_generator_config_trace_file", + str(trace), + "--trace_request_generator_config_time_scale_factor", + "1", + "--trace_request_generator_config_prefill_scale_factor", + "1", + "--trace_request_generator_config_decode_scale_factor", + "1", + "--trace_request_generator_config_max_tokens", + str(MAX_MODEL_TOKENS), + "--no-random_forrest_execution_time_predictor_config_enable_dummy_mode", + "--random_forrest_execution_time_predictor_config_linear_op_input_file", + str(profile_paths["linear"]), + "--random_forrest_execution_time_predictor_config_atten_input_file", + str(profile_paths["attention"]), + "--random_forrest_execution_time_predictor_config_moe_input_file", + str(profile_paths["moe"]), + "--random_forrest_execution_time_predictor_config_all_reduce_input_file", + str(profile_paths["all_reduce"]), + "--random_forrest_execution_time_predictor_config_prediction_max_prefill_chunk_size", + "16384", + "--random_forrest_execution_time_predictor_config_prediction_max_tokens_per_request", + str(MAX_INPUT_TOKENS + 1), + "--random_forrest_execution_time_predictor_config_prediction_max_batch_size", + "128", + "--random_forrest_execution_time_predictor_config_skip_cpu_overhead_modeling", + "--metrics_config_cache_dir", + str(cache_root / "execution"), + "--metrics_config_output_dir", + str(probe_dir / "metrics"), + "--metrics_config_run_id", + run_id, + "--metrics_config_write_metrics", + "--metrics_config_store_request_metrics", + "--no-metrics_config_store_plots", + "--no-metrics_config_enable_chrome_trace", + "--no-metrics_config_write_json_trace", + ] + + +def score_request_metrics(trace: Path, request_metrics: Path) -> dict[str, Any]: + with trace.open(encoding="utf-8", newline="") as source: + trace_rows = list(csv.DictReader(source)) + with request_metrics.open(encoding="utf-8", newline="") as source: + metric_rows = list(csv.DictReader(source)) + if len(metric_rows) != len(trace_rows): + raise ValueError( + f"request count mismatch: trace={len(trace_rows)}, metrics={len(metric_rows)}" + ) + metrics_by_id = {int(row["Request Id"]): row for row in metric_rows} + expected_ids = set(range(len(trace_rows))) + if set(metrics_by_id) != expected_ids: + raise ValueError("Frontier Request Ids do not match trace row positions") + + passed = 0 + ttfts = [] + for request_id, trace_row in enumerate(trace_rows): + ttft = float(metrics_by_id[request_id]["ttft"]) + threshold = float(trace_row["slo_ttft_ms"]) + ttfts.append(ttft) + passed += int(ttft <= threshold) + count = len(trace_rows) + pass_rate = passed / count if count else 0.0 + ordered = sorted(ttfts) + + def percentile(fraction: float) -> float | None: + if not ordered: + return None + index = round((len(ordered) - 1) * fraction) + return ordered[index] + + return { + "request_count": count, + "passed_request_count": passed, + "slo_pass_rate": pass_rate, + "feasible": pass_rate >= 0.95, + "ttft_ms": { + "min": min(ordered) if ordered else None, + "p50": percentile(0.50), + "p95": percentile(0.95), + "p99": percentile(0.99), + "max": max(ordered) if ordered else None, + }, + } + + +def find_request_metrics(probe_dir: Path) -> Path: + candidates = list((probe_dir / "metrics").glob("**/request_metrics.csv")) + if len(candidates) != 1: + raise ValueError( + f"expected one request_metrics.csv under {probe_dir}, got {candidates}" + ) + return candidates[0] + + +def run_probe( + *, + python: Path, + frontier_source: Path, + profile_root: Path, + profile_paths: dict[str, Path], + trace_record: dict[str, Any], + config: GridConfig, + probe_index: int, + output_root: Path, + cache_root: Path, +) -> dict[str, Any]: + anchor = float(trace_record["anchor"]) + probe_dir = output_root / "runs" / config.name / f"probe_{probe_index}_{anchor_filename(anchor)[:-4]}" + result_path = probe_dir / "result.json" + if result_path.is_file(): + result = json.loads(result_path.read_text()) + if result.get("status") == "completed": + return result + probe_dir.mkdir(parents=True, exist_ok=True) + trace = Path(trace_record["path"]) + run_id = f"{config.name}_probe{probe_index}_{anchor_filename(anchor)[:-4]}" + command = build_command( + python=python, + frontier_source=frontier_source, + profile_root=profile_root, + profile_paths=profile_paths, + trace=trace, + config=config, + probe_dir=probe_dir, + run_id=run_id, + cache_root=cache_root, + ) + write_json(probe_dir / "command.json", command) + environment = os.environ.copy() + environment.update( + { + "PYTHONPATH": str(frontier_source), + "WANDB_DISABLED": "true", + "VIDUR_DISABLE_WANDB": "1", + } + ) + started = time.time() + with (probe_dir / "stdout.log").open("w", encoding="utf-8") as output: + completed = subprocess.run( + command, + cwd=frontier_source, + env=environment, + stdout=output, + stderr=subprocess.STDOUT, + check=False, + ) + elapsed = time.time() - started + if completed.returncode != 0: + result = { + "status": "failed", + "returncode": completed.returncode, + "elapsed_seconds": elapsed, + "config": asdict(config), + "anchor": anchor, + "trace_sha256": trace_record["sha256"], + } + write_json(result_path, result) + raise RuntimeError(f"Frontier probe failed: {config.name}, anchor={anchor}") + + request_metrics = find_request_metrics(probe_dir) + score = score_request_metrics(trace, request_metrics) + result = { + "status": "completed", + "elapsed_seconds": elapsed, + "config": asdict(config), + "anchor": anchor, + "trace_path": str(trace), + "trace_sha256": trace_record["sha256"], + "request_metrics_path": str(request_metrics), + "request_metrics_sha256": sha256(request_metrics), + "request_rate": score["request_count"] / WINDOW_DURATION_S, + "request_rate_per_gpu": score["request_count"] + / WINDOW_DURATION_S + / config.gpu_count, + **score, + } + write_json(result_path, result) + return result + + +def selected_configs(names: list[str] | None) -> list[GridConfig]: + if not names: + return list(GRID) + by_name = {config.name: config for config in GRID} + unknown = sorted(set(names) - set(by_name)) + if unknown: + raise ValueError(f"unknown configs: {unknown}; available={sorted(by_name)}") + return [by_name[name] for name in names] + + +def run_grid(args: argparse.Namespace) -> None: + trace_manifest = json.loads(args.trace_manifest.read_text()) + if trace_manifest.get("schema") != SCHEMA: + raise ValueError(f"unexpected trace manifest schema: {trace_manifest.get('schema')}") + profile_paths = resolve_profile_paths(args.profile_root) + args.output_root.mkdir(parents=True, exist_ok=True) + cache_root = args.output_root / "cache" + configs = selected_configs(args.config) + run_manifest = { + "schema": SCHEMA, + "frontier": { + "source": str(args.frontier_source.resolve()), + "python": str(args.python.resolve()), + "fingerprint": frontier_source_fingerprint(args.frontier_source), + }, + "trace_manifest": { + "path": str(args.trace_manifest.resolve()), + "sha256": sha256(args.trace_manifest), + }, + "profiles": { + name: {"path": str(path.resolve()), "sha256": sha256(path)} + for name, path in profile_paths.items() + }, + "kv_capacity_evidence": { + "tp4": { + "path": str(args.tp4_capacity_artifact.resolve()), + "sha256": sha256(args.tp4_capacity_artifact), + "num_gpu_blocks": 26101, + }, + "tp8": { + "path": str(args.tp8_capacity_artifact.resolve()), + "sha256": sha256(args.tp8_capacity_artifact), + "num_gpu_blocks": 62351, + }, + }, + "contract": { + "metric": "maximum SLO-feasible request_rate_per_gpu", + "target_pass_rate": 0.95, + "cpu_overhead_modeling": "skipped_no_community_vllm_native_records", + "end_to_end_calibration": False, + "moe_routing_mode": "simulation", + "moe_routing_seed": MOE_ROUTING_SEED, + "kv_blocks_source": "community_vllm_measured_and_fixed_per_topology", + "network_device": "h20_dgx", + }, + "configs": [asdict(config) | {"name": config.name} for config in configs], + } + write_json(args.output_root / "run_manifest.json", run_manifest) + + all_results = {} + for config in configs: + low = SEARCH_LOW + high = SEARCH_HIGH + probes = [] + best = None + for probe_index in range(SEARCH_PROBES): + anchor = (low + high) / 2.0 + record = trace_manifest["anchors"].get(anchor_key(anchor)) + if record is None: + raise ValueError(f"trace manifest lacks anchor {anchor_key(anchor)}") + result = run_probe( + python=args.python, + frontier_source=args.frontier_source, + profile_root=args.profile_root, + profile_paths=profile_paths, + trace_record=record, + config=config, + probe_index=probe_index, + output_root=args.output_root, + cache_root=cache_root, + ) + probes.append(result) + print( + json.dumps( + { + "config": config.name, + "probe": probe_index, + "anchor": anchor, + "pass_rate": result["slo_pass_rate"], + "feasible": result["feasible"], + "elapsed_seconds": result["elapsed_seconds"], + }, + sort_keys=True, + ), + flush=True, + ) + if result["feasible"]: + low = anchor + best = result + else: + high = anchor + summary = { + "config": asdict(config) | {"name": config.name}, + "probes": probes, + "capacity_interval": [low, high], + "best_feasible": best, + } + write_json(args.output_root / "results" / f"{config.name}.json", summary) + all_results[config.name] = summary + + ranked = sorted( + all_results.values(), + key=lambda item: ( + -( + item["best_feasible"]["request_rate_per_gpu"] + if item["best_feasible"] is not None + else -1.0 + ), + item["config"]["name"], + ), + ) + freeze = { + "schema": SCHEMA, + "run_manifest_sha256": sha256(args.output_root / "run_manifest.json"), + "ranking": [ + { + "rank": index + 1, + "config": item["config"], + "capacity_interval_sampling_u": item["capacity_interval"], + "best_feasible": item["best_feasible"], + } + for index, item in enumerate(ranked) + ], + } + write_json(args.output_root / "frontier_ranking_frozen.json", freeze) + print(json.dumps(freeze, indent=2), flush=True) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + prepare = subparsers.add_parser("prepare") + prepare.add_argument("--trace", type=Path, required=True) + prepare.add_argument("--output-root", type=Path, required=True) + prepare.add_argument("--expected-trace-sha256", default=TRACE_SHA256) + + run = subparsers.add_parser("run") + run.add_argument("--frontier-source", type=Path, required=True) + run.add_argument("--python", type=Path, required=True) + run.add_argument("--profile-root", type=Path, required=True) + run.add_argument("--trace-manifest", type=Path, required=True) + run.add_argument("--output-root", type=Path, required=True) + run.add_argument("--tp4-capacity-artifact", type=Path, required=True) + run.add_argument("--tp8-capacity-artifact", type=Path, required=True) + run.add_argument("--config", action="append") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + if args.command == "prepare": + manifest = prepare_traces( + args.trace, args.output_root, args.expected_trace_sha256 + ) + print(manifest) + return + if args.command == "run": + run_grid(args) + return + raise AssertionError(args.command) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/test_frontier_prefill_grid.py b/runs/frontier-multicase-sufficiency-v0/best_effort/test_frontier_prefill_grid.py new file mode 100644 index 0000000..3ace7eb --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/test_frontier_prefill_grid.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import csv +import importlib.util +import json +import sys +from pathlib import Path + + +SCRIPT = Path(__file__).with_name("frontier_prefill_grid.py") +SPEC = importlib.util.spec_from_file_location("frontier_prefill_grid", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) + + +def test_binary_search_lattice_covers_six_probe_tree() -> None: + anchors = MODULE.binary_search_lattice() + assert len(anchors) == 63 + assert anchors[0] == 0.001953125 + assert anchors[-1] == 0.123046875 + assert 0.0625 in anchors + + +def test_prepare_preserves_stable_arrival_order(tmp_path: Path) -> None: + trace = tmp_path / "trace.jsonl" + rows = [ + {"timestamp": 1.0, "sampling_u": 0.02, "input_length": 9000, "id": "a"}, + {"timestamp": 0.5, "sampling_u": 0.03, "input_length": 100, "id": "b"}, + {"timestamp": 1.0, "sampling_u": 0.01, "input_length": 200, "id": "c"}, + {"timestamp": 0.2, "sampling_u": 0.01, "input_length": 40000, "id": "x"}, + ] + trace.write_text("".join(json.dumps(row) + "\n" for row in rows)) + manifest_path = MODULE.prepare_traces(trace, tmp_path / "out", "") + manifest = json.loads(manifest_path.read_text()) + record = manifest["anchors"][MODULE.anchor_key(0.0625)] + with Path(record["path"]).open(newline="") as source: + selected = list(csv.DictReader(source)) + assert [row["source_request_id"] for row in selected] == ["b", "a", "c"] + assert [float(row["slo_ttft_ms"]) for row in selected] == [1000, 2000, 1000] + + +def test_score_uses_trace_slo_and_request_id_mapping(tmp_path: Path) -> None: + trace = tmp_path / "trace.csv" + with trace.open("w", newline="") as output: + writer = csv.DictWriter( + output, + fieldnames=["arrived_at", "num_prefill_tokens", "num_decode_tokens", "slo_ttft_ms"], + ) + writer.writeheader() + writer.writerows( + [ + {"arrived_at": 0, "num_prefill_tokens": 10, "num_decode_tokens": 1, "slo_ttft_ms": 1000}, + {"arrived_at": 1, "num_prefill_tokens": 9000, "num_decode_tokens": 1, "slo_ttft_ms": 2000}, + ] + ) + metrics = tmp_path / "request_metrics.csv" + with metrics.open("w", newline="") as output: + writer = csv.DictWriter(output, fieldnames=["Request Id", "ttft"]) + writer.writeheader() + writer.writerows([{"Request Id": 1, "ttft": 1900}, {"Request Id": 0, "ttft": 1100}]) + score = MODULE.score_request_metrics(trace, metrics) + assert score["passed_request_count"] == 1 + assert score["slo_pass_rate"] == 0.5 + assert score["feasible"] is False From 584af7b253724a7f552615d64c950c77a0b70a68 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 19:41:07 +0800 Subject: [PATCH 040/148] Fingerprint Frontier source snapshots --- .../best_effort/frontier_prefill_grid.py | 85 +++++++++++++------ 1 file changed, 59 insertions(+), 26 deletions(-) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_prefill_grid.py b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_prefill_grid.py index ce6bec5..fc9ac9d 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_prefill_grid.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_prefill_grid.py @@ -90,39 +90,66 @@ def sha256_bytes(payload: bytes) -> str: return hashlib.sha256(payload).hexdigest() -def frontier_source_fingerprint(source: Path) -> dict[str, Any]: - commit = subprocess.run( - ["git", "rev-parse", "HEAD"], - cwd=source, - check=True, - stdout=subprocess.PIPE, - text=True, - ).stdout.strip() - status = subprocess.run( - ["git", "status", "--porcelain=v1"], - cwd=source, - check=True, - stdout=subprocess.PIPE, - text=True, - ).stdout - diff = subprocess.run( - ["git", "diff", "--binary", "HEAD"], - cwd=source, - check=True, - stdout=subprocess.PIPE, - ).stdout +def frontier_source_fingerprint( + source: Path, declared_commit: str +) -> dict[str, Any]: model_config = source / "data" / "config" / "models" / f"{MODEL}.json" if not model_config.is_file(): raise FileNotFoundError(f"Frontier model config is missing: {model_config}") - return { - "commit": commit, - "status_porcelain": status.splitlines(), - "tracked_diff_sha256": sha256_bytes(diff), + device_config = source / "data" / "config" / "device" / "h20.json" + files = sorted((source / "frontier").glob("**/*.py")) + [ + model_config, + device_config, + ] + missing = [str(path) for path in files if not path.is_file()] + if missing: + raise FileNotFoundError(f"Frontier fingerprint inputs are missing: {missing}") + digest = hashlib.sha256() + for path in files: + relative = path.relative_to(source).as_posix() + digest.update(relative.encode() + b"\0") + digest.update(bytes.fromhex(sha256(path))) + fingerprint = { + "declared_upstream_commit": declared_commit, + "python_and_config_tree_sha256": digest.hexdigest(), + "fingerprinted_file_count": len(files), "model_config": { "path": str(model_config.resolve()), "sha256": sha256(model_config), }, } + git_dir = source / ".git" + if git_dir.exists(): + commit = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=source, + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout.strip() + status = subprocess.run( + ["git", "status", "--porcelain=v1"], + cwd=source, + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout + diff = subprocess.run( + ["git", "diff", "--binary", "HEAD"], + cwd=source, + check=True, + stdout=subprocess.PIPE, + ).stdout + fingerprint.update( + { + "git_commit": commit, + "status_porcelain": status.splitlines(), + "tracked_diff_sha256": sha256_bytes(diff), + } + ) + else: + fingerprint["git_metadata"] = "absent_source_snapshot" + return fingerprint def anchor_key(anchor: float) -> str: @@ -550,7 +577,9 @@ def run_grid(args: argparse.Namespace) -> None: "frontier": { "source": str(args.frontier_source.resolve()), "python": str(args.python.resolve()), - "fingerprint": frontier_source_fingerprint(args.frontier_source), + "fingerprint": frontier_source_fingerprint( + args.frontier_source, args.frontier_commit + ), }, "trace_manifest": { "path": str(args.trace_manifest.resolve()), @@ -675,6 +704,10 @@ def parse_args() -> argparse.Namespace: run = subparsers.add_parser("run") run.add_argument("--frontier-source", type=Path, required=True) + run.add_argument( + "--frontier-commit", + default="d9cfeb6d8791fbf2f295dd9744c56a666171776e", + ) run.add_argument("--python", type=Path, required=True) run.add_argument("--profile-root", type=Path, required=True) run.add_argument("--trace-manifest", type=Path, required=True) From 6e619b75d27e0ef69eb9056061a4f229e0d897cd Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 19:53:00 +0800 Subject: [PATCH 041/148] Replay raw completion prompts without chat wrapping --- src/aituner/http_client.py | 12 ++++--- src/aituner/spec.py | 6 ++-- src/aituner/trace.py | 55 +++++++++++++++++++++----------- src/aituner/worker.py | 7 +++- tests/test_core_flow.py | 65 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 118 insertions(+), 27 deletions(-) diff --git a/src/aituner/http_client.py b/src/aituner/http_client.py index 186cf0d..0adfa80 100644 --- a/src/aituner/http_client.py +++ b/src/aituner/http_client.py @@ -254,10 +254,11 @@ def stream_chat_completion( base_url: str, body: dict[str, Any], timeout_s: float, + api_path: str = "/v1/chat/completions", ) -> StreamMetrics: data = json.dumps(body).encode("utf-8") request = urllib.request.Request( - url=_openai_url(base_url, "/v1/chat/completions"), + url=_openai_url(base_url, api_path), headers=_auth_headers(None), data=data, method="POST", @@ -285,10 +286,11 @@ def stream_chat_completion( choices = payload.get("choices") if not isinstance(choices, list) or not choices: continue - delta = choices[0].get("delta", {}) - if not isinstance(delta, dict): - continue - content = delta.get("content") + choice = choices[0] + delta = choice.get("delta", {}) + content = delta.get("content") if isinstance(delta, dict) else None + if not isinstance(content, str): + content = choice.get("text") if isinstance(content, str) and content: now = time.monotonic() if first_token_at is None: diff --git a/src/aituner/spec.py b/src/aituner/spec.py index a9c5720..a881e8e 100644 --- a/src/aituner/spec.py +++ b/src/aituner/spec.py @@ -411,8 +411,10 @@ class TraceSpec: synthetic_prompt_cap = data.get("synthetic_prompt_cap_tokens") completion_tokens_override = data.get("completion_tokens_override") request_mode = str(data.get("request_mode") or "chat").strip().lower() - if request_mode not in {"chat", "decode_only"}: - raise SpecError("trace.request_mode must be one of: chat, decode_only.") + if request_mode not in {"chat", "decode_only", "raw_completion"}: + raise SpecError( + "trace.request_mode must be one of: chat, decode_only, raw_completion." + ) if completion_tokens_override is not None: completion_tokens_override = _require_int( completion_tokens_override, diff --git a/src/aituner/trace.py b/src/aituner/trace.py index cabb3a5..9e39323 100644 --- a/src/aituner/trace.py +++ b/src/aituner/trace.py @@ -40,6 +40,7 @@ class TraceRequest: body: dict[str, Any] prompt_tokens_hint: int | None completion_tokens_hint: int | None + api_path: str = "/v1/chat/completions" metadata: dict[str, Any] = field(default_factory=dict) @@ -186,26 +187,41 @@ def load_trace_requests(study: StudySpec, *, study_spec_path: Path) -> tuple[Win prompt_tokens_hint = _coerce_prompt_tokens(row) if not _matches_input_length_filter(study, prompt_tokens_hint=prompt_tokens_hint): continue - try: - messages = _coerce_messages(row) - except TraceError: - capped_prompt_tokens = prompt_tokens_hint or 0 - if study.trace.synthetic_prompt_cap_tokens is not None: - capped_prompt_tokens = min( - capped_prompt_tokens, study.trace.synthetic_prompt_cap_tokens + api_path = "/v1/chat/completions" + if study.trace.request_mode == "raw_completion": + prompt = row.get("prompt") + if not isinstance(prompt, str) or not prompt: + raise TraceError( + f"trace row {idx} is missing prompt required by raw_completion" ) - messages = [ - { - "role": "user", - "content": _synthetic_prompt_from_tokens(capped_prompt_tokens), - } - ] - body: dict[str, Any] = { - "model": study.model.served_model_name, - "messages": messages, - "stream": True, - "stream_options": {"include_usage": True}, - } + body: dict[str, Any] = { + "model": study.model.served_model_name, + "prompt": prompt, + "stream": True, + "stream_options": {"include_usage": True}, + } + api_path = "/v1/completions" + else: + try: + messages = _coerce_messages(row) + except TraceError: + capped_prompt_tokens = prompt_tokens_hint or 0 + if study.trace.synthetic_prompt_cap_tokens is not None: + capped_prompt_tokens = min( + capped_prompt_tokens, study.trace.synthetic_prompt_cap_tokens + ) + messages = [ + { + "role": "user", + "content": _synthetic_prompt_from_tokens(capped_prompt_tokens), + } + ] + body = { + "model": study.model.served_model_name, + "messages": messages, + "stream": True, + "stream_options": {"include_usage": True}, + } completion_tokens = ( study.trace.completion_tokens_override if study.trace.completion_tokens_override is not None @@ -225,6 +241,7 @@ def load_trace_requests(study: StudySpec, *, study_spec_path: Path) -> tuple[Win body=body, prompt_tokens_hint=prompt_tokens_hint, completion_tokens_hint=completion_tokens, + api_path=api_path, metadata={ "hash_ids": row.get("hash_ids") if isinstance(row.get("hash_ids"), list) else None, "turn": row.get("turn"), diff --git a/src/aituner/worker.py b/src/aituner/worker.py index 0439537..d8f0ca6 100644 --- a/src/aituner/worker.py +++ b/src/aituner/worker.py @@ -107,7 +107,12 @@ def _run_one_request( timeout_s: float, ) -> RequestOutcome: try: - metrics = stream_chat_completion(base_url=base_url, body=request.body, timeout_s=timeout_s) + metrics = stream_chat_completion( + base_url=base_url, + body=request.body, + timeout_s=timeout_s, + api_path=request.api_path, + ) expected_completion_tokens = request.completion_tokens_hint actual_completion_tokens = metrics.completion_tokens completion_tokens_source = getattr(metrics, "completion_tokens_source", "") diff --git a/tests/test_core_flow.py b/tests/test_core_flow.py index 465244a..1e9d0a6 100644 --- a/tests/test_core_flow.py +++ b/tests/test_core_flow.py @@ -5586,6 +5586,42 @@ class CoreFlowTests(unittest.TestCase): self.assertEqual(requests[2].body["min_tokens"], 1) self.assertEqual(requests[2].body["max_tokens"], 1) + def test_raw_completion_mode_preserves_trace_prompt_and_endpoint(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + study_path = _write_study_assets( + tmp_path, + trace_overrides={"request_mode": "raw_completion"}, + ) + trace_path = tmp_path / "trace_windows" / "traces" / "chat_w1.jsonl" + rows = [json.loads(line) for line in trace_path.read_text().splitlines()] + for idx, row in enumerate(rows): + row["prompt"] = f"<|im_start|>user\nraw prompt {idx}<|im_end|>" + trace_path.write_text( + "".join(json.dumps(row) + "\n" for row in rows), + encoding="utf-8", + ) + + study = load_study_spec(study_path) + _, requests = load_trace_requests(study, study_spec_path=study_path) + + self.assertEqual(study.trace.request_mode, "raw_completion") + self.assertEqual(requests[0].api_path, "/v1/completions") + self.assertEqual(requests[0].body["prompt"], rows[0]["prompt"]) + self.assertNotIn("messages", requests[0].body) + + def test_raw_completion_mode_requires_prompt(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + study_path = _write_study_assets( + Path(tmp), + trace_overrides={"request_mode": "raw_completion"}, + ) + study = load_study_spec(study_path) + with self.assertRaisesRegex( + ValueError, "missing prompt required by raw_completion" + ): + load_trace_requests(study, study_spec_path=study_path) + def test_run_one_request_fails_fixed_length_completion_mismatch(self) -> None: request = TraceRequest( row_id="r1", @@ -8815,6 +8851,35 @@ class CoreFlowTests(unittest.TestCase): self.assertIsNone(metrics.completion_tokens) self.assertEqual(metrics.completion_tokens_source, "none") + def test_stream_chat_completion_reads_raw_completion_text(self) -> None: + class FakeResponse: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + return False + + def __iter__(self): + return iter( + [ + b'data: {"choices": [{"text": "x"}]}\n', + b'data: {"choices": [], "usage": {"completion_tokens": 1}}\n', + b"data: [DONE]\n", + ] + ) + + with mock.patch("aituner.http_client._urlopen", return_value=FakeResponse()): + metrics = stream_chat_completion( + base_url="http://127.0.0.1:8000", + body={"model": "m", "prompt": "raw"}, + timeout_s=1.0, + api_path="/v1/completions", + ) + + self.assertIsNotNone(metrics.ttft_ms) + self.assertEqual(metrics.completion_tokens, 1) + self.assertEqual(metrics.streamed_chunk_count, 1) + def test_stream_chat_completion_marks_same_instant_multitoken_tpot_unmeasurable(self) -> None: class FakeResponse: def __enter__(self): From 927561998402bded9d9a85335fa43e92fdee652d Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 20:00:41 +0800 Subject: [PATCH 042/148] Add reproducible community vLLM prefill grid --- .../best_effort/community_prefill_grid.py | 448 ++++++++++++++++++ .../test_community_prefill_grid.py | 64 +++ 2 files changed, 512 insertions(+) create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/community_prefill_grid.py create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/test_community_prefill_grid.py diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/community_prefill_grid.py b/runs/frontier-multicase-sufficiency-v0/best_effort/community_prefill_grid.py new file mode 100644 index 0000000..ce347ac --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/community_prefill_grid.py @@ -0,0 +1,448 @@ +#!/usr/bin/env python3 +"""Run the community-vLLM side of the frozen Qwen235B prefill grid.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import random +import subprocess +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + + +SCHEMA = "community-vllm-qwen235b-prefill-grid-v1" +MODEL = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8" +SERVED_MODEL = "qwen3-235b-community-prefill" +TRACE = ( + "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/" + "thinking_w20260327_1000.jsonl" +) +WINDOWS = "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/windows.json" +TRACE_SHA256 = "f878e9af18f94dcfaced94a8e1e6b20a2f7d97d64aa862448025660dbbd965b2" +ORDER_SEED = 20260715 + + +@dataclass(frozen=True) +class GridConfig: + tp: int + mns: int + mbt: int + expert_parallel: bool + num_gpu_blocks: int + + @property + def name(self) -> str: + return f"tp{self.tp}_mns{self.mns}_mbt{self.mbt}" + + +GRID = tuple( + GridConfig( + tp=tp, + mns=mns, + mbt=mbt, + expert_parallel=tp == 8, + num_gpu_blocks=26101 if tp == 4 else 62351, + ) + for tp in (4, 8) + for mns in (64, 128) + for mbt in (8192, 16384) +) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + os.replace(temporary, path) + + +def search_spec() -> dict[str, Any]: + return { + "low": 0.0, + "high": 0.125, + "tolerance": 0.001, + "max_probes": 6, + "sample_seed": 20260325, + "inherit_incumbent_floor": False, + "auto_high": { + "enabled": False, + "max_sampling_u": 1.0, + "require_human_confirmation_beyond_trace": True, + }, + } + + +def study_payload( + *, + tp: int, + repo: Path, + python: Path, + vllm: Path, + model: Path, + trace: Path, + windows: Path, + port: int, +) -> dict[str, Any]: + base_flags: dict[str, Any] = { + "host": "127.0.0.1", + "port": port, + "served-model-name": SERVED_MODEL, + "tensor-parallel-size": tp, + "disable-custom-all-reduce": True, + "quantization": "fp8", + "gpu-memory-utilization": 0.80, + "num-gpu-blocks-override": 26101 if tp == 4 else 62351, + "kv-cache-dtype": "auto", + "max-model-len": 40960, + "max-num-batched-tokens": 8192, + "max-num-seqs": 64, + "enable-prefix-caching": False, + "enable-chunked-prefill": True, + "enforce-eager": True, + "disable-log-requests": True, + } + if tp == 8: + base_flags["enable-expert-parallel"] = True + return { + "study_id": f"community-qwen235b-prefill-tp{tp}-v1", + "hardware": { + "gpu_count": tp, + "gpu_model": "NVIDIA H20", + "host_candidates": ["dash0"], + }, + "model": {"model_id": str(model), "served_model_name": SERVED_MODEL}, + "engine": { + "engine_name": "vllm", + "engine_version": "0.10.2-community", + "exec_path": str(vllm), + "cwd": str(repo), + "host": "127.0.0.1", + "port": port, + "ready_timeout_s": 1800, + "request_timeout_s": 1800, + "healthcheck_path": "/v1/models", + "launch_args": ["serve", str(model)], + "base_envs": { + "CUDA_VISIBLE_DEVICES": ",".join(str(index) for index in range(tp)), + }, + "base_flags": base_flags, + "tunable_envs": [], + "tunable_flags": ["max-num-seqs", "max-num-batched-tokens"], + "python_executable": str(python), + }, + "trace": { + "windows_path": str(windows), + "window_id": "thinking_w20260327_1000", + "trace_file_override": str(trace), + "request_mode": "raw_completion", + "completion_tokens_override": 1, + "u_field": "sampling_u", + "timestamp_field": "timestamp", + "max_concurrency": 64, + "input_length_filter": { + "min_input_tokens": 0, + "max_input_tokens": 32768, + }, + "replay_time_scale": 1.0, + "early_stop_max_lag_s": 180.0, + "early_stop_max_elapsed_s": 1200.0, + "restart_engine_after_early_stop": False, + "adaptive_stop": {"enabled": False}, + }, + "slo": { + "target_pass_rate": 0.95, + "ttft_rule": { + "kind": "step_ms", + "buckets": [ + {"max_input_tokens": 8191, "threshold_ms": 1000}, + {"max_input_tokens": 32767, "threshold_ms": 2000}, + {"threshold_ms": 2000}, + ], + }, + }, + "search": search_spec(), + "llm": {"use_harness": False}, + } + + +def git_fingerprint(repo: Path) -> dict[str, Any]: + def run(*args: str) -> str: + return subprocess.run( + ["git", *args], + cwd=repo, + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout.strip() + + return { + "commit": run("rev-parse", "HEAD"), + "status_porcelain": run("status", "--porcelain=v1").splitlines(), + } + + +def prepare(args: argparse.Namespace) -> None: + repo = args.repo.resolve() + python = args.python.resolve() + vllm = args.vllm.resolve() + model = args.model.resolve() + trace = args.trace.resolve() + windows = args.windows.resolve() + for path in (repo, python, vllm, model, trace, windows): + if not path.exists(): + raise FileNotFoundError(path) + actual_trace_sha = sha256(trace) + if actual_trace_sha != args.expected_trace_sha256: + raise ValueError( + f"trace SHA256 mismatch: expected={args.expected_trace_sha256}, " + f"actual={actual_trace_sha}" + ) + output = args.output_root.resolve() + output.mkdir(parents=True, exist_ok=True) + studies: dict[int, dict[str, str]] = {} + for tp, port in ((4, args.port), (8, args.port)): + study_dir = output / "studies" / f"tp{tp}" + study_path = study_dir / "study.json" + pointer_path = study_dir / "study_spec.source" + write_json( + study_path, + study_payload( + tp=tp, + repo=repo, + python=python, + vllm=vllm, + model=model, + trace=trace, + windows=windows, + port=port, + ), + ) + pointer_path.write_text(str(study_path) + "\n") + studies[tp] = { + "study_path": str(study_path), + "study_sha256": sha256(study_path), + "pointer_path": str(pointer_path), + } + + order = list(GRID) + random.Random(args.order_seed).shuffle(order) + trials = [] + for index, config in enumerate(order, start=1): + trial_dir = output / "trials" / config.name + trial_path = trial_dir / "trial_spec.json" + trial = { + "study_id": f"community-qwen235b-prefill-tp{config.tp}-v1", + "trial_id": config.name, + "config_patch": { + "env_patch": {}, + "flag_patch": { + "max-num-seqs": config.mns, + "max-num-batched-tokens": config.mbt, + }, + }, + "search": search_spec(), + "study_spec_path": studies[config.tp]["pointer_path"], + "artifact_dir": str(trial_dir), + "probe_log_path": str(trial_dir / "probe_history.json"), + "engine_log_path": str(trial_dir / "engine.log"), + "result_path": str(trial_dir / "result.json"), + "search_evidence": { + "enabled": False, + "original_high": 0.125, + "effective_high": 0.125, + "trace_max_sampling_u": None, + "max_sampling_u": 1.0, + "require_human_confirmation_beyond_trace": True, + "reason": "auto_high_disabled", + }, + } + write_json(trial_path, trial) + trials.append( + { + "execution_index": index, + "config": asdict(config) | {"name": config.name}, + "trial_spec_path": str(trial_path), + "trial_spec_sha256": sha256(trial_path), + } + ) + manifest = { + "schema": SCHEMA, + "created_unix_s": time.time(), + "repository": git_fingerprint(repo), + "runtime": {"python": str(python), "vllm": str(vllm)}, + "model": { + "path": str(model), + "config_sha256": sha256(model / "config.json"), + }, + "trace": {"path": str(trace), "sha256": actual_trace_sha}, + "contract": { + "metric": "maximum SLO-feasible request_rate_per_gpu", + "request_mode": "raw_completion", + "completion_tokens_override": 1, + "target_pass_rate": 0.95, + "search": search_spec(), + "prefix_caching": False, + "chunked_prefill": True, + "cuda_graphs": False, + "custom_all_reduce": False, + "kv_blocks_source": "community_vllm_measured_and_fixed_per_topology", + }, + "order": {"method": "python_random_shuffle", "seed": args.order_seed}, + "studies": studies, + "trials": trials, + } + write_json(output / "run_manifest.json", manifest) + print(output / "run_manifest.json") + + +def run(args: argparse.Namespace) -> None: + manifest = json.loads(args.manifest.read_text()) + if manifest.get("schema") != SCHEMA: + raise ValueError(f"unexpected manifest schema: {manifest.get('schema')}") + repo = args.repo.resolve() + env = os.environ.copy() + env["PYTHONPATH"] = str(repo / "src") + for trial in manifest["trials"]: + trial_path = Path(trial["trial_spec_path"]) + trial_dir = trial_path.parent + result_path = trial_dir / "result.json" + if result_path.is_file(): + result = json.loads(result_path.read_text()) + if result.get("status") == "completed": + print(json.dumps({"config": trial["config"]["name"], "skipped": True})) + continue + command = [ + str(args.python.resolve()), + "-m", + "aituner.cli", + "worker", + "run-trial", + "--trial-spec", + str(trial_path), + ] + write_json(trial_dir / "worker_command.json", command) + started = time.time() + with (trial_dir / "worker.log").open("w", encoding="utf-8") as output: + completed = subprocess.run( + command, + cwd=repo, + env=env, + stdout=output, + stderr=subprocess.STDOUT, + check=False, + ) + record = { + "config": trial["config"]["name"], + "returncode": completed.returncode, + "elapsed_seconds": time.time() - started, + } + print(json.dumps(record), flush=True) + if completed.returncode != 0: + raise RuntimeError(f"community trial failed: {record}") + assemble(args.manifest) + + +def capacity_interval(probes: list[dict[str, Any]]) -> list[float]: + low, high = 0.0, 0.125 + for probe in probes: + midpoint = float(probe["threshold"]) + if probe["feasible"]: + low = midpoint + else: + high = midpoint + return [low, high] + + +def assemble(manifest_path: Path) -> Path: + manifest = json.loads(manifest_path.read_text()) + records = [] + for trial in manifest["trials"]: + result_path = Path(trial["trial_spec_path"]).parent / "result.json" + if not result_path.is_file(): + raise FileNotFoundError(result_path) + result = json.loads(result_path.read_text()) + if result.get("status") != "completed": + raise ValueError(f"trial did not complete: {result_path}") + tp = int(trial["config"]["tp"]) + request_rate = result.get("best_request_rate") + records.append( + { + "config": trial["config"], + "result_path": str(result_path), + "result_sha256": sha256(result_path), + "capacity_interval_sampling_u": capacity_interval(result["probes"]), + "best_sampling_u": result.get("best_sampling_u"), + "best_request_rate": request_rate, + "best_request_rate_per_gpu": ( + float(request_rate) / tp if request_rate is not None else None + ), + "best_pass_rate": result.get("best_pass_rate"), + } + ) + records.sort( + key=lambda item: ( + -(item["best_request_rate_per_gpu"] or -1.0), + item["config"]["name"], + ) + ) + freeze = { + "schema": SCHEMA, + "run_manifest_sha256": sha256(manifest_path), + "ranking": [dict(record, rank=index + 1) for index, record in enumerate(records)], + } + path = manifest_path.parent / "community_ranking_frozen.json" + write_json(path, freeze) + print(path) + return path + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + prepare_parser = subparsers.add_parser("prepare") + prepare_parser.add_argument("--repo", type=Path, required=True) + prepare_parser.add_argument("--python", type=Path, required=True) + prepare_parser.add_argument("--vllm", type=Path, required=True) + prepare_parser.add_argument("--model", type=Path, default=Path(MODEL)) + prepare_parser.add_argument("--trace", type=Path, default=Path(TRACE)) + prepare_parser.add_argument("--windows", type=Path, default=Path(WINDOWS)) + prepare_parser.add_argument("--expected-trace-sha256", default=TRACE_SHA256) + prepare_parser.add_argument("--output-root", type=Path, required=True) + prepare_parser.add_argument("--order-seed", type=int, default=ORDER_SEED) + prepare_parser.add_argument("--port", type=int, default=18918) + run_parser = subparsers.add_parser("run") + run_parser.add_argument("--repo", type=Path, required=True) + run_parser.add_argument("--python", type=Path, required=True) + run_parser.add_argument("--manifest", type=Path, required=True) + assemble_parser = subparsers.add_parser("assemble") + assemble_parser.add_argument("--manifest", type=Path, required=True) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + if args.command == "prepare": + prepare(args) + elif args.command == "run": + run(args) + elif args.command == "assemble": + assemble(args.manifest) + else: + raise AssertionError(args.command) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/test_community_prefill_grid.py b/runs/frontier-multicase-sufficiency-v0/best_effort/test_community_prefill_grid.py new file mode 100644 index 0000000..0587fad --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/test_community_prefill_grid.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + + +SCRIPT = Path(__file__).with_name("community_prefill_grid.py") +SPEC = importlib.util.spec_from_file_location("community_prefill_grid", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) + + +def test_grid_matches_frozen_frontier_surface() -> None: + assert len(MODULE.GRID) == 8 + assert {item.name for item in MODULE.GRID} == { + f"tp{tp}_mns{mns}_mbt{mbt}" + for tp in (4, 8) + for mns in (64, 128) + for mbt in (8192, 16384) + } + assert {item.num_gpu_blocks for item in MODULE.GRID if item.tp == 4} == {26101} + assert {item.num_gpu_blocks for item in MODULE.GRID if item.tp == 8} == {62351} + + +def test_study_uses_raw_prompt_and_fixed_serving_contract(tmp_path: Path) -> None: + payload = MODULE.study_payload( + tp=8, + repo=tmp_path, + python=tmp_path / "python", + vllm=tmp_path / "vllm", + model=tmp_path / "model", + trace=tmp_path / "trace.jsonl", + windows=tmp_path / "windows.json", + port=18918, + ) + assert payload["trace"]["request_mode"] == "raw_completion" + assert payload["trace"]["completion_tokens_override"] == 1 + assert payload["trace"]["replay_time_scale"] == 1.0 + assert payload["engine"]["base_flags"]["enable-expert-parallel"] is True + assert payload["engine"]["base_flags"]["disable-custom-all-reduce"] is True + assert payload["engine"]["base_flags"]["num-gpu-blocks-override"] == 62351 + assert payload["engine"]["base_flags"]["enable-prefix-caching"] is False + + +def test_capacity_interval_replays_binary_decisions() -> None: + probes = [ + {"threshold": 0.0625, "feasible": False}, + {"threshold": 0.03125, "feasible": False}, + {"threshold": 0.015625, "feasible": True}, + {"threshold": 0.0234375, "feasible": False}, + {"threshold": 0.01953125, "feasible": True}, + {"threshold": 0.021484375, "feasible": True}, + ] + assert MODULE.capacity_interval(probes) == [0.021484375, 0.0234375] + + +def test_manifest_json_round_trip(tmp_path: Path) -> None: + path = tmp_path / "x.json" + MODULE.write_json(path, {"schema": MODULE.SCHEMA}) + assert json.loads(path.read_text()) == {"schema": MODULE.SCHEMA} From 76e0de33ee9978d25b2d3a2d47356a3dab8e10b3 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 20:15:09 +0800 Subject: [PATCH 043/148] Model critical EP lane for Frontier prefill --- ...rontier_moe_ep_prefill_critical_lane.patch | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_prefill_critical_lane.patch diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_prefill_critical_lane.patch b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_prefill_critical_lane.patch new file mode 100644 index 0000000..cc6d4ee --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_prefill_critical_lane.patch @@ -0,0 +1,76 @@ +diff --git a/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py b/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py +--- a/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py ++++ b/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py +@@ -386,0 +387,58 @@ ++ def _get_critical_prefill_ep_lane_tokens( ++ self, ++ total_routed_tokens: int, ++ layer_id: int, ++ ) -> Dict[int, int]: ++ """Return the routed-token allocation for the slowest prefill EP lane. ++ ++ A monolithic prefill starts with a global ``tokens * topk`` routing ++ population, but each EP rank executes only the assignments for its local ++ experts. The ranks synchronize after the MoE block, so the layer cost is ++ governed by the slowest lane rather than by either the global population ++ or the average lane. ++ """ ++ total_experts = int(self._replica_config.total_expert_num) ++ lane_count = int(self._moe_ep_size) ++ if total_experts <= 0 or lane_count <= 1 or total_experts % lane_count != 0: ++ raise ValueError( ++ "Prefill EP routing requires total_expert_num to be positive and " ++ f"divisible by moe_ep_size > 1; got total_expert_num={total_experts}, " ++ f"moe_ep_size={lane_count}" ++ ) ++ ++ experts_per_lane = total_experts // lane_count ++ global_tokens = self._get_global_per_expert_tokens( ++ total_routed_tokens=total_routed_tokens, ++ layer_id=layer_id, ++ ) ++ lane_tokens: List[Dict[int, int]] = [ ++ {local_expert_id: 0 for local_expert_id in range(experts_per_lane)} ++ for _ in range(lane_count) ++ ] ++ for global_expert_id, token_count in global_tokens.items(): ++ lane_id = int(global_expert_id) // experts_per_lane ++ local_expert_id = int(global_expert_id) % experts_per_lane ++ lane_tokens[lane_id][local_expert_id] = int(token_count) ++ ++ def _lane_compute_score(allocation: Dict[int, int]) -> float: ++ features = self._build_moe_load_imbalance_features(allocation) ++ score = 0.0 ++ for operation in ("moe_shuffling", "moe_grouped_gemm"): ++ prediction = self._predictions.get(operation) ++ if isinstance(prediction, dict) and prediction.get( ++ "_on_demand_prediction", False ++ ): ++ score += float( ++ self._get_on_demand_prediction(operation, features) ++ ) ++ return score ++ ++ return max( ++ lane_tokens, ++ key=lambda allocation: ( ++ _lane_compute_score(allocation), ++ sum(allocation.values()), ++ tuple(allocation.values()), ++ ), ++ ) ++ +@@ -1023,0 +1082,13 @@ ++ ++ from frontier.entities import EPBatchGroup ++ ++ is_monolithic_prefill_ep = ( ++ self._moe_ep_size > 1 ++ and not isinstance(batch, EPBatchGroup) ++ and not bool(getattr(batch, "is_pure_decode_batch", False)) ++ ) ++ if is_monolithic_prefill_ep: ++ return self._get_critical_prefill_ep_lane_tokens( ++ total_routed_tokens=num_tokens * self._router_topk, ++ layer_id=layer_id, ++ ) From d64f12eb5b4c46a676943e3ecb8b4b632b8adb5a Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 20:23:21 +0800 Subject: [PATCH 044/148] Preserve global token count in Frontier EP profiles --- .../frontier_moe_ep_feature_contract.patch | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_feature_contract.patch diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_feature_contract.patch b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_feature_contract.patch new file mode 100644 index 0000000..70a1c7a --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_feature_contract.patch @@ -0,0 +1,75 @@ +diff --git a/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py b/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py +--- a/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py ++++ b/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py +@@ -424 +424,4 @@ +- features = self._build_moe_load_imbalance_features(allocation) ++ features = self._build_moe_load_imbalance_features( ++ allocation, ++ num_tokens=total_routed_tokens // self._router_topk, ++ ) +@@ -1365,0 +1369 @@ ++ num_tokens: Optional[int] = None, +@@ -1384,3 +1388,6 @@ +- approx_num_tokens = max( +- 1, int(round(total_routed_tokens / float(self._router_topk))) +- ) ++ if num_tokens is None: ++ num_tokens = max( ++ 1, int(round(total_routed_tokens / float(self._router_topk))) ++ ) ++ elif num_tokens <= 0: ++ raise ValueError(f"num_tokens must be positive, got {num_tokens}") +@@ -1389 +1396 @@ +- num_tokens=approx_num_tokens, ++ num_tokens=int(num_tokens), +@@ -1475,0 +1483,3 @@ ++ profile_num_tokens = int( ++ self._get_effective_moe_total_tokens(batch) ++ ) +@@ -1478,0 +1489 @@ ++ profile_num_tokens, +@@ -1486 +1497,2 @@ +- per_expert_tokens ++ per_expert_tokens, ++ num_tokens=profile_num_tokens, +@@ -1778,4 +1790,4 @@ +- # The grouped_gemm predictor expects pre-routing num_tokens metadata, while runtime +- # allocation is post-routing (already expanded by router_topk). Recover the +- # approximate pre-routing token count to keep feature semantics aligned with +- # the profiling dataset contract. ++ # The grouped_gemm predictor keeps global pre-routing num_tokens and ++ # lane-local routed assignments as independent features. Inferring the ++ # former from the latter is valid for EP=1 but undercounts by EP size ++ # for expert-parallel execution. +@@ -1784,2 +1796,7 @@ +- approx_num_tokens = max( +- 1, int(round(total_routed_tokens / float(self._router_topk))) ++ profile_num_tokens = ( ++ int(self._get_effective_moe_total_tokens(batch)) ++ if batch is not None ++ else max( ++ 1, ++ int(round(total_routed_tokens / float(self._router_topk))), ++ ) +@@ -1804 +1821 @@ +- approx_num_tokens, ++ profile_num_tokens, +@@ -1811 +1828,4 @@ +- features = self._build_moe_load_imbalance_features(per_expert_tokens) ++ features = self._build_moe_load_imbalance_features( ++ per_expert_tokens, ++ num_tokens=profile_num_tokens, ++ ) +@@ -1851,2 +1871,7 @@ +- approx_num_tokens = max( +- 1, int(round(total_routed_tokens / float(self._router_topk))) ++ profile_num_tokens = ( ++ int(self._get_effective_moe_total_tokens(batch)) ++ if batch is not None ++ else max( ++ 1, ++ int(round(total_routed_tokens / float(self._router_topk))), ++ ) +@@ -1854 +1879 @@ +- rounded_tokens = self._round_to_valid_key(approx_num_tokens) ++ rounded_tokens = self._round_to_valid_key(profile_num_tokens) From 2f730417e08f4eeee5e9bf58952f4efe9c36e125 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 20:26:27 +0800 Subject: [PATCH 045/148] Revert "Preserve global token count in Frontier EP profiles" This reverts commit d64f12eb5b4c46a676943e3ecb8b4b632b8adb5a. --- .../frontier_moe_ep_feature_contract.patch | 75 ------------------- 1 file changed, 75 deletions(-) delete mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_feature_contract.patch diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_feature_contract.patch b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_feature_contract.patch deleted file mode 100644 index 70a1c7a..0000000 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_feature_contract.patch +++ /dev/null @@ -1,75 +0,0 @@ -diff --git a/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py b/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py ---- a/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py -+++ b/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py -@@ -424 +424,4 @@ -- features = self._build_moe_load_imbalance_features(allocation) -+ features = self._build_moe_load_imbalance_features( -+ allocation, -+ num_tokens=total_routed_tokens // self._router_topk, -+ ) -@@ -1365,0 +1369 @@ -+ num_tokens: Optional[int] = None, -@@ -1384,3 +1388,6 @@ -- approx_num_tokens = max( -- 1, int(round(total_routed_tokens / float(self._router_topk))) -- ) -+ if num_tokens is None: -+ num_tokens = max( -+ 1, int(round(total_routed_tokens / float(self._router_topk))) -+ ) -+ elif num_tokens <= 0: -+ raise ValueError(f"num_tokens must be positive, got {num_tokens}") -@@ -1389 +1396 @@ -- num_tokens=approx_num_tokens, -+ num_tokens=int(num_tokens), -@@ -1475,0 +1483,3 @@ -+ profile_num_tokens = int( -+ self._get_effective_moe_total_tokens(batch) -+ ) -@@ -1478,0 +1489 @@ -+ profile_num_tokens, -@@ -1486 +1497,2 @@ -- per_expert_tokens -+ per_expert_tokens, -+ num_tokens=profile_num_tokens, -@@ -1778,4 +1790,4 @@ -- # The grouped_gemm predictor expects pre-routing num_tokens metadata, while runtime -- # allocation is post-routing (already expanded by router_topk). Recover the -- # approximate pre-routing token count to keep feature semantics aligned with -- # the profiling dataset contract. -+ # The grouped_gemm predictor keeps global pre-routing num_tokens and -+ # lane-local routed assignments as independent features. Inferring the -+ # former from the latter is valid for EP=1 but undercounts by EP size -+ # for expert-parallel execution. -@@ -1784,2 +1796,7 @@ -- approx_num_tokens = max( -- 1, int(round(total_routed_tokens / float(self._router_topk))) -+ profile_num_tokens = ( -+ int(self._get_effective_moe_total_tokens(batch)) -+ if batch is not None -+ else max( -+ 1, -+ int(round(total_routed_tokens / float(self._router_topk))), -+ ) -@@ -1804 +1821 @@ -- approx_num_tokens, -+ profile_num_tokens, -@@ -1811 +1828,4 @@ -- features = self._build_moe_load_imbalance_features(per_expert_tokens) -+ features = self._build_moe_load_imbalance_features( -+ per_expert_tokens, -+ num_tokens=profile_num_tokens, -+ ) -@@ -1851,2 +1871,7 @@ -- approx_num_tokens = max( -- 1, int(round(total_routed_tokens / float(self._router_topk))) -+ profile_num_tokens = ( -+ int(self._get_effective_moe_total_tokens(batch)) -+ if batch is not None -+ else max( -+ 1, -+ int(round(total_routed_tokens / float(self._router_topk))), -+ ) -@@ -1854 +1879 @@ -- rounded_tokens = self._round_to_valid_key(approx_num_tokens) -+ rounded_tokens = self._round_to_valid_key(profile_num_tokens) From 6307620e2009ff92615e61e5c844783e658e76d6 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 20:36:10 +0800 Subject: [PATCH 046/148] Prevent client-side clipping in vLLM grid --- .../best_effort/community_prefill_grid.py | 15 +++++++++++---- .../best_effort/test_community_prefill_grid.py | 2 ++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/community_prefill_grid.py b/runs/frontier-multicase-sufficiency-v0/best_effort/community_prefill_grid.py index ce347ac..74eb149 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/community_prefill_grid.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/community_prefill_grid.py @@ -15,7 +15,7 @@ from pathlib import Path from typing import Any -SCHEMA = "community-vllm-qwen235b-prefill-grid-v1" +SCHEMA = "community-vllm-qwen235b-prefill-grid-v2" MODEL = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8" SERVED_MODEL = "qwen3-235b-community-prefill" TRACE = ( @@ -25,6 +25,7 @@ TRACE = ( WINDOWS = "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/windows.json" TRACE_SHA256 = "f878e9af18f94dcfaced94a8e1e6b20a2f7d97d64aa862448025660dbbd965b2" ORDER_SEED = 20260715 +CLIENT_MAX_CONCURRENCY = 256 @dataclass(frozen=True) @@ -117,7 +118,7 @@ def study_payload( if tp == 8: base_flags["enable-expert-parallel"] = True return { - "study_id": f"community-qwen235b-prefill-tp{tp}-v1", + "study_id": f"community-qwen235b-prefill-tp{tp}-v2", "hardware": { "gpu_count": tp, "gpu_model": "NVIDIA H20", @@ -151,7 +152,11 @@ def study_payload( "completion_tokens_override": 1, "u_field": "sampling_u", "timestamp_field": "timestamp", - "max_concurrency": 64, + # Keep the request generator above the largest server-side MNS so + # MNS=128 is exercised by vLLM rather than clipped by the client. + # The extra headroom also leaves an explicit engine waiting queue + # under overload instead of moving that queue into the replay loop. + "max_concurrency": CLIENT_MAX_CONCURRENCY, "input_length_filter": { "min_input_tokens": 0, "max_input_tokens": 32768, @@ -244,7 +249,7 @@ def prepare(args: argparse.Namespace) -> None: trial_dir = output / "trials" / config.name trial_path = trial_dir / "trial_spec.json" trial = { - "study_id": f"community-qwen235b-prefill-tp{config.tp}-v1", + "study_id": f"community-qwen235b-prefill-tp{config.tp}-v2", "trial_id": config.name, "config_patch": { "env_patch": {}, @@ -298,6 +303,8 @@ def prepare(args: argparse.Namespace) -> None: "chunked_prefill": True, "cuda_graphs": False, "custom_all_reduce": False, + "client_max_concurrency": CLIENT_MAX_CONCURRENCY, + "max_server_mns": max(config.mns for config in GRID), "kv_blocks_source": "community_vllm_measured_and_fixed_per_topology", }, "order": {"method": "python_random_shuffle", "seed": args.order_seed}, diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/test_community_prefill_grid.py b/runs/frontier-multicase-sufficiency-v0/best_effort/test_community_prefill_grid.py index 0587fad..5357a24 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/test_community_prefill_grid.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/test_community_prefill_grid.py @@ -40,6 +40,8 @@ def test_study_uses_raw_prompt_and_fixed_serving_contract(tmp_path: Path) -> Non assert payload["trace"]["request_mode"] == "raw_completion" assert payload["trace"]["completion_tokens_override"] == 1 assert payload["trace"]["replay_time_scale"] == 1.0 + assert payload["trace"]["max_concurrency"] == 256 + assert payload["trace"]["max_concurrency"] > max(item.mns for item in MODULE.GRID) assert payload["engine"]["base_flags"]["enable-expert-parallel"] is True assert payload["engine"]["base_flags"]["disable-custom-all-reduce"] is True assert payload["engine"]["base_flags"]["num-gpu-blocks-override"] == 62351 From ca88c06d061fc234230eeecd431bb7a3da4676c8 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 21:08:19 +0800 Subject: [PATCH 047/148] Preserve trace arrivals in Frontier grid --- .../best_effort/frontier_prefill_grid.py | 4 ++++ .../best_effort/test_frontier_prefill_grid.py | 23 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_prefill_grid.py b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_prefill_grid.py index fc9ac9d..b42bfa0 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_prefill_grid.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_prefill_grid.py @@ -319,6 +319,10 @@ def build_command( "frontier.main", "--simulation_mode", "offline", + # Frontier's offline default rewrites every request arrival to time 0. + # Preserve the trace arrival process so the simulator and serving run + # exercise the same queueing workload. + "--offline_use_generated_request_arrivals", "--sys_arch", "co-location", "--cluster_config_num_replicas", diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/test_frontier_prefill_grid.py b/runs/frontier-multicase-sufficiency-v0/best_effort/test_frontier_prefill_grid.py index 3ace7eb..0e41e8a 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/test_frontier_prefill_grid.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/test_frontier_prefill_grid.py @@ -64,3 +64,26 @@ def test_score_uses_trace_slo_and_request_id_mapping(tmp_path: Path) -> None: assert score["passed_request_count"] == 1 assert score["slo_pass_rate"] == 0.5 assert score["feasible"] is False + + +def test_build_command_preserves_trace_arrivals(tmp_path: Path) -> None: + config = MODULE.GRID[0] + command = MODULE.build_command( + python=tmp_path / "python", + frontier_source=tmp_path / "frontier", + profile_root=tmp_path / "profiles", + profile_paths={ + "linear": tmp_path / "linear.csv", + "attention": tmp_path / "attention.csv", + "moe": tmp_path / "moe.csv", + "all_reduce": tmp_path / "all_reduce.csv", + }, + trace=tmp_path / "trace.csv", + config=config, + probe_dir=tmp_path / "probe", + run_id="arrival-contract", + cache_root=tmp_path / "cache", + ) + + assert command[command.index("--simulation_mode") + 1] == "offline" + assert "--offline_use_generated_request_arrivals" in command From ed500fa0162673fb514a310d78855a16e5de33a4 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 21:47:25 +0800 Subject: [PATCH 048/148] Add fixed-cohort Frontier rank protocol --- .../best_effort/fixed_cohort_rank.py | 1404 +++++++++++++++++ .../best_effort/test_fixed_cohort_rank.py | 143 ++ 2 files changed, 1547 insertions(+) create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py new file mode 100644 index 0000000..78d4f58 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -0,0 +1,1404 @@ +#!/usr/bin/env python3 +"""Blind Frontier/community-vLLM rank comparison on one fixed prompt cohort. + +The earlier grid selected a different ``sampling_u`` subset at every load. That +confounds offered load with prompt mix and makes binary-search monotonicity an +untested assumption. This runner selects one length-stratified cohort, changes +load only by scaling its complete arrival timeline, evaluates every discrete +load, freezes Frontier first, and only then permits the community-vLLM run. +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +import os +import random +import statistics +import subprocess +import sys +import time +from dataclasses import asdict, replace +from pathlib import Path +from typing import Any, Iterable + + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +for search_path in (SCRIPT_DIR, REPO_ROOT / "src"): + if str(search_path) not in sys.path: + sys.path.insert(0, str(search_path)) + +import community_prefill_grid as community_grid # noqa: E402 +import frontier_prefill_grid as frontier_grid # noqa: E402 +from aituner.engine import build_launch_recipe # noqa: E402 +from aituner.spec import ConfigPatch, load_study_spec # noqa: E402 +from aituner.trace import TraceRequest, load_trace_requests # noqa: E402 +from aituner.worker import ( # noqa: E402 + _ignore_sigterm_if_main, + _install_sigterm_as_keyboardinterrupt, + _replay_requests, + _restore_sigterm, + _terminate_process_tree, + _wait_for_server_or_exit, +) + + +PROTOCOL_SCHEMA = "qwen235b-prefill-fixed-cohort-protocol-v1" +FRONTIER_SCHEMA = "frontier-qwen235b-prefill-fixed-cohort-v1" +COMMUNITY_SCHEMA = "community-qwen235b-prefill-fixed-cohort-v1" +COMPARISON_SCHEMA = "frontier-community-qwen235b-rank-comparison-v1" +COHORT_SIZE = 64 +COHORT_SEED = 2026071501 +CONFIG_ORDER_SEED = 2026071502 +RATE_ORDER_SEED = 2026071503 +INPUT_BINS = (0, 1024, 2048, 4096, 8192, 16384, 32769) +OFFERED_RATES = (0.15, 0.25, 0.35, 0.50, 0.75, 1.00, 1.50) +TARGET_PASS_RATE = 0.95 +EXPECTED_QUANT_SIGNATURE = "method=fp8|act=dynamic|serialized=True|block=128x128" + +SLO_VARIANTS: dict[str, dict[str, Any]] = { + "linear_8k_primary": { + "description": "TTFT <= 1000 ms + input_tokens / 8000 tokens/s", + "kind": "linear_ms", + "intercept_ms": 1000.0, + "per_token_ms": 0.125, + }, + "linear_6k": { + "description": "TTFT <= 1000 ms + input_tokens / 6000 tokens/s", + "kind": "linear_ms", + "intercept_ms": 1000.0, + "per_token_ms": 1.0 / 6.0, + }, + "linear_10k": { + "description": "TTFT <= 1000 ms + input_tokens / 10000 tokens/s", + "kind": "linear_ms", + "intercept_ms": 1000.0, + "per_token_ms": 0.1, + }, + "legacy_step_1s_2s": { + "description": "Original strict 1 s/2 s step SLO (null-capacity sensitivity)", + "kind": "step_ms", + "buckets": [ + {"max_input_tokens": 8191, "threshold_ms": 1000.0}, + {"threshold_ms": 2000.0}, + ], + }, +} +PRIMARY_SLO = "linear_8k_primary" + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def sha256_text(value: str) -> str: + return hashlib.sha256(value.encode()).hexdigest() + + +def order_hash(values: Iterable[object]) -> str: + return sha256_text("\n".join(str(value) for value in values)) + + +def write_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + os.replace(temporary, path) + + +def rate_key(rate: float) -> str: + return f"r_{rate:.3f}".replace(".", "p") + + +def _percentile(values: list[float], fraction: float) -> float | None: + if not values: + return None + ordered = sorted(values) + index = min(len(ordered) - 1, max(0, math.ceil(fraction * len(ordered)) - 1)) + return float(ordered[index]) + + +def _latency_summary(values: list[float]) -> dict[str, Any]: + return { + "count": len(values), + "mean": statistics.fmean(values) if values else None, + "p50": _percentile(values, 0.50), + "p95": _percentile(values, 0.95), + "p99": _percentile(values, 0.99), + "max": max(values) if values else None, + } + + +def slo_threshold_ms(variant: dict[str, Any], input_tokens: int) -> float: + if variant["kind"] == "linear_ms": + return float(variant["intercept_ms"]) + float(variant["per_token_ms"]) * input_tokens + if variant["kind"] != "step_ms": + raise ValueError(f"unsupported SLO kind: {variant['kind']}") + for bucket in variant["buckets"]: + ceiling = bucket.get("max_input_tokens") + if ceiling is None or input_tokens <= int(ceiling): + return float(bucket["threshold_ms"]) + raise AssertionError("step SLO must have a terminal bucket") + + +def _input_bin(input_tokens: int) -> int: + for index, (low, high) in enumerate(zip(INPUT_BINS, INPUT_BINS[1:])): + if low <= input_tokens < high: + return index + raise ValueError(f"input length outside protocol bins: {input_tokens}") + + +def _allocate_quotas(counts: list[int], cohort_size: int) -> list[int]: + total = sum(counts) + if cohort_size <= 0 or cohort_size > total: + raise ValueError(f"invalid cohort size {cohort_size} for {total} rows") + exact = [cohort_size * count / total for count in counts] + quotas = [min(count, math.floor(value)) for count, value in zip(counts, exact)] + remaining = cohort_size - sum(quotas) + order = sorted( + range(len(counts)), + key=lambda index: (-(exact[index] - math.floor(exact[index])), index), + ) + while remaining: + advanced = False + for index in order: + if quotas[index] >= counts[index]: + continue + quotas[index] += 1 + remaining -= 1 + advanced = True + if not remaining: + break + if not advanced: + raise AssertionError("could not allocate complete cohort") + return quotas + + +def load_source_rows(trace: Path) -> list[dict[str, Any]]: + rows = [] + with trace.open(encoding="utf-8") as source: + for source_index, line in enumerate(source): + if not line.strip(): + continue + raw = json.loads(line) + input_tokens = int(raw["input_length"]) + if not 0 <= input_tokens <= 32768: + continue + prompt = raw.get("prompt") + if not isinstance(prompt, str) or not prompt: + raise ValueError(f"row {source_index} lacks raw prompt") + rows.append( + { + "source_row_index": source_index, + "source_request_id": str( + raw.get("request_id") or raw.get("id") or source_index + ), + "source_arrival_s": float(raw["timestamp"]), + "sampling_u": float(raw["sampling_u"]), + "input_tokens": input_tokens, + "prompt_sha256": sha256_text(prompt), + "input_bin": _input_bin(input_tokens), + } + ) + return rows + + +def select_cohort( + rows: list[dict[str, Any]], *, cohort_size: int, seed: int +) -> tuple[list[dict[str, Any]], list[int]]: + by_bin = [[] for _ in range(len(INPUT_BINS) - 1)] + for row in rows: + by_bin[int(row["input_bin"])].append(row) + quotas = _allocate_quotas([len(items) for items in by_bin], cohort_size) + selected = [] + for bin_index, (items, quota) in enumerate(zip(by_bin, quotas)): + ranked = sorted( + items, + key=lambda row: sha256_text( + f"{seed}|{bin_index}|{row['source_row_index']}|{row['prompt_sha256']}" + ), + ) + selected.extend(ranked[:quota]) + selected.sort(key=lambda row: (row["source_arrival_s"], row["source_row_index"])) + if len(selected) != cohort_size: + raise AssertionError("cohort selection returned wrong size") + return selected, quotas + + +def prepare_protocol(args: argparse.Namespace) -> Path: + trace = args.trace.resolve() + actual_sha = sha256(trace) + if args.expected_trace_sha256 and actual_sha != args.expected_trace_sha256: + raise ValueError( + f"trace SHA256 mismatch: expected={args.expected_trace_sha256}, actual={actual_sha}" + ) + rows = load_source_rows(trace) + cohort, quotas = select_cohort(rows, cohort_size=args.cohort_size, seed=args.seed) + first_arrival = float(cohort[0]["source_arrival_s"]) + source_span = float(cohort[-1]["source_arrival_s"]) - first_arrival + if source_span <= 0: + raise ValueError("selected cohort has no arrival span") + + output_root = args.output_root.resolve() + trace_dir = output_root / "traces" + trace_dir.mkdir(parents=True, exist_ok=True) + fields = [ + "arrived_at", + "num_prefill_tokens", + "num_decode_tokens", + "source_row_index", + "source_request_id", + "source_arrival_s", + "sampling_u", + "slo_ttft_ms", + ] + rates = {} + for rate in args.rate: + duration_s = len(cohort) / rate + target = trace_dir / f"{rate_key(rate)}.csv" + with target.open("w", encoding="utf-8", newline="") as output: + writer = csv.DictWriter(output, fieldnames=fields) + writer.writeheader() + for row in cohort: + arrival_s = ( + (float(row["source_arrival_s"]) - first_arrival) + / source_span + * duration_s + ) + writer.writerow( + { + "arrived_at": f"{arrival_s:.12f}", + "num_prefill_tokens": row["input_tokens"], + "num_decode_tokens": 1, + "source_row_index": row["source_row_index"], + "source_request_id": row["source_request_id"], + "source_arrival_s": f"{row['source_arrival_s']:.12f}", + "sampling_u": f"{row['sampling_u']:.12f}", + "slo_ttft_ms": f"{slo_threshold_ms(SLO_VARIANTS[PRIMARY_SLO], int(row['input_tokens'])):.6f}", + } + ) + rates[rate_key(rate)] = { + "offered_request_rate": rate, + "duration_s": duration_s, + "path": str(target.resolve()), + "sha256": sha256(target), + "request_count": len(cohort), + "request_rate_recomputed": len(cohort) / duration_s, + "source_row_order_sha256": order_hash( + row["source_row_index"] for row in cohort + ), + "input_length_order_sha256": order_hash( + row["input_tokens"] for row in cohort + ), + } + + bin_counts = [0] * (len(INPUT_BINS) - 1) + for row in rows: + bin_counts[int(row["input_bin"])] += 1 + manifest = { + "schema": PROTOCOL_SCHEMA, + "created_unix_s": time.time(), + "source": { + "path": str(trace), + "sha256": actual_sha, + "eligible_request_count": len(rows), + }, + "selection": { + "method": "length_stratified_smallest_sha256", + "seed": args.seed, + "cohort_size": len(cohort), + "input_bin_edges": list(INPUT_BINS), + "source_bin_counts": bin_counts, + "cohort_bin_quotas": quotas, + "stable_order": ["source_arrival_s", "source_row_index"], + "cohort_source_row_order_sha256": order_hash( + row["source_row_index"] for row in cohort + ), + "source_arrival_span_s": source_span, + }, + "load_contract": { + "only_mutated_variable": "uniform_arrival_timeline_scale", + "request_count_per_point": len(cohort), + "completion_tokens_override": 1, + "target_pass_rate": TARGET_PASS_RATE, + "binary_search": False, + "monotonicity_assumed": False, + "offered_rates": list(args.rate), + }, + "slo_variants": SLO_VARIANTS, + "primary_slo": PRIMARY_SLO, + "cohort": cohort, + "rates": rates, + } + manifest_path = output_root / "protocol_manifest.json" + write_json(manifest_path, manifest) + audit_protocol(manifest_path) + print(manifest_path) + return manifest_path + + +def audit_protocol(manifest_path: Path) -> dict[str, Any]: + manifest = json.loads(manifest_path.read_text()) + if manifest.get("schema") != PROTOCOL_SCHEMA: + raise ValueError(f"unexpected protocol schema: {manifest.get('schema')}") + expected_ids = [str(row["source_row_index"]) for row in manifest["cohort"]] + expected_lengths = [int(row["input_tokens"]) for row in manifest["cohort"]] + rate_checks = [] + for record in manifest["rates"].values(): + path = Path(record["path"]) + if sha256(path) != record["sha256"]: + raise ValueError(f"trace hash mismatch: {path}") + with path.open(newline="") as source: + rows = list(csv.DictReader(source)) + ids = [row["source_row_index"] for row in rows] + lengths = [int(row["num_prefill_tokens"]) for row in rows] + arrivals = [float(row["arrived_at"]) for row in rows] + if ids != expected_ids or lengths != expected_lengths: + raise ValueError(f"cohort/order drift in {path}") + if arrivals != sorted(arrivals) or abs(arrivals[0]) > 1e-9: + raise ValueError(f"invalid arrival sequence in {path}") + recomputed = len(rows) / (arrivals[-1] - arrivals[0]) + if not math.isclose( + recomputed, float(record["offered_request_rate"]), rel_tol=1e-9 + ): + raise ValueError(f"offered-rate mismatch in {path}: {recomputed}") + rate_checks.append( + {"rate": record["offered_request_rate"], "recomputed_rate": recomputed} + ) + result = { + "status": "passed", + "protocol_manifest_sha256": sha256(manifest_path), + "cohort_size": len(expected_ids), + "rate_checks": rate_checks, + } + write_json(manifest_path.parent / "protocol_audit.json", result) + return result + + +def _read_trace(path: Path) -> list[dict[str, Any]]: + with path.open(encoding="utf-8", newline="") as source: + return list(csv.DictReader(source)) + + +def score_requests(requests: list[dict[str, Any]]) -> dict[str, Any]: + scores = {} + for name, variant in SLO_VARIANTS.items(): + passed = 0 + for request in requests: + threshold = slo_threshold_ms(variant, int(request["input_tokens"])) + passed += int( + bool(request["success"]) + and request.get("ttft_ms") is not None + and float(request["ttft_ms"]) <= threshold + ) + count = len(requests) + pass_rate = passed / count if count else 0.0 + scores[name] = { + "request_count": count, + "passed_request_count": passed, + "slo_pass_rate": pass_rate, + "target_pass_rate": TARGET_PASS_RATE, + "feasible": pass_rate >= TARGET_PASS_RATE, + } + ttfts = [ + float(request["ttft_ms"]) + for request in requests + if request.get("ttft_ms") is not None + ] + return {"scores": scores, "ttft_ms": _latency_summary(ttfts)} + + +def _capacity_record( + results: list[dict[str, Any]], *, config: dict[str, Any], slo_name: str +) -> dict[str, Any]: + ordered = sorted(results, key=lambda item: float(item["offered_request_rate"])) + feasible = [ + float(item["offered_request_rate"]) + for item in ordered + if item["scores"][slo_name]["feasible"] + ] + vector = [bool(item["scores"][slo_name]["feasible"]) for item in ordered] + violations = [] + for lower_index in range(len(vector)): + for higher_index in range(lower_index + 1, len(vector)): + if not vector[lower_index] and vector[higher_index]: + violations.append( + [ + float(ordered[lower_index]["offered_request_rate"]), + float(ordered[higher_index]["offered_request_rate"]), + ] + ) + capacity = max(feasible) if feasible else None + tp = int(config["tp"]) + return { + "config": config, + "slo": slo_name, + "tested_rates": [float(item["offered_request_rate"]) for item in ordered], + "pass_rates": [float(item["scores"][slo_name]["slo_pass_rate"]) for item in ordered], + "feasibility": vector, + "maximum_tested_feasible_request_rate": capacity, + "maximum_tested_feasible_request_rate_per_gpu": ( + capacity / tp if capacity is not None else None + ), + "lower_censored": capacity is None, + "upper_censored": bool(feasible) and capacity == max(item["offered_request_rate"] for item in ordered), + "monotonicity_violations": violations, + } + + +def rank_surface( + config_results: list[dict[str, Any]], *, slo_name: str +) -> list[dict[str, Any]]: + records = [ + _capacity_record(item["loads"], config=item["config"], slo_name=slo_name) + for item in config_results + ] + records.sort( + key=lambda item: ( + -( + item["maximum_tested_feasible_request_rate_per_gpu"] + if item["maximum_tested_feasible_request_rate_per_gpu"] is not None + else -1.0 + ), + item["config"]["name"], + ) + ) + previous_value = object() + previous_rank = 0 + for index, record in enumerate(records, start=1): + value = record["maximum_tested_feasible_request_rate_per_gpu"] + if value != previous_value: + previous_rank = index + previous_value = value + record["rank"] = previous_rank + return records + + +def _validate_profile_contract(profile_paths: dict[str, Path]) -> dict[str, Any]: + payload = json.loads(profile_paths["manifest"].read_text()) + contract = payload.get("contract") or {} + if contract.get("quant_signature") != EXPECTED_QUANT_SIGNATURE: + raise ValueError(f"unexpected FP8 profile contract: {contract}") + if contract.get("profiling_precision") != "BF16": + raise ValueError(f"unexpected output/accumulation precision: {contract}") + for name, path in ( + ("linear_op.csv", profile_paths["linear"]), + ("attention.csv", profile_paths["attention"]), + ("moe.csv", profile_paths["moe"]), + ): + expected = payload["outputs"][name]["sha256"] + if sha256(path) != expected: + raise ValueError(f"profile output hash mismatch: {path}") + return contract + + +def run_frontier_load( + *, + args: argparse.Namespace, + profile_paths: dict[str, Path], + config: Any, + rate_record: dict[str, Any], +) -> dict[str, Any]: + rate = float(rate_record["offered_request_rate"]) + load_dir = args.output_root / "runs" / config.name / rate_key(rate) + result_path = load_dir / "result.json" + if result_path.is_file(): + result = json.loads(result_path.read_text()) + if result.get("status") == "completed": + return result + load_dir.mkdir(parents=True, exist_ok=True) + trace = Path(rate_record["path"]) + command = frontier_grid.build_command( + python=args.python, + frontier_source=args.frontier_source, + profile_root=args.profile_root, + profile_paths=profile_paths, + trace=trace, + config=config, + probe_dir=load_dir, + run_id=f"{config.name}_{rate_key(rate)}", + cache_root=args.output_root / "cache", + ) + write_json(load_dir / "command.json", command) + environment = os.environ.copy() + environment.update( + { + "PYTHONPATH": str(args.frontier_source), + "WANDB_DISABLED": "true", + "VIDUR_DISABLE_WANDB": "1", + } + ) + started = time.time() + with (load_dir / "stdout.log").open("w", encoding="utf-8") as output: + completed = subprocess.run( + command, + cwd=args.frontier_source, + env=environment, + stdout=output, + stderr=subprocess.STDOUT, + check=False, + ) + if completed.returncode != 0: + raise RuntimeError( + f"Frontier failed: config={config.name}, rate={rate}, rc={completed.returncode}" + ) + request_metrics = frontier_grid.find_request_metrics(load_dir) + trace_rows = _read_trace(trace) + with request_metrics.open(encoding="utf-8", newline="") as source: + metric_rows = list(csv.DictReader(source)) + metrics_by_id = {int(row["Request Id"]): row for row in metric_rows} + if set(metrics_by_id) != set(range(len(trace_rows))): + raise ValueError(f"Frontier Request Id mismatch: {request_metrics}") + requests = [ + { + "request_id": row["source_row_index"], + "input_tokens": int(row["num_prefill_tokens"]), + "success": True, + "ttft_ms": float(metrics_by_id[index]["ttft"]), + } + for index, row in enumerate(trace_rows) + ] + score = score_requests(requests) + result = { + "status": "completed", + "config": asdict(config) | {"name": config.name}, + "offered_request_rate": rate, + "request_rate_per_gpu": rate / config.gpu_count, + "elapsed_seconds": time.time() - started, + "trace_path": str(trace), + "trace_sha256": sha256(trace), + "request_metrics_path": str(request_metrics), + "request_metrics_sha256": sha256(request_metrics), + "requests": requests, + **score, + } + write_json(result_path, result) + return result + + +def run_frontier(args: argparse.Namespace) -> Path: + protocol = json.loads(args.protocol_manifest.read_text()) + if protocol.get("schema") != PROTOCOL_SCHEMA: + raise ValueError("invalid fixed-cohort protocol manifest") + audit_protocol(args.protocol_manifest) + profile_paths = frontier_grid.resolve_profile_paths(args.profile_root) + profile_contract = _validate_profile_contract(profile_paths) + configs = frontier_grid.selected_configs(args.config) + args.output_root.mkdir(parents=True, exist_ok=True) + run_manifest = { + "schema": FRONTIER_SCHEMA, + "created_unix_s": time.time(), + "protocol_manifest": { + "path": str(args.protocol_manifest.resolve()), + "sha256": sha256(args.protocol_manifest), + }, + "frontier": { + "source": str(args.frontier_source.resolve()), + "python": str(args.python.resolve()), + "fingerprint": frontier_grid.frontier_source_fingerprint( + args.frontier_source, args.frontier_commit + ), + }, + "profiles": { + "contract": profile_contract, + "files": { + name: {"path": str(path.resolve()), "sha256": sha256(path)} + for name, path in profile_paths.items() + }, + }, + "kv_capacity_evidence": { + "tp4": { + "path": str(args.tp4_capacity_artifact.resolve()), + "sha256": sha256(args.tp4_capacity_artifact), + "num_gpu_blocks": 26101, + }, + "tp8": { + "path": str(args.tp8_capacity_artifact.resolve()), + "sha256": sha256(args.tp8_capacity_artifact), + "num_gpu_blocks": 62351, + }, + }, + "best_effort_contract": { + "arrival_semantics": "preserve_materialized_trace_arrivals", + "quantization": "block_fp8_w8a8_dynamic_with_bf16_output_accumulation", + "execution_profiles": "community_vllm_0.10.2_serving_entrypoints", + "moe_ep_prefill": "global_routing_then_slowest_local_ep_lane", + "kv_blocks": "measured_community_vllm_and_fixed_per_topology", + "cpu_overhead_modeling": "disabled_no_native_community_vllm_records", + "end_to_end_action_specific_calibration": False, + "real_serving_results_observed_before_freeze": False, + }, + "configs": [asdict(config) | {"name": config.name} for config in configs], + } + write_json(args.output_root / "run_manifest.json", run_manifest) + + config_results = [] + rate_records = sorted( + protocol["rates"].values(), key=lambda item: item["offered_request_rate"] + ) + for config in configs: + loads = [] + for record in rate_records: + result = run_frontier_load( + args=args, + profile_paths=profile_paths, + config=config, + rate_record=record, + ) + loads.append(result) + print( + json.dumps( + { + "system": "Frontier", + "config": config.name, + "rate": result["offered_request_rate"], + "pass_rate": result["scores"][PRIMARY_SLO]["slo_pass_rate"], + "feasible": result["scores"][PRIMARY_SLO]["feasible"], + "elapsed_seconds": result["elapsed_seconds"], + }, + sort_keys=True, + ), + flush=True, + ) + summary = {"config": asdict(config) | {"name": config.name}, "loads": loads} + write_json(args.output_root / "results" / f"{config.name}.json", summary) + config_results.append(summary) + freeze = { + "schema": FRONTIER_SCHEMA, + "status": "frozen_before_community_run", + "created_unix_s": time.time(), + "run_manifest_sha256": sha256(args.output_root / "run_manifest.json"), + "protocol_manifest_sha256": sha256(args.protocol_manifest), + "primary_slo": PRIMARY_SLO, + "rankings": { + name: rank_surface(config_results, slo_name=name) for name in SLO_VARIANTS + }, + "config_results": config_results, + } + path = args.output_root / "frontier_ranking_frozen.json" + write_json(path, freeze) + print(path) + return path + + +def _community_study_payload( + *, + tp: int, + repo: Path, + python: Path, + vllm: Path, + model: Path, + trace: Path, + windows: Path, + port: int, +) -> dict[str, Any]: + payload = community_grid.study_payload( + tp=tp, + repo=repo, + python=python, + vllm=vllm, + model=model, + trace=trace, + windows=windows, + port=port, + ) + payload["study_id"] = f"community-qwen235b-prefill-fixed-cohort-tp{tp}-v1" + payload["slo"] = { + "target_pass_rate": TARGET_PASS_RATE, + "ttft_rule": { + key: value + for key, value in SLO_VARIANTS[PRIMARY_SLO].items() + if key != "description" + }, + } + # The dedicated runner supplies exact fixed-cohort arrivals and disables + # SLO-based early stopping; these trace values only define raw prompt loading. + payload["trace"]["early_stop_max_lag_s"] = None + payload["trace"]["early_stop_max_elapsed_s"] = None + return payload + + +def _validate_frontier_freeze(path: Path, protocol_manifest: Path) -> dict[str, Any]: + freeze = json.loads(path.read_text()) + if freeze.get("schema") != FRONTIER_SCHEMA: + raise ValueError(f"unexpected Frontier freeze schema: {freeze.get('schema')}") + if freeze.get("status") != "frozen_before_community_run": + raise ValueError("Frontier result is not marked frozen") + if freeze.get("protocol_manifest_sha256") != sha256(protocol_manifest): + raise ValueError("Frontier and community protocol manifests differ") + if len(freeze.get("config_results") or []) != len(frontier_grid.GRID): + raise ValueError("Frontier freeze is incomplete") + expected_rates = set(OFFERED_RATES) + for item in freeze["config_results"]: + rates = {float(load["offered_request_rate"]) for load in item["loads"]} + if rates != expected_rates: + raise ValueError(f"Frontier freeze lacks rates for {item['config']['name']}") + return freeze + + +def prepare_community(args: argparse.Namespace) -> Path: + protocol = json.loads(args.protocol_manifest.read_text()) + if protocol.get("schema") != PROTOCOL_SCHEMA: + raise ValueError("invalid fixed-cohort protocol manifest") + _validate_frontier_freeze(args.frontier_freeze, args.protocol_manifest) + repo = args.repo.resolve() + python = args.python.resolve() + vllm = args.vllm.resolve() + model = args.model.resolve() + windows = args.windows.resolve() + trace = Path(protocol["source"]["path"]) + for path in (repo, python, vllm, model, windows, trace): + if not path.exists(): + raise FileNotFoundError(path) + if sha256(trace) != protocol["source"]["sha256"]: + raise ValueError("raw prompt trace changed after protocol freeze") + output_root = args.output_root.resolve() + output_root.mkdir(parents=True, exist_ok=True) + studies = {} + for tp in (4, 8): + path = output_root / "studies" / f"tp{tp}" / "study.json" + write_json( + path, + _community_study_payload( + tp=tp, + repo=repo, + python=python, + vllm=vllm, + model=model, + trace=trace, + windows=windows, + port=args.port, + ), + ) + studies[str(tp)] = {"path": str(path), "sha256": sha256(path)} + + configs = list(community_grid.GRID) + random.Random(args.config_order_seed).shuffle(configs) + records = [] + for execution_index, config in enumerate(configs, start=1): + rates = list(OFFERED_RATES) + random.Random(f"{args.rate_order_seed}|{config.name}").shuffle(rates) + records.append( + { + "execution_index": execution_index, + "config": asdict(config) | {"name": config.name}, + "rate_order": rates, + "study_path": studies[str(config.tp)]["path"], + "result_path": str(output_root / "results" / f"{config.name}.json"), + "engine_log_path": str(output_root / "runs" / config.name / "engine.log"), + } + ) + total_replay_seconds = sum( + float(record["duration_s"]) for record in protocol["rates"].values() + ) * len(configs) + manifest = { + "schema": COMMUNITY_SCHEMA, + "created_unix_s": time.time(), + "repository": community_grid.git_fingerprint(repo), + "protocol_manifest": { + "path": str(args.protocol_manifest.resolve()), + "sha256": sha256(args.protocol_manifest), + }, + "frontier_freeze": { + "path": str(args.frontier_freeze.resolve()), + "sha256": sha256(args.frontier_freeze), + }, + "runtime": {"python": str(python), "vllm": str(vllm)}, + "model": { + "path": str(model), + "config_sha256": sha256(model / "config.json"), + }, + "studies": studies, + "execution_contract": { + "request_mode": "raw_completion", + "completion_tokens_override": 1, + "prefix_caching": False, + "client_max_concurrency": community_grid.CLIENT_MAX_CONCURRENCY, + "slo_early_stop": False, + "one_server_launch_per_config": True, + "rate_order_randomized_within_config": True, + "config_order_seed": args.config_order_seed, + "rate_order_seed": args.rate_order_seed, + "estimated_replay_wall_seconds_excluding_8_model_loads": total_replay_seconds, + }, + "configs": records, + } + path = output_root / "run_manifest.json" + write_json(path, manifest) + print(path) + return path + + +def _materialize_real_requests( + *, + all_requests: list[TraceRequest], + rate_record: dict[str, Any], + cohort: list[dict[str, Any]], +) -> list[TraceRequest]: + by_id = {request.row_id: request for request in all_requests} + expected_prompt_hash = { + str(row["source_row_index"]): row["prompt_sha256"] for row in cohort + } + trace_rows = _read_trace(Path(rate_record["path"])) + requests = [] + for row in trace_rows: + request_id = row["source_row_index"] + if request_id not in by_id: + raise ValueError(f"cohort request is absent from raw trace loader: {request_id}") + request = by_id[request_id] + prompt = request.body.get("prompt") + if not isinstance(prompt, str) or sha256_text(prompt) != expected_prompt_hash[request_id]: + raise ValueError(f"raw prompt fingerprint changed: request={request_id}") + requests.append(replace(request, arrival_s=float(row["arrived_at"]))) + return requests + + +def _real_request_records( + requests: list[TraceRequest], outcomes: list[Any] +) -> list[dict[str, Any]]: + outcomes_by_id = {outcome.request_id: outcome for outcome in outcomes} + records = [] + for request in requests: + outcome = outcomes_by_id.get(request.row_id) + records.append( + { + "request_id": request.row_id, + "arrival_s": request.arrival_s, + "input_tokens": int(request.prompt_tokens_hint or 0), + "success": bool(outcome and outcome.success), + "ttft_ms": outcome.ttft_ms if outcome is not None else None, + "tpot_ms": outcome.tpot_ms if outcome is not None else None, + "completion_tokens": outcome.completion_tokens if outcome is not None else None, + "completion_tokens_source": ( + outcome.completion_tokens_source if outcome is not None else "" + ), + "error": outcome.error if outcome is not None else "missing_outcome", + } + ) + return records + + +def run_community_config( + *, + manifest: dict[str, Any], + record: dict[str, Any], + protocol: dict[str, Any], +) -> dict[str, Any]: + result_path = Path(record["result_path"]) + if result_path.is_file(): + result = json.loads(result_path.read_text()) + if result.get("status") == "completed": + return result + config = record["config"] + study_path = Path(record["study_path"]) + study = load_study_spec(study_path) + _, all_requests = load_trace_requests(study, study_spec_path=study_path) + recipe = build_launch_recipe( + study.engine, + ConfigPatch( + flag_patch={ + "max-num-seqs": int(config["mns"]), + "max-num-batched-tokens": int(config["mbt"]), + } + ), + ) + run_dir = Path(record["engine_log_path"]).parent + run_dir.mkdir(parents=True, exist_ok=True) + write_json(run_dir / "engine_command.json", recipe.argv) + marker_env = { + "AITUNER_STUDY_ID": study.study_id, + "AITUNER_TRIAL_ID": f"fixed-cohort-{config['name']}", + } + loads = [] + load_by_rate = { + float(item["offered_request_rate"]): item + for item in protocol["rates"].values() + } + started = time.time() + with Path(record["engine_log_path"]).open("w", encoding="utf-8") as engine_log: + process = subprocess.Popen( # noqa: S603 + recipe.argv, + cwd=recipe.cwd, + env={**recipe.env, **marker_env}, + stdout=engine_log, + stderr=subprocess.STDOUT, + text=True, + start_new_session=True, + ) + previous_sigterm = _install_sigterm_as_keyboardinterrupt() + try: + _wait_for_server_or_exit( + process, + base_url=recipe.base_url, + healthcheck_path=recipe.healthcheck_path, + ready_timeout_s=recipe.ready_timeout_s, + ) + for rate in record["rate_order"]: + rate = float(rate) + load_dir = run_dir / rate_key(rate) + load_result_path = load_dir / "result.json" + if load_result_path.is_file(): + load_result = json.loads(load_result_path.read_text()) + if load_result.get("status") == "completed": + loads.append(load_result) + continue + load_dir.mkdir(parents=True, exist_ok=True) + rate_record = load_by_rate[rate] + requests = _materialize_real_requests( + all_requests=all_requests, + rate_record=rate_record, + cohort=protocol["cohort"], + ) + load_started = time.time() + outcomes, early_stopped, early_stop_reason = _replay_requests( + requests, + base_url=recipe.base_url, + timeout_s=recipe.request_timeout_s, + max_concurrency=study.trace.max_concurrency, + # target=0 makes the mathematical SLO early-stop bound + # unreachable while retaining the existing replay engine. + target_pass_rate=0.0, + max_lag_s=None, + max_elapsed_s=max(float(rate_record["duration_s"]) + 900.0, 1200.0), + evaluate_outcome=lambda outcome: type( + "NoEarlyStopEvaluation", (), {"passed": bool(outcome.success)} + )(), + ) + request_records = _real_request_records(requests, outcomes) + score = score_requests(request_records) + load_result = { + "status": "completed", + "config": config, + "offered_request_rate": rate, + "request_rate_per_gpu": rate / int(config["tp"]), + "elapsed_seconds": time.time() - load_started, + "trace_path": rate_record["path"], + "trace_sha256": rate_record["sha256"], + "early_stopped": early_stopped, + "early_stop_reason": early_stop_reason, + "requests": request_records, + **score, + } + if early_stopped: + raise RuntimeError( + f"complete replay stopped early: {config['name']} rate={rate}: " + f"{early_stop_reason}" + ) + write_json(load_result_path, load_result) + loads.append(load_result) + print( + json.dumps( + { + "system": "community_vllm", + "config": config["name"], + "rate": rate, + "pass_rate": score["scores"][PRIMARY_SLO]["slo_pass_rate"], + "feasible": score["scores"][PRIMARY_SLO]["feasible"], + "elapsed_seconds": load_result["elapsed_seconds"], + }, + sort_keys=True, + ), + flush=True, + ) + time.sleep(2.0) + finally: + _ignore_sigterm_if_main() + _terminate_process_tree(process, timeout_s=30.0, marker_env=marker_env) + _restore_sigterm(previous_sigterm) + loads.sort(key=lambda item: float(item["offered_request_rate"])) + result = { + "schema": COMMUNITY_SCHEMA, + "status": "completed", + "config": config, + "elapsed_seconds": time.time() - started, + "loads": loads, + } + write_json(result_path, result) + return result + + +def run_community(args: argparse.Namespace) -> Path: + manifest = json.loads(args.manifest.read_text()) + if manifest.get("schema") != COMMUNITY_SCHEMA: + raise ValueError(f"unexpected community manifest schema: {manifest.get('schema')}") + protocol_path = Path(manifest["protocol_manifest"]["path"]) + frontier_freeze = Path(manifest["frontier_freeze"]["path"]) + if sha256(protocol_path) != manifest["protocol_manifest"]["sha256"]: + raise ValueError("protocol changed after community manifest preparation") + if sha256(frontier_freeze) != manifest["frontier_freeze"]["sha256"]: + raise ValueError("Frontier freeze changed after community manifest preparation") + _validate_frontier_freeze(frontier_freeze, protocol_path) + protocol = json.loads(protocol_path.read_text()) + config_results = [] + for record in sorted(manifest["configs"], key=lambda item: item["execution_index"]): + result = run_community_config( + manifest=manifest, + record=record, + protocol=protocol, + ) + config_results.append(result) + print( + json.dumps( + { + "config": result["config"]["name"], + "config_completed": True, + "elapsed_seconds": result["elapsed_seconds"], + }, + sort_keys=True, + ), + flush=True, + ) + freeze = { + "schema": COMMUNITY_SCHEMA, + "status": "completed", + "created_unix_s": time.time(), + "run_manifest_sha256": sha256(args.manifest), + "protocol_manifest_sha256": sha256(protocol_path), + "frontier_freeze_sha256": sha256(frontier_freeze), + "primary_slo": PRIMARY_SLO, + "rankings": { + name: rank_surface(config_results, slo_name=name) for name in SLO_VARIANTS + }, + "config_results": config_results, + } + path = args.manifest.parent / "community_ranking_frozen.json" + write_json(path, freeze) + print(path) + return path + + +def _rank_map(ranking: list[dict[str, Any]]) -> dict[str, int]: + return {item["config"]["name"]: int(item["rank"]) for item in ranking} + + +def _capacity_map(ranking: list[dict[str, Any]]) -> dict[str, float | None]: + return { + item["config"]["name"]: item["maximum_tested_feasible_request_rate_per_gpu"] + for item in ranking + } + + +def _pearson(left: list[float], right: list[float]) -> float | None: + if len(left) < 2: + return None + left_mean = statistics.fmean(left) + right_mean = statistics.fmean(right) + numerator = sum((a - left_mean) * (b - right_mean) for a, b in zip(left, right)) + denominator = math.sqrt( + sum((a - left_mean) ** 2 for a in left) + * sum((b - right_mean) ** 2 for b in right) + ) + return numerator / denominator if denominator else None + + +def _average_ranks(capacities: dict[str, float | None]) -> dict[str, float]: + ordered = sorted( + capacities, + key=lambda name: ( + -( + capacities[name] + if capacities[name] is not None + else float("-inf") + ), + name, + ), + ) + ranks = {} + start = 0 + while start < len(ordered): + value = capacities[ordered[start]] + end = start + 1 + while end < len(ordered) and capacities[ordered[end]] == value: + end += 1 + average_rank = ((start + 1) + end) / 2.0 + for name in ordered[start:end]: + ranks[name] = average_rank + start = end + return ranks + + +def _pairwise_accuracy( + frontier_capacity: dict[str, float | None], real_capacity: dict[str, float | None] +) -> dict[str, Any]: + names = sorted(set(frontier_capacity) & set(real_capacity)) + comparable = 0 + correct = 0 + frontier_ties = 0 + real_ties = 0 + for left_index, left in enumerate(names): + for right in names[left_index + 1 :]: + f_left = frontier_capacity[left] + f_right = frontier_capacity[right] + r_left = real_capacity[left] + r_right = real_capacity[right] + if None in (f_left, f_right, r_left, r_right): + continue + f_sign = (f_left > f_right) - (f_left < f_right) + r_sign = (r_left > r_right) - (r_left < r_right) + frontier_ties += int(f_sign == 0) + real_ties += int(r_sign == 0) + if f_sign == 0 or r_sign == 0: + continue + comparable += 1 + correct += int(f_sign == r_sign) + return { + "comparable_non_tied_pairs": comparable, + "correct_pairs": correct, + "accuracy": correct / comparable if comparable else None, + "frontier_tied_pairs": frontier_ties, + "real_tied_pairs": real_ties, + } + + +def _request_residuals( + frontier_results: list[dict[str, Any]], real_results: list[dict[str, Any]] +) -> dict[str, Any]: + def index(results: list[dict[str, Any]]) -> dict[tuple[str, float, str], dict[str, Any]]: + indexed = {} + for config in results: + name = config["config"]["name"] + for load in config["loads"]: + rate = float(load["offered_request_rate"]) + for request in load["requests"]: + indexed[(name, rate, str(request["request_id"]))] = request + return indexed + + frontier_index = index(frontier_results) + real_index = index(real_results) + keys = sorted(set(frontier_index) & set(real_index)) + errors = [] + absolute_percentage_errors = [] + by_bin: dict[str, list[float]] = {} + for key in keys: + frontier_request = frontier_index[key] + real_request = real_index[key] + if not real_request["success"] or real_request.get("ttft_ms") is None: + continue + predicted = float(frontier_request["ttft_ms"]) + observed = float(real_request["ttft_ms"]) + error = predicted - observed + errors.append(error) + if observed > 0: + absolute_percentage_errors.append(abs(error) / observed) + input_tokens = int(real_request["input_tokens"]) + bin_index = _input_bin(input_tokens) + label = f"[{INPUT_BINS[bin_index]},{INPUT_BINS[bin_index + 1]})" + by_bin.setdefault(label, []).append(error) + return { + "matched_successful_requests": len(errors), + "signed_error_ms_mean": statistics.fmean(errors) if errors else None, + "mae_ms": statistics.fmean(abs(value) for value in errors) if errors else None, + "mape": ( + statistics.fmean(absolute_percentage_errors) + if absolute_percentage_errors + else None + ), + "by_input_bin": { + label: { + "count": len(values), + "signed_error_ms_mean": statistics.fmean(values), + "mae_ms": statistics.fmean(abs(value) for value in values), + } + for label, values in sorted(by_bin.items()) + }, + } + + +def _pass_rate_residuals( + frontier_results: list[dict[str, Any]], real_results: list[dict[str, Any]] +) -> dict[str, Any]: + def index( + results: list[dict[str, Any]], slo_name: str + ) -> dict[tuple[str, float], float]: + return { + (config["config"]["name"], float(load["offered_request_rate"])): float( + load["scores"][slo_name]["slo_pass_rate"] + ) + for config in results + for load in config["loads"] + } + + variants = {} + for slo_name in SLO_VARIANTS: + frontier_index = index(frontier_results, slo_name) + real_index = index(real_results, slo_name) + keys = sorted(set(frontier_index) & set(real_index)) + errors = [frontier_index[key] - real_index[key] for key in keys] + variants[slo_name] = { + "matched_config_load_points": len(errors), + "signed_error_mean": statistics.fmean(errors) if errors else None, + "mae": statistics.fmean(abs(value) for value in errors) if errors else None, + "max_absolute_error": max((abs(value) for value in errors), default=None), + } + return variants + + +def compare(args: argparse.Namespace) -> Path: + frontier = json.loads(args.frontier_freeze.read_text()) + community = json.loads(args.community_freeze.read_text()) + if frontier.get("schema") != FRONTIER_SCHEMA: + raise ValueError("unexpected Frontier freeze") + if community.get("schema") != COMMUNITY_SCHEMA or community.get("status") != "completed": + raise ValueError("unexpected/incomplete community freeze") + if frontier["protocol_manifest_sha256"] != community["protocol_manifest_sha256"]: + raise ValueError("comparison inputs used different protocols") + variants = {} + for slo_name in SLO_VARIANTS: + frontier_ranking = frontier["rankings"][slo_name] + real_ranking = community["rankings"][slo_name] + frontier_ranks = _rank_map(frontier_ranking) + real_ranks = _rank_map(real_ranking) + names = sorted(set(frontier_ranks) & set(real_ranks)) + frontier_capacity = _capacity_map(frontier_ranking) + real_capacity = _capacity_map(real_ranking) + frontier_average_ranks = _average_ranks(frontier_capacity) + real_average_ranks = _average_ranks(real_capacity) + best_frontier = max( + (value for value in frontier_capacity.values() if value is not None), + default=None, + ) + best_real = max( + (value for value in real_capacity.values() if value is not None), + default=None, + ) + frontier_top_set = [ + name for name, value in frontier_capacity.items() if value == best_frontier + ] + real_top_set = [name for name, value in real_capacity.items() if value == best_real] + selected_real_capacities = [ + real_capacity[name] + for name in frontier_top_set + if real_capacity[name] is not None + ] + regret_values = [ + (best_real - value) / best_real + for value in selected_real_capacities + if best_real not in (None, 0) + ] + variants[slo_name] = { + "frontier_top1_set": frontier_top_set, + "real_top1_set": real_top_set, + "top1_set_intersection": sorted(set(frontier_top_set) & set(real_top_set)), + "top1_set_match": set(frontier_top_set) == set(real_top_set), + "top1_regret_fraction_best_tie_break": min(regret_values) + if regret_values + else None, + "top1_regret_fraction_worst_tie_break": max(regret_values) + if regret_values + else None, + "spearman_rank_correlation": _pearson( + [frontier_average_ranks[name] for name in names], + [real_average_ranks[name] for name in names], + ), + "pairwise_ordering": _pairwise_accuracy(frontier_capacity, real_capacity), + "frontier_ranking": frontier_ranking, + "real_ranking": real_ranking, + } + result = { + "schema": COMPARISON_SCHEMA, + "created_unix_s": time.time(), + "frontier_freeze": { + "path": str(args.frontier_freeze.resolve()), + "sha256": sha256(args.frontier_freeze), + }, + "community_freeze": { + "path": str(args.community_freeze.resolve()), + "sha256": sha256(args.community_freeze), + }, + "primary_slo": PRIMARY_SLO, + "variants": variants, + "request_level_ttft_residuals": _request_residuals( + frontier["config_results"], community["config_results"] + ), + "pass_rate_surface_residuals": _pass_rate_residuals( + frontier["config_results"], community["config_results"] + ), + } + write_json(args.output, result) + print(args.output) + return args.output + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + + prepare = subparsers.add_parser("prepare-protocol") + prepare.add_argument("--trace", type=Path, required=True) + prepare.add_argument("--expected-trace-sha256", default=frontier_grid.TRACE_SHA256) + prepare.add_argument("--output-root", type=Path, required=True) + prepare.add_argument("--cohort-size", type=int, default=COHORT_SIZE) + prepare.add_argument("--seed", type=int, default=COHORT_SEED) + prepare.add_argument("--rate", action="append", type=float, default=[]) + + audit = subparsers.add_parser("audit-protocol") + audit.add_argument("--manifest", type=Path, required=True) + + frontier = subparsers.add_parser("run-frontier") + frontier.add_argument("--frontier-source", type=Path, required=True) + frontier.add_argument( + "--frontier-commit", default="d9cfeb6d8791fbf2f295dd9744c56a666171776e" + ) + frontier.add_argument("--python", type=Path, required=True) + frontier.add_argument("--profile-root", type=Path, required=True) + frontier.add_argument("--protocol-manifest", type=Path, required=True) + frontier.add_argument("--output-root", type=Path, required=True) + frontier.add_argument("--tp4-capacity-artifact", type=Path, required=True) + frontier.add_argument("--tp8-capacity-artifact", type=Path, required=True) + frontier.add_argument("--config", action="append") + + community = subparsers.add_parser("prepare-community") + community.add_argument("--repo", type=Path, required=True) + community.add_argument("--python", type=Path, required=True) + community.add_argument("--vllm", type=Path, required=True) + community.add_argument("--model", type=Path, required=True) + community.add_argument("--windows", type=Path, required=True) + community.add_argument("--protocol-manifest", type=Path, required=True) + community.add_argument("--frontier-freeze", type=Path, required=True) + community.add_argument("--output-root", type=Path, required=True) + community.add_argument("--port", type=int, default=18918) + community.add_argument("--config-order-seed", type=int, default=CONFIG_ORDER_SEED) + community.add_argument("--rate-order-seed", type=int, default=RATE_ORDER_SEED) + + run_real = subparsers.add_parser("run-community") + run_real.add_argument("--manifest", type=Path, required=True) + + compare_parser = subparsers.add_parser("compare") + compare_parser.add_argument("--frontier-freeze", type=Path, required=True) + compare_parser.add_argument("--community-freeze", type=Path, required=True) + compare_parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + if args.command == "prepare-protocol": + if not args.rate: + args.rate = list(OFFERED_RATES) + if sorted(set(args.rate)) != list(args.rate): + raise ValueError("--rate values must be unique and ascending") + prepare_protocol(args) + elif args.command == "audit-protocol": + print(json.dumps(audit_protocol(args.manifest), indent=2)) + elif args.command == "run-frontier": + run_frontier(args) + elif args.command == "prepare-community": + prepare_community(args) + elif args.command == "run-community": + run_community(args) + elif args.command == "compare": + compare(args) + else: + raise AssertionError(args.command) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py new file mode 100644 index 0000000..966a912 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +import argparse +import csv +import importlib.util +import json +import sys +from pathlib import Path + + +SCRIPT = Path(__file__).with_name("fixed_cohort_rank.py") +SPEC = importlib.util.spec_from_file_location("fixed_cohort_rank", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) + + +def _write_trace(path: Path) -> None: + lengths = [100, 1500, 3000, 6000, 12000, 24000] + rows = [] + for index in range(120): + rows.append( + { + "timestamp": index * 0.5, + "sampling_u": (index + 1) / 121, + "input_length": lengths[index % len(lengths)], + "output_length": 10, + "prompt": f"raw prompt {index}", + } + ) + path.write_text("".join(json.dumps(row) + "\n" for row in rows)) + + +def test_prepare_protocol_keeps_identical_cohort_and_scales_only_arrivals( + tmp_path: Path, +) -> None: + trace = tmp_path / "trace.jsonl" + _write_trace(trace) + manifest_path = MODULE.prepare_protocol( + argparse.Namespace( + trace=trace, + expected_trace_sha256="", + output_root=tmp_path / "protocol", + cohort_size=24, + seed=7, + rate=[0.2, 0.4], + ) + ) + manifest = json.loads(manifest_path.read_text()) + assert manifest["selection"]["cohort_bin_quotas"] == [4, 4, 4, 4, 4, 4] + assert manifest["load_contract"]["binary_search"] is False + assert manifest["load_contract"]["monotonicity_assumed"] is False + + materialized = [] + for record in manifest["rates"].values(): + with Path(record["path"]).open(newline="") as source: + rows = list(csv.DictReader(source)) + materialized.append(rows) + duration = float(rows[-1]["arrived_at"]) - float(rows[0]["arrived_at"]) + assert len(rows) / duration == record["offered_request_rate"] + assert [row["source_row_index"] for row in materialized[0]] == [ + row["source_row_index"] for row in materialized[1] + ] + assert [row["num_prefill_tokens"] for row in materialized[0]] == [ + row["num_prefill_tokens"] for row in materialized[1] + ] + assert MODULE.audit_protocol(manifest_path)["status"] == "passed" + + +def test_score_requests_applies_all_slos_to_same_outcomes() -> None: + requests = [ + {"input_tokens": 100, "success": True, "ttft_ms": 900.0}, + {"input_tokens": 16000, "success": True, "ttft_ms": 2500.0}, + {"input_tokens": 24000, "success": False, "ttft_ms": None}, + ] + score = MODULE.score_requests(requests) + assert score["scores"]["linear_8k_primary"]["passed_request_count"] == 2 + assert score["scores"]["legacy_step_1s_2s"]["passed_request_count"] == 1 + assert score["ttft_ms"]["count"] == 2 + + +def test_rank_surface_records_nonmonotone_feasibility_without_binary_search() -> None: + config_results = [ + { + "config": {"name": "a", "tp": 4}, + "loads": [ + { + "offered_request_rate": 0.2, + "scores": {"linear_8k_primary": {"feasible": False, "slo_pass_rate": 0.9}}, + }, + { + "offered_request_rate": 0.4, + "scores": {"linear_8k_primary": {"feasible": True, "slo_pass_rate": 0.95}}, + }, + ], + }, + { + "config": {"name": "b", "tp": 8}, + "loads": [ + { + "offered_request_rate": 0.2, + "scores": {"linear_8k_primary": {"feasible": True, "slo_pass_rate": 1.0}}, + }, + { + "offered_request_rate": 0.4, + "scores": {"linear_8k_primary": {"feasible": False, "slo_pass_rate": 0.9}}, + }, + ], + }, + ] + ranking = MODULE.rank_surface(config_results, slo_name="linear_8k_primary") + a = next(item for item in ranking if item["config"]["name"] == "a") + assert a["maximum_tested_feasible_request_rate"] == 0.4 + assert a["monotonicity_violations"] == [[0.2, 0.4]] + + +def test_average_ranks_use_midrank_for_capacity_ties() -> None: + assert MODULE._average_ranks({"a": 0.2, "b": 0.2, "c": 0.1}) == { + "a": 1.5, + "b": 1.5, + "c": 3.0, + } + + +def test_community_study_uses_affine_primary_slo(tmp_path: Path) -> None: + payload = MODULE._community_study_payload( + tp=8, + repo=tmp_path, + python=tmp_path / "python", + vllm=tmp_path / "vllm", + model=tmp_path / "model", + trace=tmp_path / "trace.jsonl", + windows=tmp_path / "windows.json", + port=18918, + ) + assert payload["slo"]["ttft_rule"] == { + "kind": "linear_ms", + "intercept_ms": 1000.0, + "per_token_ms": 0.125, + } + assert payload["trace"]["request_mode"] == "raw_completion" + assert payload["trace"]["completion_tokens_override"] == 1 From 5fadc86eddea2ec951b676d7202306f0bd395caf Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 22:07:40 +0800 Subject: [PATCH 049/148] Cache exact Frontier EP lane predictions --- .../best_effort/fixed_cohort_rank.py | 1 + .../frontier_moe_ep_prefill_exact_cache.patch | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_prefill_exact_cache.patch diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py index 78d4f58..ca29fd9 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -631,6 +631,7 @@ def run_frontier(args: argparse.Namespace) -> Path: "quantization": "block_fp8_w8a8_dynamic_with_bf16_output_accumulation", "execution_profiles": "community_vllm_0.10.2_serving_entrypoints", "moe_ep_prefill": "global_routing_then_slowest_local_ep_lane", + "moe_ep_critical_lane_cache": "exact_(total_routed_tokens,layer_id)_memoization", "kv_blocks": "measured_community_vllm_and_fixed_per_topology", "cpu_overhead_modeling": "disabled_no_native_community_vllm_records", "end_to_end_action_specific_calibration": False, diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_prefill_exact_cache.patch b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_prefill_exact_cache.patch new file mode 100644 index 0000000..0bb9203 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_prefill_exact_cache.patch @@ -0,0 +1,17 @@ +diff --git a/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py b/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py +--- a/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py ++++ b/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py +@@ -1,4 +1,5 @@ + from typing import Any, Dict, List, Optional, TYPE_CHECKING, Union ++from functools import lru_cache + import os + + import numpy as np +@@ -384,6 +385,7 @@ + allocation_ratios=allocation_ratios, + ) + ++ @lru_cache(maxsize=None) + def _get_critical_prefill_ep_lane_tokens( + self, + total_routed_tokens: int, From ae933f2256fe42d2c9d6ca7387cf2f3c7a198797 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 22:10:36 +0800 Subject: [PATCH 050/148] Record Frontier cache equivalence evidence --- .../best_effort/fixed_cohort_rank.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py index ca29fd9..cb97cf0 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -591,6 +591,15 @@ def run_frontier(args: argparse.Namespace) -> Path: audit_protocol(args.protocol_manifest) profile_paths = frontier_grid.resolve_profile_paths(args.profile_root) profile_contract = _validate_profile_contract(profile_paths) + cache_equivalence = json.loads(args.cache_equivalence_artifact.read_text()) + if ( + cache_equivalence.get("schema") + != "frontier-ep-critical-lane-cache-equivalence-v1" + or not cache_equivalence.get("byte_identical") + ): + raise ValueError("EP critical-lane cache lacks byte-identical evidence") + if Path(cache_equivalence["new_source"]).resolve() != args.frontier_source.resolve(): + raise ValueError("cache equivalence was measured on a different Frontier source") configs = frontier_grid.selected_configs(args.config) args.output_root.mkdir(parents=True, exist_ok=True) run_manifest = { @@ -614,6 +623,12 @@ def run_frontier(args: argparse.Namespace) -> Path: for name, path in profile_paths.items() }, }, + "ep_critical_lane_cache_equivalence": { + "path": str(args.cache_equivalence_artifact.resolve()), + "sha256": sha256(args.cache_equivalence_artifact), + "request_metrics_byte_identical": True, + "request_metrics_sha256": cache_equivalence["new_sha256"], + }, "kv_capacity_evidence": { "tp4": { "path": str(args.tp4_capacity_artifact.resolve()), @@ -1354,6 +1369,7 @@ def parse_args() -> argparse.Namespace: frontier.add_argument("--output-root", type=Path, required=True) frontier.add_argument("--tp4-capacity-artifact", type=Path, required=True) frontier.add_argument("--tp8-capacity-artifact", type=Path, required=True) + frontier.add_argument("--cache-equivalence-artifact", type=Path, required=True) frontier.add_argument("--config", action="append") community = subparsers.add_parser("prepare-community") From 5dbcd9f51c8e6ecc1144610ab60c35c5a2675694 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 22:11:44 +0800 Subject: [PATCH 051/148] Parallelize independent Frontier configs --- .../best_effort/fixed_cohort_rank.py | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py index cb97cf0..4206272 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -21,6 +21,7 @@ import statistics import subprocess import sys import time +from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import asdict, replace from pathlib import Path from typing import Any, Iterable @@ -525,7 +526,7 @@ def run_frontier_load( config=config, probe_dir=load_dir, run_id=f"{config.name}_{rate_key(rate)}", - cache_root=args.output_root / "cache", + cache_root=args.output_root / "cache" / config.name, ) write_json(load_dir / "command.json", command) environment = os.environ.copy() @@ -652,15 +653,20 @@ def run_frontier(args: argparse.Namespace) -> Path: "end_to_end_action_specific_calibration": False, "real_serving_results_observed_before_freeze": False, }, + "orchestration": { + "max_parallel_configs": args.max_parallel_configs, + "cache_isolation": "one_cache_root_per_config", + "loads_within_config": "sequential", + }, "configs": [asdict(config) | {"name": config.name} for config in configs], } write_json(args.output_root / "run_manifest.json", run_manifest) - config_results = [] rate_records = sorted( protocol["rates"].values(), key=lambda item: item["offered_request_rate"] ) - for config in configs: + + def run_config(config: Any) -> dict[str, Any]: loads = [] for record in rate_records: result = run_frontier_load( @@ -686,7 +692,15 @@ def run_frontier(args: argparse.Namespace) -> Path: ) summary = {"config": asdict(config) | {"name": config.name}, "loads": loads} write_json(args.output_root / "results" / f"{config.name}.json", summary) - config_results.append(summary) + return summary + + results_by_name = {} + with ThreadPoolExecutor(max_workers=args.max_parallel_configs) as pool: + futures = {pool.submit(run_config, config): config for config in configs} + for future in as_completed(futures): + config = futures[future] + results_by_name[config.name] = future.result() + config_results = [results_by_name[config.name] for config in configs] freeze = { "schema": FRONTIER_SCHEMA, "status": "frozen_before_community_run", @@ -1370,6 +1384,7 @@ def parse_args() -> argparse.Namespace: frontier.add_argument("--tp4-capacity-artifact", type=Path, required=True) frontier.add_argument("--tp8-capacity-artifact", type=Path, required=True) frontier.add_argument("--cache-equivalence-artifact", type=Path, required=True) + frontier.add_argument("--max-parallel-configs", type=int, default=1) frontier.add_argument("--config", action="append") community = subparsers.add_parser("prepare-community") From dadcdbd35185258074bbea1b8056d107e85d567e Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 22:32:40 +0800 Subject: [PATCH 052/148] Warm community rank runs before measurement --- .../best_effort/fixed_cohort_rank.py | 97 ++++++++++++++++--- .../best_effort/test_fixed_cohort_rank.py | 28 ++++++ 2 files changed, 114 insertions(+), 11 deletions(-) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py index 4206272..95ab0e2 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -56,6 +56,7 @@ COHORT_SIZE = 64 COHORT_SEED = 2026071501 CONFIG_ORDER_SEED = 2026071502 RATE_ORDER_SEED = 2026071503 +WARMUP_SEED = 2026071504 INPUT_BINS = (0, 1024, 2048, 4096, 8192, 16384, 32769) OFFERED_RATES = (0.15, 0.25, 0.35, 0.50, 0.75, 1.00, 1.50) TARGET_PASS_RATE = 0.95 @@ -854,6 +855,12 @@ def prepare_community(args: argparse.Namespace) -> Path: "client_max_concurrency": community_grid.CLIENT_MAX_CONCURRENCY, "slo_early_stop": False, "one_server_launch_per_config": True, + "warmup": { + "method": "one_disjoint_request_per_input_length_bin", + "request_count": len(INPUT_BINS) - 1, + "seed": WARMUP_SEED, + "latencies_discarded": True, + }, "rate_order_randomized_within_config": True, "config_order_seed": args.config_order_seed, "rate_order_seed": args.rate_order_seed, @@ -891,6 +898,51 @@ def _materialize_real_requests( return requests +def _select_warmup_requests( + all_requests: list[TraceRequest], *, cohort_ids: set[str] +) -> list[TraceRequest]: + by_bin = [[] for _ in range(len(INPUT_BINS) - 1)] + for request in all_requests: + if request.row_id in cohort_ids or request.prompt_tokens_hint is None: + continue + input_tokens = int(request.prompt_tokens_hint) + if not 0 <= input_tokens <= 32768: + continue + by_bin[_input_bin(input_tokens)].append(request) + selected = [] + for bin_index, candidates in enumerate(by_bin): + if not candidates: + raise ValueError(f"no disjoint warmup request in input bin {bin_index}") + selected.append( + min( + candidates, + key=lambda request: sha256_text( + f"{WARMUP_SEED}|{bin_index}|{request.row_id}" + ), + ) + ) + return [replace(request, arrival_s=0.0) for request in selected] + + +def _run_real_replay( + requests: list[TraceRequest], *, recipe: Any, study: Any, max_elapsed_s: float +) -> tuple[list[Any], bool, str]: + return _replay_requests( + requests, + base_url=recipe.base_url, + timeout_s=recipe.request_timeout_s, + max_concurrency=study.trace.max_concurrency, + # target=0 makes the mathematical SLO early-stop bound unreachable + # while retaining the existing replay engine. + target_pass_rate=0.0, + max_lag_s=None, + max_elapsed_s=max_elapsed_s, + evaluate_outcome=lambda outcome: type( + "NoEarlyStopEvaluation", (), {"passed": bool(outcome.success)} + )(), + ) + + def _real_request_records( requests: list[TraceRequest], outcomes: list[Any] ) -> list[dict[str, Any]]: @@ -971,6 +1023,37 @@ def run_community_config( healthcheck_path=recipe.healthcheck_path, ready_timeout_s=recipe.ready_timeout_s, ) + cohort_ids = { + str(row["source_row_index"]) for row in protocol["cohort"] + } + warmup_requests = _select_warmup_requests( + all_requests, cohort_ids=cohort_ids + ) + warmup_started = time.time() + warmup_outcomes, warmup_early_stopped, warmup_stop_reason = _run_real_replay( + warmup_requests, + recipe=recipe, + study=study, + max_elapsed_s=1200.0, + ) + warmup_records = _real_request_records(warmup_requests, warmup_outcomes) + warmup_result = { + "status": "completed" + if not warmup_early_stopped + and all(item["success"] for item in warmup_records) + else "failed", + "elapsed_seconds": time.time() - warmup_started, + "early_stopped": warmup_early_stopped, + "early_stop_reason": warmup_stop_reason, + "cohort_disjoint": not bool( + cohort_ids & {item["request_id"] for item in warmup_records} + ), + "requests": warmup_records, + } + write_json(run_dir / "warmup.json", warmup_result) + if warmup_result["status"] != "completed": + raise RuntimeError(f"server warmup failed: {warmup_result}") + time.sleep(2.0) for rate in record["rate_order"]: rate = float(rate) load_dir = run_dir / rate_key(rate) @@ -988,19 +1071,11 @@ def run_community_config( cohort=protocol["cohort"], ) load_started = time.time() - outcomes, early_stopped, early_stop_reason = _replay_requests( + outcomes, early_stopped, early_stop_reason = _run_real_replay( requests, - base_url=recipe.base_url, - timeout_s=recipe.request_timeout_s, - max_concurrency=study.trace.max_concurrency, - # target=0 makes the mathematical SLO early-stop bound - # unreachable while retaining the existing replay engine. - target_pass_rate=0.0, - max_lag_s=None, + recipe=recipe, + study=study, max_elapsed_s=max(float(rate_record["duration_s"]) + 900.0, 1200.0), - evaluate_outcome=lambda outcome: type( - "NoEarlyStopEvaluation", (), {"passed": bool(outcome.success)} - )(), ) request_records = _real_request_records(requests, outcomes) score = score_requests(request_records) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py index 966a912..0dbbd11 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py @@ -141,3 +141,31 @@ def test_community_study_uses_affine_primary_slo(tmp_path: Path) -> None: } assert payload["trace"]["request_mode"] == "raw_completion" assert payload["trace"]["completion_tokens_override"] == 1 + + +def test_warmup_is_disjoint_and_covers_each_input_bin() -> None: + lengths = [100, 1500, 3000, 6000, 12000, 24000] + requests = [] + cohort_ids = set() + for bin_index, length in enumerate(lengths): + for suffix in ("cohort", "warmup"): + row_id = f"{bin_index}-{suffix}" + if suffix == "cohort": + cohort_ids.add(row_id) + requests.append( + MODULE.TraceRequest( + row_id=row_id, + arrival_s=10.0, + sampling_u=0.5, + body={"prompt": row_id}, + prompt_tokens_hint=length, + completion_tokens_hint=1, + ) + ) + selected = MODULE._select_warmup_requests(requests, cohort_ids=cohort_ids) + assert len(selected) == len(MODULE.INPUT_BINS) - 1 + assert not cohort_ids & {request.row_id for request in selected} + assert [MODULE._input_bin(int(request.prompt_tokens_hint)) for request in selected] == list( + range(len(MODULE.INPUT_BINS) - 1) + ) + assert {request.arrival_s for request in selected} == {0.0} From 1fabea26ef9e65c3afaed1e3bea4a8aef6fce37b Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 22:36:54 +0800 Subject: [PATCH 053/148] Batch exact Frontier EP lane predictions --- ..._moe_ep_prefill_batched_lane_predict.patch | 82 +++++++++++++++++++ ...ier_moe_ep_prefill_use_batched_lanes.patch | 56 +++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_prefill_batched_lane_predict.patch create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_prefill_use_batched_lanes.patch diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_prefill_batched_lane_predict.patch b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_prefill_batched_lane_predict.patch new file mode 100644 index 0000000..088aae6 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_prefill_batched_lane_predict.patch @@ -0,0 +1,82 @@ +diff --git a/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py b/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py +--- a/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py ++++ b/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py +@@ -385,6 +385,78 @@ + allocation_ratios=allocation_ratios, + ) + ++ def _get_on_demand_predictions_batch( ++ self, ++ model_name: str, ++ feature_rows: List[Dict[str, float]], ++ ) -> List[float]: ++ """Predict independent feature rows in one exact forest invocation. ++ ++ ``RandomForestRegressor.predict`` applies the same trees to every row. ++ Batching the eight EP lanes removes repeated pandas/joblib call overhead; ++ it does not combine lanes or alter their features. Results populate the ++ same runtime cache used by the scalar prediction path. ++ """ ++ if not feature_rows: ++ return [] ++ ++ model_info = self._predictions.get(model_name) ++ if ( ++ not isinstance(model_info, dict) ++ or not model_info.get("_on_demand_prediction", False) ++ ): ++ raise ValueError( ++ f"Model {model_name} is not configured for on-demand prediction" ++ ) ++ model = model_info.get("_model") ++ feature_names = model_info.get("_feature_names", []) ++ if model is None: ++ raise ValueError(f"Model {model_name} has no trained model available") ++ ++ family_name = self._measurement_family_name(self._active_measurement_type) ++ runtime_cache = self._runtime_cache[family_name][model_name] ++ results: List[Optional[float]] = [None] * len(feature_rows) ++ misses: Dict[tuple, tuple] = {} ++ for index, features in enumerate(feature_rows): ++ missing = [name for name in feature_names if name not in features] ++ if missing: ++ raise ValueError( ++ f"On-demand prediction missing required features for {model_name}: " ++ f"{missing}. Provided keys: {sorted(list(features.keys()))}" ++ ) ++ cache_key = tuple(features[key] for key in sorted(features.keys())) ++ if cache_key in runtime_cache: ++ results[index] = float(runtime_cache[cache_key]) ++ continue ++ if cache_key not in misses: ++ misses[cache_key] = (features, []) ++ misses[cache_key][1].append(index) ++ ++ if misses: ++ miss_keys = list(misses) ++ frame = pd.DataFrame( ++ [ ++ [misses[key][0][name] for name in feature_names] ++ for key in miss_keys ++ ], ++ columns=feature_names, ++ ) ++ try: ++ predictions = model.predict(frame) ++ except Exception as error: ++ raise ValueError( ++ f"Batched on-demand prediction failed for {model_name}: {error}" ++ ) from error ++ for cache_key, raw_prediction in zip(miss_keys, predictions): ++ prediction = max(0.0, float(raw_prediction)) ++ runtime_cache[cache_key] = prediction ++ for index in misses[cache_key][1]: ++ results[index] = prediction ++ ++ if any(result is None for result in results): ++ raise AssertionError("batched on-demand prediction left an empty result") ++ return [float(result) for result in results] ++ + @lru_cache(maxsize=None) + def _get_critical_prefill_ep_lane_tokens( + self, diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_prefill_use_batched_lanes.patch b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_prefill_use_batched_lanes.patch new file mode 100644 index 0000000..5b23cff --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/frontier_moe_ep_prefill_use_batched_lanes.patch @@ -0,0 +1,56 @@ +diff --git a/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py b/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py +--- a/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py ++++ b/frontier/execution_time_predictor/sklearn_moe_execution_time_predictor.py +@@ -494,27 +494,35 @@ + local_expert_id = int(global_expert_id) % experts_per_lane + lane_tokens[lane_id][local_expert_id] = int(token_count) + +- def _lane_compute_score(allocation: Dict[int, int]) -> float: +- features = self._build_moe_load_imbalance_features(allocation) +- score = 0.0 +- for operation in ("moe_shuffling", "moe_grouped_gemm"): +- prediction = self._predictions.get(operation) +- if isinstance(prediction, dict) and prediction.get( +- "_on_demand_prediction", False +- ): +- score += float( +- self._get_on_demand_prediction(operation, features) ++ lane_features = [ ++ self._build_moe_load_imbalance_features(allocation) ++ for allocation in lane_tokens ++ ] ++ lane_scores = [0.0] * lane_count ++ for operation in ("moe_shuffling", "moe_grouped_gemm"): ++ prediction = self._predictions.get(operation) ++ if isinstance(prediction, dict) and prediction.get( ++ "_on_demand_prediction", False ++ ): ++ operation_predictions = self._get_on_demand_predictions_batch( ++ operation, lane_features ++ ) ++ lane_scores = [ ++ score + operation_prediction ++ for score, operation_prediction in zip( ++ lane_scores, operation_predictions + ) +- return score ++ ] + +- return max( +- lane_tokens, +- key=lambda allocation: ( +- _lane_compute_score(allocation), +- sum(allocation.values()), +- tuple(allocation.values()), ++ critical_lane_index = max( ++ range(lane_count), ++ key=lambda lane_index: ( ++ lane_scores[lane_index], ++ sum(lane_tokens[lane_index].values()), ++ tuple(lane_tokens[lane_index].values()), + ), + ) ++ return lane_tokens[critical_lane_index] + + def predict_monolithic_decode_shared_domain_lane_moe_times_ms( + self, From 33c91109d739f73d756a6c63f6f174b083651b51 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 22:45:40 +0800 Subject: [PATCH 054/148] Require batched Frontier EP equivalence --- .../best_effort/fixed_cohort_rank.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py index 95ab0e2..cc1cf73 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -593,15 +593,14 @@ def run_frontier(args: argparse.Namespace) -> Path: audit_protocol(args.protocol_manifest) profile_paths = frontier_grid.resolve_profile_paths(args.profile_root) profile_contract = _validate_profile_contract(profile_paths) - cache_equivalence = json.loads(args.cache_equivalence_artifact.read_text()) + ep_equivalence = json.loads(args.ep_equivalence_artifact.read_text()) if ( - cache_equivalence.get("schema") - != "frontier-ep-critical-lane-cache-equivalence-v1" - or not cache_equivalence.get("byte_identical") + ep_equivalence.get("schema") != "frontier-ep-batched-lane-equivalence-v1" + or not ep_equivalence.get("byte_identical") ): - raise ValueError("EP critical-lane cache lacks byte-identical evidence") - if Path(cache_equivalence["new_source"]).resolve() != args.frontier_source.resolve(): - raise ValueError("cache equivalence was measured on a different Frontier source") + raise ValueError("batched EP lane prediction lacks byte-identical evidence") + if Path(ep_equivalence["new_source"]).resolve() != args.frontier_source.resolve(): + raise ValueError("EP equivalence was measured on a different Frontier source") configs = frontier_grid.selected_configs(args.config) args.output_root.mkdir(parents=True, exist_ok=True) run_manifest = { @@ -625,11 +624,11 @@ def run_frontier(args: argparse.Namespace) -> Path: for name, path in profile_paths.items() }, }, - "ep_critical_lane_cache_equivalence": { - "path": str(args.cache_equivalence_artifact.resolve()), - "sha256": sha256(args.cache_equivalence_artifact), + "ep_prediction_optimization_equivalence": { + "path": str(args.ep_equivalence_artifact.resolve()), + "sha256": sha256(args.ep_equivalence_artifact), "request_metrics_byte_identical": True, - "request_metrics_sha256": cache_equivalence["new_sha256"], + "request_metrics_sha256": ep_equivalence["new_sha256"], }, "kv_capacity_evidence": { "tp4": { @@ -649,6 +648,7 @@ def run_frontier(args: argparse.Namespace) -> Path: "execution_profiles": "community_vllm_0.10.2_serving_entrypoints", "moe_ep_prefill": "global_routing_then_slowest_local_ep_lane", "moe_ep_critical_lane_cache": "exact_(total_routed_tokens,layer_id)_memoization", + "moe_ep_lane_inference": "eight_independent_rows_in_one_exact_forest_predict_call", "kv_blocks": "measured_community_vllm_and_fixed_per_topology", "cpu_overhead_modeling": "disabled_no_native_community_vllm_records", "end_to_end_action_specific_calibration": False, @@ -1458,7 +1458,7 @@ def parse_args() -> argparse.Namespace: frontier.add_argument("--output-root", type=Path, required=True) frontier.add_argument("--tp4-capacity-artifact", type=Path, required=True) frontier.add_argument("--tp8-capacity-artifact", type=Path, required=True) - frontier.add_argument("--cache-equivalence-artifact", type=Path, required=True) + frontier.add_argument("--ep-equivalence-artifact", type=Path, required=True) frontier.add_argument("--max-parallel-configs", type=int, default=1) frontier.add_argument("--config", action="append") From 91869c531c8f1e0e28090d69ad5e50a74c76b641 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 23:17:43 +0800 Subject: [PATCH 055/148] Add blind Qwen235B fleet job --- .../best_effort/fleet_fixed_cohort.toml | 25 +++++++++++++++++++ .../best_effort/jobs_fixed_cohort.toml | 10 ++++++++ 2 files changed, 35 insertions(+) create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/fleet_fixed_cohort.toml create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/jobs_fixed_cohort.toml diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fleet_fixed_cohort.toml b/runs/frontier-multicase-sufficiency-v0/best_effort/fleet_fixed_cohort.toml new file mode 100644 index 0000000..16cfc56 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fleet_fixed_cohort.toml @@ -0,0 +1,25 @@ +version = 1 + +[paths] +state_dir = ".aituner/gpu_fleet/fixed_cohort_state" +artifacts_dir = ".aituner/gpu_fleet/fixed_cohort_artifacts" + +[ssh] +connect_timeout_sec = 10 + +[scheduler] +gpu_free_memory_mb = 1024 +gpu_free_utilization_pct = 10 +prefer_pack = true + +[sync] +mode = "rsync" +local_path = "runs/frontier-multicase-sufficiency-v0/best_effort" +exclude = ["__pycache__/", "*.pyc"] + +[[hosts]] +name = "dash0" +ssh_alias = "dash0" +enabled = true +sync_remote_path = "/home/admin/cpfs/wjh/aituner/aituner-frontier-best-effort-20260715/runs/frontier-multicase-sufficiency-v0/best_effort" +fleet_root = "/home/admin/.aituner_gpu_fleet_fixed_cohort" diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/jobs_fixed_cohort.toml b/runs/frontier-multicase-sufficiency-v0/best_effort/jobs_fixed_cohort.toml new file mode 100644 index 0000000..afd140d --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/jobs_fixed_cohort.toml @@ -0,0 +1,10 @@ +# Immutable one-job queue for the blind Qwen235B fixed-cohort rank experiment. +version = 1 + +[[jobs]] +name = "qwen235b-fixed-cohort-community-v1-blind-20260715" +gpus = 8 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd ../../.. && /tmp/wjh-frontier-vllm0102-smoke/.venv/bin/python runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py run-community --manifest /home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/run_manifest.json" +env = { PYTHONPATH = "/home/admin/cpfs/wjh/aituner/aituner-frontier-best-effort-20260715/src" } From ef5c17e6ec4ff19af8c29c4249c53ddf895f9448 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 23:20:01 +0800 Subject: [PATCH 056/148] Use writable fleet root on dash0 --- .../best_effort/fleet_fixed_cohort.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fleet_fixed_cohort.toml b/runs/frontier-multicase-sufficiency-v0/best_effort/fleet_fixed_cohort.toml index 16cfc56..7ecc003 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fleet_fixed_cohort.toml +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fleet_fixed_cohort.toml @@ -22,4 +22,4 @@ name = "dash0" ssh_alias = "dash0" enabled = true sync_remote_path = "/home/admin/cpfs/wjh/aituner/aituner-frontier-best-effort-20260715/runs/frontier-multicase-sufficiency-v0/best_effort" -fleet_root = "/home/admin/.aituner_gpu_fleet_fixed_cohort" +fleet_root = "/home/admin/cpfs/wjh/.aituner_gpu_fleet_fixed_cohort" From e47bbf3e760af6a5ed41b744c0c0f7caecdb5c88 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Wed, 15 Jul 2026 23:32:07 +0800 Subject: [PATCH 057/148] Compare Frontier rank under explicit objectives --- .../best_effort/fixed_cohort_rank.py | 159 ++++++++++++------ .../best_effort/test_fixed_cohort_rank.py | 56 ++++++ 2 files changed, 166 insertions(+), 49 deletions(-) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py index cc1cf73..2aa4300 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -1184,9 +1184,11 @@ def _rank_map(ranking: list[dict[str, Any]]) -> dict[str, int]: return {item["config"]["name"]: int(item["rank"]) for item in ranking} -def _capacity_map(ranking: list[dict[str, Any]]) -> dict[str, float | None]: +def _capacity_map( + ranking: list[dict[str, Any]], *, field: str +) -> dict[str, float | None]: return { - item["config"]["name"]: item["maximum_tested_feasible_request_rate_per_gpu"] + item["config"]["name"]: item[field] for item in ranking } @@ -1263,6 +1265,73 @@ def _pairwise_accuracy( } +def _objective_fidelity( + frontier_ranking: list[dict[str, Any]], + real_ranking: list[dict[str, Any]], + *, + field: str, + only_names: set[str] | None = None, +) -> dict[str, Any]: + frontier_capacity = _capacity_map(frontier_ranking, field=field) + real_capacity = _capacity_map(real_ranking, field=field) + if only_names is not None: + frontier_capacity = { + name: value for name, value in frontier_capacity.items() if name in only_names + } + real_capacity = { + name: value for name, value in real_capacity.items() if name in only_names + } + names = sorted(set(frontier_capacity) & set(real_capacity)) + frontier_valid = [value for value in frontier_capacity.values() if value is not None] + real_valid = [value for value in real_capacity.values() if value is not None] + if not frontier_valid or not real_valid: + return { + "rankable": False, + "reason": "one_or_both_surfaces_have_no_feasible_config", + "frontier_capacity": frontier_capacity, + "real_capacity": real_capacity, + } + frontier_average_ranks = _average_ranks(frontier_capacity) + real_average_ranks = _average_ranks(real_capacity) + best_frontier = max(frontier_valid) + best_real = max(real_valid) + frontier_top_set = [ + name for name, value in frontier_capacity.items() if value == best_frontier + ] + real_top_set = [name for name, value in real_capacity.items() if value == best_real] + selected_real_capacities = [ + real_capacity[name] + for name in frontier_top_set + if real_capacity.get(name) is not None + ] + regret_values = [ + (best_real - value) / best_real + for value in selected_real_capacities + if best_real != 0 + ] + return { + "rankable": True, + "capacity_field": field, + "frontier_top1_set": frontier_top_set, + "real_top1_set": real_top_set, + "top1_set_intersection": sorted(set(frontier_top_set) & set(real_top_set)), + "top1_set_match": set(frontier_top_set) == set(real_top_set), + "top1_regret_fraction_best_tie_break": min(regret_values) + if regret_values + else None, + "top1_regret_fraction_worst_tie_break": max(regret_values) + if regret_values + else None, + "spearman_rank_correlation": _pearson( + [frontier_average_ranks[name] for name in names], + [real_average_ranks[name] for name in names], + ), + "pairwise_ordering": _pairwise_accuracy(frontier_capacity, real_capacity), + "frontier_capacity": frontier_capacity, + "real_capacity": real_capacity, + } + + def _request_residuals( frontier_results: list[dict[str, Any]], real_results: list[dict[str, Any]] ) -> dict[str, Any]: @@ -1322,10 +1391,11 @@ def _pass_rate_residuals( ) -> dict[str, Any]: def index( results: list[dict[str, Any]], slo_name: str - ) -> dict[tuple[str, float], float]: + ) -> dict[tuple[str, float], tuple[float, bool]]: return { - (config["config"]["name"], float(load["offered_request_rate"])): float( - load["scores"][slo_name]["slo_pass_rate"] + (config["config"]["name"], float(load["offered_request_rate"])): ( + float(load["scores"][slo_name]["slo_pass_rate"]), + bool(load["scores"][slo_name]["feasible"]), ) for config in results for load in config["loads"] @@ -1336,12 +1406,24 @@ def _pass_rate_residuals( frontier_index = index(frontier_results, slo_name) real_index = index(real_results, slo_name) keys = sorted(set(frontier_index) & set(real_index)) - errors = [frontier_index[key] - real_index[key] for key in keys] + errors = [frontier_index[key][0] - real_index[key][0] for key in keys] + label_matches = [frontier_index[key][1] == real_index[key][1] for key in keys] + false_positives = sum( + frontier_index[key][1] and not real_index[key][1] for key in keys + ) + false_negatives = sum( + not frontier_index[key][1] and real_index[key][1] for key in keys + ) variants[slo_name] = { "matched_config_load_points": len(errors), "signed_error_mean": statistics.fmean(errors) if errors else None, "mae": statistics.fmean(abs(value) for value in errors) if errors else None, "max_absolute_error": max((abs(value) for value in errors), default=None), + "feasibility_label_accuracy": ( + sum(label_matches) / len(label_matches) if label_matches else None + ), + "false_feasible_points": false_positives, + "false_infeasible_points": false_negatives, } return variants @@ -1359,51 +1441,30 @@ def compare(args: argparse.Namespace) -> Path: for slo_name in SLO_VARIANTS: frontier_ranking = frontier["rankings"][slo_name] real_ranking = community["rankings"][slo_name] - frontier_ranks = _rank_map(frontier_ranking) - real_ranks = _rank_map(real_ranking) - names = sorted(set(frontier_ranks) & set(real_ranks)) - frontier_capacity = _capacity_map(frontier_ranking) - real_capacity = _capacity_map(real_ranking) - frontier_average_ranks = _average_ranks(frontier_capacity) - real_average_ranks = _average_ranks(real_capacity) - best_frontier = max( - (value for value in frontier_capacity.values() if value is not None), - default=None, - ) - best_real = max( - (value for value in real_capacity.values() if value is not None), - default=None, - ) - frontier_top_set = [ - name for name, value in frontier_capacity.items() if value == best_frontier - ] - real_top_set = [name for name, value in real_capacity.items() if value == best_real] - selected_real_capacities = [ - real_capacity[name] - for name in frontier_top_set - if real_capacity[name] is not None - ] - regret_values = [ - (best_real - value) / best_real - for value in selected_real_capacities - if best_real not in (None, 0) - ] variants[slo_name] = { - "frontier_top1_set": frontier_top_set, - "real_top1_set": real_top_set, - "top1_set_intersection": sorted(set(frontier_top_set) & set(real_top_set)), - "top1_set_match": set(frontier_top_set) == set(real_top_set), - "top1_regret_fraction_best_tie_break": min(regret_values) - if regret_values - else None, - "top1_regret_fraction_worst_tie_break": max(regret_values) - if regret_values - else None, - "spearman_rank_correlation": _pearson( - [frontier_average_ranks[name] for name in names], - [real_average_ranks[name] for name in names], + "per_gpu_efficiency": _objective_fidelity( + frontier_ranking, + real_ranking, + field="maximum_tested_feasible_request_rate_per_gpu", ), - "pairwise_ordering": _pairwise_accuracy(frontier_capacity, real_capacity), + "single_replica_raw_capacity": _objective_fidelity( + frontier_ranking, + real_ranking, + field="maximum_tested_feasible_request_rate", + ), + "within_topology_raw_capacity": { + f"tp{tp}": _objective_fidelity( + frontier_ranking, + real_ranking, + field="maximum_tested_feasible_request_rate", + only_names={ + item["config"]["name"] + for item in frontier_ranking + if int(item["config"]["tp"]) == tp + }, + ) + for tp in (4, 8) + }, "frontier_ranking": frontier_ranking, "real_ranking": real_ranking, } diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py index 0dbbd11..182d081 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py @@ -169,3 +169,59 @@ def test_warmup_is_disjoint_and_covers_each_input_bin() -> None: range(len(MODULE.INPUT_BINS) - 1) ) assert {request.arrival_s for request in selected} == {0.0} + + +def test_objective_fidelity_separates_raw_and_per_gpu_rankings() -> None: + frontier = [ + { + "config": {"name": "tp8", "tp": 8}, + "maximum_tested_feasible_request_rate": 0.4, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + }, + { + "config": {"name": "tp4", "tp": 4}, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + }, + ] + real = [ + { + "config": {"name": "tp8", "tp": 8}, + "maximum_tested_feasible_request_rate": 0.2, + "maximum_tested_feasible_request_rate_per_gpu": 0.025, + }, + { + "config": {"name": "tp4", "tp": 4}, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + }, + ] + raw = MODULE._objective_fidelity( + frontier, + real, + field="maximum_tested_feasible_request_rate", + ) + efficiency = MODULE._objective_fidelity( + frontier, + real, + field="maximum_tested_feasible_request_rate_per_gpu", + ) + assert raw["frontier_top1_set"] == ["tp8"] + assert raw["real_top1_set"] == ["tp4"] + assert abs(raw["top1_regret_fraction_worst_tie_break"] - 1 / 3) < 1e-12 + assert efficiency["frontier_top1_set"] == efficiency["real_top1_set"] == ["tp4"] + + +def test_objective_fidelity_marks_null_capacity_unrankable() -> None: + ranking = [ + { + "config": {"name": "a", "tp": 4}, + "maximum_tested_feasible_request_rate": None, + } + ] + result = MODULE._objective_fidelity( + ranking, + ranking, + field="maximum_tested_feasible_request_rate", + ) + assert result["rankable"] is False From 84584c719c84e6a0535e6f30183ab1d6274af0b7 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 00:52:13 +0800 Subject: [PATCH 058/148] Add reverse boundary repeats for Frontier comparison --- .../best_effort/fixed_cohort_rank.py | 145 ++++++++++++++++++ .../best_effort/test_fixed_cohort_rank.py | 33 ++++ 2 files changed, 178 insertions(+) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py index 2aa4300..5b8feb4 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -52,6 +52,7 @@ PROTOCOL_SCHEMA = "qwen235b-prefill-fixed-cohort-protocol-v1" FRONTIER_SCHEMA = "frontier-qwen235b-prefill-fixed-cohort-v1" COMMUNITY_SCHEMA = "community-qwen235b-prefill-fixed-cohort-v1" COMPARISON_SCHEMA = "frontier-community-qwen235b-rank-comparison-v1" +REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v1" COHORT_SIZE = 64 COHORT_SEED = 2026071501 CONFIG_ORDER_SEED = 2026071502 @@ -874,6 +875,142 @@ def prepare_community(args: argparse.Namespace) -> Path: return path +def _boundary_repeat_rates(config_result: dict[str, Any]) -> list[float]: + """Select every adjacent load pair whose primary feasibility label changes.""" + loads = sorted( + config_result["loads"], key=lambda item: float(item["offered_request_rate"]) + ) + rates = [float(item["offered_request_rate"]) for item in loads] + if len(rates) != len(set(rates)): + raise ValueError(f"duplicate offered rate in {config_result['config']['name']}") + if not rates: + raise ValueError(f"no loads in {config_result['config']['name']}") + feasibility = [bool(item["scores"][PRIMARY_SLO]["feasible"]) for item in loads] + selected: set[float] = set() + for index in range(len(loads) - 1): + if feasibility[index] != feasibility[index + 1]: + selected.update((rates[index], rates[index + 1])) + if not selected: + if len(rates) == 1: + selected.add(rates[0]) + elif all(feasibility): + selected.update(rates[-2:]) + else: + selected.update(rates[:2]) + return sorted(selected) + + +def prepare_community_repeat(args: argparse.Namespace) -> Path: + """Prepare a fresh-server reverse-order repeat of decision-boundary loads.""" + source_manifest = json.loads(args.source_manifest.read_text()) + first_pass = json.loads(args.community_freeze.read_text()) + if source_manifest.get("schema") != COMMUNITY_SCHEMA: + raise ValueError("unexpected source community manifest") + if ( + first_pass.get("schema") != COMMUNITY_SCHEMA + or first_pass.get("status") != "completed" + ): + raise ValueError("unexpected/incomplete first-pass community freeze") + if first_pass.get("run_manifest_sha256") != sha256(args.source_manifest): + raise ValueError("first-pass freeze does not match the source manifest") + protocol_path = Path(source_manifest["protocol_manifest"]["path"]) + if sha256(protocol_path) != source_manifest["protocol_manifest"]["sha256"]: + raise ValueError("source protocol changed before repeat preparation") + protocol = json.loads(protocol_path.read_text()) + duration_by_rate = { + float(record["offered_request_rate"]): float(record["duration_s"]) + for record in protocol["rates"].values() + } + source_records = { + item["config"]["name"]: item for item in source_manifest["configs"] + } + first_results = { + item["config"]["name"]: item for item in first_pass["config_results"] + } + if set(source_records) != set(first_results): + raise ValueError("source manifest and first-pass config sets differ") + + output_root = args.output_root.resolve() + output_root.mkdir(parents=True, exist_ok=True) + first_execution_order = sorted( + source_manifest["configs"], key=lambda item: int(item["execution_index"]) + ) + records = [] + selections = [] + for execution_index, source_record in enumerate( + reversed(first_execution_order), start=1 + ): + name = source_record["config"]["name"] + selected_rates = _boundary_repeat_rates(first_results[name]) + source_relative_order = [ + float(rate) + for rate in source_record["rate_order"] + if float(rate) in selected_rates + ] + if set(source_relative_order) != set(selected_rates): + raise ValueError(f"repeat rate is absent from source order: {name}") + repeat_order = list(reversed(source_relative_order)) + records.append( + { + "execution_index": execution_index, + "config": source_record["config"], + "rate_order": repeat_order, + "study_path": source_record["study_path"], + "result_path": str(output_root / "results" / f"{name}.json"), + "engine_log_path": str(output_root / "runs" / name / "engine.log"), + } + ) + selections.append( + { + "config": name, + "selected_rates": selected_rates, + "source_relative_order": source_relative_order, + "repeat_order": repeat_order, + } + ) + total_replay_seconds = sum( + duration_by_rate[float(rate)] + for record in records + for rate in record["rate_order"] + ) + manifest = { + "schema": COMMUNITY_SCHEMA, + "created_unix_s": time.time(), + "repository": community_grid.git_fingerprint(args.repo.resolve()), + "protocol_manifest": source_manifest["protocol_manifest"], + "frontier_freeze": source_manifest["frontier_freeze"], + "runtime": source_manifest["runtime"], + "model": source_manifest["model"], + "studies": source_manifest["studies"], + "repeat_of": { + "source_manifest": { + "path": str(args.source_manifest.resolve()), + "sha256": sha256(args.source_manifest), + }, + "first_pass_freeze": { + "path": str(args.community_freeze.resolve()), + "sha256": sha256(args.community_freeze), + }, + }, + "execution_contract": { + "schema": REPEAT_SELECTION_SCHEMA, + "selection": "all_adjacent_primary_slo_feasibility_transitions", + "fallback_when_no_transition": "top_two_if_all_feasible_else_bottom_two", + "config_order": "reverse_of_first_pass", + "rate_order": "reverse_of_first_pass_relative_order_within_selection", + "one_fresh_server_launch_per_config": True, + "warmup": source_manifest["execution_contract"]["warmup"], + "estimated_replay_wall_seconds_excluding_model_loads": total_replay_seconds, + "selections": selections, + }, + "configs": records, + } + path = output_root / "run_manifest.json" + write_json(path, manifest) + print(path) + return path + + def _materialize_real_requests( *, all_requests: list[TraceRequest], @@ -1539,6 +1676,12 @@ def parse_args() -> argparse.Namespace: run_real = subparsers.add_parser("run-community") run_real.add_argument("--manifest", type=Path, required=True) + repeat = subparsers.add_parser("prepare-community-repeat") + repeat.add_argument("--repo", type=Path, required=True) + repeat.add_argument("--source-manifest", type=Path, required=True) + repeat.add_argument("--community-freeze", type=Path, required=True) + repeat.add_argument("--output-root", type=Path, required=True) + compare_parser = subparsers.add_parser("compare") compare_parser.add_argument("--frontier-freeze", type=Path, required=True) compare_parser.add_argument("--community-freeze", type=Path, required=True) @@ -1562,6 +1705,8 @@ def main() -> None: prepare_community(args) elif args.command == "run-community": run_community(args) + elif args.command == "prepare-community-repeat": + prepare_community_repeat(args) elif args.command == "compare": compare(args) else: diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py index 182d081..cb9c030 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py @@ -225,3 +225,36 @@ def test_objective_fidelity_marks_null_capacity_unrankable() -> None: field="maximum_tested_feasible_request_rate", ) assert result["rankable"] is False + + +def test_boundary_repeat_rates_keep_every_transition_and_censored_fallback() -> None: + def result(feasibility: list[bool]) -> dict[str, object]: + return { + "config": {"name": "test"}, + "loads": [ + { + "offered_request_rate": rate, + "scores": {MODULE.PRIMARY_SLO: {"feasible": feasible}}, + } + for rate, feasible in zip((0.1, 0.2, 0.3, 0.4), feasibility) + ], + } + + assert MODULE._boundary_repeat_rates(result([True, True, False, False])) == [ + 0.2, + 0.3, + ] + assert MODULE._boundary_repeat_rates(result([True, False, True, False])) == [ + 0.1, + 0.2, + 0.3, + 0.4, + ] + assert MODULE._boundary_repeat_rates(result([True, True, True, True])) == [ + 0.3, + 0.4, + ] + assert MODULE._boundary_repeat_rates(result([False, False, False, False])) == [ + 0.1, + 0.2, + ] From 678388084c5df2357cfdeb896835c3ec6c6d2723 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 01:28:23 +0800 Subject: [PATCH 059/148] Add fleet job for Frontier boundary repeats --- .../best_effort/jobs_fixed_cohort_repeat.toml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/jobs_fixed_cohort_repeat.toml diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/jobs_fixed_cohort_repeat.toml b/runs/frontier-multicase-sufficiency-v0/best_effort/jobs_fixed_cohort_repeat.toml new file mode 100644 index 0000000..0655160 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/jobs_fixed_cohort_repeat.toml @@ -0,0 +1,10 @@ +# Reverse-order fresh-server boundary repeat for the frozen blind surface. +version = 1 + +[[jobs]] +name = "qwen235b-fixed-cohort-community-v2-boundary-reverse-20260715" +gpus = 8 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd ../../.. && /tmp/wjh-frontier-vllm0102-smoke/.venv/bin/python runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py run-community --manifest /home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/run_manifest.json" +env = { PYTHONPATH = "/home/admin/cpfs/wjh/aituner/aituner-frontier-best-effort-20260715/src" } From f7052edd14be0329eb20eb06d440a1ae5b8efd1a Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 01:42:13 +0800 Subject: [PATCH 060/148] Report request-level SLO fidelity for Frontier --- .../best_effort/fixed_cohort_rank.py | 94 +++++++++++++++++++ .../best_effort/test_fixed_cohort_rank.py | 33 +++++++ 2 files changed, 127 insertions(+) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py index 5b8feb4..ae9c64d 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -1565,6 +1565,97 @@ def _pass_rate_residuals( return variants +def _request_slo_classification( + frontier_results: list[dict[str, Any]], real_results: list[dict[str, Any]] +) -> dict[str, Any]: + def index( + results: list[dict[str, Any]], + ) -> dict[tuple[str, float, str], dict[str, Any]]: + return { + ( + config["config"]["name"], + float(load["offered_request_rate"]), + str(request["request_id"]), + ): request + for config in results + for load in config["loads"] + for request in load["requests"] + } + + def request_passes(request: dict[str, Any], variant: dict[str, Any]) -> bool: + return bool( + request["success"] + and request.get("ttft_ms") is not None + and float(request["ttft_ms"]) + <= slo_threshold_ms(variant, int(request["input_tokens"])) + ) + + frontier_index = index(frontier_results) + real_index = index(real_results) + keys = sorted(set(frontier_index) & set(real_index)) + variants = {} + for slo_name, variant in SLO_VARIANTS.items(): + by_load: dict[tuple[str, float], dict[str, Any]] = {} + frontier_failures: set[tuple[str, float, str]] = set() + real_failures: set[tuple[str, float, str]] = set() + for key in keys: + frontier_passes = request_passes(frontier_index[key], variant) + real_passes = request_passes(real_index[key], variant) + if not frontier_passes: + frontier_failures.add(key) + if not real_passes: + real_failures.add(key) + group = by_load.setdefault( + key[:2], + { + "matched_requests": 0, + "matching_labels": 0, + "false_slo_pass_request_ids": [], + "false_slo_fail_request_ids": [], + }, + ) + group["matched_requests"] += 1 + group["matching_labels"] += int(frontier_passes == real_passes) + if frontier_passes and not real_passes: + group["false_slo_pass_request_ids"].append(key[2]) + elif not frontier_passes and real_passes: + group["false_slo_fail_request_ids"].append(key[2]) + intersection = frontier_failures & real_failures + union = frontier_failures | real_failures + load_records = [] + for (config, rate), record in sorted(by_load.items()): + matched = int(record["matched_requests"]) + load_records.append( + { + "config": config, + "offered_request_rate": rate, + **record, + "label_accuracy": record["matching_labels"] / matched, + } + ) + matching_labels = sum(item["matching_labels"] for item in by_load.values()) + false_passes = sum( + len(item["false_slo_pass_request_ids"]) for item in by_load.values() + ) + false_fails = sum( + len(item["false_slo_fail_request_ids"]) for item in by_load.values() + ) + variants[slo_name] = { + "matched_requests": len(keys), + "matching_labels": matching_labels, + "label_accuracy": matching_labels / len(keys) if keys else None, + "false_slo_pass_requests": false_passes, + "false_slo_fail_requests": false_fails, + "frontier_failure_requests": len(frontier_failures), + "real_failure_requests": len(real_failures), + "failure_set_intersection": len(intersection), + "failure_set_union": len(union), + "failure_set_jaccard": len(intersection) / len(union) if union else 1.0, + "by_config_load": load_records, + } + return variants + + def compare(args: argparse.Namespace) -> Path: frontier = json.loads(args.frontier_freeze.read_text()) community = json.loads(args.community_freeze.read_text()) @@ -1624,6 +1715,9 @@ def compare(args: argparse.Namespace) -> Path: "pass_rate_surface_residuals": _pass_rate_residuals( frontier["config_results"], community["config_results"] ), + "request_slo_classification": _request_slo_classification( + frontier["config_results"], community["config_results"] + ), } write_json(args.output, result) print(args.output) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py index cb9c030..f0e70d3 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py @@ -258,3 +258,36 @@ def test_boundary_repeat_rates_keep_every_transition_and_censored_fallback() -> 0.1, 0.2, ] + + +def test_request_slo_classification_exposes_aggregate_error_cancellation() -> None: + def results(ttfts: list[float]) -> list[dict[str, object]]: + return [ + { + "config": {"name": "a"}, + "loads": [ + { + "offered_request_rate": 0.1, + "requests": [ + { + "request_id": str(index), + "input_tokens": 0, + "success": True, + "ttft_ms": ttft, + } + for index, ttft in enumerate(ttfts) + ], + } + ], + } + ] + + classification = MODULE._request_slo_classification( + results([900.0, 1100.0]), results([1100.0, 900.0]) + )[MODULE.PRIMARY_SLO] + assert classification["frontier_failure_requests"] == 1 + assert classification["real_failure_requests"] == 1 + assert classification["failure_set_jaccard"] == 0.0 + assert classification["label_accuracy"] == 0.0 + assert classification["false_slo_pass_requests"] == 1 + assert classification["false_slo_fail_requests"] == 1 From 993cce6143f9e9a40b0f4f9fbdb2c38f756acfe5 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 01:59:37 +0800 Subject: [PATCH 061/148] Refine Frontier rank boundaries before repeats --- .../best_effort/fixed_cohort_rank.py | 67 +++++++++++++++---- .../best_effort/test_fixed_cohort_rank.py | 33 +++++++++ 2 files changed, 88 insertions(+), 12 deletions(-) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py index ae9c64d..cf61958 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -52,7 +52,7 @@ PROTOCOL_SCHEMA = "qwen235b-prefill-fixed-cohort-protocol-v1" FRONTIER_SCHEMA = "frontier-qwen235b-prefill-fixed-cohort-v1" COMMUNITY_SCHEMA = "community-qwen235b-prefill-fixed-cohort-v1" COMPARISON_SCHEMA = "frontier-community-qwen235b-rank-comparison-v1" -REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v1" +REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v2" COHORT_SIZE = 64 COHORT_SEED = 2026071501 CONFIG_ORDER_SEED = 2026071502 @@ -768,7 +768,11 @@ def _validate_frontier_freeze(path: Path, protocol_manifest: Path) -> dict[str, raise ValueError("Frontier and community protocol manifests differ") if len(freeze.get("config_results") or []) != len(frontier_grid.GRID): raise ValueError("Frontier freeze is incomplete") - expected_rates = set(OFFERED_RATES) + protocol = json.loads(protocol_manifest.read_text()) + expected_rates = { + float(record["offered_request_rate"]) + for record in protocol["rates"].values() + } for item in freeze["config_results"]: rates = {float(load["offered_request_rate"]) for load in item["loads"]} if rates != expected_rates: @@ -913,9 +917,20 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: raise ValueError("unexpected/incomplete first-pass community freeze") if first_pass.get("run_manifest_sha256") != sha256(args.source_manifest): raise ValueError("first-pass freeze does not match the source manifest") - protocol_path = Path(source_manifest["protocol_manifest"]["path"]) - if sha256(protocol_path) != source_manifest["protocol_manifest"]["sha256"]: + source_protocol_path = Path(source_manifest["protocol_manifest"]["path"]) + if sha256(source_protocol_path) != source_manifest["protocol_manifest"]["sha256"]: raise ValueError("source protocol changed before repeat preparation") + protocol_path = ( + args.protocol_manifest.resolve() + if args.protocol_manifest is not None + else source_protocol_path + ) + frontier_freeze_path = ( + args.frontier_freeze.resolve() + if args.frontier_freeze is not None + else Path(source_manifest["frontier_freeze"]["path"]) + ) + _validate_frontier_freeze(frontier_freeze_path, protocol_path) protocol = json.loads(protocol_path.read_text()) duration_by_rate = { float(record["offered_request_rate"]): float(record["duration_s"]) @@ -941,15 +956,33 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: reversed(first_execution_order), start=1 ): name = source_record["config"]["name"] - selected_rates = _boundary_repeat_rates(first_results[name]) + boundary_rates = _boundary_repeat_rates(first_results[name]) + refinement_rates = sorted( + rate + for rate in duration_by_rate + if min(boundary_rates) < rate < max(boundary_rates) + ) + selected_rates = sorted(set(boundary_rates) | set(refinement_rates)) source_relative_order = [ float(rate) for rate in source_record["rate_order"] - if float(rate) in selected_rates + if float(rate) in boundary_rates ] - if set(source_relative_order) != set(selected_rates): + if set(source_relative_order) != set(boundary_rates): raise ValueError(f"repeat rate is absent from source order: {name}") - repeat_order = list(reversed(source_relative_order)) + reversed_boundaries = list(reversed(source_relative_order)) + if len(reversed_boundaries) == 2: + refinement_order = sorted( + refinement_rates, + reverse=reversed_boundaries[0] > reversed_boundaries[1], + ) + repeat_order = [ + reversed_boundaries[0], + *refinement_order, + reversed_boundaries[1], + ] + else: + repeat_order = [*reversed_boundaries, *reversed(refinement_rates)] records.append( { "execution_index": execution_index, @@ -963,6 +996,8 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: selections.append( { "config": name, + "boundary_rates": boundary_rates, + "added_refinement_rates": refinement_rates, "selected_rates": selected_rates, "source_relative_order": source_relative_order, "repeat_order": repeat_order, @@ -977,8 +1012,14 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: "schema": COMMUNITY_SCHEMA, "created_unix_s": time.time(), "repository": community_grid.git_fingerprint(args.repo.resolve()), - "protocol_manifest": source_manifest["protocol_manifest"], - "frontier_freeze": source_manifest["frontier_freeze"], + "protocol_manifest": { + "path": str(protocol_path), + "sha256": sha256(protocol_path), + }, + "frontier_freeze": { + "path": str(frontier_freeze_path), + "sha256": sha256(frontier_freeze_path), + }, "runtime": source_manifest["runtime"], "model": source_manifest["model"], "studies": source_manifest["studies"], @@ -994,10 +1035,10 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: }, "execution_contract": { "schema": REPEAT_SELECTION_SCHEMA, - "selection": "all_adjacent_primary_slo_feasibility_transitions", + "selection": "all_adjacent_primary_slo_feasibility_transitions_plus_between_rate_refinement", "fallback_when_no_transition": "top_two_if_all_feasible_else_bottom_two", "config_order": "reverse_of_first_pass", - "rate_order": "reverse_of_first_pass_relative_order_within_selection", + "rate_order": "reverse_first_pass_boundary_order_with_between_rate_refinement", "one_fresh_server_launch_per_config": True, "warmup": source_manifest["execution_contract"]["warmup"], "estimated_replay_wall_seconds_excluding_model_loads": total_replay_seconds, @@ -1774,6 +1815,8 @@ def parse_args() -> argparse.Namespace: repeat.add_argument("--repo", type=Path, required=True) repeat.add_argument("--source-manifest", type=Path, required=True) repeat.add_argument("--community-freeze", type=Path, required=True) + repeat.add_argument("--protocol-manifest", type=Path) + repeat.add_argument("--frontier-freeze", type=Path) repeat.add_argument("--output-root", type=Path, required=True) compare_parser = subparsers.add_parser("compare") diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py index f0e70d3..b66758a 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py @@ -291,3 +291,36 @@ def test_request_slo_classification_exposes_aggregate_error_cancellation() -> No assert classification["label_accuracy"] == 0.0 assert classification["false_slo_pass_requests"] == 1 assert classification["false_slo_fail_requests"] == 1 + + +def test_frontier_freeze_validation_uses_manifest_rates(tmp_path: Path) -> None: + protocol_path = tmp_path / "protocol.json" + protocol = { + "rates": { + "low": {"offered_request_rate": 0.2}, + "high": {"offered_request_rate": 0.4}, + } + } + protocol_path.write_text(json.dumps(protocol)) + freeze_path = tmp_path / "freeze.json" + freeze_path.write_text( + json.dumps( + { + "schema": MODULE.FRONTIER_SCHEMA, + "status": "frozen_before_community_run", + "protocol_manifest_sha256": MODULE.sha256(protocol_path), + "config_results": [ + { + "config": {"name": config.name}, + "loads": [ + {"offered_request_rate": 0.2}, + {"offered_request_rate": 0.4}, + ], + } + for config in MODULE.frontier_grid.GRID + ], + } + ) + ) + validated = MODULE._validate_frontier_freeze(freeze_path, protocol_path) + assert validated["protocol_manifest_sha256"] == MODULE.sha256(protocol_path) From 163fb3d9e8e7a5ceb9d2bb6bb9b42e3135622b10 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 02:35:41 +0800 Subject: [PATCH 062/148] Repeat boundaries for every registered SLO --- .../best_effort/fixed_cohort_rank.py | 52 +++++++++++-------- .../best_effort/test_fixed_cohort_rank.py | 7 +++ 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py index cf61958..77842c7 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -52,7 +52,7 @@ PROTOCOL_SCHEMA = "qwen235b-prefill-fixed-cohort-protocol-v1" FRONTIER_SCHEMA = "frontier-qwen235b-prefill-fixed-cohort-v1" COMMUNITY_SCHEMA = "community-qwen235b-prefill-fixed-cohort-v1" COMPARISON_SCHEMA = "frontier-community-qwen235b-rank-comparison-v1" -REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v2" +REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v3" COHORT_SIZE = 64 COHORT_SEED = 2026071501 CONFIG_ORDER_SEED = 2026071502 @@ -879,8 +879,10 @@ def prepare_community(args: argparse.Namespace) -> Path: return path -def _boundary_repeat_rates(config_result: dict[str, Any]) -> list[float]: - """Select every adjacent load pair whose primary feasibility label changes.""" +def _boundary_repeat_rates( + config_result: dict[str, Any], *, slo_name: str = PRIMARY_SLO +) -> list[float]: + """Select every adjacent load pair whose SLO feasibility label changes.""" loads = sorted( config_result["loads"], key=lambda item: float(item["offered_request_rate"]) ) @@ -889,7 +891,7 @@ def _boundary_repeat_rates(config_result: dict[str, Any]) -> list[float]: raise ValueError(f"duplicate offered rate in {config_result['config']['name']}") if not rates: raise ValueError(f"no loads in {config_result['config']['name']}") - feasibility = [bool(item["scores"][PRIMARY_SLO]["feasible"]) for item in loads] + feasibility = [bool(item["scores"][slo_name]["feasible"]) for item in loads] selected: set[float] = set() for index in range(len(loads) - 1): if feasibility[index] != feasibility[index + 1]: @@ -956,11 +958,27 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: reversed(first_execution_order), start=1 ): name = source_record["config"]["name"] - boundary_rates = _boundary_repeat_rates(first_results[name]) + boundary_rates_by_slo = { + slo_name: _boundary_repeat_rates( + first_results[name], slo_name=slo_name + ) + for slo_name in SLO_VARIANTS + } + boundary_rates = sorted( + { + rate + for rates in boundary_rates_by_slo.values() + for rate in rates + } + ) refinement_rates = sorted( - rate - for rate in duration_by_rate - if min(boundary_rates) < rate < max(boundary_rates) + { + rate + for rates in boundary_rates_by_slo.values() + for low, high in zip(rates, rates[1:]) + for rate in duration_by_rate + if low < rate < high and rate not in boundary_rates + } ) selected_rates = sorted(set(boundary_rates) | set(refinement_rates)) source_relative_order = [ @@ -971,18 +989,7 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: if set(source_relative_order) != set(boundary_rates): raise ValueError(f"repeat rate is absent from source order: {name}") reversed_boundaries = list(reversed(source_relative_order)) - if len(reversed_boundaries) == 2: - refinement_order = sorted( - refinement_rates, - reverse=reversed_boundaries[0] > reversed_boundaries[1], - ) - repeat_order = [ - reversed_boundaries[0], - *refinement_order, - reversed_boundaries[1], - ] - else: - repeat_order = [*reversed_boundaries, *reversed(refinement_rates)] + repeat_order = [*reversed_boundaries, *reversed(refinement_rates)] records.append( { "execution_index": execution_index, @@ -996,6 +1003,7 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: selections.append( { "config": name, + "boundary_rates_by_slo": boundary_rates_by_slo, "boundary_rates": boundary_rates, "added_refinement_rates": refinement_rates, "selected_rates": selected_rates, @@ -1035,10 +1043,10 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: }, "execution_contract": { "schema": REPEAT_SELECTION_SCHEMA, - "selection": "all_adjacent_primary_slo_feasibility_transitions_plus_between_rate_refinement", + "selection": "all_adjacent_pre_registered_slo_feasibility_transitions_plus_between_rate_refinement", "fallback_when_no_transition": "top_two_if_all_feasible_else_bottom_two", "config_order": "reverse_of_first_pass", - "rate_order": "reverse_first_pass_boundary_order_with_between_rate_refinement", + "rate_order": "reverse_first_pass_boundary_order_then_descending_refinement_rates", "one_fresh_server_launch_per_config": True, "warmup": source_manifest["execution_contract"]["warmup"], "estimated_replay_wall_seconds_excluding_model_loads": total_replay_seconds, diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py index b66758a..de99b50 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py @@ -259,6 +259,13 @@ def test_boundary_repeat_rates_keep_every_transition_and_censored_fallback() -> 0.2, ] + sensitivity = result([True, True, False, False]) + for load, feasible in zip(sensitivity["loads"], [True, True, True, False]): + load["scores"]["linear_6k"] = {"feasible": feasible} + assert MODULE._boundary_repeat_rates( + sensitivity, slo_name="linear_6k" + ) == [0.3, 0.4] + def test_request_slo_classification_exposes_aggregate_error_cancellation() -> None: def results(ttfts: list[float]) -> list[dict[str, object]]: From 501ceb817122747a23dc6d9fdc9d71dcb3aa8e72 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 03:09:19 +0800 Subject: [PATCH 063/148] Skip null SLOs in boundary repeats --- .../best_effort/fixed_cohort_rank.py | 26 ++++++++++++++++--- .../best_effort/test_fixed_cohort_rank.py | 7 +++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py index 77842c7..33a899f 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -52,7 +52,7 @@ PROTOCOL_SCHEMA = "qwen235b-prefill-fixed-cohort-protocol-v1" FRONTIER_SCHEMA = "frontier-qwen235b-prefill-fixed-cohort-v1" COMMUNITY_SCHEMA = "community-qwen235b-prefill-fixed-cohort-v1" COMPARISON_SCHEMA = "frontier-community-qwen235b-rank-comparison-v1" -REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v3" +REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v4" COHORT_SIZE = 64 COHORT_SEED = 2026071501 CONFIG_ORDER_SEED = 2026071502 @@ -906,6 +906,13 @@ def _boundary_repeat_rates( return sorted(selected) +def _slo_has_feasible_load(config_result: dict[str, Any], *, slo_name: str) -> bool: + return any( + bool(load["scores"][slo_name]["feasible"]) + for load in config_result["loads"] + ) + + def prepare_community_repeat(args: argparse.Namespace) -> Path: """Prepare a fresh-server reverse-order repeat of decision-boundary loads.""" source_manifest = json.loads(args.source_manifest.read_text()) @@ -958,10 +965,21 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: reversed(first_execution_order), start=1 ): name = source_record["config"]["name"] - boundary_rates_by_slo = { - slo_name: _boundary_repeat_rates( + excluded_unrankable_slos = [ + slo_name + for slo_name in SLO_VARIANTS + if not _slo_has_feasible_load( first_results[name], slo_name=slo_name ) + ] + boundary_rates_by_slo = { + slo_name: ( + [] + if slo_name in excluded_unrankable_slos + else _boundary_repeat_rates( + first_results[name], slo_name=slo_name + ) + ) for slo_name in SLO_VARIANTS } boundary_rates = sorted( @@ -1003,6 +1021,7 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: selections.append( { "config": name, + "excluded_unrankable_slos": excluded_unrankable_slos, "boundary_rates_by_slo": boundary_rates_by_slo, "boundary_rates": boundary_rates, "added_refinement_rates": refinement_rates, @@ -1044,6 +1063,7 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: "execution_contract": { "schema": REPEAT_SELECTION_SCHEMA, "selection": "all_adjacent_pre_registered_slo_feasibility_transitions_plus_between_rate_refinement", + "unrankable_slo_policy": "exclude_per_config_surfaces_with_no_feasible_load", "fallback_when_no_transition": "top_two_if_all_feasible_else_bottom_two", "config_order": "reverse_of_first_pass", "rate_order": "reverse_first_pass_boundary_order_then_descending_refinement_rates", diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py index de99b50..f764251 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py @@ -265,6 +265,13 @@ def test_boundary_repeat_rates_keep_every_transition_and_censored_fallback() -> assert MODULE._boundary_repeat_rates( sensitivity, slo_name="linear_6k" ) == [0.3, 0.4] + assert MODULE._slo_has_feasible_load( + sensitivity, slo_name="linear_6k" + ) + assert not MODULE._slo_has_feasible_load( + result([False, False, False, False]), + slo_name=MODULE.PRIMARY_SLO, + ) def test_request_slo_classification_exposes_aggregate_error_cancellation() -> None: From 3db632552b2e26f23aa4dd1adf382b65169f12d1 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 03:41:41 +0800 Subject: [PATCH 064/148] Compare Frontier fidelity across real trials --- .../best_effort/fixed_cohort_rank.py | 307 ++++++++++++++++++ .../best_effort/test_fixed_cohort_rank.py | 60 ++++ 2 files changed, 367 insertions(+) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py index 33a899f..d81db14 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -52,6 +52,7 @@ PROTOCOL_SCHEMA = "qwen235b-prefill-fixed-cohort-protocol-v1" FRONTIER_SCHEMA = "frontier-qwen235b-prefill-fixed-cohort-v1" COMMUNITY_SCHEMA = "community-qwen235b-prefill-fixed-cohort-v1" COMPARISON_SCHEMA = "frontier-community-qwen235b-rank-comparison-v1" +TRIAL_STABILITY_SCHEMA = "community-qwen235b-trial-stability-v1" REPEAT_SELECTION_SCHEMA = "community-qwen235b-boundary-repeat-selection-v4" COHORT_SIZE = 64 COHORT_SEED = 2026071501 @@ -1793,6 +1794,304 @@ def compare(args: argparse.Namespace) -> Path: return args.output +def _index_loads( + config_results: list[dict[str, Any]], +) -> dict[tuple[str, float], dict[str, Any]]: + return { + (config["config"]["name"], float(load["offered_request_rate"])): load + for config in config_results + for load in config["loads"] + } + + +def _index_requests( + config_results: list[dict[str, Any]], +) -> dict[tuple[str, float, str], dict[str, Any]]: + return { + ( + config["config"]["name"], + float(load["offered_request_rate"]), + str(request["request_id"]), + ): request + for config in config_results + for load in config["loads"] + for request in load["requests"] + } + + +def _request_passes_slo(request: dict[str, Any], variant: dict[str, Any]) -> bool: + return bool( + request["success"] + and request.get("ttft_ms") is not None + and float(request["ttft_ms"]) + <= slo_threshold_ms(variant, int(request["input_tokens"])) + ) + + +def compare_trials(args: argparse.Namespace) -> Path: + frontier = json.loads(args.frontier_freeze.read_text()) + first = json.loads(args.first_community_freeze.read_text()) + repeat = json.loads(args.repeat_community_freeze.read_text()) + if frontier.get("schema") != FRONTIER_SCHEMA: + raise ValueError("unexpected Frontier freeze") + for label, payload in (("first", first), ("repeat", repeat)): + if payload.get("schema") != COMMUNITY_SCHEMA or payload.get("status") != "completed": + raise ValueError(f"unexpected/incomplete {label} community freeze") + + frontier_loads = _index_loads(frontier["config_results"]) + first_loads = _index_loads(first["config_results"]) + repeat_loads = _index_loads(repeat["config_results"]) + shared_load_keys = sorted(set(frontier_loads) & set(first_loads) & set(repeat_loads)) + if not shared_load_keys: + raise ValueError("trials have no shared config/load points") + for key in shared_load_keys: + trace_hashes = { + index[key].get("trace_sha256") + for index in (frontier_loads, first_loads, repeat_loads) + if index[key].get("trace_sha256") is not None + } + if len(trace_hashes) > 1: + raise ValueError(f"trace mismatch at {key}") + + frontier_requests = _index_requests(frontier["config_results"]) + first_requests = _index_requests(first["config_results"]) + repeat_requests = _index_requests(repeat["config_results"]) + shared_request_keys = sorted( + set(frontier_requests) & set(first_requests) & set(repeat_requests) + ) + if not shared_request_keys: + raise ValueError("trials have no shared requests") + for key in shared_request_keys: + lengths = { + int(index[key]["input_tokens"]) + for index in (frontier_requests, first_requests, repeat_requests) + } + if len(lengths) != 1: + raise ValueError(f"input-token mismatch at {key}") + + trial_ttft_differences = [] + for key in shared_request_keys: + first_request = first_requests[key] + repeat_request = repeat_requests[key] + if ( + first_request["success"] + and repeat_request["success"] + and first_request.get("ttft_ms") is not None + and repeat_request.get("ttft_ms") is not None + ): + trial_ttft_differences.append( + float(repeat_request["ttft_ms"]) - float(first_request["ttft_ms"]) + ) + + variants = {} + for slo_name, variant in SLO_VARIANTS.items(): + request_counts = { + "matching_real_trial_labels": 0, + "frontier_correct_first": 0, + "frontier_correct_repeat": 0, + "frontier_wrong_both_stable_real_label": 0, + "real_trial_label_flips": 0, + } + first_failures = set() + repeat_failures = set() + for key in shared_request_keys: + predicted = _request_passes_slo(frontier_requests[key], variant) + first_observed = _request_passes_slo(first_requests[key], variant) + repeat_observed = _request_passes_slo(repeat_requests[key], variant) + request_counts["matching_real_trial_labels"] += int( + first_observed == repeat_observed + ) + request_counts["frontier_correct_first"] += int(predicted == first_observed) + request_counts["frontier_correct_repeat"] += int(predicted == repeat_observed) + request_counts["frontier_wrong_both_stable_real_label"] += int( + first_observed == repeat_observed and predicted != first_observed + ) + request_counts["real_trial_label_flips"] += int( + first_observed != repeat_observed + ) + if not first_observed: + first_failures.add(key) + if not repeat_observed: + repeat_failures.add(key) + + load_records = [] + matching_load_labels = 0 + frontier_correct_first_loads = 0 + frontier_correct_repeat_loads = 0 + for config_name, rate in shared_load_keys: + predicted_score = frontier_loads[(config_name, rate)]["scores"][slo_name] + first_score = first_loads[(config_name, rate)]["scores"][slo_name] + repeat_score = repeat_loads[(config_name, rate)]["scores"][slo_name] + predicted_label = bool(predicted_score["feasible"]) + first_label = bool(first_score["feasible"]) + repeat_label = bool(repeat_score["feasible"]) + matching_load_labels += int(first_label == repeat_label) + frontier_correct_first_loads += int(predicted_label == first_label) + frontier_correct_repeat_loads += int(predicted_label == repeat_label) + load_records.append( + { + "config": config_name, + "offered_request_rate": rate, + "frontier_passed_request_count": int( + predicted_score["passed_request_count"] + ), + "first_passed_request_count": int(first_score["passed_request_count"]), + "repeat_passed_request_count": int( + repeat_score["passed_request_count"] + ), + "frontier_feasible": predicted_label, + "first_feasible": first_label, + "repeat_feasible": repeat_label, + } + ) + failure_union = first_failures | repeat_failures + variants[slo_name] = { + "shared_requests": len(shared_request_keys), + **request_counts, + "real_trial_request_label_accuracy": ( + request_counts["matching_real_trial_labels"] / len(shared_request_keys) + ), + "frontier_request_label_accuracy_first": ( + request_counts["frontier_correct_first"] / len(shared_request_keys) + ), + "frontier_request_label_accuracy_repeat": ( + request_counts["frontier_correct_repeat"] / len(shared_request_keys) + ), + "real_trial_failure_set_jaccard": ( + len(first_failures & repeat_failures) / len(failure_union) + if failure_union + else 1.0 + ), + "shared_config_load_points": len(shared_load_keys), + "real_trial_load_label_accuracy": matching_load_labels / len(shared_load_keys), + "frontier_load_label_accuracy_first": ( + frontier_correct_first_loads / len(shared_load_keys) + ), + "frontier_load_label_accuracy_repeat": ( + frontier_correct_repeat_loads / len(shared_load_keys) + ), + "config_load_records": load_records, + } + + first_primary_capacity = _capacity_map( + first["rankings"][PRIMARY_SLO], + field="maximum_tested_feasible_request_rate", + ) + first_config = { + item["config"]["name"]: item["config"] for item in first["config_results"] + } + decision_records = [] + repeat_capacity_records = [] + for config_name in sorted(first_primary_capacity): + old_rates = sorted( + rate for name, rate in first_loads if name == config_name + ) + capacity = first_primary_capacity[config_name] + if capacity is None: + continue + higher_rates = [rate for rate in old_rates if rate > capacity] + next_rate = min(higher_rates) if higher_rates else None + boundary_rates = [capacity] + ([next_rate] if next_rate is not None else []) + boundary_repeated = all( + (config_name, rate) in repeat_loads for rate in boundary_rates + ) + capacity_still_feasible = bool( + boundary_repeated + and repeat_loads[(config_name, capacity)]["scores"][PRIMARY_SLO]["feasible"] + ) + next_still_infeasible = bool( + next_rate is None + or ( + boundary_repeated + and not repeat_loads[(config_name, next_rate)]["scores"][PRIMARY_SLO][ + "feasible" + ] + ) + ) + stable = boundary_repeated and capacity_still_feasible and next_still_infeasible + decision_records.append( + { + "config": config_name, + "first_capacity": capacity, + "first_next_tested_rate": next_rate, + "boundary_repeated": boundary_repeated, + "capacity_still_feasible": capacity_still_feasible, + "next_rate_still_infeasible": next_still_infeasible, + "stable_original_grid_capacity": stable, + } + ) + repeat_capacity = capacity if stable else None + repeat_capacity_records.append( + { + "config": first_config[config_name], + "maximum_tested_feasible_request_rate": repeat_capacity, + "maximum_tested_feasible_request_rate_per_gpu": ( + repeat_capacity / int(first_config[config_name]["tp"]) + if repeat_capacity is not None + else None + ), + } + ) + + result = { + "schema": TRIAL_STABILITY_SCHEMA, + "created_unix_s": time.time(), + "inputs": { + "frontier_freeze": { + "path": str(args.frontier_freeze.resolve()), + "sha256": sha256(args.frontier_freeze), + }, + "first_community_freeze": { + "path": str(args.first_community_freeze.resolve()), + "sha256": sha256(args.first_community_freeze), + }, + "repeat_community_freeze": { + "path": str(args.repeat_community_freeze.resolve()), + "sha256": sha256(args.repeat_community_freeze), + }, + }, + "shared_config_load_points": len(shared_load_keys), + "shared_requests": len(shared_request_keys), + "real_trial_ttft_ms": { + "paired_successful_requests": len(trial_ttft_differences), + "repeat_minus_first_mean": ( + statistics.fmean(trial_ttft_differences) + if trial_ttft_differences + else None + ), + "mae": ( + statistics.fmean(abs(value) for value in trial_ttft_differences) + if trial_ttft_differences + else None + ), + "absolute_error_p95": _percentile( + [abs(value) for value in trial_ttft_differences], 0.95 + ), + }, + "variants": variants, + "primary_original_grid_capacity_stability": { + "stable_configs": sum( + item["stable_original_grid_capacity"] for item in decision_records + ), + "total_configs": len(decision_records), + "records": decision_records, + "per_gpu_efficiency": _objective_fidelity( + first["rankings"][PRIMARY_SLO], + repeat_capacity_records, + field="maximum_tested_feasible_request_rate_per_gpu", + ), + "single_replica_raw_capacity": _objective_fidelity( + first["rankings"][PRIMARY_SLO], + repeat_capacity_records, + field="maximum_tested_feasible_request_rate", + ), + }, + } + write_json(args.output, result) + print(args.output) + return args.output + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() subparsers = parser.add_subparsers(dest="command", required=True) @@ -1851,6 +2150,12 @@ def parse_args() -> argparse.Namespace: compare_parser.add_argument("--frontier-freeze", type=Path, required=True) compare_parser.add_argument("--community-freeze", type=Path, required=True) compare_parser.add_argument("--output", type=Path, required=True) + + trials = subparsers.add_parser("compare-trials") + trials.add_argument("--frontier-freeze", type=Path, required=True) + trials.add_argument("--first-community-freeze", type=Path, required=True) + trials.add_argument("--repeat-community-freeze", type=Path, required=True) + trials.add_argument("--output", type=Path, required=True) return parser.parse_args() @@ -1874,6 +2179,8 @@ def main() -> None: prepare_community_repeat(args) elif args.command == "compare": compare(args) + elif args.command == "compare-trials": + compare_trials(args) else: raise AssertionError(args.command) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py index f764251..b78c9ad 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/test_fixed_cohort_rank.py @@ -338,3 +338,63 @@ def test_frontier_freeze_validation_uses_manifest_rates(tmp_path: Path) -> None: ) validated = MODULE._validate_frontier_freeze(freeze_path, protocol_path) assert validated["protocol_manifest_sha256"] == MODULE.sha256(protocol_path) + + +def test_compare_trials_separates_real_label_flips_from_stable_frontier_errors( + tmp_path: Path, +) -> None: + config = {"name": "a", "tp": 4} + + def config_result(ttfts: list[float]) -> dict[str, object]: + requests = [ + { + "request_id": str(index), + "input_tokens": 0, + "success": True, + "ttft_ms": ttft, + } + for index, ttft in enumerate(ttfts) + ] + load = { + "offered_request_rate": 0.1, + "requests": requests, + **MODULE.score_requests(requests), + } + return {"config": config, "loads": [load]} + + frontier_result = config_result([900.0, 1100.0]) + first_result = config_result([900.0, 900.0]) + repeat_result = config_result([900.0, 1100.0]) + + def freeze(path: Path, schema: str, result: dict[str, object]) -> Path: + payload = { + "schema": schema, + "status": "completed", + "config_results": [result], + "rankings": { + name: MODULE.rank_surface([result], slo_name=name) + for name in MODULE.SLO_VARIANTS + }, + } + path.write_text(json.dumps(payload)) + return path + + frontier_path = freeze(tmp_path / "frontier.json", MODULE.FRONTIER_SCHEMA, frontier_result) + first_path = freeze(tmp_path / "first.json", MODULE.COMMUNITY_SCHEMA, first_result) + repeat_path = freeze(tmp_path / "repeat.json", MODULE.COMMUNITY_SCHEMA, repeat_result) + output = tmp_path / "stability.json" + MODULE.compare_trials( + argparse.Namespace( + frontier_freeze=frontier_path, + first_community_freeze=first_path, + repeat_community_freeze=repeat_path, + output=output, + ) + ) + result = json.loads(output.read_text()) + primary = result["variants"][MODULE.PRIMARY_SLO] + assert primary["shared_requests"] == 2 + assert primary["real_trial_label_flips"] == 1 + assert primary["frontier_wrong_both_stable_real_label"] == 0 + assert primary["real_trial_load_label_accuracy"] == 0.0 + assert result["primary_original_grid_capacity_stability"]["stable_configs"] == 0 From def9e24c4d4149a3977795fde4143529fb7fe982 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 04:02:29 +0800 Subject: [PATCH 065/148] Allow targeted SLO boundary repeats --- .../best_effort/fixed_cohort_rank.py | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py index d81db14..90bdce5 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -916,6 +916,11 @@ def _slo_has_feasible_load(config_result: dict[str, Any], *, slo_name: str) -> b def prepare_community_repeat(args: argparse.Namespace) -> Path: """Prepare a fresh-server reverse-order repeat of decision-boundary loads.""" + selected_slos = list(getattr(args, "slo", [])) or list(SLO_VARIANTS) + unknown_slos = sorted(set(selected_slos) - set(SLO_VARIANTS)) + if unknown_slos: + raise ValueError(f"unknown repeat SLOs: {unknown_slos}") + selected_configs = set(getattr(args, "config", [])) source_manifest = json.loads(args.source_manifest.read_text()) first_pass = json.loads(args.community_freeze.read_text()) if source_manifest.get("schema") != COMMUNITY_SCHEMA: @@ -960,6 +965,15 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: first_execution_order = sorted( source_manifest["configs"], key=lambda item: int(item["execution_index"]) ) + if selected_configs: + unknown_configs = sorted(selected_configs - set(source_records)) + if unknown_configs: + raise ValueError(f"unknown repeat configs: {unknown_configs}") + first_execution_order = [ + item + for item in first_execution_order + if item["config"]["name"] in selected_configs + ] records = [] selections = [] for execution_index, source_record in enumerate( @@ -968,7 +982,7 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: name = source_record["config"]["name"] excluded_unrankable_slos = [ slo_name - for slo_name in SLO_VARIANTS + for slo_name in selected_slos if not _slo_has_feasible_load( first_results[name], slo_name=slo_name ) @@ -981,7 +995,7 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: first_results[name], slo_name=slo_name ) ) - for slo_name in SLO_VARIANTS + for slo_name in selected_slos } boundary_rates = sorted( { @@ -1064,6 +1078,8 @@ def prepare_community_repeat(args: argparse.Namespace) -> Path: "execution_contract": { "schema": REPEAT_SELECTION_SCHEMA, "selection": "all_adjacent_pre_registered_slo_feasibility_transitions_plus_between_rate_refinement", + "selected_slos": selected_slos, + "selected_configs": sorted(selected_configs) if selected_configs else "all", "unrankable_slo_policy": "exclude_per_config_surfaces_with_no_feasible_load", "fallback_when_no_transition": "top_two_if_all_feasible_else_bottom_two", "config_order": "reverse_of_first_pass", @@ -2145,6 +2161,8 @@ def parse_args() -> argparse.Namespace: repeat.add_argument("--protocol-manifest", type=Path) repeat.add_argument("--frontier-freeze", type=Path) repeat.add_argument("--output-root", type=Path, required=True) + repeat.add_argument("--slo", action="append", default=[]) + repeat.add_argument("--config", action="append", default=[]) compare_parser = subparsers.add_parser("compare") compare_parser.add_argument("--frontier-freeze", type=Path, required=True) From 344af3a4285ccfdc6e10e540767808907ad81ddf Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 04:16:49 +0800 Subject: [PATCH 066/148] Prepare targeted disputed-boundary trial --- .../best_effort/fixed_cohort_rank.py | 3 ++- .../best_effort/jobs_fixed_cohort_targeted.toml | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/jobs_fixed_cohort_targeted.toml diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py index 90bdce5..0afc07c 100644 --- a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py @@ -1998,7 +1998,8 @@ def compare_trials(args: argparse.Namespace) -> Path: } decision_records = [] repeat_capacity_records = [] - for config_name in sorted(first_primary_capacity): + repeated_config_names = {name for name, _ in repeat_loads} + for config_name in sorted(set(first_primary_capacity) & repeated_config_names): old_rates = sorted( rate for name, rate in first_loads if name == config_name ) diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/jobs_fixed_cohort_targeted.toml b/runs/frontier-multicase-sufficiency-v0/best_effort/jobs_fixed_cohort_targeted.toml new file mode 100644 index 0000000..74f2c95 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/jobs_fixed_cohort_targeted.toml @@ -0,0 +1,10 @@ +# Third fresh-server trial for primary-SLO boundaries disputed after refinement. +version = 1 + +[[jobs]] +name = "qwen235b-fixed-cohort-community-v3-primary-disputed-20260715" +gpus = 8 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd ../../.. && /tmp/wjh-frontier-vllm0102-smoke/.venv/bin/python runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_rank.py run-community --manifest /home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/run_manifest.json" +env = { PYTHONPATH = "/home/admin/cpfs/wjh/aituner/aituner-frontier-best-effort-20260715/src" } From 13f0713bf2c3bf30bacecab25c43285000cce62b Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 06:21:40 +0800 Subject: [PATCH 067/148] Document Frontier fixed-cohort fidelity results --- .../refined_protocol_manifest.json | 790 ++ .../fixed_cohort_evidence/report.md | 204 + .../v1_blind_comparison.json | 6384 +++++++++++++++++ .../v1_v2_trial_stability.json | 1038 +++ .../v2_refined_comparison.json | 5248 ++++++++++++++ .../v2_run_manifest.json | 583 ++ .../v2_v3_trial_stability.json | 732 ++ .../v3_run_manifest.json | 345 + .../v3_targeted_comparison.json | 3830 ++++++++++ 9 files changed, 19154 insertions(+) create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/refined_protocol_manifest.json create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/report.md create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v1_blind_comparison.json create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v1_v2_trial_stability.json create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_refined_comparison.json create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_run_manifest.json create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_v3_trial_stability.json create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v3_run_manifest.json create mode 100644 runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v3_targeted_comparison.json diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/refined_protocol_manifest.json b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/refined_protocol_manifest.json new file mode 100644 index 0000000..8671510 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/refined_protocol_manifest.json @@ -0,0 +1,790 @@ +{ + "cohort": [ + { + "input_bin": 0, + "input_tokens": 865, + "prompt_sha256": "c9e22fa0559b409b259bdb15ed6eec1bb36ecb8611ae367b2e122bc6303f56ae", + "sampling_u": 0.46149221772182225, + "source_arrival_s": 8.92599999999993, + "source_request_id": "250", + "source_row_index": 250 + }, + { + "input_bin": 4, + "input_tokens": 9154, + "prompt_sha256": "53b0a0af6cb1c0278183c61b8b6c026977b6cc46e07fca825b5bc8c5802b0e17", + "sampling_u": 0.8156596361943489, + "source_arrival_s": 12.708999999999833, + "source_request_id": "367", + "source_row_index": 367 + }, + { + "input_bin": 0, + "input_tokens": 718, + "prompt_sha256": "87d2846513ce3def0c4a7f9d3707f2eda02069f74535ba69dba6fb761bbfadf5", + "sampling_u": 0.9701010298706338, + "source_arrival_s": 25.190000000000055, + "source_request_id": "682", + "source_row_index": 682 + }, + { + "input_bin": 1, + "input_tokens": 1519, + "prompt_sha256": "7dbffc44084f1dfd9af43c2c40b5e669728a2c01b0a1e29f65ea17bd37e5bffc", + "sampling_u": 0.48115960917218603, + "source_arrival_s": 25.639999999999873, + "source_request_id": "690", + "source_row_index": 690 + }, + { + "input_bin": 1, + "input_tokens": 1968, + "prompt_sha256": "65e1015fbc28a8a2af5ffced6947478c4d68b39cd07a79c5e2fb937ecf9d64a7", + "sampling_u": 0.7394708415743882, + "source_arrival_s": 36.82400000000007, + "source_request_id": "1022", + "source_row_index": 1022 + }, + { + "input_bin": 0, + "input_tokens": 62, + "prompt_sha256": "6ba251e2a1653df7c13c70ed9aa3317af23f42d0ec14b14e46b1376b86f548b7", + "sampling_u": 0.5901445515787429, + "source_arrival_s": 39.96399999999994, + "source_request_id": "1098", + "source_row_index": 1098 + }, + { + "input_bin": 0, + "input_tokens": 702, + "prompt_sha256": "ec3dbd6ac6674268fb420b0e006f7edbbad38906f178faf4c41d5ac3aa7a7968", + "sampling_u": 0.4229843993649856, + "source_arrival_s": 60.55600000000004, + "source_request_id": "1571", + "source_row_index": 1571 + }, + { + "input_bin": 1, + "input_tokens": 1472, + "prompt_sha256": "1fc79d848cc4d3f92ee5a9ad6ecc28a3173737f77ec0b2eaca1b38026225c701", + "sampling_u": 0.38189150492188706, + "source_arrival_s": 69.46300000000019, + "source_request_id": "1792", + "source_row_index": 1792 + }, + { + "input_bin": 2, + "input_tokens": 2701, + "prompt_sha256": "084b29ad36178a06d93a1fdfbed02bc0718b42f43187d184bc06f70f03df4dfc", + "sampling_u": 0.8169083216699623, + "source_arrival_s": 69.79100000000017, + "source_request_id": "1799", + "source_row_index": 1799 + }, + { + "input_bin": 3, + "input_tokens": 7825, + "prompt_sha256": "044f618323ac49afa864bf6b62b1e06a9f3c351647a953cbe38b3173542caf51", + "sampling_u": 0.3751377474987089, + "source_arrival_s": 70.71099999999979, + "source_request_id": "1818", + "source_row_index": 1818 + }, + { + "input_bin": 0, + "input_tokens": 715, + "prompt_sha256": "e5f9aba41593f81170000f1299c71d0f09f52afc58addc2968fb862f8b740359", + "sampling_u": 0.7098663141194472, + "source_arrival_s": 75.24699999999984, + "source_request_id": "1935", + "source_row_index": 1935 + }, + { + "input_bin": 1, + "input_tokens": 1573, + "prompt_sha256": "d69852bb4dbc6b7653ed3a200e891190af5d052d20e57c2ff3abbccc6bf254aa", + "sampling_u": 0.6074460889792176, + "source_arrival_s": 84.5329999999999, + "source_request_id": "2176", + "source_row_index": 2176 + }, + { + "input_bin": 5, + "input_tokens": 29385, + "prompt_sha256": "86cb5ce843bad59ada0e7b9e1a60c1162534b037f5c1b5a7fadca5be26254047", + "sampling_u": 0.17186958800332738, + "source_arrival_s": 93.04399999999987, + "source_request_id": "2396", + "source_row_index": 2396 + }, + { + "input_bin": 1, + "input_tokens": 1939, + "prompt_sha256": "06bad3b6fc6db1c9227a8bb2d4ce349b31802e51e056b541858773debb0a388d", + "sampling_u": 0.29421530368234206, + "source_arrival_s": 98.4050000000002, + "source_request_id": "2505", + "source_row_index": 2505 + }, + { + "input_bin": 1, + "input_tokens": 1532, + "prompt_sha256": "9117e580cf5b8dcedb3c6c02318610c4780fbbf1c4413367db6f7eb60b73459b", + "sampling_u": 0.41616945652066173, + "source_arrival_s": 116.05200000000013, + "source_request_id": "2913", + "source_row_index": 2913 + }, + { + "input_bin": 0, + "input_tokens": 513, + "prompt_sha256": "6cf0a32f0a62d4a16f6abddf0bed8af587ba8f0aff335fd54c72f5772da4464a", + "sampling_u": 0.3613591846324354, + "source_arrival_s": 118.6239999999998, + "source_request_id": "2979", + "source_row_index": 2979 + }, + { + "input_bin": 1, + "input_tokens": 1508, + "prompt_sha256": "ad49c5bb6cf0ced603cc052215c18159d5c25927048ddb7efd6bf315bd84c937", + "sampling_u": 0.1650938738573754, + "source_arrival_s": 121.26600000000008, + "source_request_id": "3056", + "source_row_index": 3056 + }, + { + "input_bin": 0, + "input_tokens": 705, + "prompt_sha256": "c7b6f66c2ade7054abed59f4459bbe4ab4659a3d53654197d27ce26f650450e1", + "sampling_u": 0.8484961251249794, + "source_arrival_s": 130.1909999999998, + "source_request_id": "3301", + "source_row_index": 3301 + }, + { + "input_bin": 4, + "input_tokens": 8287, + "prompt_sha256": "e597785700703dc05da72b1045e734f326ed491276f2d9f45f663f03f1ccd509", + "sampling_u": 0.7289458530141959, + "source_arrival_s": 133.10300000000007, + "source_request_id": "3379", + "source_row_index": 3379 + }, + { + "input_bin": 1, + "input_tokens": 1756, + "prompt_sha256": "b9631ff1dfec71fbd9e3a084bccf30e40a058b74743ae9878e3dd4be77efdf83", + "sampling_u": 0.4082222424943361, + "source_arrival_s": 152.64800000000014, + "source_request_id": "3844", + "source_row_index": 3844 + }, + { + "input_bin": 0, + "input_tokens": 703, + "prompt_sha256": "a943002270d455f4e65022747cf40807044f47fb574089d95deadffed56fcf2e", + "sampling_u": 0.49508596140129646, + "source_arrival_s": 166.43499999999995, + "source_request_id": "4163", + "source_row_index": 4163 + }, + { + "input_bin": 1, + "input_tokens": 1478, + "prompt_sha256": "4b84065a6336bfb9ddd174099740689adfeb5ce294fe670885334934cf583d8b", + "sampling_u": 0.2476209159147146, + "source_arrival_s": 172.2800000000002, + "source_request_id": "4323", + "source_row_index": 4323 + }, + { + "input_bin": 0, + "input_tokens": 686, + "prompt_sha256": "b2154c41a891d72e417d3bb3d67920590349860655d93cb3ff478ad3da74e2f8", + "sampling_u": 0.7190432102580353, + "source_arrival_s": 193.67599999999993, + "source_request_id": "4838", + "source_row_index": 4838 + }, + { + "input_bin": 1, + "input_tokens": 1600, + "prompt_sha256": "9f4cafecff466decfe98d4a9544d05c6246853f24cfeb6e77c9e4d1a5cb6cb02", + "sampling_u": 0.71711230795038, + "source_arrival_s": 194.66400000000021, + "source_request_id": "4870", + "source_row_index": 4870 + }, + { + "input_bin": 2, + "input_tokens": 4000, + "prompt_sha256": "30c385ba8463097d52f838bbfa69f18a32c91b38878e15c11e6b69d6bf81abc7", + "sampling_u": 0.12696475188087025, + "source_arrival_s": 216.49299999999994, + "source_request_id": "5469", + "source_row_index": 5469 + }, + { + "input_bin": 0, + "input_tokens": 710, + "prompt_sha256": "807c8b29c95a0623c585ee7d15d82e72be5858f3683d3657ef3295e71ef6aa10", + "sampling_u": 0.32798737479046675, + "source_arrival_s": 223.39199999999983, + "source_request_id": "5648", + "source_row_index": 5648 + }, + { + "input_bin": 0, + "input_tokens": 727, + "prompt_sha256": "b872bc3174db8e4b17e182608dbb48f99196220dfd55096c2490484a6cadd595", + "sampling_u": 0.3644506007131907, + "source_arrival_s": 239.0999999999999, + "source_request_id": "6031", + "source_row_index": 6031 + }, + { + "input_bin": 4, + "input_tokens": 8786, + "prompt_sha256": "5757f2f9e6be7d4a29a79ad82408e51069044828f55b8c21b7216abbf73b0757", + "sampling_u": 0.12629800224853702, + "source_arrival_s": 260.4920000000002, + "source_request_id": "6604", + "source_row_index": 6604 + }, + { + "input_bin": 4, + "input_tokens": 9518, + "prompt_sha256": "730aee2b4ec388efbf4ca55cbfcbe31c8aa9636c8876b624b37b499d5dcb00f5", + "sampling_u": 0.29338926296423934, + "source_arrival_s": 260.91499999999996, + "source_request_id": "6611", + "source_row_index": 6611 + }, + { + "input_bin": 0, + "input_tokens": 796, + "prompt_sha256": "e0f7312dcc97af6236edac6e47a828f9e18e3e118023537d82f6cb540025191e", + "sampling_u": 0.47864598488768767, + "source_arrival_s": 274.52, + "source_request_id": "6905", + "source_row_index": 6905 + }, + { + "input_bin": 0, + "input_tokens": 228, + "prompt_sha256": "b85c14bcfaec63a99ebc382a7e20d32390a65999062e84c303441344832c3279", + "sampling_u": 0.8157098758872824, + "source_arrival_s": 274.6399999999999, + "source_request_id": "6911", + "source_row_index": 6911 + }, + { + "input_bin": 5, + "input_tokens": 18439, + "prompt_sha256": "8f7259429d8b3b6ab062c885d990fa2136e8ac061ce5b59169e5dbe7cdd5c055", + "sampling_u": 0.44219574639241527, + "source_arrival_s": 287.1959999999999, + "source_request_id": "7271", + "source_row_index": 7271 + }, + { + "input_bin": 0, + "input_tokens": 736, + "prompt_sha256": "1c56453602e159f1dfaae09b5b40785d0dbc176d9b27844362e342e3731c2032", + "sampling_u": 0.6889130017060773, + "source_arrival_s": 291.97400000000016, + "source_request_id": "7403", + "source_row_index": 7403 + }, + { + "input_bin": 4, + "input_tokens": 8557, + "prompt_sha256": "14346a1cc9d1382dd67eaf700f5b52798b77fc8a08bc1df71768d38d613e9e43", + "sampling_u": 0.3112632330750767, + "source_arrival_s": 292.692, + "source_request_id": "7422", + "source_row_index": 7422 + }, + { + "input_bin": 1, + "input_tokens": 1586, + "prompt_sha256": "4d24bf64da9142747d8ee8d87daf50f81309928c3d4792f7c345ecd9dd0c24ac", + "sampling_u": 0.23031083293934718, + "source_arrival_s": 304.69900000000007, + "source_request_id": "7741", + "source_row_index": 7741 + }, + { + "input_bin": 1, + "input_tokens": 1871, + "prompt_sha256": "5a4917df4ebb8a2e4e3dcb0f15c231cd46830af3dc63452ebf421b4433756159", + "sampling_u": 0.10577279384258167, + "source_arrival_s": 312.683, + "source_request_id": "7981", + "source_row_index": 7981 + }, + { + "input_bin": 0, + "input_tokens": 698, + "prompt_sha256": "610618e93c2f4d7bd0ca26b2611b61ffdbbbe092fe0ff3a5f7d7c91f52de7c74", + "sampling_u": 0.43535389643319383, + "source_arrival_s": 319.1579999999999, + "source_request_id": "8179", + "source_row_index": 8179 + }, + { + "input_bin": 0, + "input_tokens": 710, + "prompt_sha256": "3b397d03d7a7001ab8f3915b1b88ed8f8b859ad51769ac752ff35f567ccb1183", + "sampling_u": 0.6383291776591391, + "source_arrival_s": 321.4380000000001, + "source_request_id": "8224", + "source_row_index": 8224 + }, + { + "input_bin": 1, + "input_tokens": 1452, + "prompt_sha256": "671c93bcdd7a292d1a4b4dcce44c679369955fdb4d250d451170c63391216436", + "sampling_u": 0.12477813551209259, + "source_arrival_s": 337.8319999999999, + "source_request_id": "8612", + "source_row_index": 8612 + }, + { + "input_bin": 5, + "input_tokens": 24271, + "prompt_sha256": "a84a31f74c755d02154a7eaeec8bd71450c16813d8b7e713999ff52419f84ed6", + "sampling_u": 0.5538066537263959, + "source_arrival_s": 348.692, + "source_request_id": "8891", + "source_row_index": 8891 + }, + { + "input_bin": 3, + "input_tokens": 4607, + "prompt_sha256": "720cf238e668f4ae567ab909b2ec0185d222a14eb474ae2ef481141a885b3aad", + "sampling_u": 0.03539900309623126, + "source_arrival_s": 382.55499999999984, + "source_request_id": "9768", + "source_row_index": 9768 + }, + { + "input_bin": 0, + "input_tokens": 111, + "prompt_sha256": "8a12215d12ddcf36d1e9013ec0b4a433e4f27c5c2a774ead0f66c93bb31a8063", + "sampling_u": 0.9241501254905812, + "source_arrival_s": 402.8620000000001, + "source_request_id": "10305", + "source_row_index": 10305 + }, + { + "input_bin": 0, + "input_tokens": 714, + "prompt_sha256": "e1dd5099fb8565866691241945b15ee40a5ea8ad9045d10e1dd7774b668d9fa4", + "sampling_u": 0.4533435244095546, + "source_arrival_s": 405.0459999999998, + "source_request_id": "10383", + "source_row_index": 10383 + }, + { + "input_bin": 1, + "input_tokens": 1792, + "prompt_sha256": "9d94bba8c056a775a5d00ecfd6567e83030c99dc20f71ce53d787683ca2450e9", + "sampling_u": 0.8983379345538031, + "source_arrival_s": 405.08100000000013, + "source_request_id": "10385", + "source_row_index": 10385 + }, + { + "input_bin": 0, + "input_tokens": 697, + "prompt_sha256": "7a66c3cec2e1727fa78ed1a7dbe1d39d2c877cc2835c0b10479b396204749984", + "sampling_u": 0.43645445582603326, + "source_arrival_s": 413.16499999999996, + "source_request_id": "10618", + "source_row_index": 10618 + }, + { + "input_bin": 0, + "input_tokens": 709, + "prompt_sha256": "a71447eebb513ece438e0a81a0a9788fbbf34a71d9a66987e6f476cf76f0b7d5", + "sampling_u": 0.4730229506348141, + "source_arrival_s": 424.991, + "source_request_id": "10911", + "source_row_index": 10911 + }, + { + "input_bin": 0, + "input_tokens": 699, + "prompt_sha256": "e04aaa160224e82157f9b0c89180dc341e2c8eebbd518ea8d3327eb73ebfac1c", + "sampling_u": 0.5350250700640652, + "source_arrival_s": 430.51800000000003, + "source_request_id": "11058", + "source_row_index": 11058 + }, + { + "input_bin": 2, + "input_tokens": 3130, + "prompt_sha256": "ba93e2babcf7a0491ba97930f5b3bfb3cc2170dc3cd7b3ca3c3f9fe2905dc72e", + "sampling_u": 0.6597844956490088, + "source_arrival_s": 434.3449999999998, + "source_request_id": "11166", + "source_row_index": 11166 + }, + { + "input_bin": 0, + "input_tokens": 711, + "prompt_sha256": "80c05739d748df64d97f92742f28e384365b26f4f5f1e999666ea2f7f187fd16", + "sampling_u": 0.7240395487416784, + "source_arrival_s": 441.3699999999999, + "source_request_id": "11373", + "source_row_index": 11373 + }, + { + "input_bin": 1, + "input_tokens": 1536, + "prompt_sha256": "ee1209838aca67d2392bb9156b74d69de22d166d97c88b56501ca9cfcc70578e", + "sampling_u": 0.9806533325226275, + "source_arrival_s": 463.8670000000002, + "source_request_id": "11911", + "source_row_index": 11911 + }, + { + "input_bin": 1, + "input_tokens": 1580, + "prompt_sha256": "a4e5fdb5d6119e9207384fd3d32ac027469b3ecca278738c7078026551999786", + "sampling_u": 0.8508070546796023, + "source_arrival_s": 471.08899999999994, + "source_request_id": "12100", + "source_row_index": 12100 + }, + { + "input_bin": 1, + "input_tokens": 1466, + "prompt_sha256": "09a6942491467ee3597eef41c4f36de46bd88b0533af8c1084bb9e15cb150832", + "sampling_u": 0.03536418626599094, + "source_arrival_s": 488.5859999999998, + "source_request_id": "12587", + "source_row_index": 12587 + }, + { + "input_bin": 0, + "input_tokens": 750, + "prompt_sha256": "6a53081dfa161f2b1e50e03885b0d2868c7390f91130d0a1cd95b4b75cdb1126", + "sampling_u": 0.4344415561103455, + "source_arrival_s": 488.6289999999999, + "source_request_id": "12588", + "source_row_index": 12588 + }, + { + "input_bin": 0, + "input_tokens": 736, + "prompt_sha256": "3514d193ff83b6d6589d52f24072b0159fa00d7b44ca8c6ec269515db4cfba89", + "sampling_u": 0.7257534252552246, + "source_arrival_s": 497.2449999999999, + "source_request_id": "12824", + "source_row_index": 12824 + }, + { + "input_bin": 0, + "input_tokens": 90, + "prompt_sha256": "915b3aee8d1288262c7f363f9035af19236e45540b748a9ced17c2d23f428f34", + "sampling_u": 0.7669962160527245, + "source_arrival_s": 520.1750000000002, + "source_request_id": "13350", + "source_row_index": 13350 + }, + { + "input_bin": 0, + "input_tokens": 697, + "prompt_sha256": "af93c55d3eb4f5161eb69410e73039e1d500a92190bff884e1b4040e491c8ed5", + "sampling_u": 0.594626944330893, + "source_arrival_s": 525.469, + "source_request_id": "13491", + "source_row_index": 13491 + }, + { + "input_bin": 1, + "input_tokens": 1470, + "prompt_sha256": "dd86d891c322f43310d4207c02f27a7d5e76885c420793164c76020f7ff166c4", + "sampling_u": 0.9612563305576178, + "source_arrival_s": 526.116, + "source_request_id": "13510", + "source_row_index": 13510 + }, + { + "input_bin": 1, + "input_tokens": 1592, + "prompt_sha256": "2ab08103a5ec0e01aea2f74f9215c52222ef8722a2fb9327bd9187a644df1c4b", + "sampling_u": 0.7977149220573079, + "source_arrival_s": 529.1809999999996, + "source_request_id": "13612", + "source_row_index": 13612 + }, + { + "input_bin": 5, + "input_tokens": 26776, + "prompt_sha256": "d09d69f9110f28c69d0767bdd501a8bdba167775ae1c6e6c53d15cf0fdcf5f6e", + "sampling_u": 0.9888861232028356, + "source_arrival_s": 546.9939999999997, + "source_request_id": "14026", + "source_row_index": 14026 + }, + { + "input_bin": 1, + "input_tokens": 1589, + "prompt_sha256": "973120a2be961f8f44523988a44f59b2071e82081a292cf37f10fb72d2874535", + "sampling_u": 0.8325378218001509, + "source_arrival_s": 549.6970000000001, + "source_request_id": "14094", + "source_row_index": 14094 + }, + { + "input_bin": 1, + "input_tokens": 1550, + "prompt_sha256": "8d899d80a6b9c47f414f6d65ed1638b5fc3bc782eaa19b2592878f35ff5484e7", + "sampling_u": 0.7285184471546545, + "source_arrival_s": 556.8310000000001, + "source_request_id": "14316", + "source_row_index": 14316 + }, + { + "input_bin": 2, + "input_tokens": 2501, + "prompt_sha256": "4a88c92c72a1fd9a653a7dbbd169eff30774f1b95cb5482f60c45532923d9f47", + "sampling_u": 0.38599257949920635, + "source_arrival_s": 558.3209999999999, + "source_request_id": "14359", + "source_row_index": 14359 + }, + { + "input_bin": 0, + "input_tokens": 741, + "prompt_sha256": "9c2341411206e2d28d6479ef20d5ce9e90daac574e72c4066130e2601731624c", + "sampling_u": 0.5930037066322071, + "source_arrival_s": 583.5829999999996, + "source_request_id": "15069", + "source_row_index": 15069 + }, + { + "input_bin": 3, + "input_tokens": 6044, + "prompt_sha256": "614a0d743b4cec35549b490110bc71d8fc3a22cd07c68de7281fc98270f81ee8", + "sampling_u": 0.4726761733329724, + "source_arrival_s": 594.8950000000004, + "source_request_id": "15377", + "source_row_index": 15377 + } + ], + "created_unix_s": 1784139549.1404486, + "load_contract": { + "binary_search": false, + "completion_tokens_override": 1, + "monotonicity_assumed": false, + "offered_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "only_mutated_variable": "uniform_arrival_timeline_scale", + "request_count_per_point": 64, + "target_pass_rate": 0.95 + }, + "primary_slo": "linear_8k_primary", + "rates": { + "r_0p150": { + "duration_s": 426.6666666666667, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 0.15, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_0p150.csv", + "request_count": 64, + "request_rate_recomputed": 0.15, + "sha256": "f35ce4424c17b2ee16144bb59011e467a07d3f57895716165fcdc73ba19839d9", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + }, + "r_0p200": { + "duration_s": 320.0, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 0.2, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_0p200.csv", + "request_count": 64, + "request_rate_recomputed": 0.2, + "sha256": "b81a48706a5c41222477aef8da357ae14c7b0bc19d35ea5ecc0c12b86d32590c", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + }, + "r_0p250": { + "duration_s": 256.0, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 0.25, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_0p250.csv", + "request_count": 64, + "request_rate_recomputed": 0.25, + "sha256": "721d622abb5d0a9080d6577063af910906b4e9928b402f48804ee55b25f108fc", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + }, + "r_0p300": { + "duration_s": 213.33333333333334, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 0.3, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_0p300.csv", + "request_count": 64, + "request_rate_recomputed": 0.3, + "sha256": "a5c8a7b93f314bf4365763fc0157597c582229993bb79b41b697ef3247495e49", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + }, + "r_0p350": { + "duration_s": 182.85714285714286, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 0.35, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_0p350.csv", + "request_count": 64, + "request_rate_recomputed": 0.35, + "sha256": "ec9debce28b76981f4f5363f0c77faa963c550a5d3ecbed327a304dc0d495242", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + }, + "r_0p400": { + "duration_s": 160.0, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 0.4, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_0p400.csv", + "request_count": 64, + "request_rate_recomputed": 0.4, + "sha256": "aea1cae38b09ddb0ad1a7d6da03bd6ad76afdaee3b2879b7cb65eba9b3ef5bab", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + }, + "r_0p450": { + "duration_s": 142.22222222222223, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 0.45, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_0p450.csv", + "request_count": 64, + "request_rate_recomputed": 0.44999999999999996, + "sha256": "672197deca834d9b9c5aea963448805a03e86495721d5eb270ccec2416faada2", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + }, + "r_0p500": { + "duration_s": 128.0, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 0.5, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_0p500.csv", + "request_count": 64, + "request_rate_recomputed": 0.5, + "sha256": "7cd2d16904e0fafb26a67718c4f2d7b6752934864219ab96813b8f62b47019d4", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + }, + "r_0p750": { + "duration_s": 85.33333333333333, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 0.75, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_0p750.csv", + "request_count": 64, + "request_rate_recomputed": 0.75, + "sha256": "e8b00f14b6c85ac8bd1c8fe3b83797f5c61903d70d8bbf6acfe9a1bd384db443", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + }, + "r_1p000": { + "duration_s": 64.0, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 1.0, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_1p000.csv", + "request_count": 64, + "request_rate_recomputed": 1.0, + "sha256": "7071e0247414af9214a1bf06f05a4865965568ba812bbae6713648e994a40a34", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + }, + "r_1p500": { + "duration_s": 42.666666666666664, + "input_length_order_sha256": "d796fc955a9e7278512b56ce0544afd465d689e0060920c18839e85632360645", + "offered_request_rate": 1.5, + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/traces/r_1p500.csv", + "request_count": 64, + "request_rate_recomputed": 1.5, + "sha256": "86b03ef1fe87ad3c589f1585a279cd3136975009316e36e0910fe8340b972eee", + "source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5" + } + }, + "schema": "qwen235b-prefill-fixed-cohort-protocol-v1", + "selection": { + "cohort_bin_quotas": [ + 27, + 21, + 4, + 3, + 5, + 4 + ], + "cohort_size": 64, + "cohort_source_row_order_sha256": "d3945dc4a79840a0a157a73f10f2836006a3f0bbc5ecdbd657886865497affa5", + "input_bin_edges": [ + 0, + 1024, + 2048, + 4096, + 8192, + 16384, + 32769 + ], + "method": "length_stratified_smallest_sha256", + "seed": 2026071501, + "source_arrival_span_s": 585.9690000000005, + "source_bin_counts": [ + 6549, + 5111, + 916, + 743, + 1194, + 896 + ], + "stable_order": [ + "source_arrival_s", + "source_row_index" + ] + }, + "slo_variants": { + "legacy_step_1s_2s": { + "buckets": [ + { + "max_input_tokens": 8191, + "threshold_ms": 1000.0 + }, + { + "threshold_ms": 2000.0 + } + ], + "description": "Original strict 1 s/2 s step SLO (null-capacity sensitivity)", + "kind": "step_ms" + }, + "linear_10k": { + "description": "TTFT <= 1000 ms + input_tokens / 10000 tokens/s", + "intercept_ms": 1000.0, + "kind": "linear_ms", + "per_token_ms": 0.1 + }, + "linear_6k": { + "description": "TTFT <= 1000 ms + input_tokens / 6000 tokens/s", + "intercept_ms": 1000.0, + "kind": "linear_ms", + "per_token_ms": 0.16666666666666666 + }, + "linear_8k_primary": { + "description": "TTFT <= 1000 ms + input_tokens / 8000 tokens/s", + "intercept_ms": 1000.0, + "kind": "linear_ms", + "per_token_ms": 0.125 + } + }, + "source": { + "eligible_request_count": 15409, + "path": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/thinking_w20260327_1000.jsonl", + "sha256": "f878e9af18f94dcfaced94a8e1e6b20a2f7d97d64aa862448025660dbbd965b2" + } +} diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/report.md b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/report.md new file mode 100644 index 0000000..dfe9855 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/report.md @@ -0,0 +1,204 @@ +# Qwen3-235B FP8：Frontier 与 community vLLM config rank 对比 + +## 结论 + +在本实验限定的 Qwen3-235B-A22B-FP8、H20、prefill-only workload 上，经过 best-effort 补齐 FP8/MoE serving path 后，Frontier **能够选对满足 SLO 的 per-GPU throughput 最优配置**：`TP=4, MBT=16384`，`MNS=64/128` 并列。细化网格上的 Spearman rank correlation 为 `0.9487`，20 个可比较非 tie config pair 全部同序，选择 regret 为 `0`。 + +但这不等于 Frontier 已精确模拟系统。原始粗网格给出 56/56 load labels 正确、Spearman=1 的表象；加入 decision-boundary rates 后，Frontier 在 33 个被测试边界点中产生 6 个 false-infeasible，均为两个 MNS 配置上可复现的稳定误差。它系统性地把三个 action family 的最大经验 SLO throughput 低估一个 `0.05 req/s` 档位： + +| TP | MBT | Frontier capacity | real capacity | real v2→v3 transition | 结论 | +|---:|---:|---:|---:|---|---| +| 4 | 8192 | 0.15 | 0.20 | `0.20: 61→61`, `0.25: 60→60` | 稳定低估 0.05 | +| 4 | 16384 | 0.25 | 0.30 | `0.30: 61→61`, `0.35: 59→59` | 稳定低估 0.05 | +| 8 | 8192 | 0.40 | 0.45 | `0.45: 61→61`, `0.50: 60→60` | 稳定低估 0.05 | +| 8 | 16384 | 0.45 | 0.45 | v2: `0.45: 61`, `0.50: 60` | 边界正确 | + +表中 pass count 分母均为 64，经验可行门槛为 61。每一行的两个 MNS 结果一致；v3 对前三行的两个 MNS 都做了 fresh-server 复测。 + +因此，对这个 case 最准确的判断是： + +- Frontier 有较好的 **decision/ordinal fidelity**，不能再用“它找不到最优 config”作为该 case 的研究 premise。 +- Frontier 没有达到 **capacity calibration fidelity**，也没有完全学对 action effect；它适合筛选，但不能在 SLO knee 附近代替真机 capacity certification。 +- 粗网格上的“完全匹配”并不足以证明 simulator solve 了问题；它会同时掩盖绝对容量误差和错误的 action differential。 + +## 固定实验口径 + +| 项目 | 固定值 | +|---|---| +| model | `models/Qwen/Qwen3-235B-A22B-FP8` | +| real runtime | community vLLM 0.10.2 | +| hardware | dash0，8×NVIDIA H20 | +| workload | prefill-only，raw completion，output tokens=1，prefix cache disabled | +| config surface | `TP∈{4,8} × MNS∈{64,128} × MBT∈{8192,16384}` | +| cohort | 64 个固定 prompt；所有 config/rate 的 request ID、顺序、输入长度完全相同 | +| input length | p50=1478，p95=18439，max=29385，mean=3511.55 tokens | +| rates | 0.15, 0.20, 0.25, 0.30, 0.35, 0.40, 0.45, 0.50, 0.75, 1.00, 1.50 req/s | +| primary SLO | `TTFT ≤ 1000 ms + input_tokens / 8000 tokens/s`，至少 95% requests 通过 | +| capacity | 所有 64 requests 完整结束后，最大被测试可行 offered rate；不做 early stop/二分单调假设 | +| primary objective | capacity / TP，即 per-GPU request throughput | + +cohort 只通过 uniform arrival scaling 改变 load，不随 rate 重新抽样。该设计消除了 workload composition confound，但 64-request 固定 cohort 的结果仍是经验容量，不代表总体 workload distribution 的置信界。 + +## Frontier best-effort 公平性 + +这里比较的不是未经修改的 stock Frontier。为了避免“故意把 baseline 调差”的质疑，我们在真机 ranking 冻结前完成并冻结了以下补齐： + +- 使用 community vLLM 0.10.2 在 H20 上得到的真实 FP8 operator profiles；quantization contract 为 FP8 W8A8 dynamic activation、BF16 output/accumulation，而不是用 BF16 compute profile 冒充 FP8。 +- 补齐 Qwen MoE community serving path、MoE plan 与 serving entrypoint。 +- 按真实 expert route 构造 EP local lanes，并以最慢 lane 作为同步 critical path。 +- 修正 TP/EP-aware profile cache key，避免不同 topology 错用缓存结果。 +- 使用 batched lane prediction；与逐 lane exact path 做 byte-identical equivalence 后才用于完整 sweep，模拟运行加速约 2.57×。 +- KV block capacity 使用真机测量值：TP4=26101，TP8=62351。 +- Frontier refined 8×11=88 个结果在真机运行前冻结；旧 rates 的 56 个结果与先前 freeze byte-identical。没有用真机 v2/v3 结果回调 simulator 参数。 + +相关实现与审计入口包括： + +- `fixed_cohort_rank.py` +- `frontier_fp8_profile_contract.patch` +- `frontier_moe_serving_path.patch` +- `frontier_moe_ep_prefill_exact_cache.patch` +- `frontier_moe_ep_prefill_use_batched_lanes.patch` +- `frontier_moe_ep_prefill_batched_lane_predict.patch` + +这说明结果反映的是“给 Frontier 当前抽象最大的合理帮助以后，它能做到什么”，不是 stock baseline 的缺功能测试。 + +## 三阶段实验结果 + +### v1：blind 全表面、粗网格 + +Frontier 先冻结，再运行 community vLLM 的 8 configs × 7 rates： + +- 56/56 config-load primary feasibility labels 正确。 +- raw capacity 与 per-GPU efficiency 的 top set 均完全匹配。 +- Spearman=1，20/20 非 tie pair 同序，top-1 regret=0。 +- 但 3584 个 request 的 TTFT MAE 为 124.20 ms，MAPE 为 20.98%，平均 `predicted-real=+93.20 ms`。 +- request-level SLO label accuracy 为 96.82%；Frontier/real failure-set Jaccard 仅 0.7569（103 false fail、11 false pass）。 + +因此，aggregate capacity 完全匹配时，具体失败请求和 request latency mechanism 已明显不匹配。 + +### v2：反向 config/rate 顺序，加入细化边界 + +v2 只重跑预注册 SLO transition 及中间 refinement rates，共 8 configs、33 loads、2112 requests： + +- 0 server/request failure，0 early stop，0 monotonicity violation；所有 trace hash 匹配 refined protocol。 +- per-GPU top set 仍精确匹配：`TP4, MBT16384, MNS64/128`。 +- Frontier/real per-GPU capacity Spearman=0.9487，20/20 非 tie pair 同序，regret=0。 +- 33 个边界点中 27 个 feasibility label 正确；6 个错误全部是 false-infeasible，没有 false-feasible。 +- pass-rate MAE=1.278 percentage points，平均 `Frontier-real=-1.278 pp`。 +- 2112 个 request 的 TTFT MAE=83.15 ms，MAPE=16.34%，平均 `predicted-real=+53.02 ms`。 +- request SLO label accuracy=98.34%，failure-set Jaccard=0.7535。 + +v1 与 v2 的 19 个共同 config-load 点包含 1216 个相同 requests： + +- 真机 trial-to-trial TTFT MAE=14.05 ms，absolute-error p95=57.94 ms，平均 v2-v1=+1.75 ms。 +- primary request label 只有 1/1216 flip；19/19 load labels 和 8/8 原始网格 capacity 全部稳定。 +- Frontier 有 23 个 request label 在两次真机 trial 中都稳定预测错误。 + +这排除了 config order 和普通 runtime noise 是主要 residual 来源。 + +### v3：只复测 v2 的全部争议 transition + +v3 的选择规则是 v2 中所有 Frontier/real primary capacity 不同的 config；它是 data-dependent replication，不作为新的 blind primary result。共 6 configs、12 loads、768 requests： + +- 0 failure/early-stop/monotonicity violation，trace hash 全部匹配。 +- v2 与 v3 的 768/768 primary request labels 完全一致，failure-set Jaccard=1.0。 +- 12/12 config-load labels、6/6 capacity boundaries 完全一致。 +- 真机 trial-to-trial TTFT MAE=14.98 ms，absolute-error p95=62.52 ms，平均 v3-v2=+1.55 ms。 +- Frontier 在该争议集合上仍有 6/12 false-infeasible；18 个 request labels 在 v2/v3 都稳定预测错误。 + +这证明细化边界误差不是一次随机跨阈值。 + +## 系统层面的理解 + +### 1. 绝对误差与 action differential 必须分开 + +如果只是一个全局 latency scale 偏差，所有 topology 下 MBT 的相对收益应基本保持。数据并非如此： + +- TP4：Frontier 预测 MBT `8192→16384` 的 capacity 增益为 `0.15→0.25`，即 +0.10 req/s；真机为 `0.20→0.30`，同样 +0.10。 +- TP8：Frontier 预测 `0.40→0.45`，即 +0.05；真机为 `0.45→0.45`，实际增益为 0。 + +也就是说，Frontier 在 TP4 学对了 MBT differential、错了 absolute offset；在 TP8 则连 differential 都错了。该误差依赖 TP/EP topology，不能靠一个全局 calibration factor 修复。这正是 execution configuration 与 scheduler state/action effect 相互耦合的证据。 + +### 2. 决策正确不代表 simulator 对系统有正确解释 + +per-GPU objective 下，TP4/MBT16384 与其他配置有足够大的 margin，所以即便 capacity 每类偏差 0.05，最优 config 仍不变。raw single-replica objective 下，Frontier 只把 TP8/MBT16384 列为 top set，而真机四个 TP8 config 全部在 0.45 并列；虽然从 Frontier top set 中任选仍是 zero regret，但它虚构了 TP8 上 MBT 的收益。 + +因此需要同时报告： + +1. selected-config regret； +2. rank/order fidelity； +3. capacity error； +4. action differential error； +5. request/failure-set fidelity。 + +只报其中任意一层都会得到偏颇结论。 + +### 3. residual 是状态/长度相关尾部误差，不是 profile 全局比例错误 + +v2 中 Frontier 的平均 signed TTFT error 随输入长度显著变化： + +| input tokens | mean `predicted-real` | MAE | +|---|---:|---:| +| [0,1024) | -3.61 ms | 40.74 ms | +| [1024,2048) | +44.45 ms | 57.63 ms | +| [2048,4096) | +91.21 ms | 109.73 ms | +| [4096,8192) | +138.80 ms | 140.80 ms | +| [8192,16384) | +237.37 ms | 237.37 ms | +| [16384,32769) | +147.36 ms | 240.89 ms | + +短请求甚至略偏乐观，中长请求明显偏保守；与此同时 aggregate load label 只由 1–2 个请求是否跨 SLO 阈值决定。这更像 batch composition、critical-lane/EP synchronization 和 queueing state 共同造成的 residual，而不是 operator profile 全局慢了固定百分比。 + +### 4. MNS 在该 workload 上是非活跃维度，但不能外推 + +在所有共同测试点中，MNS=64/128 的 pass counts 基本逐点相同，Frontier 也预测 tie。对这个 prefill-only、固定 cohort、当前 concurrency 范围，MBT/token budget 和 topology 是活跃因素，MNS 没有形成独立瓶颈。但 decode/mixed workload 的 sequence residency、KV pressure 和 iteration cadence 不同，不能据此宣称 MNS 普遍无效。 + +## 对研究方向的判断 + +这个 case 不支持以下论文 premise： + +> Frontier 类 simulator 无法为 Qwen3-235B FP8 选择最优配置，所以必须由一个新 tuner 替代。 + +在公平补齐 FP8/MoE path 后，它已经能以 zero regret 选出该 workload 的 per-GPU 最优配置。继续用这一 case 证明“simulator 排序失败”会被 reviewer 用本组数据直接反驳。 + +本 case 真正揭示、且更有 systems 内容的问题是： + +> 一个 simulator 可以在 coarse grid 上拥有完美 rank，并在最终 objective 上 zero regret，同时仍错误表达 topology-dependent action differential 和 SLO knee；我们如何判断一个模拟决策何时足够可信，何时必须回到真机？ + +可继续发展的研究问题包括： + +- decision-aware uncertainty：不追求每条 TTFT 精确，而是给 config ordering/capacity boundary 提供可校验置信区间。 +- counterexample-guided measurement：只在模拟 action margin 小或 differential 不稳定的 boundary/state 上调用真机，而不是全表面 sweep。 +- topology-aware residual model：显式建模 TP/EP、batch composition、critical lane 与输入长度共同造成的尾部偏差，避免全局比例校准。 +- distinguish safe screening from certification：Frontier 当前在该 case 是保守筛选器（0 false-feasible），但不是精确 capacity certifier。 + +下一步要判断这是否足以构成更一般的 research premise，应转向已计划的 Qwen3-235B decode-only 和 mixed prefill/decode case。它们会激活 KV state、replica residency、iteration cadence 和 communication overlap；如果只有这些 case 出现稳定排序/regret 错误，研究问题就应围绕 simulator 在 stateful execution/scheduling coupling 下的决策失效，而不是泛化地宣称 simulator 不工作。 + +## 证据与复现 + +本目录保存紧凑 comparison、trial-stability 与 manifest；完整 request-level freeze 保留在 dash0： + +`/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1` + +关键 artifact SHA256： + +| artifact | SHA256 | +|---|---| +| refined protocol manifest | `695c0500b9a5d51cdd4e39b722e3c9db15f8bed48795454f4d0d6442d24929cb` | +| Frontier refined freeze | `318b3c7742c8371191e2dbd429f0e0e63c1a42f4806a599e018b0548cb9d6336` | +| v1 blind comparison | `fc87c54179b891e2148a738dbdfeed328595fa32198f2248c892c890625c9fe2` | +| v2 run manifest | `cf1a722043f42ff8f435e3521d0b2b2892a8ea0922e1e4870ec10fba26c0f998` | +| v2 community freeze | `5a9c59fc4082db47bec7b59432e9ebe9f907f70baa95a9f0d3480e85f8992ff2` | +| v2 comparison | `d21f2dd846c4d8361668f312f0c063f2e8f949ec7a9d3b19100c813b3da3b586` | +| v1-v2 stability | `a522189618ad26df68f18d2d9b67c38b1fe59c5c8f13af633eced0b092f8e06d` | +| v3 run manifest | `0f8908671b8903e516f0dc8e68f556483a15ff7ec51f7cfbc18a9c74b3a5ba61` | +| v3 community freeze | `4edd25edde3d7508d8feb9b024b5bca4ba6a5f25335ac84008f09c524ed7bc34` | +| v3 comparison | `e04f40ead995e007d64b2f6f0f2731999204907770f05b4c2d037d224e49c573` | +| v2-v3 stability | `7be2ff27dd0393c99f0512bd6a0b7b35eb404ee0bfd22bd3ec33a4b8ce14748b` | + +## 限制 + +- 只测试了一个模型、H20、community vLLM 0.10.2 和 prefill-only workload。 +- 固定 64-request cohort 适合 simulator-vs-real paired comparison，但不足以给总体 workload distribution 的 95% SLO capacity 做统计认证。 +- v3 是在观察 v2 后选择全部争议点的 replication，不是 blind discovery dataset。 +- operator profiles、KV capacity 和 MoE route contract 本身使用了真机信息；本结果评估的是 best-effort simulator 的决策能力与剩余误差，不是 zero-measurement transfer。 +- 没有测试 decode、mixed phase、prefix caching、multi-replica placement、异构 GPU 或跨节点通信。 diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v1_blind_comparison.json b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v1_blind_comparison.json new file mode 100644 index 0000000..001eafa --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v1_blind_comparison.json @@ -0,0 +1,6384 @@ +{ + "community_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/community_ranking_frozen.json", + "sha256": "055fe2cf1e0c01a0503145a272097b8ceda257f1b86bdae39abf17277d9a18d9" + }, + "created_unix_s": 1784139490.8308787, + "frontier_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/frontier-v3-batched-par4/frontier_ranking_frozen.json", + "sha256": "9a971f25cc97975845968cc776241c7d44a49dee2ec2f015f227bcff82e66971" + }, + "pass_rate_surface_residuals": { + "legacy_step_1s_2s": { + "false_feasible_points": 0, + "false_infeasible_points": 0, + "feasibility_label_accuracy": 1.0, + "mae": 0.041294642857142856, + "matched_config_load_points": 56, + "max_absolute_error": 0.15625, + "signed_error_mean": -0.03962053571428571 + }, + "linear_10k": { + "false_feasible_points": 2, + "false_infeasible_points": 0, + "feasibility_label_accuracy": 0.9642857142857143, + "mae": 0.033482142857142856, + "matched_config_load_points": 56, + "max_absolute_error": 0.140625, + "signed_error_mean": -0.029575892857142856 + }, + "linear_6k": { + "false_feasible_points": 0, + "false_infeasible_points": 1, + "feasibility_label_accuracy": 0.9821428571428571, + "mae": 0.022600446428571428, + "matched_config_load_points": 56, + "max_absolute_error": 0.109375, + "signed_error_mean": -0.020926339285714284 + }, + "linear_8k_primary": { + "false_feasible_points": 0, + "false_infeasible_points": 0, + "feasibility_label_accuracy": 1.0, + "mae": 0.026227678571428572, + "matched_config_load_points": 56, + "max_absolute_error": 0.125, + "signed_error_mean": -0.025669642857142856 + } + }, + "primary_slo": "linear_8k_primary", + "request_level_ttft_residuals": { + "by_input_bin": { + "[0,1024)": { + "count": 1512, + "mae_ms": 80.67140471814413, + "signed_error_ms_mean": 40.081426915862 + }, + "[1024,2048)": { + "count": 1176, + "mae_ms": 113.2322821016931, + "signed_error_ms_mean": 95.35554067785822 + }, + "[16384,32769)": { + "count": 224, + "mae_ms": 248.44144323616518, + "signed_error_ms_mean": 151.67347724381298 + }, + "[2048,4096)": { + "count": 224, + "mae_ms": 146.4061790530529, + "signed_error_ms_mean": 116.84675686840379 + }, + "[4096,8192)": { + "count": 168, + "mae_ms": 150.66978579178092, + "signed_error_ms_mean": 148.27281627513304 + }, + "[8192,16384)": { + "count": 280, + "mae_ms": 272.270894330094, + "signed_error_ms_mean": 272.270894330094 + } + }, + "mae_ms": 124.19937765168956, + "mape": 0.20982454623810715, + "matched_successful_requests": 3584, + "signed_error_ms_mean": 93.2018552795005 + }, + "request_slo_classification": { + "legacy_step_1s_2s": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "1098" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1799", + "3056", + "6611", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "3379", + "6611", + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "4323", + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "2505" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "6611", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1799", + "6611", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "1799", + "3379", + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1799", + "1935", + "4163", + "4323", + "8612" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1799", + "3056", + "6611", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "3379", + "6611", + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "4323", + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "2505" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "6611", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "1799", + "6611", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "1799", + "3379", + "6611", + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "1799", + "1935", + "4163", + "4323", + "8612" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "14359", + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1799", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1799", + "1818", + "2979", + "7422", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1799", + "1818", + "1935", + "8179" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "7271", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1799", + "1818", + "3056", + "6611", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "15377", + "1799", + "3379", + "3844", + "6611", + "6905", + "6911", + "8179", + "8224" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.84375, + "matched_requests": 64, + "matching_labels": 54, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "14359", + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1799", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1799", + "1818", + "2979", + "7422", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1799", + "1818", + "1935", + "8179" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "7271", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "1799", + "1818", + "3056", + "6611", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "15377", + "1799", + "3379", + "3844", + "6611", + "6905", + "6911", + "8179", + "8224" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.84375, + "matched_requests": 64, + "matching_labels": 54, + "offered_request_rate": 1.5 + } + ], + "failure_set_intersection": 581, + "failure_set_jaccard": 0.7926330150068213, + "failure_set_union": 733, + "false_slo_fail_requests": 147, + "false_slo_pass_requests": 5, + "frontier_failure_requests": 728, + "label_accuracy": 0.9575892857142857, + "matched_requests": 3584, + "matching_labels": 3432, + "real_failure_requests": 586 + }, + "linear_10k": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "1098", + "8891" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7741" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "3379", + "6611", + "7981" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "4323", + "6611", + "6905", + "6911", + "9768" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.890625, + "matched_requests": 64, + "matching_labels": 57, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6604" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "7403", + "8891" + ], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "6604", + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "3056", + "6604", + "6611", + "7741" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "3379", + "6604", + "7981" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "6604", + "6905", + "6911", + "8612" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.90625, + "matched_requests": 64, + "matching_labels": 58, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7741" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "3379", + "6611", + "7981" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "4323", + "6611", + "6905", + "6911", + "9768" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.890625, + "matched_requests": 64, + "matching_labels": 57, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6604" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "7403", + "8891" + ], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "6604", + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "3056", + "6604", + "6611", + "7741" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "3379", + "6604", + "7981" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "6604", + "6905", + "8612" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "2505" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "2979", + "7271", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1935", + "3379", + "7271", + "7981", + "8179" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "14359", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "2396", + "3056", + "6611", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "15069", + "2396", + "3379", + "6611", + "7271", + "7981", + "8179", + "8224" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.859375, + "matched_requests": 64, + "matching_labels": 55, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "2505" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "2979", + "7271", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1935", + "3379", + "7271", + "7981", + "8179" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "14359", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "2396", + "3056", + "6611", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "15069", + "2396", + "3379", + "6611", + "7271", + "7981", + "8179", + "8224" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.859375, + "matched_requests": 64, + "matching_labels": 55, + "offered_request_rate": 1.5 + } + ], + "failure_set_intersection": 415, + "failure_set_jaccard": 0.7167530224525043, + "failure_set_union": 579, + "false_slo_fail_requests": 135, + "false_slo_pass_requests": 29, + "frontier_failure_requests": 550, + "label_accuracy": 0.9542410714285714, + "matched_requests": 3584, + "matching_labels": 3420, + "real_failure_requests": 444 + }, + "linear_6k": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "1098" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "4323", + "6905", + "6911", + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.90625, + "matched_requests": 64, + "matching_labels": 58, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "3056", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "15377", + "1935", + "4163", + "6611", + "6905", + "6911", + "9768" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.890625, + "matched_requests": 64, + "matching_labels": 57, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "4323", + "6905", + "6911", + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.90625, + "matched_requests": 64, + "matching_labels": 58, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "3056", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "15377", + "1935", + "4163", + "6611", + "6905", + "6911", + "9768" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.890625, + "matched_requests": 64, + "matching_labels": 57, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "14316" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "2979", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7422", + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14094" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "2505" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7422", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "3379", + "7981", + "8179", + "8224" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "2913", + "2979", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7422", + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14094" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "2505" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7422", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "3379", + "7981", + "8179", + "8224" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.5 + } + ], + "failure_set_intersection": 306, + "failure_set_jaccard": 0.7906976744186046, + "failure_set_union": 387, + "false_slo_fail_requests": 78, + "false_slo_pass_requests": 3, + "frontier_failure_requests": 384, + "label_accuracy": 0.9773995535714286, + "matched_requests": 3584, + "matching_labels": 3503, + "real_failure_requests": 309 + }, + "linear_8k_primary": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "1098" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "14026", + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7741" + ], + "false_slo_pass_request_ids": [ + "14026" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "3379", + "7981" + ], + "false_slo_pass_request_ids": [ + "14026" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "4323", + "6611", + "6905", + "6911" + ], + "false_slo_pass_request_ids": [ + "14026" + ], + "label_accuracy": 0.890625, + "matched_requests": 64, + "matching_labels": 57, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [ + "2396" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "3056", + "7271", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "6611", + "7271", + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "6611", + "6905", + "6911", + "7271", + "9768" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.890625, + "matched_requests": 64, + "matching_labels": 57, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "14026", + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7741" + ], + "false_slo_pass_request_ids": [ + "14026" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "3379", + "7981" + ], + "false_slo_pass_request_ids": [ + "14026" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1935", + "4163", + "4323", + "6611", + "6905", + "6911" + ], + "false_slo_pass_request_ids": [ + "14026" + ], + "label_accuracy": 0.890625, + "matched_requests": 64, + "matching_labels": 57, + "offered_request_rate": 1.5 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [ + "2396" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "3056", + "7271", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.75 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "6611", + "7271", + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 1.0 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "15377", + "1935", + "4163", + "6611", + "6905", + "6911", + "7271", + "9768" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.875, + "matched_requests": 64, + "matching_labels": 56, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "14094" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "2979", + "7422", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "3056", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "3379", + "7981", + "8179", + "8224" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "14094" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "2979", + "7422", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7981" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 1.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "3056", + "7741" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 1.0 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "15069", + "3379", + "7981", + "8179", + "8224" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 1.5 + } + ], + "failure_set_intersection": 355, + "failure_set_jaccard": 0.7569296375266524, + "failure_set_union": 469, + "false_slo_fail_requests": 103, + "false_slo_pass_requests": 11, + "frontier_failure_requests": 458, + "label_accuracy": 0.9681919642857143, + "matched_requests": 3584, + "matching_labels": 3470, + "real_failure_requests": 366 + } + }, + "schema": "frontier-community-qwen235b-rank-comparison-v1", + "variants": { + "legacy_step_1s_2s": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625, + 0.84375, + 0.8125, + 0.734375, + 0.6875, + 0.515625 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.8125, + 0.8125, + 0.734375, + 0.671875, + 0.5 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625, + 0.84375, + 0.8125, + 0.734375, + 0.6875, + 0.515625 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.8125, + 0.8125, + 0.734375, + 0.671875, + 0.5 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.921875, + 0.890625, + 0.84375, + 0.828125, + 0.765625, + 0.6875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625, + 0.875, + 0.828125, + 0.828125, + 0.734375, + 0.578125 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.921875, + 0.890625, + 0.84375, + 0.828125, + 0.765625, + 0.6875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625, + 0.875, + 0.828125, + 0.828125, + 0.734375, + 0.578125 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "frontier_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "real_ranking": [ + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.875, + 0.84375, + 0.796875, + 0.734375, + 0.578125 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.90625, + 0.890625, + 0.859375, + 0.84375, + 0.78125, + 0.734375, + 0.578125 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625, + 0.875, + 0.84375, + 0.796875, + 0.734375, + 0.578125 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.90625, + 0.890625, + 0.859375, + 0.84375, + 0.78125, + 0.75, + 0.578125 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 128, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.921875, + 0.921875, + 0.875, + 0.859375, + 0.84375, + 0.75 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.921875, + 0.921875, + 0.875, + 0.859375, + 0.8125, + 0.734375 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 64, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.921875, + 0.921875, + 0.875, + 0.859375, + 0.84375, + 0.75 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.921875, + 0.921875, + 0.875, + 0.859375, + 0.8125, + 0.734375 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "frontier_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "within_topology_raw_capacity": { + "tp4": { + "frontier_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "tp8": { + "frontier_capacity": { + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + } + } + }, + "linear_10k": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.921875, + 0.890625, + 0.84375, + 0.75 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.953125, + 0.890625, + 0.890625, + 0.796875, + 0.6875 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.921875, + 0.890625, + 0.84375, + 0.75 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.953125, + 0.890625, + 0.890625, + 0.796875, + 0.6875 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375, + 0.859375, + 0.84375, + 0.796875, + 0.734375, + 0.5625 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375, + 0.859375, + 0.84375, + 0.796875, + 0.734375, + 0.5625 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.90625, + 0.828125, + 0.828125, + 0.765625, + 0.703125, + 0.53125 + ], + "rank": 7, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.90625, + 0.828125, + 0.828125, + 0.765625, + 0.703125, + 0.53125 + ], + "rank": 7, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.0375, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": 0.0375, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": 0.04375, + "tp8_mns128_mbt8192": 0.04375, + "tp8_mns64_mbt16384": 0.04375, + "tp8_mns64_mbt8192": 0.04375 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": 0.04375, + "tp8_mns128_mbt8192": 0.04375, + "tp8_mns64_mbt16384": 0.04375, + "tp8_mns64_mbt8192": 0.04375 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.9428090415820635, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "top1_set_match": true + }, + "real_ranking": [ + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 128, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.9375, + 0.921875, + 0.890625, + 0.828125 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.9375, + 0.90625, + 0.875, + 0.828125 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 64, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.9375, + 0.921875, + 0.890625, + 0.828125 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.9375, + 0.90625, + 0.875, + 0.828125 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.90625, + 0.875, + 0.84375, + 0.8125, + 0.765625, + 0.671875 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.90625, + 0.875, + 0.84375, + 0.8125, + 0.75, + 0.625 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.921875, + 0.875, + 0.84375, + 0.8125, + 0.765625, + 0.671875 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.90625, + 0.875, + 0.84375, + 0.8125, + 0.75, + 0.609375 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.15, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": 0.15, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.9428090415820635, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "top1_set_match": true + }, + "within_topology_raw_capacity": { + "tp4": { + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.15, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": 0.15, + "tp4_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "tp8": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "top1_set_match": true + } + } + }, + "linear_6k": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.9375, + 0.90625, + 0.859375, + 0.8125, + 0.640625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.921875, + 0.90625, + 0.84375, + 0.796875, + 0.625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.9375, + 0.90625, + 0.859375, + 0.8125, + 0.640625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.921875, + 0.90625, + 0.84375, + 0.796875, + 0.625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.96875, + 0.9375, + 0.921875, + 0.875, + 0.8125 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.953125, + 0.921875, + 0.90625, + 0.859375, + 0.75 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.96875, + 0.9375, + 0.921875, + 0.875, + 0.8125 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.953125, + 0.921875, + 0.90625, + 0.859375, + 0.75 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.0625, + "tp4_mns128_mbt8192": 0.0625, + "tp4_mns64_mbt16384": 0.0625, + "tp4_mns64_mbt8192": 0.0625, + "tp8_mns128_mbt16384": 0.04375, + "tp8_mns128_mbt8192": 0.04375, + "tp8_mns64_mbt16384": 0.04375, + "tp8_mns64_mbt8192": 0.04375 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 12, + "correct_pairs": 12, + "frontier_tied_pairs": 12, + "real_tied_pairs": 13 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.0625, + "tp4_mns128_mbt8192": 0.0625, + "tp4_mns64_mbt16384": 0.0625, + "tp4_mns64_mbt8192": 0.0625, + "tp8_mns128_mbt16384": 0.0625, + "tp8_mns128_mbt8192": 0.04375, + "tp8_mns64_mbt16384": 0.04375, + "tp8_mns64_mbt8192": 0.04375 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192", + "tp8_mns128_mbt16384" + ], + "spearman_rank_correlation": 0.7745966692414834, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "top1_set_match": false + }, + "real_ranking": [ + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.953125, + 0.9375, + 0.921875, + 0.875, + 0.828125, + 0.734375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.953125, + 0.9375, + 0.921875, + 0.875, + 0.828125, + 0.734375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.9375, + 0.921875, + 0.875, + 0.828125, + 0.734375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.953125, + 0.9375, + 0.921875, + 0.875, + 0.828125, + 0.734375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 128, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.5, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.96875, + 0.953125, + 0.921875, + 0.90625, + 0.84375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.96875, + 0.9375, + 0.921875, + 0.890625, + 0.828125 + ], + "rank": 6, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 64, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.96875, + 0.9375, + 0.921875, + 0.921875, + 0.84375 + ], + "rank": 6, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.96875, + 0.9375, + 0.921875, + 0.890625, + 0.828125 + ], + "rank": 6, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.25, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.25, + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 16, + "correct_pairs": 16, + "frontier_tied_pairs": 12, + "real_tied_pairs": 9 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.25, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.25, + "tp8_mns128_mbt16384": 0.5, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384" + ], + "spearman_rank_correlation": 0.9561828874675149, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.30000000000000004, + "top1_set_intersection": [ + "tp8_mns128_mbt16384" + ], + "top1_set_match": false + }, + "within_topology_raw_capacity": { + "tp4": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.25, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.25 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.25, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.25 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "top1_set_match": true + }, + "tp8": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 3 + }, + "rankable": true, + "real_capacity": { + "tp8_mns128_mbt16384": 0.5, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.30000000000000004, + "top1_set_intersection": [ + "tp8_mns128_mbt16384" + ], + "top1_set_match": false + } + } + }, + "linear_8k_primary": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.890625, + 0.859375, + 0.828125, + 0.765625, + 0.609375 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.890625, + 0.859375, + 0.828125, + 0.765625, + 0.609375 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.9375, + 0.90625, + 0.859375, + 0.8125 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.953125, + 0.921875, + 0.90625, + 0.84375, + 0.75 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.9375, + 0.90625, + 0.859375, + 0.8125 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.953125, + 0.921875, + 0.90625, + 0.84375, + 0.75 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.9375, + 0.859375, + 0.859375, + 0.796875, + 0.734375, + 0.578125 + ], + "rank": 7, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.9375, + 0.859375, + 0.859375, + 0.796875, + 0.734375, + 0.578125 + ], + "rank": 7, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.0625, + "tp4_mns128_mbt8192": 0.0375, + "tp4_mns64_mbt16384": 0.0625, + "tp4_mns64_mbt8192": 0.0375, + "tp8_mns128_mbt16384": 0.04375, + "tp8_mns128_mbt8192": 0.04375, + "tp8_mns64_mbt16384": 0.04375, + "tp8_mns64_mbt8192": 0.04375 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 20, + "correct_pairs": 20, + "frontier_tied_pairs": 8, + "real_tied_pairs": 8 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.0625, + "tp4_mns128_mbt8192": 0.0375, + "tp4_mns64_mbt16384": 0.0625, + "tp4_mns64_mbt8192": 0.0375, + "tp8_mns128_mbt16384": 0.04375, + "tp8_mns128_mbt8192": 0.04375, + "tp8_mns64_mbt16384": 0.04375, + "tp8_mns64_mbt8192": 0.04375 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "spearman_rank_correlation": 1.0, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "top1_set_match": true + }, + "real_ranking": [ + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.953125, + 0.921875, + 0.90625, + 0.84375, + 0.78125, + 0.6875 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.921875, + 0.90625, + 0.84375, + 0.78125, + 0.6875 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 128, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.96875, + 0.9375, + 0.921875, + 0.90625, + 0.828125 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.9375, + 0.921875, + 0.875, + 0.828125 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 64, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.96875, + 0.9375, + 0.921875, + 0.90625, + 0.828125 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.9375, + 0.921875, + 0.875, + 0.828125 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.9375, + 0.90625, + 0.875, + 0.84375, + 0.796875, + 0.6875 + ], + "rank": 7, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.9375, + 0.90625, + 0.875, + 0.84375, + 0.796875, + 0.703125 + ], + "rank": 7, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.25, + 0.35, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.15, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.15, + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 20, + "correct_pairs": 20, + "frontier_tied_pairs": 8, + "real_tied_pairs": 8 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.15, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.15, + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 1.0, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "top1_set_match": true + }, + "within_topology_raw_capacity": { + "tp4": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.15, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.15 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 4, + "correct_pairs": 4, + "frontier_tied_pairs": 2, + "real_tied_pairs": 2 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.15, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.15 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "spearman_rank_correlation": 1.0, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "top1_set_match": true + }, + "tp8": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "top1_set_match": true + } + } + } + } +} diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v1_v2_trial_stability.json b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v1_v2_trial_stability.json new file mode 100644 index 0000000..b64a7d5 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v1_v2_trial_stability.json @@ -0,0 +1,1038 @@ +{ + "created_unix_s": 1784150483.6645162, + "inputs": { + "first_community_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/community_ranking_frozen.json", + "sha256": "055fe2cf1e0c01a0503145a272097b8ceda257f1b86bdae39abf17277d9a18d9" + }, + "frontier_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/frontier-v4-refined-rates-batched-par4/frontier_ranking_frozen.json", + "sha256": "318b3c7742c8371191e2dbd429f0e0e63c1a42f4806a599e018b0548cb9d6336" + }, + "repeat_community_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/community_ranking_frozen.json", + "sha256": "5a9c59fc4082db47bec7b59432e9ebe9f907f70baa95a9f0d3480e85f8992ff2" + } + }, + "primary_original_grid_capacity_stability": { + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.0625, + "tp4_mns128_mbt8192": 0.0375, + "tp4_mns64_mbt16384": 0.0625, + "tp4_mns64_mbt8192": 0.0375, + "tp8_mns128_mbt16384": 0.04375, + "tp8_mns128_mbt8192": 0.04375, + "tp8_mns64_mbt16384": 0.04375, + "tp8_mns64_mbt8192": 0.04375 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 20, + "correct_pairs": 20, + "frontier_tied_pairs": 8, + "real_tied_pairs": 8 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.0625, + "tp4_mns128_mbt8192": 0.0375, + "tp4_mns64_mbt16384": 0.0625, + "tp4_mns64_mbt8192": 0.0375, + "tp8_mns128_mbt16384": 0.04375, + "tp8_mns128_mbt8192": 0.04375, + "tp8_mns64_mbt16384": 0.04375, + "tp8_mns64_mbt8192": 0.04375 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "spearman_rank_correlation": 1.0, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "top1_set_match": true + }, + "records": [ + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp4_mns128_mbt16384", + "first_capacity": 0.25, + "first_next_tested_rate": 0.35, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp4_mns128_mbt8192", + "first_capacity": 0.15, + "first_next_tested_rate": 0.25, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp4_mns64_mbt16384", + "first_capacity": 0.25, + "first_next_tested_rate": 0.35, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp4_mns64_mbt8192", + "first_capacity": 0.15, + "first_next_tested_rate": 0.25, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp8_mns128_mbt16384", + "first_capacity": 0.35, + "first_next_tested_rate": 0.5, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp8_mns128_mbt8192", + "first_capacity": 0.35, + "first_next_tested_rate": 0.5, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp8_mns64_mbt16384", + "first_capacity": 0.35, + "first_next_tested_rate": 0.5, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp8_mns64_mbt8192", + "first_capacity": 0.35, + "first_next_tested_rate": 0.5, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.15, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.15, + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 20, + "correct_pairs": 20, + "frontier_tied_pairs": 8, + "real_tied_pairs": 8 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.15, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.15, + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 1.0, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "top1_set_match": true + }, + "stable_configs": 8, + "total_configs": 8 + }, + "real_trial_ttft_ms": { + "absolute_error_p95": 57.944677013438195, + "mae": 14.052099088554602, + "paired_successful_requests": 1216, + "repeat_minus_first_mean": 1.7543344055363712 + }, + "schema": "community-qwen235b-trial-stability-v1", + "shared_config_load_points": 19, + "shared_requests": 1216, + "variants": { + "legacy_step_1s_2s": { + "config_load_records": [ + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 57, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 54, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.15, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 57, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 57 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 55, + "frontier_feasible": false, + "frontier_passed_request_count": 52, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 55 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 54, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.15, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 57, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 57 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 55, + "frontier_feasible": false, + "frontier_passed_request_count": 52, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 55 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 54, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 55, + "frontier_feasible": false, + "frontier_passed_request_count": 53, + "offered_request_rate": 0.75, + "repeat_feasible": false, + "repeat_passed_request_count": 55 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 56, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 53, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp8_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp8_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 54, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 56, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 53, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + } + ], + "frontier_correct_first": 1177, + "frontier_correct_repeat": 1178, + "frontier_load_label_accuracy_first": 1.0, + "frontier_load_label_accuracy_repeat": 1.0, + "frontier_request_label_accuracy_first": 0.9679276315789473, + "frontier_request_label_accuracy_repeat": 0.96875, + "frontier_wrong_both_stable_real_label": 38, + "matching_real_trial_labels": 1215, + "real_trial_failure_set_jaccard": 0.9925373134328358, + "real_trial_label_flips": 1, + "real_trial_load_label_accuracy": 1.0, + "real_trial_request_label_accuracy": 0.9991776315789473, + "shared_config_load_points": 19, + "shared_requests": 1216 + }, + "linear_10k": { + "config_load_records": [ + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 55, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.15, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 53, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 55, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.15, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 53, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.75, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 61, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns64_mbt16384", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 61, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + } + ], + "frontier_correct_first": 1173, + "frontier_correct_repeat": 1174, + "frontier_load_label_accuracy_first": 1.0, + "frontier_load_label_accuracy_repeat": 1.0, + "frontier_request_label_accuracy_first": 0.9646381578947368, + "frontier_request_label_accuracy_repeat": 0.9654605263157895, + "frontier_wrong_both_stable_real_label": 42, + "matching_real_trial_labels": 1215, + "real_trial_failure_set_jaccard": 0.9891304347826086, + "real_trial_label_flips": 1, + "real_trial_load_label_accuracy": 1.0, + "real_trial_request_label_accuracy": 0.9991776315789473, + "shared_config_load_points": 19, + "shared_requests": 1216 + }, + "linear_6k": { + "config_load_records": [ + { + "config": "tp4_mns128_mbt16384", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.25, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 63, + "frontier_feasible": true, + "frontier_passed_request_count": 63, + "offered_request_rate": 0.15, + "repeat_feasible": true, + "repeat_passed_request_count": 63 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.25, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.25, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 63, + "frontier_feasible": true, + "frontier_passed_request_count": 63, + "offered_request_rate": 0.15, + "repeat_feasible": true, + "repeat_passed_request_count": 63 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.25, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.75, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 61, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns64_mbt16384", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 61, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + } + ], + "frontier_correct_first": 1206, + "frontier_correct_repeat": 1208, + "frontier_load_label_accuracy_first": 0.9473684210526315, + "frontier_load_label_accuracy_repeat": 1.0, + "frontier_request_label_accuracy_first": 0.9917763157894737, + "frontier_request_label_accuracy_repeat": 0.993421052631579, + "frontier_wrong_both_stable_real_label": 8, + "matching_real_trial_labels": 1214, + "real_trial_failure_set_jaccard": 0.9655172413793104, + "real_trial_label_flips": 2, + "real_trial_load_label_accuracy": 0.9473684210526315, + "real_trial_request_label_accuracy": 0.9983552631578947, + "shared_config_load_points": 19, + "shared_requests": 1216 + }, + "linear_8k_primary": { + "config_load_records": [ + { + "config": "tp4_mns128_mbt16384", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.25, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.15, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 55, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.25, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.15, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 55, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.75, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 61, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns64_mbt16384", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 61, + "offered_request_rate": 0.35, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + } + ], + "frontier_correct_first": 1192, + "frontier_correct_repeat": 1193, + "frontier_load_label_accuracy_first": 1.0, + "frontier_load_label_accuracy_repeat": 1.0, + "frontier_request_label_accuracy_first": 0.9802631578947368, + "frontier_request_label_accuracy_repeat": 0.9810855263157895, + "frontier_wrong_both_stable_real_label": 23, + "matching_real_trial_labels": 1215, + "real_trial_failure_set_jaccard": 0.9852941176470589, + "real_trial_label_flips": 1, + "real_trial_load_label_accuracy": 1.0, + "real_trial_request_label_accuracy": 0.9991776315789473, + "shared_config_load_points": 19, + "shared_requests": 1216 + } + } +} diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_refined_comparison.json b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_refined_comparison.json new file mode 100644 index 0000000..9574186 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_refined_comparison.json @@ -0,0 +1,5248 @@ +{ + "community_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/community_ranking_frozen.json", + "sha256": "5a9c59fc4082db47bec7b59432e9ebe9f907f70baa95a9f0d3480e85f8992ff2" + }, + "created_unix_s": 1784150358.538842, + "frontier_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/frontier-v4-refined-rates-batched-par4/frontier_ranking_frozen.json", + "sha256": "318b3c7742c8371191e2dbd429f0e0e63c1a42f4806a599e018b0548cb9d6336" + }, + "pass_rate_surface_residuals": { + "legacy_step_1s_2s": { + "false_feasible_points": 0, + "false_infeasible_points": 0, + "feasibility_label_accuracy": 1.0, + "mae": 0.028409090909090908, + "matched_config_load_points": 33, + "max_absolute_error": 0.046875, + "signed_error_mean": -0.026515151515151516 + }, + "linear_10k": { + "false_feasible_points": 0, + "false_infeasible_points": 8, + "feasibility_label_accuracy": 0.7575757575757576, + "mae": 0.022727272727272728, + "matched_config_load_points": 33, + "max_absolute_error": 0.046875, + "signed_error_mean": -0.020833333333333332 + }, + "linear_6k": { + "false_feasible_points": 0, + "false_infeasible_points": 2, + "feasibility_label_accuracy": 0.9393939393939394, + "mae": 0.006628787878787879, + "matched_config_load_points": 33, + "max_absolute_error": 0.015625, + "signed_error_mean": -0.004734848484848485 + }, + "linear_8k_primary": { + "false_feasible_points": 0, + "false_infeasible_points": 6, + "feasibility_label_accuracy": 0.8181818181818182, + "mae": 0.01278409090909091, + "matched_config_load_points": 33, + "max_absolute_error": 0.046875, + "signed_error_mean": -0.01278409090909091 + } + }, + "primary_slo": "linear_8k_primary", + "request_level_ttft_residuals": { + "by_input_bin": { + "[0,1024)": { + "count": 891, + "mae_ms": 40.73985496630227, + "signed_error_ms_mean": -3.610411376010397 + }, + "[1024,2048)": { + "count": 693, + "mae_ms": 57.62684905459171, + "signed_error_ms_mean": 44.446875608364664 + }, + "[16384,32769)": { + "count": 132, + "mae_ms": 240.89426466203582, + "signed_error_ms_mean": 147.35987190047473 + }, + "[2048,4096)": { + "count": 132, + "mae_ms": 109.72593289069287, + "signed_error_ms_mean": 91.2075693704322 + }, + "[4096,8192)": { + "count": 99, + "mae_ms": 140.80238805214807, + "signed_error_ms_mean": 138.7995225585974 + }, + "[8192,16384)": { + "count": 165, + "mae_ms": 237.3733609970516, + "signed_error_ms_mean": 237.3733609970516 + } + }, + "mae_ms": 83.15460427483131, + "mape": 0.1633638442645967, + "matched_successful_requests": 2112, + "signed_error_ms_mean": 53.022475287000866 + }, + "request_slo_classification": { + "legacy_step_1s_2s": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "2505" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14316", + "1818", + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "6611", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "2505" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14316", + "1818", + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "6611", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "14359", + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1799", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "7271", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14316", + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14316", + "14359", + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "14359", + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "1818", + "7271", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14316", + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14316", + "14359", + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + } + ], + "failure_set_intersection": 225, + "failure_set_jaccard": 0.7785467128027682, + "failure_set_union": 289, + "false_slo_fail_requests": 60, + "false_slo_pass_requests": 4, + "frontier_failure_requests": 285, + "label_accuracy": 0.9696969696969697, + "matched_requests": 2112, + "matching_labels": 2048, + "real_failure_requests": 229 + }, + "linear_10k": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7271", + "7403", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6604" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "7403", + "8891" + ], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "6604", + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7271", + "7403", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6604" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "7403", + "8891" + ], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "6604", + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "14316", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "14359", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "14316", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "14359", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + } + ], + "failure_set_intersection": 129, + "failure_set_jaccard": 0.6172248803827751, + "failure_set_union": 209, + "false_slo_fail_requests": 62, + "false_slo_pass_requests": 18, + "frontier_failure_requests": 191, + "label_accuracy": 0.9621212121212122, + "matched_requests": 2112, + "matching_labels": 2032, + "real_failure_requests": 147 + }, + "linear_6k": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14316" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14316" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + } + ], + "failure_set_intersection": 91, + "failure_set_jaccard": 0.8666666666666667, + "failure_set_union": 105, + "false_slo_fail_requests": 12, + "false_slo_pass_requests": 2, + "frontier_failure_requests": 103, + "label_accuracy": 0.9933712121212122, + "matched_requests": 2112, + "matching_labels": 2098, + "real_failure_requests": 93 + }, + "linear_8k_primary": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [ + "2396" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [ + "2396" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.15 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7271" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.75 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14316" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.35 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.4 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14316" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + } + ], + "failure_set_intersection": 107, + "failure_set_jaccard": 0.7535211267605634, + "failure_set_union": 142, + "false_slo_fail_requests": 31, + "false_slo_pass_requests": 4, + "frontier_failure_requests": 138, + "label_accuracy": 0.9834280303030303, + "matched_requests": 2112, + "matching_labels": 2077, + "real_failure_requests": 111 + } + }, + "schema": "frontier-community-qwen235b-rank-comparison-v1", + "variants": { + "legacy_step_1s_2s": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625, + 0.90625, + 0.875, + 0.84375, + 0.8125, + 0.8125, + 0.8125, + 0.734375, + 0.6875, + 0.515625 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.890625, + 0.84375, + 0.8125, + 0.8125, + 0.8125, + 0.8125, + 0.734375, + 0.671875, + 0.5 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625, + 0.90625, + 0.875, + 0.84375, + 0.8125, + 0.8125, + 0.8125, + 0.734375, + 0.6875, + 0.515625 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.890625, + 0.84375, + 0.8125, + 0.8125, + 0.8125, + 0.8125, + 0.734375, + 0.671875, + 0.5 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.9375, + 0.921875, + 0.921875, + 0.890625, + 0.875, + 0.859375, + 0.84375, + 0.828125, + 0.765625, + 0.6875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.921875, + 0.90625, + 0.90625, + 0.875, + 0.859375, + 0.84375, + 0.828125, + 0.828125, + 0.734375, + 0.578125 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.9375, + 0.921875, + 0.921875, + 0.890625, + 0.875, + 0.859375, + 0.84375, + 0.828125, + 0.765625, + 0.6875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.921875, + 0.90625, + 0.90625, + 0.875, + 0.859375, + 0.84375, + 0.828125, + 0.828125, + 0.734375, + 0.578125 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "frontier_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "real_ranking": [ + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.90625, + 0.90625, + 0.875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.90625, + 0.90625, + 0.890625, + 0.890625, + 0.859375 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.90625, + 0.90625, + 0.875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.90625, + 0.90625, + 0.890625, + 0.890625, + 0.859375 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 128, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.875, + 0.875, + 0.859375 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5, + 0.75 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.890625, + 0.875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 64, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.875, + 0.875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.890625, + 0.875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "frontier_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "within_topology_raw_capacity": { + "tp4": { + "frontier_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "tp8": { + "frontier_capacity": { + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + } + } + }, + "linear_10k": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.9375, + 0.9375, + 0.921875, + 0.890625, + 0.84375, + 0.75 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.9375, + 0.90625, + 0.890625, + 0.890625, + 0.796875, + 0.6875 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.9375, + 0.9375, + 0.921875, + 0.890625, + 0.84375, + 0.75 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.9375, + 0.90625, + 0.890625, + 0.890625, + 0.796875, + 0.6875 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375, + 0.9375, + 0.875, + 0.859375, + 0.84375, + 0.84375, + 0.84375, + 0.796875, + 0.734375, + 0.5625 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375, + 0.9375, + 0.875, + 0.859375, + 0.84375, + 0.84375, + 0.84375, + 0.796875, + 0.734375, + 0.5625 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.90625, + 0.90625, + 0.859375, + 0.828125, + 0.828125, + 0.828125, + 0.828125, + 0.765625, + 0.703125, + 0.53125 + ], + "rank": 7, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.90625, + 0.90625, + 0.859375, + 0.828125, + 0.828125, + 0.828125, + 0.828125, + 0.765625, + 0.703125, + 0.53125 + ], + "rank": 7, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.0375, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": 0.0375, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": 0.04375, + "tp8_mns128_mbt8192": 0.04375, + "tp8_mns64_mbt16384": 0.04375, + "tp8_mns64_mbt8192": 0.04375 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": 0.05625, + "tp8_mns128_mbt8192": 0.05625, + "tp8_mns64_mbt16384": 0.05625, + "tp8_mns64_mbt8192": 0.05625 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.9428090415820635, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "top1_set_match": true + }, + "real_ranking": [ + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 128, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5, + 0.75 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.96875, + 0.953125, + 0.9375 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 64, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125, + 0.953125, + 0.9375 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.96875, + 0.953125, + 0.9375 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.921875, + 0.875 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.921875, + 0.90625, + 0.890625, + 0.875 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.921875, + 0.875 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.921875, + 0.90625, + 0.890625, + 0.875 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.15, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": 0.15, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.9428090415820635, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "top1_set_match": true + }, + "within_topology_raw_capacity": { + "tp4": { + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.15, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": 0.15, + "tp4_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "tp8": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "top1_set_match": true + } + } + }, + "linear_6k": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.9375, + 0.9375, + 0.90625, + 0.90625, + 0.859375, + 0.8125, + 0.640625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.921875, + 0.90625, + 0.90625, + 0.90625, + 0.84375, + 0.796875, + 0.625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.9375, + 0.9375, + 0.90625, + 0.90625, + 0.859375, + 0.8125, + 0.640625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.921875, + 0.90625, + 0.90625, + 0.90625, + 0.84375, + 0.796875, + 0.625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.875, + 0.8125 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.875, + 0.8125 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.4, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.90625, + 0.859375, + 0.75 + ], + "rank": 7, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.4, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.90625, + 0.859375, + 0.75 + ], + "rank": 7, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.075, + "tp4_mns128_mbt8192": 0.075, + "tp4_mns64_mbt16384": 0.075, + "tp4_mns64_mbt8192": 0.075, + "tp8_mns128_mbt16384": 0.05625, + "tp8_mns128_mbt8192": 0.05, + "tp8_mns64_mbt16384": 0.05625, + "tp8_mns64_mbt8192": 0.05 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 16, + "correct_pairs": 16, + "frontier_tied_pairs": 8, + "real_tied_pairs": 12 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.075, + "tp4_mns128_mbt8192": 0.075, + "tp4_mns64_mbt16384": 0.075, + "tp4_mns64_mbt8192": 0.075, + "tp8_mns128_mbt16384": 0.05625, + "tp8_mns128_mbt8192": 0.05625, + "tp8_mns64_mbt16384": 0.05625, + "tp8_mns64_mbt8192": 0.05625 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.9428090415820635, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "top1_set_match": true + }, + "real_ranking": [ + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.96875, + 0.9375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.953125, + 0.953125, + 0.9375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.96875, + 0.9375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.953125, + 0.953125, + 0.9375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 128, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5, + 0.75 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.96875, + 0.953125, + 0.9375 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 64, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125, + 0.953125, + 0.9375 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.96875, + 0.953125, + 0.9375 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.3, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.3, + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.4, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.4 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 16, + "correct_pairs": 16, + "frontier_tied_pairs": 8, + "real_tied_pairs": 12 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.3, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.3, + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.9428090415820635, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "top1_set_match": false + }, + "within_topology_raw_capacity": { + "tp4": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.3, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.3 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.3, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.3 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "top1_set_match": true + }, + "tp8": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.4, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.4 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 2, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "top1_set_match": false + } + } + }, + "linear_8k_primary": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.921875, + 0.890625, + 0.890625, + 0.875, + 0.859375, + 0.828125, + 0.765625, + 0.609375 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.921875, + 0.890625, + 0.890625, + 0.875, + 0.859375, + 0.828125, + 0.765625, + 0.609375 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.90625, + 0.859375, + 0.8125 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.90625, + 0.859375, + 0.8125 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.4, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.90625, + 0.84375, + 0.75 + ], + "rank": 5, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.4, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.90625, + 0.84375, + 0.75 + ], + "rank": 5, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.9375, + 0.9375, + 0.90625, + 0.859375, + 0.859375, + 0.859375, + 0.859375, + 0.796875, + 0.734375, + 0.578125 + ], + "rank": 7, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.9375, + 0.9375, + 0.90625, + 0.859375, + 0.859375, + 0.859375, + 0.859375, + 0.796875, + 0.734375, + 0.578125 + ], + "rank": 7, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.0625, + "tp4_mns128_mbt8192": 0.0375, + "tp4_mns64_mbt16384": 0.0625, + "tp4_mns64_mbt8192": 0.0375, + "tp8_mns128_mbt16384": 0.05625, + "tp8_mns128_mbt8192": 0.05, + "tp8_mns64_mbt16384": 0.05625, + "tp8_mns64_mbt8192": 0.05 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 20, + "correct_pairs": 20, + "frontier_tied_pairs": 4, + "real_tied_pairs": 8 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.075, + "tp4_mns128_mbt8192": 0.05, + "tp4_mns64_mbt16384": 0.075, + "tp4_mns64_mbt8192": 0.05, + "tp8_mns128_mbt16384": 0.05625, + "tp8_mns128_mbt8192": 0.05625, + "tp8_mns64_mbt16384": 0.05625, + "tp8_mns64_mbt8192": 0.05625 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "spearman_rank_correlation": 0.9486832980505138, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "top1_set_match": true + }, + "real_ranking": [ + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125, + 0.921875 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125, + 0.921875 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 128, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5, + 0.75 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.96875, + 0.953125, + 0.9375 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 64, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125, + 0.953125, + 0.9375 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.96875, + 0.953125, + 0.9375 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.2, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125, + 0.9375, + 0.921875, + 0.90625 + ], + "rank": 7, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.2, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125, + 0.9375, + 0.921875, + 0.90625 + ], + "rank": 7, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.15, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.15, + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.4, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.4 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 20, + "correct_pairs": 20, + "frontier_tied_pairs": 4, + "real_tied_pairs": 8 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.2, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.2, + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.9486832980505138, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "top1_set_match": false + }, + "within_topology_raw_capacity": { + "tp4": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.15, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.15 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 4, + "correct_pairs": 4, + "frontier_tied_pairs": 2, + "real_tied_pairs": 2 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.2, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.2 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "spearman_rank_correlation": 1.0, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "top1_set_match": true + }, + "tp8": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.4, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.4 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 2, + "real_tied_pairs": 6 + }, + "rankable": true, + "real_capacity": { + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "top1_set_match": false + } + } + } + } +} diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_run_manifest.json b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_run_manifest.json new file mode 100644 index 0000000..189ada2 --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_run_manifest.json @@ -0,0 +1,583 @@ +{ + "configs": [ + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 128, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/runs/tp8_mns128_mbt16384/engine.log", + "execution_index": 1, + "rate_order": [ + 0.75, + 0.5, + 0.35, + 0.45, + 0.4 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/results/tp8_mns128_mbt16384.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp8/study.json" + }, + { + "config": { + "expert_parallel": true, + "mbt": 16384, + "mns": 64, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/runs/tp8_mns64_mbt16384/engine.log", + "execution_index": 2, + "rate_order": [ + 0.5, + 0.35, + 0.45, + 0.4 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/results/tp8_mns64_mbt16384.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp8/study.json" + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/runs/tp4_mns64_mbt16384/engine.log", + "execution_index": 3, + "rate_order": [ + 0.35, + 0.25, + 0.3 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/results/tp4_mns64_mbt16384.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp4/study.json" + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/runs/tp8_mns64_mbt8192/engine.log", + "execution_index": 4, + "rate_order": [ + 0.5, + 0.35, + 0.45, + 0.4 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/results/tp8_mns64_mbt8192.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp8/study.json" + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/runs/tp8_mns128_mbt8192/engine.log", + "execution_index": 5, + "rate_order": [ + 0.5, + 0.35, + 0.45, + 0.4 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/results/tp8_mns128_mbt8192.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp8/study.json" + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/runs/tp4_mns128_mbt8192/engine.log", + "execution_index": 6, + "rate_order": [ + 0.15, + 0.35, + 0.25, + 0.3, + 0.2 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/results/tp4_mns128_mbt8192.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp4/study.json" + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/runs/tp4_mns64_mbt8192/engine.log", + "execution_index": 7, + "rate_order": [ + 0.25, + 0.35, + 0.15, + 0.3, + 0.2 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/results/tp4_mns64_mbt8192.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp4/study.json" + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/runs/tp4_mns128_mbt16384/engine.log", + "execution_index": 8, + "rate_order": [ + 0.35, + 0.25, + 0.3 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/results/tp4_mns128_mbt16384.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp4/study.json" + } + ], + "created_unix_s": 1784142580.9845052, + "execution_contract": { + "config_order": "reverse_of_first_pass", + "estimated_replay_wall_seconds_excluding_model_loads": 6639.746031746032, + "fallback_when_no_transition": "top_two_if_all_feasible_else_bottom_two", + "one_fresh_server_launch_per_config": true, + "rate_order": "reverse_first_pass_boundary_order_then_descending_refinement_rates", + "schema": "community-qwen235b-boundary-repeat-selection-v4", + "selection": "all_adjacent_pre_registered_slo_feasibility_transitions_plus_between_rate_refinement", + "selections": [ + { + "added_refinement_rates": [ + 0.4, + 0.45 + ], + "boundary_rates": [ + 0.35, + 0.5, + 0.75 + ], + "boundary_rates_by_slo": { + "legacy_step_1s_2s": [], + "linear_10k": [ + 0.35, + 0.5 + ], + "linear_6k": [ + 0.5, + 0.75 + ], + "linear_8k_primary": [ + 0.35, + 0.5 + ] + }, + "config": "tp8_mns128_mbt16384", + "excluded_unrankable_slos": [ + "legacy_step_1s_2s" + ], + "repeat_order": [ + 0.75, + 0.5, + 0.35, + 0.45, + 0.4 + ], + "selected_rates": [ + 0.35, + 0.4, + 0.45, + 0.5, + 0.75 + ], + "source_relative_order": [ + 0.35, + 0.5, + 0.75 + ] + }, + { + "added_refinement_rates": [ + 0.4, + 0.45 + ], + "boundary_rates": [ + 0.35, + 0.5 + ], + "boundary_rates_by_slo": { + "legacy_step_1s_2s": [], + "linear_10k": [ + 0.35, + 0.5 + ], + "linear_6k": [ + 0.35, + 0.5 + ], + "linear_8k_primary": [ + 0.35, + 0.5 + ] + }, + "config": "tp8_mns64_mbt16384", + "excluded_unrankable_slos": [ + "legacy_step_1s_2s" + ], + "repeat_order": [ + 0.5, + 0.35, + 0.45, + 0.4 + ], + "selected_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "source_relative_order": [ + 0.35, + 0.5 + ] + }, + { + "added_refinement_rates": [ + 0.3 + ], + "boundary_rates": [ + 0.25, + 0.35 + ], + "boundary_rates_by_slo": { + "legacy_step_1s_2s": [], + "linear_10k": [], + "linear_6k": [ + 0.25, + 0.35 + ], + "linear_8k_primary": [ + 0.25, + 0.35 + ] + }, + "config": "tp4_mns64_mbt16384", + "excluded_unrankable_slos": [ + "linear_10k", + "legacy_step_1s_2s" + ], + "repeat_order": [ + 0.35, + 0.25, + 0.3 + ], + "selected_rates": [ + 0.25, + 0.3, + 0.35 + ], + "source_relative_order": [ + 0.25, + 0.35 + ] + }, + { + "added_refinement_rates": [ + 0.4, + 0.45 + ], + "boundary_rates": [ + 0.35, + 0.5 + ], + "boundary_rates_by_slo": { + "legacy_step_1s_2s": [], + "linear_10k": [ + 0.35, + 0.5 + ], + "linear_6k": [ + 0.35, + 0.5 + ], + "linear_8k_primary": [ + 0.35, + 0.5 + ] + }, + "config": "tp8_mns64_mbt8192", + "excluded_unrankable_slos": [ + "legacy_step_1s_2s" + ], + "repeat_order": [ + 0.5, + 0.35, + 0.45, + 0.4 + ], + "selected_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "source_relative_order": [ + 0.35, + 0.5 + ] + }, + { + "added_refinement_rates": [ + 0.4, + 0.45 + ], + "boundary_rates": [ + 0.35, + 0.5 + ], + "boundary_rates_by_slo": { + "legacy_step_1s_2s": [], + "linear_10k": [ + 0.35, + 0.5 + ], + "linear_6k": [ + 0.35, + 0.5 + ], + "linear_8k_primary": [ + 0.35, + 0.5 + ] + }, + "config": "tp8_mns128_mbt8192", + "excluded_unrankable_slos": [ + "legacy_step_1s_2s" + ], + "repeat_order": [ + 0.5, + 0.35, + 0.45, + 0.4 + ], + "selected_rates": [ + 0.35, + 0.4, + 0.45, + 0.5 + ], + "source_relative_order": [ + 0.35, + 0.5 + ] + }, + { + "added_refinement_rates": [ + 0.2, + 0.3 + ], + "boundary_rates": [ + 0.15, + 0.25, + 0.35 + ], + "boundary_rates_by_slo": { + "legacy_step_1s_2s": [], + "linear_10k": [], + "linear_6k": [ + 0.25, + 0.35 + ], + "linear_8k_primary": [ + 0.15, + 0.25 + ] + }, + "config": "tp4_mns128_mbt8192", + "excluded_unrankable_slos": [ + "linear_10k", + "legacy_step_1s_2s" + ], + "repeat_order": [ + 0.15, + 0.35, + 0.25, + 0.3, + 0.2 + ], + "selected_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35 + ], + "source_relative_order": [ + 0.25, + 0.35, + 0.15 + ] + }, + { + "added_refinement_rates": [ + 0.2, + 0.3 + ], + "boundary_rates": [ + 0.15, + 0.25, + 0.35 + ], + "boundary_rates_by_slo": { + "legacy_step_1s_2s": [], + "linear_10k": [], + "linear_6k": [ + 0.25, + 0.35 + ], + "linear_8k_primary": [ + 0.15, + 0.25 + ] + }, + "config": "tp4_mns64_mbt8192", + "excluded_unrankable_slos": [ + "linear_10k", + "legacy_step_1s_2s" + ], + "repeat_order": [ + 0.25, + 0.35, + 0.15, + 0.3, + 0.2 + ], + "selected_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35 + ], + "source_relative_order": [ + 0.15, + 0.35, + 0.25 + ] + }, + { + "added_refinement_rates": [ + 0.3 + ], + "boundary_rates": [ + 0.25, + 0.35 + ], + "boundary_rates_by_slo": { + "legacy_step_1s_2s": [], + "linear_10k": [], + "linear_6k": [ + 0.25, + 0.35 + ], + "linear_8k_primary": [ + 0.25, + 0.35 + ] + }, + "config": "tp4_mns128_mbt16384", + "excluded_unrankable_slos": [ + "linear_10k", + "legacy_step_1s_2s" + ], + "repeat_order": [ + 0.35, + 0.25, + 0.3 + ], + "selected_rates": [ + 0.25, + 0.3, + 0.35 + ], + "source_relative_order": [ + 0.25, + 0.35 + ] + } + ], + "unrankable_slo_policy": "exclude_per_config_surfaces_with_no_feasible_load", + "warmup": { + "latencies_discarded": true, + "method": "one_disjoint_request_per_input_length_bin", + "request_count": 6, + "seed": 2026071504 + } + }, + "frontier_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/frontier-v4-refined-rates-batched-par4/frontier_ranking_frozen.json", + "sha256": "318b3c7742c8371191e2dbd429f0e0e63c1a42f4806a599e018b0548cb9d6336" + }, + "model": { + "config_sha256": "702c46d431bb984db9035a1225186bbfdb52c0d19c82104df4a37cd005e0369e", + "path": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8" + }, + "protocol_manifest": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/protocol_manifest.json", + "sha256": "695c0500b9a5d51cdd4e39b722e3c9db15f8bed48795454f4d0d6442d24929cb" + }, + "repeat_of": { + "first_pass_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/community_ranking_frozen.json", + "sha256": "055fe2cf1e0c01a0503145a272097b8ceda257f1b86bdae39abf17277d9a18d9" + }, + "source_manifest": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/run_manifest.json", + "sha256": "6da81a34140d801cb6055e089bed7a12adb14c09dc87ca6dfa7f79df4ac34920" + } + }, + "repository": { + "commit": "501ceb817122747a23dc6d9fdc9d71dcb3aa8e72", + "status_porcelain": [] + }, + "runtime": { + "python": "/usr/bin/python3.12", + "vllm": "/tmp/wjh-frontier-vllm0102-smoke/.venv/bin/vllm" + }, + "schema": "community-qwen235b-prefill-fixed-cohort-v1", + "studies": { + "4": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp4/study.json", + "sha256": "ea0d7d31e4a5102cf530e900754debb1d8b18f860748ead88ebf7de57d92faa3" + }, + "8": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp8/study.json", + "sha256": "48c4d917b5531f742a05bab3bcd5c880f2eaddf9992b7629e9f48d8e00fd1906" + } + } +} diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_v3_trial_stability.json b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_v3_trial_stability.json new file mode 100644 index 0000000..94ea5de --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_v3_trial_stability.json @@ -0,0 +1,732 @@ +{ + "created_unix_s": 1784153851.3812342, + "inputs": { + "first_community_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/community_ranking_frozen.json", + "sha256": "5a9c59fc4082db47bec7b59432e9ebe9f907f70baa95a9f0d3480e85f8992ff2" + }, + "frontier_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/frontier-v4-refined-rates-batched-par4/frontier_ranking_frozen.json", + "sha256": "318b3c7742c8371191e2dbd429f0e0e63c1a42f4806a599e018b0548cb9d6336" + }, + "repeat_community_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/community_ranking_frozen.json", + "sha256": "4edd25edde3d7508d8feb9b024b5bca4ba6a5f25335ac84008f09c524ed7bc34" + } + }, + "primary_original_grid_capacity_stability": { + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.075, + "tp4_mns128_mbt8192": 0.05, + "tp4_mns64_mbt16384": 0.075, + "tp4_mns64_mbt8192": 0.05, + "tp8_mns128_mbt16384": 0.05625, + "tp8_mns128_mbt8192": 0.05625, + "tp8_mns64_mbt16384": 0.05625, + "tp8_mns64_mbt8192": 0.05625 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 12, + "correct_pairs": 12, + "frontier_tied_pairs": 3, + "real_tied_pairs": 3 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.075, + "tp4_mns128_mbt8192": 0.05, + "tp4_mns64_mbt16384": 0.075, + "tp4_mns64_mbt8192": 0.05, + "tp8_mns128_mbt8192": 0.05625, + "tp8_mns64_mbt8192": 0.05625 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "spearman_rank_correlation": 1.0, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "top1_set_match": true + }, + "records": [ + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp4_mns128_mbt16384", + "first_capacity": 0.3, + "first_next_tested_rate": 0.35, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp4_mns128_mbt8192", + "first_capacity": 0.2, + "first_next_tested_rate": 0.25, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp4_mns64_mbt16384", + "first_capacity": 0.3, + "first_next_tested_rate": 0.35, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp4_mns64_mbt8192", + "first_capacity": 0.2, + "first_next_tested_rate": 0.25, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp8_mns128_mbt8192", + "first_capacity": 0.45, + "first_next_tested_rate": 0.5, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + }, + { + "boundary_repeated": true, + "capacity_still_feasible": true, + "config": "tp8_mns64_mbt8192", + "first_capacity": 0.45, + "first_next_tested_rate": 0.5, + "next_rate_still_infeasible": true, + "stable_original_grid_capacity": true + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.2, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.2, + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 12, + "correct_pairs": 12, + "frontier_tied_pairs": 3, + "real_tied_pairs": 3 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.2, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.2, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.9933992677987828, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "top1_set_match": false + }, + "stable_configs": 6, + "total_configs": 6 + }, + "real_trial_ttft_ms": { + "absolute_error_p95": 62.524845998268574, + "mae": 14.980625974885697, + "paired_successful_requests": 768, + "repeat_minus_first_mean": 1.5463623839574818 + }, + "schema": "community-qwen235b-trial-stability-v1", + "shared_config_load_points": 12, + "shared_requests": 768, + "variants": { + "legacy_step_1s_2s": { + "config_load_records": [ + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 56, + "offered_request_rate": 0.3, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 54, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.2, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 57, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 57 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 56, + "offered_request_rate": 0.3, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 54, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.2, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 57, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 57 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 57, + "frontier_feasible": false, + "frontier_passed_request_count": 54, + "offered_request_rate": 0.45, + "repeat_feasible": false, + "repeat_passed_request_count": 57 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 53, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 57, + "frontier_feasible": false, + "frontier_passed_request_count": 54, + "offered_request_rate": 0.45, + "repeat_feasible": false, + "repeat_passed_request_count": 57 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 53, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + } + ], + "frontier_correct_first": 742, + "frontier_correct_repeat": 742, + "frontier_load_label_accuracy_first": 1.0, + "frontier_load_label_accuracy_repeat": 1.0, + "frontier_request_label_accuracy_first": 0.9661458333333334, + "frontier_request_label_accuracy_repeat": 0.9661458333333334, + "frontier_wrong_both_stable_real_label": 26, + "matching_real_trial_labels": 768, + "real_trial_failure_set_jaccard": 1.0, + "real_trial_label_flips": 0, + "real_trial_load_label_accuracy": 1.0, + "real_trial_request_label_accuracy": 1.0, + "shared_config_load_points": 12, + "shared_requests": 768 + }, + "linear_10k": { + "config_load_records": [ + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 56, + "offered_request_rate": 0.3, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 55, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.2, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 56, + "offered_request_rate": 0.3, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 56, + "frontier_feasible": false, + "frontier_passed_request_count": 55, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 56 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.2, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 58, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 58 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.45, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 58, + "offered_request_rate": 0.45, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + } + ], + "frontier_correct_first": 726, + "frontier_correct_repeat": 726, + "frontier_load_label_accuracy_first": 0.8333333333333334, + "frontier_load_label_accuracy_repeat": 0.8333333333333334, + "frontier_request_label_accuracy_first": 0.9453125, + "frontier_request_label_accuracy_repeat": 0.9453125, + "frontier_wrong_both_stable_real_label": 42, + "matching_real_trial_labels": 768, + "real_trial_failure_set_jaccard": 1.0, + "real_trial_label_flips": 0, + "real_trial_load_label_accuracy": 1.0, + "real_trial_request_label_accuracy": 1.0, + "shared_config_load_points": 12, + "shared_requests": 768 + }, + "linear_6k": { + "config_load_records": [ + { + "config": "tp4_mns128_mbt16384", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 61, + "offered_request_rate": 0.3, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.2, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.25, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 61, + "offered_request_rate": 0.3, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 62, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.2, + "repeat_feasible": true, + "repeat_passed_request_count": 62 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": true, + "frontier_passed_request_count": 62, + "offered_request_rate": 0.25, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.45, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.45, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + } + ], + "frontier_correct_first": 760, + "frontier_correct_repeat": 760, + "frontier_load_label_accuracy_first": 0.8333333333333334, + "frontier_load_label_accuracy_repeat": 0.8333333333333334, + "frontier_request_label_accuracy_first": 0.9895833333333334, + "frontier_request_label_accuracy_repeat": 0.9895833333333334, + "frontier_wrong_both_stable_real_label": 8, + "matching_real_trial_labels": 768, + "real_trial_failure_set_jaccard": 1.0, + "real_trial_label_flips": 0, + "real_trial_load_label_accuracy": 1.0, + "real_trial_request_label_accuracy": 1.0, + "shared_config_load_points": 12, + "shared_requests": 768 + }, + "linear_8k_primary": { + "config_load_records": [ + { + "config": "tp4_mns128_mbt16384", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.3, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp4_mns128_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.2, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp4_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.3, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp4_mns64_mbt16384", + "first_feasible": false, + "first_passed_request_count": 59, + "frontier_feasible": false, + "frontier_passed_request_count": 57, + "offered_request_rate": 0.35, + "repeat_feasible": false, + "repeat_passed_request_count": 59 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.2, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp4_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.25, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.45, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp8_mns128_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": true, + "first_passed_request_count": 61, + "frontier_feasible": false, + "frontier_passed_request_count": 60, + "offered_request_rate": 0.45, + "repeat_feasible": true, + "repeat_passed_request_count": 61 + }, + { + "config": "tp8_mns64_mbt8192", + "first_feasible": false, + "first_passed_request_count": 60, + "frontier_feasible": false, + "frontier_passed_request_count": 59, + "offered_request_rate": 0.5, + "repeat_feasible": false, + "repeat_passed_request_count": 60 + } + ], + "frontier_correct_first": 750, + "frontier_correct_repeat": 750, + "frontier_load_label_accuracy_first": 0.5, + "frontier_load_label_accuracy_repeat": 0.5, + "frontier_request_label_accuracy_first": 0.9765625, + "frontier_request_label_accuracy_repeat": 0.9765625, + "frontier_wrong_both_stable_real_label": 18, + "matching_real_trial_labels": 768, + "real_trial_failure_set_jaccard": 1.0, + "real_trial_label_flips": 0, + "real_trial_load_label_accuracy": 1.0, + "real_trial_request_label_accuracy": 1.0, + "shared_config_load_points": 12, + "shared_requests": 768 + } + } +} diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v3_run_manifest.json b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v3_run_manifest.json new file mode 100644 index 0000000..99aab5f --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v3_run_manifest.json @@ -0,0 +1,345 @@ +{ + "configs": [ + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/runs/tp4_mns128_mbt16384/engine.log", + "execution_index": 1, + "rate_order": [ + 0.3, + 0.35 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/results/tp4_mns128_mbt16384.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp4/study.json" + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/runs/tp4_mns64_mbt8192/engine.log", + "execution_index": 2, + "rate_order": [ + 0.2, + 0.25 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/results/tp4_mns64_mbt8192.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp4/study.json" + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/runs/tp4_mns128_mbt8192/engine.log", + "execution_index": 3, + "rate_order": [ + 0.2, + 0.25 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/results/tp4_mns128_mbt8192.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp4/study.json" + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/runs/tp8_mns128_mbt8192/engine.log", + "execution_index": 4, + "rate_order": [ + 0.45, + 0.5 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/results/tp8_mns128_mbt8192.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp8/study.json" + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/runs/tp8_mns64_mbt8192/engine.log", + "execution_index": 5, + "rate_order": [ + 0.45, + 0.5 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/results/tp8_mns64_mbt8192.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp8/study.json" + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "engine_log_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/runs/tp4_mns64_mbt16384/engine.log", + "execution_index": 6, + "rate_order": [ + 0.3, + 0.35 + ], + "result_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/results/tp4_mns64_mbt16384.json", + "study_path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp4/study.json" + } + ], + "created_unix_s": 1784150504.0091515, + "execution_contract": { + "config_order": "reverse_of_first_pass", + "estimated_replay_wall_seconds_excluding_model_loads": 2484.8253968253966, + "fallback_when_no_transition": "top_two_if_all_feasible_else_bottom_two", + "one_fresh_server_launch_per_config": true, + "rate_order": "reverse_first_pass_boundary_order_then_descending_refinement_rates", + "schema": "community-qwen235b-boundary-repeat-selection-v4", + "selected_configs": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "selected_slos": [ + "linear_8k_primary" + ], + "selection": "all_adjacent_pre_registered_slo_feasibility_transitions_plus_between_rate_refinement", + "selections": [ + { + "added_refinement_rates": [], + "boundary_rates": [ + 0.3, + 0.35 + ], + "boundary_rates_by_slo": { + "linear_8k_primary": [ + 0.3, + 0.35 + ] + }, + "config": "tp4_mns128_mbt16384", + "excluded_unrankable_slos": [], + "repeat_order": [ + 0.3, + 0.35 + ], + "selected_rates": [ + 0.3, + 0.35 + ], + "source_relative_order": [ + 0.35, + 0.3 + ] + }, + { + "added_refinement_rates": [], + "boundary_rates": [ + 0.2, + 0.25 + ], + "boundary_rates_by_slo": { + "linear_8k_primary": [ + 0.2, + 0.25 + ] + }, + "config": "tp4_mns64_mbt8192", + "excluded_unrankable_slos": [], + "repeat_order": [ + 0.2, + 0.25 + ], + "selected_rates": [ + 0.2, + 0.25 + ], + "source_relative_order": [ + 0.25, + 0.2 + ] + }, + { + "added_refinement_rates": [], + "boundary_rates": [ + 0.2, + 0.25 + ], + "boundary_rates_by_slo": { + "linear_8k_primary": [ + 0.2, + 0.25 + ] + }, + "config": "tp4_mns128_mbt8192", + "excluded_unrankable_slos": [], + "repeat_order": [ + 0.2, + 0.25 + ], + "selected_rates": [ + 0.2, + 0.25 + ], + "source_relative_order": [ + 0.25, + 0.2 + ] + }, + { + "added_refinement_rates": [], + "boundary_rates": [ + 0.45, + 0.5 + ], + "boundary_rates_by_slo": { + "linear_8k_primary": [ + 0.45, + 0.5 + ] + }, + "config": "tp8_mns128_mbt8192", + "excluded_unrankable_slos": [], + "repeat_order": [ + 0.45, + 0.5 + ], + "selected_rates": [ + 0.45, + 0.5 + ], + "source_relative_order": [ + 0.5, + 0.45 + ] + }, + { + "added_refinement_rates": [], + "boundary_rates": [ + 0.45, + 0.5 + ], + "boundary_rates_by_slo": { + "linear_8k_primary": [ + 0.45, + 0.5 + ] + }, + "config": "tp8_mns64_mbt8192", + "excluded_unrankable_slos": [], + "repeat_order": [ + 0.45, + 0.5 + ], + "selected_rates": [ + 0.45, + 0.5 + ], + "source_relative_order": [ + 0.5, + 0.45 + ] + }, + { + "added_refinement_rates": [], + "boundary_rates": [ + 0.3, + 0.35 + ], + "boundary_rates_by_slo": { + "linear_8k_primary": [ + 0.3, + 0.35 + ] + }, + "config": "tp4_mns64_mbt16384", + "excluded_unrankable_slos": [], + "repeat_order": [ + 0.3, + 0.35 + ], + "selected_rates": [ + 0.3, + 0.35 + ], + "source_relative_order": [ + 0.35, + 0.3 + ] + } + ], + "unrankable_slo_policy": "exclude_per_config_surfaces_with_no_feasible_load", + "warmup": { + "latencies_discarded": true, + "method": "one_disjoint_request_per_input_length_bin", + "request_count": 6, + "seed": 2026071504 + } + }, + "frontier_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/frontier-v4-refined-rates-batched-par4/frontier_ranking_frozen.json", + "sha256": "318b3c7742c8371191e2dbd429f0e0e63c1a42f4806a599e018b0548cb9d6336" + }, + "model": { + "config_sha256": "702c46d431bb984db9035a1225186bbfdb52c0d19c82104df4a37cd005e0369e", + "path": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8" + }, + "protocol_manifest": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/protocol/protocol_manifest.json", + "sha256": "695c0500b9a5d51cdd4e39b722e3c9db15f8bed48795454f4d0d6442d24929cb" + }, + "repeat_of": { + "first_pass_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/community_ranking_frozen.json", + "sha256": "5a9c59fc4082db47bec7b59432e9ebe9f907f70baa95a9f0d3480e85f8992ff2" + }, + "source_manifest": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v2-boundary-reverse/run_manifest.json", + "sha256": "cf1a722043f42ff8f435e3521d0b2b2892a8ea0922e1e4870ec10fba26c0f998" + } + }, + "repository": { + "commit": "344af3a4285ccfdc6e10e540767808907ad81ddf", + "status_porcelain": [] + }, + "runtime": { + "python": "/usr/bin/python3.12", + "vllm": "/tmp/wjh-frontier-vllm0102-smoke/.venv/bin/vllm" + }, + "schema": "community-qwen235b-prefill-fixed-cohort-v1", + "studies": { + "4": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp4/study.json", + "sha256": "ea0d7d31e4a5102cf530e900754debb1d8b18f860748ead88ebf7de57d92faa3" + }, + "8": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v1-blind/studies/tp8/study.json", + "sha256": "48c4d917b5531f742a05bab3bcd5c880f2eaddf9992b7629e9f48d8e00fd1906" + } + } +} diff --git a/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v3_targeted_comparison.json b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v3_targeted_comparison.json new file mode 100644 index 0000000..485ca5e --- /dev/null +++ b/runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v3_targeted_comparison.json @@ -0,0 +1,3830 @@ +{ + "community_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/community-v3-primary-disputed/community_ranking_frozen.json", + "sha256": "4edd25edde3d7508d8feb9b024b5bca4ba6a5f25335ac84008f09c524ed7bc34" + }, + "created_unix_s": 1784153851.1556726, + "frontier_freeze": { + "path": "/home/admin/cpfs/wjh/frontier-community-qwen235-smoke-20260715/rankable-fixed-cohort-v1/refinement-v1/frontier-v4-refined-rates-batched-par4/frontier_ranking_frozen.json", + "sha256": "318b3c7742c8371191e2dbd429f0e0e63c1a42f4806a599e018b0548cb9d6336" + }, + "pass_rate_surface_residuals": { + "legacy_step_1s_2s": { + "false_feasible_points": 0, + "false_infeasible_points": 0, + "feasibility_label_accuracy": 1.0, + "mae": 0.028645833333333332, + "matched_config_load_points": 12, + "max_absolute_error": 0.046875, + "signed_error_mean": -0.028645833333333332 + }, + "linear_10k": { + "false_feasible_points": 0, + "false_infeasible_points": 2, + "feasibility_label_accuracy": 0.8333333333333334, + "mae": 0.028645833333333332, + "matched_config_load_points": 12, + "max_absolute_error": 0.046875, + "signed_error_mean": -0.028645833333333332 + }, + "linear_6k": { + "false_feasible_points": 0, + "false_infeasible_points": 2, + "feasibility_label_accuracy": 0.8333333333333334, + "mae": 0.010416666666666666, + "matched_config_load_points": 12, + "max_absolute_error": 0.015625, + "signed_error_mean": -0.005208333333333333 + }, + "linear_8k_primary": { + "false_feasible_points": 0, + "false_infeasible_points": 6, + "feasibility_label_accuracy": 0.5, + "mae": 0.018229166666666668, + "matched_config_load_points": 12, + "max_absolute_error": 0.03125, + "signed_error_mean": -0.018229166666666668 + } + }, + "primary_slo": "linear_8k_primary", + "request_level_ttft_residuals": { + "by_input_bin": { + "[0,1024)": { + "count": 324, + "mae_ms": 50.51891975082174, + "signed_error_ms_mean": -9.882795155125912 + }, + "[1024,2048)": { + "count": 252, + "mae_ms": 59.564509081632316, + "signed_error_ms_mean": 41.897250452193944 + }, + "[16384,32769)": { + "count": 48, + "mae_ms": 280.18396195154355, + "signed_error_ms_mean": 186.5034642541033 + }, + "[2048,4096)": { + "count": 48, + "mae_ms": 94.21679883383375, + "signed_error_ms_mean": 67.31957197829648 + }, + "[4096,8192)": { + "count": 36, + "mae_ms": 110.9597465520623, + "signed_error_ms_mean": 108.56515721844958 + }, + "[8192,16384)": { + "count": 60, + "mae_ms": 241.64142363249, + "signed_error_ms_mean": 241.64142363249 + } + }, + "mae_ms": 88.33679570229081, + "mape": 0.16159539870054299, + "matched_successful_requests": 768, + "signed_error_ms_mean": 49.40939882898548 + }, + "request_slo_classification": { + "legacy_step_1s_2s": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6611" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14316", + "14359", + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14316", + "14359", + "1818" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359", + "1818", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + } + ], + "failure_set_intersection": 82, + "failure_set_jaccard": 0.7592592592592593, + "failure_set_union": 108, + "false_slo_fail_requests": 24, + "false_slo_pass_requests": 2, + "frontier_failure_requests": 106, + "label_accuracy": 0.9661458333333334, + "matched_requests": 768, + "matching_labels": 742, + "real_failure_requests": 84 + }, + "linear_10k": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7271", + "7403", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "7403", + "8891" + ], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7271", + "7403", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.921875, + "matched_requests": 64, + "matching_labels": 59, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "6611", + "7422" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "8891" + ], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "6604", + "6611" + ], + "false_slo_pass_request_ids": [ + "7403", + "8891" + ], + "label_accuracy": 0.9375, + "matched_requests": 64, + "matching_labels": 60, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "14316", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "14359", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "14316", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026", + "14359", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.953125, + "matched_requests": 64, + "matching_labels": 61, + "offered_request_rate": 0.5 + } + ], + "failure_set_intersection": 52, + "failure_set_jaccard": 0.5531914893617021, + "failure_set_union": 94, + "false_slo_fail_requests": 32, + "false_slo_pass_requests": 10, + "frontier_failure_requests": 84, + "label_accuracy": 0.9453125, + "matched_requests": 768, + "matching_labels": 726, + "real_failure_requests": 62 + }, + "linear_6k": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [], + "label_accuracy": 1.0, + "matched_requests": 64, + "matching_labels": 64, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14316" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14316" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + } + ], + "failure_set_intersection": 34, + "failure_set_jaccard": 0.8095238095238095, + "failure_set_union": 42, + "false_slo_fail_requests": 6, + "false_slo_pass_requests": 2, + "frontier_failure_requests": 40, + "label_accuracy": 0.9895833333333334, + "matched_requests": 768, + "matching_labels": 760, + "real_failure_requests": 36 + }, + "linear_8k_primary": { + "by_config_load": [ + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns128_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7403" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.3 + }, + { + "config": "tp4_mns64_mbt16384", + "false_slo_fail_request_ids": [ + "7271", + "7422" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.35 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.2 + }, + { + "config": "tp4_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14026" + ], + "false_slo_pass_request_ids": [ + "7403" + ], + "label_accuracy": 0.96875, + "matched_requests": 64, + "matching_labels": 62, + "offered_request_rate": 0.25 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14316" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns128_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14316" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.45 + }, + { + "config": "tp8_mns64_mbt8192", + "false_slo_fail_request_ids": [ + "14359" + ], + "false_slo_pass_request_ids": [], + "label_accuracy": 0.984375, + "matched_requests": 64, + "matching_labels": 63, + "offered_request_rate": 0.5 + } + ], + "failure_set_intersection": 42, + "failure_set_jaccard": 0.7, + "failure_set_union": 60, + "false_slo_fail_requests": 16, + "false_slo_pass_requests": 2, + "frontier_failure_requests": 58, + "label_accuracy": 0.9765625, + "matched_requests": 768, + "matching_labels": 750, + "real_failure_requests": 44 + } + }, + "schema": "frontier-community-qwen235b-rank-comparison-v1", + "variants": { + "legacy_step_1s_2s": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625, + 0.90625, + 0.875, + 0.84375, + 0.8125, + 0.8125, + 0.8125, + 0.734375, + 0.6875, + 0.515625 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.890625, + 0.84375, + 0.8125, + 0.8125, + 0.8125, + 0.8125, + 0.734375, + 0.671875, + 0.5 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625, + 0.90625, + 0.875, + 0.84375, + 0.8125, + 0.8125, + 0.8125, + 0.734375, + 0.6875, + 0.515625 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.890625, + 0.890625, + 0.84375, + 0.8125, + 0.8125, + 0.8125, + 0.8125, + 0.734375, + 0.671875, + 0.5 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.9375, + 0.921875, + 0.921875, + 0.890625, + 0.875, + 0.859375, + 0.84375, + 0.828125, + 0.765625, + 0.6875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.921875, + 0.90625, + 0.90625, + 0.875, + 0.859375, + 0.84375, + 0.828125, + 0.828125, + 0.734375, + 0.578125 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.9375, + 0.921875, + 0.921875, + 0.890625, + 0.875, + 0.859375, + 0.84375, + 0.828125, + 0.765625, + 0.6875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.921875, + 0.90625, + 0.90625, + 0.875, + 0.859375, + 0.84375, + 0.828125, + 0.828125, + 0.734375, + 0.578125 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "frontier_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "real_ranking": [ + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.90625, + 0.875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.90625, + 0.890625 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.2, + 0.25 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.90625, + 0.875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.90625, + 0.890625 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.2, + 0.25 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.890625, + 0.875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.890625, + 0.875 + ], + "rank": 1, + "slo": "legacy_step_1s_2s", + "tested_rates": [ + 0.45, + 0.5 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "frontier_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "within_topology_raw_capacity": { + "tp4": { + "frontier_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "tp8": { + "frontier_capacity": { + "tp8_mns128_mbt16384": null, + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt16384": null, + "tp8_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp8_mns128_mbt8192": null, + "tp8_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + } + } + }, + "linear_10k": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.9375, + 0.9375, + 0.921875, + 0.890625, + 0.84375, + 0.75 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.9375, + 0.90625, + 0.890625, + 0.890625, + 0.796875, + 0.6875 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.9375, + 0.9375, + 0.921875, + 0.890625, + 0.84375, + 0.75 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.35, + "maximum_tested_feasible_request_rate_per_gpu": 0.04375, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.9375, + 0.90625, + 0.890625, + 0.890625, + 0.796875, + 0.6875 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375, + 0.9375, + 0.875, + 0.859375, + 0.84375, + 0.84375, + 0.84375, + 0.796875, + 0.734375, + 0.5625 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375, + 0.9375, + 0.875, + 0.859375, + 0.84375, + 0.84375, + 0.84375, + 0.796875, + 0.734375, + 0.5625 + ], + "rank": 5, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.90625, + 0.90625, + 0.859375, + 0.828125, + 0.828125, + 0.828125, + 0.828125, + 0.765625, + 0.703125, + 0.53125 + ], + "rank": 7, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.9375, + 0.90625, + 0.90625, + 0.859375, + 0.828125, + 0.828125, + 0.828125, + 0.828125, + 0.765625, + 0.703125, + 0.53125 + ], + "rank": 7, + "slo": "linear_10k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.0375, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": 0.0375, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": 0.04375, + "tp8_mns128_mbt8192": 0.04375, + "tp8_mns64_mbt16384": 0.04375, + "tp8_mns64_mbt8192": 0.04375 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 1, + "real_tied_pairs": 1 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt8192": 0.05625, + "tp8_mns64_mbt8192": 0.05625 + }, + "real_top1_set": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.917662935482247, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "top1_set_match": false + }, + "real_ranking": [ + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375 + ], + "rank": 1, + "slo": "linear_10k", + "tested_rates": [ + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.875 + ], + "rank": 3, + "slo": "linear_10k", + "tested_rates": [ + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625 + ], + "rank": 3, + "slo": "linear_10k", + "tested_rates": [ + 0.2, + 0.25 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.875 + ], + "rank": 3, + "slo": "linear_10k", + "tested_rates": [ + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + false, + false + ], + "lower_censored": true, + "maximum_tested_feasible_request_rate": null, + "maximum_tested_feasible_request_rate_per_gpu": null, + "monotonicity_violations": [], + "pass_rates": [ + 0.921875, + 0.90625 + ], + "rank": 3, + "slo": "linear_10k", + "tested_rates": [ + 0.2, + 0.25 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.15, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": 0.15, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 1, + "real_tied_pairs": 1 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.917662935482247, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "top1_set_match": false + }, + "within_topology_raw_capacity": { + "tp4": { + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.15, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": 0.15, + "tp4_mns64_mbt8192": null + }, + "rankable": false, + "real_capacity": { + "tp4_mns128_mbt16384": null, + "tp4_mns128_mbt8192": null, + "tp4_mns64_mbt16384": null, + "tp4_mns64_mbt8192": null + }, + "reason": "one_or_both_surfaces_have_no_feasible_config" + }, + "tp8": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp8_mns128_mbt16384": 0.35, + "tp8_mns128_mbt8192": 0.35, + "tp8_mns64_mbt16384": 0.35, + "tp8_mns64_mbt8192": 0.35 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns128_mbt8192", + "tp8_mns64_mbt16384", + "tp8_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 1, + "real_tied_pairs": 1 + }, + "rankable": true, + "real_capacity": { + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "top1_set_match": false + } + } + }, + "linear_6k": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.9375, + 0.9375, + 0.90625, + 0.90625, + 0.859375, + 0.8125, + 0.640625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.921875, + 0.90625, + 0.90625, + 0.90625, + 0.84375, + 0.796875, + 0.625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.9375, + 0.9375, + 0.90625, + 0.90625, + 0.859375, + 0.8125, + 0.640625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + true, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.921875, + 0.90625, + 0.90625, + 0.90625, + 0.84375, + 0.796875, + 0.625 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.875, + 0.8125 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 1.0, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.875, + 0.8125 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.4, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.90625, + 0.859375, + 0.75 + ], + "rank": 7, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.4, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.90625, + 0.859375, + 0.75 + ], + "rank": 7, + "slo": "linear_6k", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.075, + "tp4_mns128_mbt8192": 0.075, + "tp4_mns64_mbt16384": 0.075, + "tp4_mns64_mbt8192": 0.075, + "tp8_mns128_mbt16384": 0.05625, + "tp8_mns128_mbt8192": 0.05, + "tp8_mns64_mbt16384": 0.05625, + "tp8_mns64_mbt8192": 0.05 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 8, + "correct_pairs": 8, + "frontier_tied_pairs": 7, + "real_tied_pairs": 3 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.075, + "tp4_mns128_mbt8192": 0.0625, + "tp4_mns64_mbt16384": 0.075, + "tp4_mns64_mbt8192": 0.0625, + "tp8_mns128_mbt8192": 0.05625, + "tp8_mns64_mbt8192": 0.05625 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "spearman_rank_correlation": 0.8660254037844387, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.16666666666666663, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "top1_set_match": false + }, + "real_ranking": [ + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.9375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.9375 + ], + "rank": 1, + "slo": "linear_6k", + "tested_rates": [ + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125 + ], + "rank": 3, + "slo": "linear_6k", + "tested_rates": [ + 0.2, + 0.25 + ], + "upper_censored": true + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.953125 + ], + "rank": 3, + "slo": "linear_6k", + "tested_rates": [ + 0.2, + 0.25 + ], + "upper_censored": true + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375 + ], + "rank": 5, + "slo": "linear_6k", + "tested_rates": [ + 0.45, + 0.5 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.3, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.3, + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.4, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.4 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 8, + "correct_pairs": 8, + "frontier_tied_pairs": 7, + "real_tied_pairs": 3 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.25, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.25, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 0.8660254037844387, + "top1_regret_fraction_best_tie_break": null, + "top1_regret_fraction_worst_tie_break": null, + "top1_set_intersection": [], + "top1_set_match": false + }, + "within_topology_raw_capacity": { + "tp4": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.3, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.3 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns128_mbt8192", + "tp4_mns64_mbt16384", + "tp4_mns64_mbt8192" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 6, + "real_tied_pairs": 2 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.25, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.25 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.16666666666666663, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "top1_set_match": false + }, + "tp8": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.4, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.4 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 1, + "real_tied_pairs": 1 + }, + "rankable": true, + "real_capacity": { + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": null, + "top1_regret_fraction_worst_tie_break": null, + "top1_set_intersection": [], + "top1_set_match": false + } + } + }, + "linear_8k_primary": { + "frontier_ranking": [ + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.921875, + 0.890625, + 0.890625, + 0.875, + 0.859375, + 0.828125, + 0.765625, + 0.609375 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + true, + true, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.25, + "maximum_tested_feasible_request_rate_per_gpu": 0.0625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.96875, + 0.96875, + 0.921875, + 0.890625, + 0.890625, + 0.875, + 0.859375, + 0.828125, + 0.765625, + 0.609375 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.90625, + 0.859375, + 0.8125 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 16384, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt16384", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + true, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.90625, + 0.859375, + 0.8125 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.4, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.90625, + 0.84375, + 0.75 + ], + "rank": 5, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 8, + "moe_tp": 1, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + true, + true, + true, + true, + true, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.4, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.984375, + 0.984375, + 0.96875, + 0.96875, + 0.953125, + 0.953125, + 0.9375, + 0.921875, + 0.90625, + 0.84375, + 0.75 + ], + "rank": 5, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 128, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.9375, + 0.9375, + 0.90625, + 0.859375, + 0.859375, + 0.859375, + 0.859375, + 0.796875, + 0.734375, + 0.578125 + ], + "rank": 7, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + }, + { + "config": { + "mbt": 8192, + "mns": 64, + "moe_ep": 1, + "moe_tp": 4, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.15, + "maximum_tested_feasible_request_rate_per_gpu": 0.0375, + "monotonicity_violations": [], + "pass_rates": [ + 0.96875, + 0.9375, + 0.9375, + 0.90625, + 0.859375, + 0.859375, + 0.859375, + 0.859375, + 0.796875, + 0.734375, + 0.578125 + ], + "rank": 7, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.15, + 0.2, + 0.25, + 0.3, + 0.35, + 0.4, + 0.45, + 0.5, + 0.75, + 1.0, + 1.5 + ], + "upper_censored": false + } + ], + "per_gpu_efficiency": { + "capacity_field": "maximum_tested_feasible_request_rate_per_gpu", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.0625, + "tp4_mns128_mbt8192": 0.0375, + "tp4_mns64_mbt16384": 0.0625, + "tp4_mns64_mbt8192": 0.0375, + "tp8_mns128_mbt16384": 0.05625, + "tp8_mns128_mbt8192": 0.05, + "tp8_mns64_mbt16384": 0.05625, + "tp8_mns64_mbt8192": 0.05 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 12, + "correct_pairs": 12, + "frontier_tied_pairs": 3, + "real_tied_pairs": 3 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.075, + "tp4_mns128_mbt8192": 0.05, + "tp4_mns64_mbt16384": 0.075, + "tp4_mns64_mbt8192": 0.05, + "tp8_mns128_mbt8192": 0.05625, + "tp8_mns64_mbt8192": 0.05625 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "spearman_rank_correlation": 0.9819805060619656, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "top1_set_match": true + }, + "real_ranking": [ + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 128, + "name": "tp4_mns128_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.921875 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 16384, + "mns": 64, + "name": "tp4_mns64_mbt16384", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.3, + "maximum_tested_feasible_request_rate_per_gpu": 0.075, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.921875 + ], + "rank": 1, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.3, + 0.35 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 128, + "name": "tp8_mns128_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": true, + "mbt": 8192, + "mns": 64, + "name": "tp8_mns64_mbt8192", + "num_gpu_blocks": 62351, + "tp": 8 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.45, + "maximum_tested_feasible_request_rate_per_gpu": 0.05625, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375 + ], + "rank": 3, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.45, + 0.5 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 128, + "name": "tp4_mns128_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.2, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375 + ], + "rank": 5, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.2, + 0.25 + ], + "upper_censored": false + }, + { + "config": { + "expert_parallel": false, + "mbt": 8192, + "mns": 64, + "name": "tp4_mns64_mbt8192", + "num_gpu_blocks": 26101, + "tp": 4 + }, + "feasibility": [ + true, + false + ], + "lower_censored": false, + "maximum_tested_feasible_request_rate": 0.2, + "maximum_tested_feasible_request_rate_per_gpu": 0.05, + "monotonicity_violations": [], + "pass_rates": [ + 0.953125, + 0.9375 + ], + "rank": 5, + "slo": "linear_8k_primary", + "tested_rates": [ + 0.2, + 0.25 + ], + "upper_censored": false + } + ], + "single_replica_raw_capacity": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.15, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.15, + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.4, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.4 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 12, + "correct_pairs": 12, + "frontier_tied_pairs": 3, + "real_tied_pairs": 3 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.2, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.2, + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": 1.0, + "top1_regret_fraction_best_tie_break": null, + "top1_regret_fraction_worst_tie_break": null, + "top1_set_intersection": [], + "top1_set_match": false + }, + "within_topology_raw_capacity": { + "tp4": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp4_mns128_mbt16384": 0.25, + "tp4_mns128_mbt8192": 0.15, + "tp4_mns64_mbt16384": 0.25, + "tp4_mns64_mbt8192": 0.15 + }, + "frontier_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": 1.0, + "comparable_non_tied_pairs": 4, + "correct_pairs": 4, + "frontier_tied_pairs": 2, + "real_tied_pairs": 2 + }, + "rankable": true, + "real_capacity": { + "tp4_mns128_mbt16384": 0.3, + "tp4_mns128_mbt8192": 0.2, + "tp4_mns64_mbt16384": 0.3, + "tp4_mns64_mbt8192": 0.2 + }, + "real_top1_set": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "spearman_rank_correlation": 1.0, + "top1_regret_fraction_best_tie_break": 0.0, + "top1_regret_fraction_worst_tie_break": 0.0, + "top1_set_intersection": [ + "tp4_mns128_mbt16384", + "tp4_mns64_mbt16384" + ], + "top1_set_match": true + }, + "tp8": { + "capacity_field": "maximum_tested_feasible_request_rate", + "frontier_capacity": { + "tp8_mns128_mbt16384": 0.45, + "tp8_mns128_mbt8192": 0.4, + "tp8_mns64_mbt16384": 0.45, + "tp8_mns64_mbt8192": 0.4 + }, + "frontier_top1_set": [ + "tp8_mns128_mbt16384", + "tp8_mns64_mbt16384" + ], + "pairwise_ordering": { + "accuracy": null, + "comparable_non_tied_pairs": 0, + "correct_pairs": 0, + "frontier_tied_pairs": 1, + "real_tied_pairs": 1 + }, + "rankable": true, + "real_capacity": { + "tp8_mns128_mbt8192": 0.45, + "tp8_mns64_mbt8192": 0.45 + }, + "real_top1_set": [ + "tp8_mns128_mbt8192", + "tp8_mns64_mbt8192" + ], + "spearman_rank_correlation": null, + "top1_regret_fraction_best_tie_break": null, + "top1_regret_fraction_worst_tie_break": null, + "top1_set_intersection": [], + "top1_set_match": false + } + } + } + } +} From e9a4be61530bfb1712e45c175130adec4dcc7ab5 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 21:00:27 +0800 Subject: [PATCH 068/148] Add vLLM 0.20 per-TP FlashAttention profile smoke --- .../experiment-card.md | 43 ++++ .../fleet.toml | 24 +++ .../jobs_smoke.toml | 15 ++ .../mock-profile-ablation.png | Bin 0 -> 208385 bytes .../plot_mock_ablation.py | 41 ++++ .../profile_vllm020_flashattn.py | 190 ++++++++++++++++++ .../run_flashattn_smoke.sh | 58 ++++++ 7 files changed, 371 insertions(+) create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/experiment-card.md create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/fleet.toml create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/mock-profile-ablation.png create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/plot_mock_ablation.py create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_flashattn.py create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/run_flashattn_smoke.sh diff --git a/runs/frontier-qwen30-vllm020-profile-v1/experiment-card.md b/runs/frontier-qwen30-vllm020-profile-v1/experiment-card.md new file mode 100644 index 0000000..a51365c --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/experiment-card.md @@ -0,0 +1,43 @@ +# 实验 EXP-SIMFID-Q30-P020：同栈 per-TP operator profile 能否恢复 Frontier ranking？ + +> **状态：** 运行中（用户已批准 5-step campaign） + +## Claim 与决策 + +- **Parent claim：** Frontier 在 Qwen3-30B-A3B 的 TP×MNS surface 上出现 25.63% selection regret，原因可能不是 simulator scheduler 本身，而是 execution profile 与真实 serving stack 不一致。 +- **目的：** 区分 profile provenance mismatch 与 simulator composition/schema mismatch。 +- **Competing hypotheses：** H1：换成与真实 serving 相同的 per-TP operator profiles 后，profile-only ranking 达到 regret ≤5% 且 Kendall τ-b ≥0.8；H2：即使 operator profile 同栈，Frontier 的 operator composition 或 mixed-state abstraction 仍无法保持 ranking。 +- **事前预测：** 若 H1 成立，新 profile-only top set 应从旧结果的 TP4 移到真实 top family TP2，且不需要 end-to-end scalar calibration；若 H2 成立，operator microbench 单项可对齐，但组成后的 config rank 仍错，残差应随 phase、batch state 或 TP action 系统变化。 +- **判定规则：** `P-020` profile-only regret ≤5% 且 τ-b ≥0.8 → 支持 H1；否则进入 stage/action-conditioned residual 分解，禁止用本 surface 的 E2E measurement 拟合 scale。 + +## Setup + +- **自变量：** execution profile root：`P-old`（历史 profile）与 `P-020`（本实验冻结 profile）。`P-cal` 历史 per-TP scalar calibration 只作上界参照。 +- **控制变量：** Frontier simulator code/commit、trace fixtures、KV capacity、SLO、12-cell config surface、random seed 与 analysis code保持完全相同。 +- **System context：** Qwen3-30B-A3B，BF16 weights/activations/KV；community vLLM 0.20.0；dash0 8×NVIDIA H20；TP∈{1,2,4}，DP=PP=EP=1；MBT=8192；MNS∈{8,16,32,64}；prefix caching、chunked prefill 与 async scheduling均启用。 +- **真实 runtime contract：** engine log 已确认 TP1/2/4 均为 FlashAttention 3 attention + FlashInfer CUTLASS unquantized MoE；TP2/4 使用 FlashInfer TRT-LLM all-reduce。旧 profile log 明确为 vLLM 0.11.1，attention backend 为 FlashInfer。 +- **Workload 或 trace：** 历史 ground truth `chat_w20260311_1000`，source 600 s window、replay time scale 0.1、input 0–8192、fixed output 128、prefix cache on。当前本地 recovered artifact 无法重建 exact source requests；本次只替换 frozen simulator profile，不重跑或改写 ground truth。 +- **Baselines：** old profile-only；new `P-020` profile-only；historical frozen per-TP calibrated。 +- **Metrics：** real/sim capacity per GPU、selected top set、worst selected-config regret、Kendall τ-b、informative-pair direction、逐 TP operator-time ratio、profile coverage/extrapolation ratio。 + +## 预期产物与 review + +- **预期数据：** TP1/2/4 attention、KV-cache update、linear、MoE、collective raw profiles；冻结 manifest/sha256；12-cell simulation outputs；三 baseline comparison table。 +- **Figure prototype：** `mock-profile-ablation.png`；x 为 12 configs，y 为 normalized capacity；series 为 real、old profile-only、new profile-only、calibrated。图中数值明确标为 mock，不进入结论。 +- **人工 review：** 已批准。用户明确要求推进 5 个步骤，并要求 smoke 通过后直接完整运行。 +- **Review 意见：** 不能把 TP–MNS 耦合拆成互不相关问题；profile 只冻结 execution counterfactual 的证据，最终仍以完整 config ranking 判断。 + +## 复现信息 + +- **Code：** AITuner 当前 branch `codex/fidelity-prefix-pilot-20260714`；Frontier canonical commit `d9cfeb6d8791fbf2f295dd9744c56a666171776e`；vLLM commit/tag `88d34c6409e9fb3c7b8ca0c04756f061d2099eb1` / `0.20.0`。 +- **Environment：** `/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1`；hardware/driver/package freeze 写入每个 profile artifact。 +- **产物路径：** remote `/home/admin/cpfs/wjh/frontier_qwen30_vllm020_profiles/`; local harvest `runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/`。 +- **已知 deviation：** 历史 real ground truth 来自 dash1；本次 profile 按用户要求在 dash0 生成。driver 也已从历史 570.133.20 漂移到 dash0 当前 580.95.05。结果只能归因于“profile stack alignment 在当前 dash0 上的效果”，不能声称严格复现旧 dash1 execution latency。 + +## 结果 + +- **观察事实：** 待完整 profile 与 simulation。 +- **异常：** Frontier 原生 attention profiler 只实现 `FLASHINFER`/`NO_OP`，不能调用真实 serving 的 `FLASH_ATTN`；其 MoE wrapper声明并硬编码 vLLM 0.10.x API。vLLM 0.20.0 自带的 attention benchmark可调用实际 FlashAttention backend，但 mixed prefill+decode 在 FA3 中是一个 fused varlen call，而 Frontier schema将其拆成 `attn_prefill` 与 `attn_decode` 两项。 +- **Interpretation 与剩余 alternatives：** 这既是 profiler compatibility gap，也是潜在 representational gap。先用真实 kernel smoke 判断能否无损物化 Frontier profile；不能无损时同时保留 fused total 与 schema projection，避免误把 projection error 当 kernel error。 +- **Claim update：** unchanged。 +- **下一步：** 通过 vLLM 0.20 exact-kernel smoke；冻结 trace-derived shape support；完整 TP1/2/4 profiling；profile-only rerun。 diff --git a/runs/frontier-qwen30-vllm020-profile-v1/fleet.toml b/runs/frontier-qwen30-vllm020-profile-v1/fleet.toml new file mode 100644 index 0000000..e560549 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/fleet.toml @@ -0,0 +1,24 @@ +version = 1 + +[paths] +state_dir = "runs/frontier-qwen30-vllm020-profile-v1/fleet-state" +artifacts_dir = "runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts" + +[ssh] +connect_timeout_sec = 10 + +[scheduler] +gpu_free_memory_mb = 1024 +gpu_free_utilization_pct = 10 +prefer_pack = true + +[sync] +mode = "scp" +local_path = "runs/frontier-qwen30-vllm020-profile-v1" + +[[hosts]] +name = "dash0" +ssh_alias = "dash0" +enabled = true +sync_remote_path = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet" +fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-qwen30-vllm020-profile-v1" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml new file mode 100644 index 0000000..eee4f5a --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml @@ -0,0 +1,15 @@ +version = 1 + +[[jobs]] +name = "qwen30-vllm020-flashattn-smoke-20260716-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 900 bash run_flashattn_smoke.sh" +artifacts = ["artifacts/flashattn-smoke"] + +[jobs.env] +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/flashattn-smoke" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" +MODEL = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/mock-profile-ablation.png b/runs/frontier-qwen30-vllm020-profile-v1/mock-profile-ablation.png new file mode 100644 index 0000000000000000000000000000000000000000..0457cd24648e13cbbdb468478e89fe6c226bf385 GIT binary patch literal 208385 zcmd43c|6u#+dldgp`xgSWN4Pktb{^I63R?wnWqp^GDIqwnv9t$Lo$^@Ql?Om6lF}N zl$pvr?ql8e^Se#cdd1v=W!m#aju|KYR8yYZd^%G6!UQ<1r3T? z;X_e$>lx|sFBNKcw%`vbx1)M)nrE%t&YQVfQmSTd&JJhY9Bj?`JS<($*`9S06BH2^ zJg|q)#?8(7oRpA|<9~mL;8|B|p&g11`goNU&Pw{{D2gkG{GY~n2;4MiPSkk|5z z8)|pg(`s2_7@PVO%C~ugfPlaTkAQaWX4An|`^=XwO`Fae=^7PreaL%Kc3Qfrly2Rr zZ7DsAi=F$=Tbo)%Z&nyzxR`Lk!(-&YuLQ4O8xra+g(cB|vdgp#H)CUC z<5ds4x{9veB#+wHCkCE8dlrz8!1JNDwyU$#&#C=oSGeq=oU`+j`h6A*v|nZz zepUvv2lu+WyFWem>!ZVjk&cQA6JC4sEv2!Nr3In3VVR5LjGCz`@(Z&QW|o%A4<0PHUTZUuJXY985zOlfvX<6 z_EtNeKmYb)yfiBpSH|=+_?N-ONyIe=VAH*`3)zqw-8fmW?9bxg_znb^Zo&XO9#zSuG?aw8x}arljaF*_cA5oh8z85%I?~=V%-jv($+lt&5Zuva)11= zpkrXzDk;gaH2-%EP8|c~IX^8Tpp*G1xAwx{pJp?Edanlsy?<^v%Y6v@+E#G(T2Ro6 z9jdWcc4?-THaF|_I8JZCnPtO<*5!V~?=DM=3l2Hf_}z7Gy1O#ib7Fw4`tGhf!~6a! zYiX^w`WP34J?9e<5o!4I{osS>hLV&%)1FK0fT5$GFf zFS>?>qNS801Of-XX8byP?RxRLYioF`@*Uf#Wb2!|wH}>X^tfKEk6r3^Z*FftsGH!u zN~(X+*~MihRg`6IZ$-G9iC*!g1I8sj%wAqzfwC(}06*{#`Zs3B-+cP~SzcLr@tvH)qhnhw3vEF~Jtqxqp{wdF17yC$2A)%q4i@m+XoxXAG+qdufjT`0I)nHnN z)tzsytzl$jlq>QWe&Wy~8p11UY}vMJ*Dj9zPMaDt^|$)yJhxGQZreaFVfV!^B7$x0 z+O;g3HifKW7cA}Q;9ap|MRVb~ozL^~o=wzTl4lY2nfJ^zEY$dxfR}xEGy8rJYP1>nx**%`OII(iR(`EU1XKbFf0^peQq0&YgtoXU%wWM=D9E{p_X_k6g%5f6D1UV&}$#o zIzounR837SdjHvk)0W(uHfae)@%-9(GC{${hTp-_Q9Ck`HB{WYrR#hqX_!N8oUdAo$p&%#c zH?1Ep<^Czx^2A*M?Ug@&{+ynl5i>5n6m)0j$!^5i^mzZes&L-$$J!ZocbA7=cSSms z(ByNL*L|#&e%AG#jEs!W_0lzJMMo8uv*s9)Xj70~P47dp_UAQk%F1YLZEZal=L=7B zh9=AJ?X8Krh9zo9JEJfx)q@d9a*`;|+%aVxW zs;ZSw%-&RVZaR1FT%4rKGUkoq?{vI4Q<9TQo0{0=CUd>H4_(q@n$3OwyyEl2W7?C$ ziwkplB|Zn4n3&cJo>WqjYs+_Z`Sp>Tl6!x9`%|mhjnD0y)XZiOjGOuRsDy-svERQx zzgQ})tgaSl5tnqlkG%s;IXO?}>r+m+;1%X)2XiF7CLOXT0{*MUUBAx1H7hfig)G0f?6>xI8o3WP z$sceeE0VU^a%*d7Xne}h6%y&oFeq@Eo*OM;qxYuiZ`l6v-u^W;k$cz4a!J^&H-3FF z<;gsj0IbzuBthw%YjzAr6YKhAb|%ymZfqF0(>t^|cr6 zr#frz?U(djd{s5v%Xa?!c|4KN!t4tq+xjG-bSd|N&JQtSdif5VX&1KrH&}Qe^B?Fi zd!K2OB+Kd3roy03)>f$PyjwZQh&IoE%LEW)<)#lbdQ&O^f=2$&o;_=Id~`&r_F+VXcK)(zi?sH0E-vrYO}@Xn5HK;=L`&hFp4vAF zHn+5-8JQ zGP4|(-}%pv<9P)}>rfZ;^6c0)irY}DR;`MTj!we)ix#(Ci~5wM!v&bwedy1hKN_b; z0Dh2WwjtLo!*ccw43t+@(c@DbzPHa&`22{WjPIgh@5}D)?k5NpYz7Uf0r&53W@2Vm zPgUV`8*F4AY|1(2=H`}q5l?5%dwHDc9zyRZf{lFWzP`S9-X-$#^5z#TFXwZ%shl|R z=4b7_u7QD-TwGinTwDZ&p#t!jo16a_xO(M^DT>*n(|H0PK725BatZ?ou!yPExR|ND zZEZ@!@e@%e(q4szhE^^uPRV|Gti5W(h7H2|_FcVmXI<*aL_xJXSt`g;Kp}z1QTNp! zDn5Do^fHj9imIx5@udr%o}QgR3(A9Do}O}BZoON+4~r(%P#EFDYyGW2BT?4(D!v(; zZ`YWao;#{>ENl~Nn?{7TSiwg#?c}*5TW<03^D|P;W1aG3H;~cR7pUICX)yotNd1vk z`r4Ij0(7*rw0HODuXS>A>h{)*`0_<-sQD=^DMmqSx0{-pQb&#)xy!G<40T!#_~of% z+rHlnY`LT!_DTp#TUs8kD@6VaPIPR`-`ntXOVOz7y$62>fA_rQ5HY9Qv}u#6jm>oe zUy(YSpIL{uHk4skxe?6Q9zNWP2Q_nWU}9ikcu`c8!St%mb2U=3eAnW;{&D}71Nn6F za@d*5L-UJsqXJDLRv$Q?4t=_RB*H0U{N+S{!_N81wzJ&}wx%pZj_``*cz34`&7HUPu@yS-saHyd#@0KR>fRX;|W8WXTo#on3r0<1$j4 zaJb%7SFgN!^{TV8bLn-~tz&(kx8}740n;KE3tPNnk(8DWuDgShi+n~;KxUMXi9nXn zYaZpqG!h?Un?8EVhKGm0#q$VX9Jl$BWx$S-a5XzykTfhmeq6!1>O%EsY28oIz};ZF zf56@-dQ-p7zCM0Nf4=87pAS=fT3M&HZR1hdR!~F*Pb zeSCb(x!KvH9HQQToK+0r*j!OrxdN@sGRh3cbLHTr`Dyh}U%#?ZjHqSo{s93J19iZ} zm4CJ#3tKN6<&%a5J#1l-+}hSTG9s8?U|F*vZ_sjLVSM203ByB9-z4+e7SsVn4kO@h zwS{wgbzvZKGW5f>&6x230>P-Ji^wrA$`@84Z?}R&FPk6Ks+$|_5(0x%-70R{K z#@04H6S>A{v!tY+Wry{L=*>#V5{`I7#gW;r{{EO=AMNep;#$rg9y%|Jkpfqs(BG2o zU~5nO(tjr{Ev?$|PO;Sak<8~M*Z}{mEXPPk9BPZlyS^2etwa*lxS*18 z@#jtNL+TT*=g;d-XKwQvvd}hZDXx8lgWR19w6}Y>NUJX4VC5fF!rJFD`|G+g-;IR( z`BC@no3aIRRB>cXiZ8K#`}QsO+qL1g0`>xZ&|I?@%{Uuh64{& zbS%Zb%O+dY6=oV=n23w}J>QyZ8Jv`plQZdZPu4W6 zf$Uf27{AsibmW!wSsyE5AK9DF%qgm%q^zvbtXS1!hA!JbKR>^EA!nd9I;$$vpu+Yh04-4A>=$PRqEui&?kDR~WLxmUqrk;A~W*aegyOt* zy(R`yTra+j16C#Yy3#~J`oMuMl>3dA#TrdLZ^MDigjCVyZ8BohwmB=VM@&EgWRP{ zGLs|i!K_;qEqsr-DzVVTNuO8CK0BU>UXgGe>FMc+=c8$A@tvr(1_lNS$Bw{5N@=e)BE!6JRt#RMEtDHm3Dx`kj4wYz%5*8Ib9L{s_ zQ<6OGdO<@Dpe6F{VVe&4fBJL^33A!~v)>g`lp{p*{-W@ZEPMI#Wr{*eGv438kZ0d? zH716WG|a%N;mv4RE1MVaVXFVshlq^{IyE;piI$=={+%}@f-iJ69wC* zC3ibE0PKP{*#I1$a8n*y>;I{o%zL!lS4s- zH?T4CU0T?0RhPibw{Wou)$U7%?iv&tB_*ZOqVuE9gN=I&=MF1k3;Nm$gfuiYX(;Us zoui?ul<%R^*m>v|^M?l;Jb!wz2ZR28^sv` z>W?6klJl~Dmj+#qNl6v@kHDrJlOAvy(ZXiS@L56uGq-|~LBD+Tp<-xKTG};KP&51- zdINI4!uC0~CIRiYjI6ntke=*4H|aS$z6)tF1YamcUPh#sBgV&m)^fe_S=fjJrWKpF z`_Lht9Xoc|e1CNSU?~Wnjo{F-vNF%{K31@SfuL<1+VYuE!prdty#wJ~`&S#ibSt~T zw!8D!F9u4lz=_+z!Qn@B#2$T(;o*(wf1}IX@JPVa#ns-<<9+Is90)Zr?wZ02tL^Q& zD}u%*CISNjXz7_asAI>Dp@JdJtTK;57?OJXYKOF> z5kGvvv;ZQIq<=?(-8W|aRr2lLOx+xMb z(f96Uyza0B&Shd)!*eAsFVAx+6daZ9tXvtI*Ub=bs2 zwx;Ags(KIrM$)C3KOBq_W2PRcG^xysD-01&|Z^{W>`?+~r* zAf%Z24tkB<+AkSyvBF?1Q^jH@mmB1EMhBW1hF19HlSMS-o_cJ&s zV9$qoPtB2;$+{8-W_o&>4-5l%x+~~wrkPX;q%_fAtcoWT2W#)a`K6pwDV!y_)ixjn zC29Q76$vhO8);VwwS*V~Tz}NzJt!knvCn6wm%Y#u)rmD~_*Kl9CuqYtlsoaYv&ns>7qVIj~-nCV7P2i_Jn^X(oV3x=Rjjt z#^k;qN2_vxZh~tNRDs}sfeydFzVKJ^)m8My8mP0E%U$vP>;Tlusl%v?Y*tS_Xa1x_ z>(?M}cuo$*ycj|dn(qr0zYtaS38zq@>7^#>QyVX>>yQTFV&$kT z_O0%2`!;FL62k3KEI9vxyLGeqhcz!zUQN6b>Xm}3}a>Ji*QrEslh%~4oV`y7N#0#7G>v@1YUpX3l z{P^+m(#y1!uJ(!d!?*zxyP3!4sIdNF+g zEXF6$$}TPFbt5K^d6evKW9NLfEgab0%m8(Qm10~QqySF6uv`?d+g1NQ6b7+~_ntB! z`y;73NZF_s%bB9 zuV24@+o5&PzXWP0l}K1$2>y>8(0*re>tpdT#FSazr~AvOvg&GOL5|J4b{&t~VF8t* z``6=TC5<^ILW1_F68W?I%vWx&o_v2h-uMUDtQI>H)|N7=#n;)XKKXdUQh*~)o%ZS- z8CxYJwEF<%Qp-5`yl*J)-@o4s4PtqgK>>QG`I&Q16E0tt&!2mE{XwzqAB~-3C+_I# z$giz06~g%d4TCNfskJ(y{@uOO z9F)>`%-NrPfxmXI9@uqH`MC1FPx2RES#0An)j&EW4+9{~!Y~i=S4-`efV6WCU^A-9 zDccqkpP8e3q#awItJFw)4Dqivv}B``vTxk;G%JKgK{wlwH7a7S!9D|;>{vRV%C~QW z8=M-^F_5gGP;SaIdqbaBZP|-%$XptFq_*CU{j?=?T_x7aR5^soi2SXuOF$g2 zINMPiVGw!m*OQkPQ%8{0w2jjaU7F_W>+i?Xf+0zF*pF4=*GvteII;RDJ&Pn{F}3+k6&ZUwFqYjQrPmX==@1bFaIR{&M49g%ALSVq7uAU4bL1HEkMu& z_vWANNU$v&nsWZ}jx}OI_WVeko{qZkvmuF0WK3mR-G>h!R`SN*-KD8$7W0a6{xOn7 zb80TNAFqT`^q%`%OT3^qjy~{=xiknk4L!w%QXzNfNydhwIDxKNMx-s%NW7HoNdEGG z_ln!g>&SU7t=yF4#c}@f^70?m7Tb|fFPS_#b>Cs)SpN54b&0)^?-zYN+Uk^%*{6NJ zy>Jnhkf^;gL6su8R`z@4O4n|sz!`KQ!+~)$PPJ8q4z%Lz*|YQWf9oD;XKWzA!F{kX zsKNF9?8dnQJ~ZtNHxIr3M1H<5r9nLQ-aYHs!wTHhsTc0vj+b)918givH1IU!-vRPz zxOF)NID8?pVWhCIQ2y942C5?6Nff}Anw=b8MZItz+(0x|v}}air~ClPG7d)A4>o3j zR7!T6xw2v%upx72iETzQ9^094-ExVCyib8gjTjv0C@Gmf)RTXFiuLB9g`1DllU?6{ z=c)MX24ozQs#BxXI@Q-_<6^gWSHI6^01UKD5&7zQcG^&H1a`L5Q-+1EM~A%LhHSWY2V+CXL>VO9`{nGx#3Q5E>t6l#EdweK<@f&1&J2eWy1MLW#V9}Q=vGu6 z!yVGY-%(9mKlSjQ0N>Hs*_q^4`1tQmd@W9~0KBOU>SMhJs=(I#iP;JJ!t|lP%1+z{ z8uaw^AV}AMV5BH?Kb4?RD+lwSjRKzU*dLlAaA$|A#>)?&nPk{|dU{+yo)`8aA5k|C zS>`_Wpi0Zjmkqb((Nc!5JXrw)@x+j2!AELlEDa1aKivvWlTC29P9_IF1T+$~0wfij zrEcW()_ZG!)N~taR<}$S{KyBp46W?!ZUEQPP|Ij(Z{56^l$-U9 zg@fZddR-a{4b=*EcJ>rYJZ`MLuf4dY5@|KHITLFE3P1b0{VAd)6-8m@J~9QN zWfNq5!@wBd2V|%Gknxu(ExomA@hzSIuAObjxrX-;9W_4Kw1%{7?JwOEG6!)y0F46_ zw!HttJOAtfniBMmY+xzgLKSrZcZlP%9C+Qem#C#cyWNNzPykQ9mG(5<7I~)-gTiVK zw%)E_5S%C)igaxvz7Bf;AzttBnPWzgY=*C3hg#fqKtS*-3>4AkVAOcr&O^yu_@l5X z$q9B1BHX8|pbHiO+e#2Lj;P)X=ldLz3NxdSFU_B1K=FZ3BF>U~-MV!UZMtz%lxs5e zpV6ZqI127(gOKqaALATj;;u+pB)Tw(ay`#&{^CYN|eG zufC%JNHG#pM#YynDc4@9v@mqWL8Bmta);(nQ7WfSCdgdQ$k^>SUTSDt8vpE>7;wCH ze#5H^=cZ?8t#TTcM)6jz!|CSM*4JOY zHtBXopFe*l>!}tec^NzdI|vomvcX)$Zq&kR`y>daPYMd&qD(wJ+i?iuzNjWu4jQ0x zFxcVKBQ2-+H!TrgUHIEd2(E9DZ5bWIBAS1?4(a5^hP^kiN~XXXa6o|>I0E|J^&s`f zO9M+9QKKC%yWi;>OGn1K%(lncyTLTAmb*wD!#9cdKdy2eY`(3XqRh5OKbH>0nP}Bg zZhd@$ZMl}KfWW>C+$$_396@@n$}_84q9alLQ_t|RgKLu8bRM1BTsSJvY5PRi+nsUbg75DXTiKz&^XxUZ*_mSI^>w5qE&`a?!-!V!94|;;Jdmh z6I7gPbVg5IdK8D+i`bDU`s!0wEka;1eE;+IXX2_zUL$E8p?=H=(btDGE*ZTMrX;TY zPK0zM4d>w_M`$P_LL(?iv;8!CXz^pl!YT?tSUn>ywPZR!H&>&`QI!Zs0J(3#X%OK^ zB~DVrfb+M7WyJ6*r8PSS@V*|S-$TK#;duH)l*WVz>0_S{Dko zh~~QgQ$;vK;Vc1*x>oGMvi@I@4lH^)Aa8R&FzxW_xcxejX35b&LE%c3zN@xlTT}5_ zYM?_}i#r%dkd4+<$mr!Z_VK8Q%tgvdkrL;>1Rn4bW&7N<_)cNT|y-q z9~b28O_miWR8`s1&VO!b&^+*BKYA!4?#W$zqUS^B^PYzmtfZo{a<%SznMo922i5yA zq_5C2Zb<)aaXigcBNd%DX_lMRpINB}>UU|W@4S~drO>lM`3!7K%}~&sVpq8vzP_k^spE;+kwQy?1Jia+C(aJ|Ewg@0oqGV9@X7P%3aCNdKy*k7FbNE1Om-P;ljfUmJ6mD~<49q> z)9KS2XQxJm_Vf4s`gILTw<+{{yMl)ft(^L~4dw5h!iVOF4g&4ME@%H%I3r;8vEE4c zad)n_FRX^jQG)S39^&m$EfKtf;t~>-uP*!*hBNJ3i7zkO*D`d9E+gL#z~8rd#}0b5 zjFq8W`w7Ksb?Fv!^UUdvy(tC+;EYe;RG33@QWpEo#I~j#H5Kc)Cr=PsFBV{188Ps2?>pq z{5ji58nW2dV|B0VO#^vjhVDKeFqE~-aV!*xchqs6YrZXY?k5wWsHR##gF0iq@@UVW z`=hdXVd>pFC1PVh`0CilcZjtth#|C5Rrm6G-6?RLYq0|KqbP1}?o9_=>3}@7^+tfz zHjazh8yg#UySsstP#(=G{OzDf(%ON)jtlGkiHoam_Z>CeDfkCvJjdil#N74u^xml+ z-zr_27Its{*)6c2p{H1n^@=nQlx}|z9)2?0!sC=8+9~K^@>mr(GW!Vno}Zt0P4Vs* z4@XZ7iR5(l^Yqo4nwmsgpI-{meEM$WjIEAcJGbM1s;@&_0!QiQ z5B$8Mu;o^b$5dWeF!T$wr#X}LcO4WDx7d!l+Cf`*;rV-u-*~4%|9CK3g>tY7Uz~m+ zs_dd}DQ)A|NWMOO!O7{KxaTxbGvk^yb?OEK@3M+(tX?))nEX`m$<$OnabnfSj~^k5 zF%hv2b$OGNn-Y^st)yzS!>JcL-fr*xhR~o%KA9+m^3!DCqkAL_hVOhQYsgFlMm8%_C zBUkUvB>K*wp*_Z8b@9?syIZufO-clf8G<+LT?tNrG#7Fg$86Dm-;9oaq+@_~RJTxu z5Gl>OV36f7rKeP5#aRQ6Lr2K!w|aMrehdZ=p1IiDwHI%HEqxXo96Wt!r*{DiNUvWX zq&o35SeaA$iy-J)jH9%WAS*Pyw~e~{LZ-(>xeNiaeK1tvazWtcG^vFgk`x^cY@&kTirBZ zf+PGCUSW|pKT#IMmhV2Ec;`)dxfXo`SUBOg`Zj+C^k!l$a-P=rH)`Kn@Ua|JK0bP< zFi-YRck@OWPhlJ?8Va3LRk8QnDFA$`6peG|@Nkxnw7RVx(c+g>lx_yq(6ROkPradsrP z#eVhhS)AuYvsjpKOz56!gMa5L9`MG(Hv}o!4*>1~dL$xfLQjxzN<%a|ySgr;UimKZ zpk1lCdtdGr_~%ZzXDI5-nKM9Gdo?-1hZoBCaf(?znt$6BGSX2J4*Dv?h=+1>ckk-y z2>`O9Ra&!X*HQd>I-_F<2GFaLUrn4;{_@X-cj|*Nj$CDUXSI}4OIzC+pdZQ)#-;J$ zHr-9a8rKR6_Cxqi@|f{YMg!j0{Pdw`n9c27@^*kF-JK2y*sk8*h+caz%#Vy;OZkn5 zT+(NL)f-69yfF!Yo2q~I4l4JthjQVl6+JNfz0VlwT?O#w(wYTf?<(h*y6}t5OT>Bgt z;mxD^^)~~9r9U46BL4ca;3uSnl!~c(`ICGL)1wV(Yy8H~oW}{f@SR=R8ip{wgMWi^ z%LcGUd-R_0)#QsGC=lO#@ADw|eK>qWPdCUcEA(3ZE08pPG~+1$AOG6Ez8fD<>RacG zj4Y6N0uH8IJw89aKK&yB^UO+6CDctHS#7p?aB_P^lq$b}Hx_*O^sZatkH*D!!)NUA z2?P#CK?fnWyuVEmH%)AAE=wp@nrp*WNBF zB)QC_e#j7p7n`Q+ctv^Z+~fwC*qE3yxYZ3D-wm|p?Itb+$cT^TD_sG9;25g-AOW}L z%^E?z2^gV+m*n8$J@+NYgcnwBt(I#r8I6y$a{+KIW{Z`eiYv$*%U$<;r!?bcE|xoN zE%JZnt7r4jWs1CMffE(>->tBOs9MIOaZLhLiDLpSQ3* zkXTIh2iUbSo%ny@zsOAmb2+i~k(V$4C3`p7H0q-gHbKO6Z0T zYqn_{xE~?nQTjxw*N1r+UuI>wTZGOX3n~pF+jGA-yP75_c3q$ z0N)|;vUL?pf#RyB@pNa#=nhB`t(P*}w=YLbuq?^;Jg+F3{}U`}^V0LT`8*sk%A@ko zAP;xi;|8+e3ZmKibONmL$O`EZPc@OGHS4|N_!uF+-{Dwq3~(nB9I*wChF^h@ zHcf%fz(v_YHeVkw!*(26HSmDQ+3zxibtmCs{m1Zc^$m*s^|Iq&4`M`X-_bXHh?O{D zruYsR>8GSHCk0Nb^21H^V!NN{mylTa{ko{ANV4)(CIo~ew^ChEzPC#7iFD~1#?lhG|EhcOyJ#6d4(pKAz*T_>7-?3#@)N?A-fCLJ(K>G1~eEPa}@3A73|I95A_^u zY=KD9*Bchl=e!V73O~wsknxoIlP(;t-j64bVjhSQ??9D8!N8+ASqQA#xbX&x;-iVF zM{WkPa020Z`1e{nIfbE-qET97EDbACswlFQJVO6B(9)iNxtt3pTzp(eE!k492Zxb! zcf|_gRL5RPy8d89-oJ(?eL(EN`T2@&ZsLmzQzn}vohf)G7;Ywf9-Qj%)s0!PjMB+A z41%`_zBw9nz#P&Zdo~CfUI)`(*3!bMqoYHNJzzlz>YkpP`{gO82opGn!()J{guaI% zVL702C3t#bg9>fEO}`J+pCVJ{JMy|i9VNWvyCD^Ib$9#2c2){|!NrRg8CR`(0}hw? zjDPEXfpZSC8?VdLBSB@OiwGu@Hpnt$s)h_-2xvb}0;NJ&U_xww$R`1Vrl<<9;cEG~ zD2}`y2%WD2_E@`A)q6i3uZh0!ni51!TmuV)0wVNFHkdIJm3S#OILDe_d+lGGDp?Am z*<}(O5fS0ZBkMKE1OiSD$5 zm0?x}10$DFKp!A1As6gL1IKJ+WQ4A6D;f@hw*piw0z6Yljm2yQ9}q4$4q`SUCVQ;) z8#toTEfa?z;kA;Il2mAxYd_Yy3@;cNeu|eqU*6if5hZDzSR@$k>g7+vIK}93U@LGe zf1}+YpolC8*4h+oG3q24kin7(k@JYr8A(Y=Q@{n+L3bFJ0TM;r2eg2i9^e2upF4L1 z)*>@gQ$JA1rT_=n(IbEwY^&+)?7*Klm-xzJwXVUUnS@q`W1j;@TwEL(<3Qdd#67>( zBNo_K=zV>c9CGY|naC_0z6DIQrWOPO&>q)etA`%>Q>xD5ENu~!6UK|IEqlLCRwz%L0YJd$^o4%H3Y2~J8>OWu~D@~Kg2m>fs;@|;a z8Ug@)2240+r4E1|yn+7=ZrCkQ4=(-p!k2W--!K76N=ef;nA)4(atgEMiS(}3>4&)5af{upc`Pf;Y6+6M*ruopwVet zk6yqZ;+mUA8tj7Kiv1?>&a3qCAMC6fa3B!-vPG%Az`E~y_H+~;HcNjJLiPw}98D+z$2pRt% z7A(kPn_-PY@gp&G9~~0@=$ZcAv$73j;~Vg%WD*j-t%snmIZ$>0zDZOgb{H_bC2wJ| zjSM^Wt`Uvhe^zkO=zEsIelquoktg$;7J!&UScMj0Z0qbpjH)>%vP41vk>-hg&39@i zqR6AXf18ps%LrTc`TS}>O3vBM?Slu`KURMI6^x85_@LV(bthci+}1;!xvy@5pA~3s zYs>grjBg?XK3$lUi)kfp{+gMYnRG+z!-xGpfpL8L3fa~fhPS-A`GK#ypOSeTiB@Av zJpL9Nd;5$$UW$=SnUDcq%OcdEa+urcAcEz!O6=dy4)JLx1T#y!6Zvs3!G}~xLC+x5 z%i3@eML33Xh#VgFsb0BywNC3Vbj>$vFeH=h(uQ-kVQ_AGntx9Z%>0o+%E7IU_J0<@ z(=fs!x)wvkUs{`v-dNANI6EkWJSpfm~8ksRp_>YClAE$~K zno|a2FwNPl2H{e4@6`DC-J-})o`a`tWid@fVxTvl8#wbxUY@pn8_`e*#cC#vNOf&G zt-YXO0Ww~B5G!XbkQf=c;{$A0pPHI_nauik#=+O7eJBayLV^oY2y^PIhFb{_w5xa z$gY)tU}0QY^QbLV%JI0ROo1r;@+TsjjNQW|+-uO)8}$}Lp1eOsrVluee?G9~D{Pl3 z!xrC!Vtq_74HNTm6hc$pz|Wrt;vzCXZa<1R3ntTC8?Pg2EP7444Wf zW2Rx~EtZ}RBUvdEpfxiZpWgf`BFN}ZT*YM|hx%WNj!3sG#NHWvAoVM1hvl6+cgRqU zRpF@Dc;Ef87@YweBY#o@XZ06U@2jYVXF>G~s-a)Qlj(L4P)s`p z0Q#{(PD_2xyL$ENa%2K+Cj%|5yPZFOrZ#%eu)vHEkGuij%UD-54>Oq1BoVQTPcs%{@6sfB*h{7@ML!k_J-)$$uYm z>&T1*2D+hyd+mYoZ3Pg6X5n|7>s2`ZuD_T`7y>pb0=e>H%YI_cdzqu^ueMaf^G>py^*nW`_}S6=q!& zV9-4U;TY}Y5m1g8FM9+f6+d$g22(y@1#IBQTqCHB9};DGk*vtjDyZtqXp+IL9Cm1W z|9%CI6+uu)@PzfZs#cQkZ^$41{@P&PSCv>tYOfSPE;sDg^;j>U#fX|)g}wRxme)54 zHU;t^;JcF^J`6y&{{}!4IrS~U#OOl1(-M6&qK!tp)FI5uc+aj!!B>Dkn=nHd^OM6z zCWB4L)Idh`ps}u8xpF;bEv)9BOMHD@E+(V2-6t-cCNb|>HxbQm$O=zj4~bYn4r2@B zl?@}n7+S%nhK3^;vFiHy^D4}Zk0QKN5Oa)7OclUFpVKvW5+M@KPkdWIUY-P|cEj;X z-K38j>!QVw!20|%_V%lMeSJwzhhtkVM%2<2tr^lY_N){o0xU6n=IQ0L6P@Fxb4c*t-eC<~kbD5$JxVeZ^pDPHuw#5Wu` zx(rwo+=Cx3a)4%OhG{#LFEWlp#CI|tg8$s*n1-z)w+_H3d=+Occ-Lso?isX6=2>NSjXlSA#tOC8P!qgE)D+z!>bEQZw1u!jJMi})Em$sAS zgSj|vC|t)NkC52{3|byBFyI0aOW3HY5Y7Og{WAztbNbj2;@>BR0_^Zn)B>PPGBrww z0^fxRb~IqglTU9(z%t!~`pY5Z#t*ry(zZP!C^&ds$1fdT*MP9F)%zTp{cw`a-`(8Y zT@0t00wycSnImQapgzqR6$4aTQkp^39>YGEp);_Ws}73WWPohbod;1US77ucx~k@U zM=tPPkUVqad@!OtKw1@Yr2|BcH($Q2M<0xdt_^cr;v-LULYpD2#`%$NW$;gd&kaG; zkkG|xm6O*f=qC!YfGSyB-4^cQH8t*7=BL)DX+M_QcXpBT2Ipp_`FnD@- z^^A4BnU{`6s_f(9W4*$3ZpA;N@2tJKICMSpGk=}k-2)R7O>k$+_2R9FDRQ?0Rf<6$ zOykb9ZN_wy8Qb2|>%lVP(v>$rwYf~|AD#^tzgb&Z*_*+jOZ-Hvtdx_O{DKXSJ@P={P!LN{M4U`iKDpVh5FCM2vhxV zHelv~3%J#tw;S^u9Ka~{Z$Qp)I0$+N)Db+e-#pA*EdI}klBg{sevmUreG z7x_m#fcluwsek`Uet)nDJ-m|7`OE)#O}P*MpZD!3K_dB6w2o*`|Ic5^hphh3y0+)v z=ODkYdLmSvE#=rh&ng>< zX`?N$%pl$oDIamq4EauVp6DV?Iwbz+zdjitroG`p6mkGO6KC6y$E#%plnV2r)A ztxdXPR_?~}x~-_up#ytQKZo6(4*sm_dtv{x?4S1&$NKVUSm2P5q1m~9`53;Bif&&( zTP%V15Vx)0c2-JJA*N#c+url+xvCpG$vp}d7FWr_UIoFU^m1St_zhz=i! z^QmjEaz&-3_qP@Pekj@ZiTMqQ8PrmBlu$S!9`4`2f8cxm*>*DE24{2+aLe`Z@N0M@ zTrGnU`M3_Ft!!ZB0|BYY9R!#h)J^TJKc^d?B-3Iaw*|G9IE56$c|w8y-arkvyii>X zbYF_;a&R$kK-}SurvJ$;<8078eLQmmbaI6N57#5<;>oHosC)}NONUYII(=qP{3xT> z2n+?Sg3+>5$lHBIQE{;$i60lP{P^PUyFiAfkUakuahb%U(2!3HB!zlkQB~EAMn>=T zMTy6{IbnEg%M2r2<>NVK3ilH?;rl>M+|S7gM^{2_n<4j+V5(|;<1o}yuv%A27H3xh zL@GKs?7=-ltQ$6f4)P7bTpqr&M$|JCOcN>Zf+ocwAYFhjIsvq&d%|Tgcyk4S!5U55 z4b%oqU50X9gMLDcH`}(+0m{%)P?^gY44k$|)mvmSOJOc+tM@MH0@M}`!m0W>wS;9gSpbUAj_U{D7 zp9vEPyP+(Bs0fJzbxku{xv;-jwX$$7#Z~345)wiNPHQ zBdYG;2O|?NYRu~tniQ=J9cD1QgnlOB(vkJ!i-m*d6B_DCSJ~BN@e>(fY6;;3v?mM) z!u*I@u_(|Zegs#)sZnxvb^G7hBFs)3K<` z+uKi=qgl3T%;dz#!dnImy$?Cwp2%aMO4!i>Gf=qeKrqqk@c4q%{2yWeGYw*qbZ_a| z`!Y>1kZ9ickoY>W{4Gv{z{zCaVNWMm9s|{Auo>{=I-b77ID=G{E6~nNFc54B6aAL&+_=Z~IklJfmefKgrQOXnxW#{zOS4KvL@0JppdK!6_ zCmzG6$tS}#6vAR+N5DK05-?=~;eaC2+1pDd2gyYxq@N`>s{|=5L&7Tj0}fmP43jDH zB03`k>H{=mE7Qa$r>1cEmOLpobzu1S@*K-etZ_Q#LAQK48M%XHfSj6;kPzar!VPLX zXTQICuXB%cW^V2w=ze}U`El0{2-ZMs-Wf8>tE#H4cTDQ&3lu=A6U~cFNAW=nb`l~U z#ELo0WXjJfcioPSy#XsC>M9}d1!|mdfeET9apYY{3d2wabu=+`8q)Cg^lu>+KA4$Q)%Ab!+RoNWiS zB|q#D^xyIS$n;Dnd-`4p)za(S1+1)T#*GnU&D)qdT#8E7=J+#EZYV#9q~t;%vzUK5 zc}4`#e_c@YKw$+@lSxnBI*FDZ_J=_7Y4J%AP1xT61S4tU!0X7g>~N>20{s0e%D$mi ziO`z@fiV---6O8K13f#W19CKwD#_hfFgBb1 zd;24~pb6aP^ukm}W&fiuM7BnrJiO&rC+;h(eQx-&gs3d|bhcmiv0AF`Ym3s7ubce} ze;iEulv1LAjwYT_g_ z_)Fl85!w2Q{kHXVRaI32O-r*A_MmmN9!}H@KyZO?GZ1x`w6hR7%W&ZVA*nGn!{MJa z&$$~mOMtp)nG2#s$Rl=_UGaA?9gZwT5CLl0F+DvFw4E#)He5y9{SbsCtk^l<77R9W zSV2edA-}2Jf$b!I*f9b%L=eN3G`@>7I|0x3sAbKM_iJYAJza)3u0n@C9VQR!>1J|%WIQpc4C2}k9qX?{EDGRNT z8wp+nr0K>3LCy6;vrk6k!336q8UP5ib36kRHDUO$JQzG%hD31`eX3Z5eSPu@zj6CI zVhgmp_up}JedRuSpIfbhGIPkjp*hZpm-&`2H)z%>@_N1&YwgMcFf zD5Mh51#i)@!CDmx+U`b+vFHF08@We9R#p}o6AggF^q<~UMBxJ>dk9|Pz(#Q$#Ri&E zBKYAyv6NdOs*-_poN=?o`5B^e-oA5(%-)l$HOR$bxI-0p2aMqo62hK=lsnbxp+;{q z3QCJ4Gaxczc*HNFx?$Xoj6D(C1;+Hhd>4W*&i~pq1|$BlWT^Dht<&}gwAG!A#qTCu+1q>@Ep3t zNP(u!tryL$!QE-e>T~2WxfeO>RaJL73gt8kxlfOoAqSg+y~94q!5)K&3v&4tRSIzQ z1kFUs#1f!A9qV3#YdSRohjLF6F`8kz$u)H6)D1G**QAd#C`fOMY4&tS@0 zn;>n9B6KL!ZFKCHq2$kl7A9UuYHhcb8#*p-AhK34Y{X+qIIdh|%eaSh-+v|W%@Z*X>qIs5u`MsievmFcPZQQsv^ z4oMFjK3qda?BH;W6IO|chd5cJl|hz(OdTz$nkfMB$mZ)~HbnCkcw)$%VgU1(%X9I8 zkAdu^NtPWXlR5wX`aE@VMrOZOXGE-rOi2+%aXgp!72wCYe6L;NpyT5$%p>sT#?Y6* zV@R&Hy@Zh;h)QoDo1MZaBx*u8)=dv&H*s>n8<+&gIOcVN4ZXoLlS@vK-Hzh%X_BqG zaNFWqn42yW!xJVtUU<)WWINqSrS-$z#_|}$A(I9~)`tv20+0;4k*ihU;nXX1-Hyre z>v#gv4g6ok{dqW+>-PqX--c4sOrs=a7SU)-qcUZl5@jlt22xR^K?4<;g_K!}A|!Jq z8KS5pqGU*Ao~QiIrM*A<`~LSkj`xq(v5$6p<9Y7qzOQRtYn|&n&qcu-du7$RKWDLy zETQ=Z#O`jIxsbygtKbIohM%c^zDsZxjKfs7(rAhjc8@OMDw&RtzQMLfw zdebcGwf(M1-}R7d#RdMo$ZY_5PeSfOIWq*CTWp80of?2_K84>QP7#0Wk*)QpQwv1I(&c*uUcD!A0(#48}bGX$k4N4}UyviZtTbK~hTa{4OU$lxf)g7)ciq!^?D?lRLwyAj=J0R;tKEDD&2gcup#?m9#YjMe*R;G~o~ z<>ez8IN7TyhI%=bvAubnh+4LK16Wp!8y-!(2GSw^l)ZjU>54$i3(vHDkq)Eil!SW^13vpF}mm-t!j{gHv2VERVFh@zL z1}Zp=%U6moS01`9Vv*RAXIk#C{b-20t*jmFeFeTSc2_Lj-Pm-T7Xs5+pvkA z;|Ea~Hta{nOE@h8*H!En3w%loH!TBfq|0apG~oRv_&z=DU-K_A72#jF`*!l!<^}!k z+ro47Xom!wUaosd1+>+L>YLV_;<-U=9M)39%a0}DQsmCdNDt0VUk z3rox6Fw{9Nc9h@c+!8RI7z%nBmXCMb5+GX`oE^u$=CRPZfW~HnQxFakG;}u4VyBKc zjyvCJofQD4k)y*DEm9)A1F*?~%9!a0x9q~g!eIaClm^6IM~(-@J2e74D(n72%d~6X z4R|ihy{bn}ua{>om7L*v3QvW1jsI;~2O(n5fVV>=l{n*+m<`+ffYDGl>_D!bl(+rX&FxOcn+!R{G>`qvM}npcK4^ zN{wt1H}nM{_osk{8i`FgCU51oi%Dp4vvk&f{e@=_jh?a%murZ$bq{0waRLDB^$z+LV7sPm1e%8J)~IEB^@nr|%1J%`Qn&aqtmsaY88?d^sRx4%?%d`^#VgryPg z5FRB5vjASlh!erZz%cKD2gAsmGsL3BEZ5B4&Hixw{ypHYkxV>*boT8HFiJZ|3H4x6yA-@mh=7fA^e5_IE_zBBGm;^5e? z?Q`|VMtbwRn=Ud8q5^F3?E@SWO*2@T4KBk^YF)&>A3`i*DlR^=f+~s2?d;ybUKjiz zx+HGE2MGTg9PC=ofS{%o>LD$B-kcnkw+fucKk)X&@^=RmN-kljz)AcLC?K${-@z9y zQ{aBg;2+BgldgGRWdNW9H(b;d(Nk8MX~Zh!ECJ6^%eJOM4vDV8vl-L^ysxj;BxfJe zYvcv63n#HZ->(YM1*`df{q#Pf20K$S?sT`N&RNg)>NL3B<-?2Xn@p?TnT4^2Z&dM7 z$lo9T?{Iz2o+H)p9hxU^WJ(0w-yq zSsC#9*ijezTO%WFR0?$Mz@`R3={kIHv_cr5wmb@xf=%{7Apj3Dt-9HbLK+;*FLIx8XmwlYt>pIL z{`KTdslT~$>Yh2OaRq4wyjvlT3K%x==ZUC>QUkRW2a!?|5>Mm`3JZf}F0=cWaKuty z)JKS0T_6EH*ps=gq`N}-3-ZL&30;IE`X-Mj9d5kn3+dgN3rayXu{2^ zp7CH(#p*VIAjF%%Qa=w`9{6>fAVHJF{>aaw@v)9`svVJ}Ni6{aFIHQ2J*p(SNnaM$ zX2fhj)=lLi)~$|_=07o{Y?w!Tt$);Sd)r55+hK#I`b+hfRq71eW5LOejq6$;gW~+4 z+bB?J7tTLV$L#EEf4+?s%DmNfMXhNG!)A^rPB2lshGM6}QDic&>bcYWj_|u@lp*Ey zpBx%5;9{oF{nFe!zDF@mSIBbK5zU)(TOAE`7=`X zA5`&U1<}I3S8-rYnsK9Q$w)Sv(iby^Sa0Ui~KE~ExXS1^)0kZE43k`kAm{#XW(&s;G zUCJpmc2qOg=>&M3J9bO~KaqfI5ZkDF1(;d9#=08$ieKH9=n-a6Dx?Bss(<2oEKsgu z{{X{gWCTP3g${%$E|eX9sj2$d0DBRyx?|ye%n?q`L56is;S^7Qku4$*=Z?EJmPSHH; ziC+g3X%&R1V01>{RjB$o_&#pgjGJuUi?81F@~Zk#HOLUz5v>l6&!0aJ_EkZsn#YhL z;)rT6x#l7E+g}XHAXmvLGiFFWVsp!oj{AtVxaqSKQvtv)SaV<&+$33nL8SgH)~tVd ze(mtP(O-X#CZ=a}R4bKc$f>k-=YANUu2?cq+UI(9CF*{3Di#x;i%>=A-waLCkN~0j zv`vdi^|ePU9l9Q}mIRVPl)X5eXpaqFl9I2RCl6$0H8(XypZIKUOlS!5)~?QPTcWM4 z0*h?Elp>?wLQe=Sa1ahyLA-&MFS!j_^-9B!Ngns}WU8C)%bXEi{?J!{opx*Y?+44} zJ-9_#`OoNXa6Ww+^ycLy_Vqv0)CfR^2ME!Jh*1IM#8bH3iq}il>PDTFRQMq`r~VmKAY|1KtF;J&J*}r6w4Z4I@9F71lKwtF{!Mh& z^y1G4)}^W&FP^(b)cg_064O0<=m|`@;D*evc+#1j94Fm=Pj4r71w)x;=^e$zU{d%` z8)j`>y}L=Mb+4A|45rztb8YMhA9-9-b#kk?6Y~)|uJ-jzC;%*{} z!nb6}xqCz(6~>X9&)e?$+Q)PUTmWw?vN09UyzU^RDUJK-f5& zR5`Q`Lx~@C;+ab8FaU&aKYj#Rhj`Q?MBA==O45pfS9spGwP{bv0L;QCB>Is$Rn-N_ z$?>5MIQ?d#t`Dh`UB5GZ42y!?>7L7=q;GKQPEFhM5&1B~5U#Xi$JGsQB7eV+yN!0h z9Of%Jin1MFXDT~~4h{`1BMZ7<@av=*-Tn=>XjZv}E-l1*e1Uo%(tdYT#{|I~{4kA+ zi`|<~WWq7*?_;aqNQlyt)D=eS?+Qs=e?QIs@tTh>hFR9$d+6U&sy@^Pd;5z()d1RE zM-T5>IGEv!!Ixs(kTf2wZO4(w!T@IB3_J%x0_0`o+r3iyS8~zAO9iSa`*|R_jfgtn zj)9Y;f}A2H^yKufcNb|N`P*x_xc2^B-o@8nNX=}~xRT?81rmWy7Imutiu(sF2KYza zICKo{(o~1DtkLwD^Nf?Im2$CIl>v(iq|#1oPk!(wR|s#$lje^BKrZ|4x-9~g-cr<*NX!i zwgEMV)2pL)!pj$;75jvc`cz<$Y&*&yq{ErO-P(1eZ9SxPpG}?|Sms&>wml-;WH;x+ z$6{a_J_q_l&mP-~8WdtB!a_oXL^2FnbteV>o3nr#f{`P)w(~yFcyvzhqQvTR{;AUW ze{S_@zcNYT+E*_I;QVPu1`E?%7UtR5*ekIb=`NXWFf>O*b+y+HYu>wS)+60M?j{t8 z@<{;cp^A%P^X>&qY0=jN62^$6A9yDqeN|E(blT&8MxBg|jlTh-L6N5sFtTTncj2bB z5urtzX;SeQ?(~%)QO@d*=iCde!mNvrY2X27Jr!#RQsIa@OBjgA?lcJ;qJi-@&|bfX zM*3G8?CUiA zHh28b?$X{5aPhbS(J{6C5P&(n`BtQZHz)mwh-uWCf#zQuJ z1KQ$w91fDi&kdCtD+4-uV*iQv9y?M7vR%haYU{EBf`Z84tV?jQpV$aIe}XC=m<&au zbhXI)PM$D8T%})_BXvNOzYJiAWyd+QK*gD$0@Jt9*)fHM5tYV(dcr8I7@P=XW&((xQJ%%F0ApJCi{&GuOa84!9tgBrcklj=T8FeIpbr{mW?GCEl*iPNEnmORR_YAYmV9+QI)z8Q2ymEbLG+ zqdLt40~a)g(?H`^%qa+7XU&#Dnat#+;wFhd#&ZSfQc%zFPhnM1RNRL(heVW!5Lhmk z;dKFF31}^?>W;#ip(2y#>;ZH}uqHaz+&O+44+n_#OC4iX)#q@jC2Btv!qjTQh%A!L zfR&kFS}IXf^9cmm-rv=8;r>nBRH*)l)e2ah*)el6)R(4dHRiCg2CjzGkK8E8bsG^~ z#PsQ-AngZ`4U46`Yno_nA`M5owgBxr;?jZ~>RI>DALz-%Ll-*{{T)3eX{Ms^MdcZPe~(fVzSxDE z>6xQAFoxq8wgc~5qbwW9l8J)}B5vU4>O4y8v82dpkGcxD`sh`IYQtYnVoV8oud<(< zshRQjOH70;)AUx^9@_c#$%W~+4IUMBB%WQ1RBk(jW1Yx7EYUQfl#)8Ypy(n2i2&qE z+8BsPrl7e-3?!I4Su4B&h+91vYm|ZUZtx>^pCmpp4 z6E$5=Ew3kR5`rLAw(C(y(+OyUCTAbeZ0gn^Q<#lUfZE?r%A5~-bSqK`Z7|H-?O=bc z$93t=2AC*P#~s`fTA`3*EZ`qq=>r-o*ivxB^}=HQhRA~4fP4>-Sh?e>Jai}9tOzj{ zk&sYta`w{@j7v(IX>M*#lRl7ayg)FA2c8?96X@g<{0n?9WNiBAIe31Y$5Nr*HwaG8 zKYdyN(Oo`LF8eh%RxluY=fkcLRy9QRA%ArwOm|T*{KJ61424tk>BY&B^d!N&F-S6- zj<-c~gctE1u0z|g{i(_RhkEwIOo)aSVnP=~OTo^r z&9qkR4Q*3Y^5hRr%2gGWLf&*)JoizEudqN3a})=7Rv`P`aH^e`WQCCUe#OyFeLQ6T z7hnXi7y53oBwuVH5@2@1x}yP+5-sMnILG!KJ?cfmPD)cW7PdDxPlnYBzRdyQ$QB~S zz#Vq7GUo+1q&RGYQkJ}byCd|c%lL{5$xZ@45`4xu&I~^TZEqEZXfeheGV`~vRZ^cYZ< zzC*7WM8my!iZn|RI8xJ3(DWKW!Uh3HPfkBVj39IhZRoEIj*RR#;e#!%joMT$uB{+y z$oO}`xgzDcNXAwFz8 zh4k)+JP=COKhtinU%i^RdfuQ4inDD*0rwIeD`IWHVp%TGS*`v89#jL&9thMU;rd_B zbj0C*><4BS=?Dkx_;twf^3@zg1V1XN*f-^YhyW7Or00-+0(rZL-6bK=d(a;tvNunmTp?n9po_tDwKb zXCIMdDS`JOrn0LANNhsPw0BVi2n(M4o4%8x<21txw7S)|Hquxe-kYt zeE)w#9R7cM!SB|K<0x$lN_~z~rMB1yVo0 zK>`$L{OiqYITps7^k+}w3vY zi_xt(2bwinTxUtUx}L7inq)CL+MOxhdlA8s8wVPsm-pW#Fa2hAXSZwPnUVCbyHDo~ z1~h&A1EHoHrb1vS-*vDFYJYuQFDBgLh*Si8G-QxN3O^S9yB^T5B3CpSDjVT`Ju&u7 zEr8g?&}?WU%b~HZu3Sp5A0sl5p&`%?!u9Z7;_5sB}2fbNR6}iDA8PaGPHt4w+*>rm3#SVfVVD{cO*_3~eMakml_F8(jYG*&iXX zzV%D)vc5dEXjXSpbDxJE%V){ru|JIhK>T_;>X*XBh=_S)1p_wHHMTl1h-lCW^7%t{VzSlAiQcC=eQA`tNH*8W?&Z}ylXT1*$1jY?4s3wP9oRg7! z!X1tat*Do10?}lV(RQ8lw?u!a{T#oT>yb1#7cskeSBpJA?5WN|IeQrnFMDANO#*E+ z2x;K0#GYqs>W5FkQNlfZ1boNV@s1Ako@r4%?1x!QBdZ5F=HXskhv6mI^E7Ras#_c# z%};&}H3R4&Rf-q`uU#&N>fG2h`jt36j>9XMykC(;l>VbMnb=4!l|fye@8@?*bq*&t zC;evkblCDeXcBo#-G#fV;9(+Q%nY#QU(}#VAXULpmX+3`)Rwm^5 z0b}oJ{ECY$;GC{BAkhJaWCVB@7X1NmVF0cw6ns8TGmBv_$BQ0;%^V_6gJs47bPPo&!JR?az5>ga^L(sF8HC7&=H4WJ z7n9o%tkE zYI~~(n*?OTBszoUkP~1b{OqQ{(UQyDy#8=%ur|V3jCwFDdiy93Um)+MNxeYv0dDYo z#9W6t>Ea?HQ)r!lGf8Ay&1)Z#3*JZxqjo65Hhx~LWE+M~!PpU#9KPFT1JGs|cti+8 zCok*)4A&i9OT;9^s7{o@RA>X1!T`BVV8$N9Oqs`Lacik%Pr^T30!`rqnHYZ5@ySxv z)dR^8Y2QHNN)#J3iOBd7G%D)-qaf11nC+$iUvPi2OGJYAcNhbQHU}!WRZfkeu*?Fi2A|BK#isoM}w?8 zF)ErKHt5KtO2x*yj#sxzSXwb+Vo>!S?LHc1msK|(tbUSFJ7-rl%3r^Fl~Ocv(uO0~ zBrH!_R4BEm90rOeYIhDFhH*m<`TC2Oznw0i8Jfq1CkfpJDMqOS4HtzJt|QZtF2j3) z1kZFGSfm#H>7iIw81r1zdtE^Md(VyJVK9tP8zFYjAl)q3c(|MUKyOg33CJE0gnJl3 z1fuW*wFT^R9u?I6zTeaxp<|MkC14PcP>Fv<+DEkWb&U!qG)Y@apPvHlgvJY_$N09Q z;zD4cVZbl_M(}cQ6(sg0Jqt180V$9JZLu>tKU9hUgJFU8leRL58dQBF%ZPrY`O0$7 zLhBZjBZ|ATVGalNhBErp7=q`8RG9ocQ3kpfJ5LZM|IWe$$OOFq)%VkgC~%i4jo1OUXFHzND|W zmn?^wJoH!B{vUGak1NaHu4i%Ud~vBAwrZ`!&uTU1>JJ2&C;&tay3e{=8y zaE949-7y;nh1Kenfv$lV;20b*Y4T)FxLsN-M2rL|q<(fBz(vA?BoA1q8Vz7o()>O~ z8J9Y@o^2NR{}||=r{0%-xLN?5;n~1rjE*?}mofv0yUoC;}Xb)+7XJf;#?#@M`n z1QYc92lt4YB)v!V_OfF&8lY`$8ft2D$t!}Hg>@bIBoxMqj52tOm*0|gOi;j=^@tkJ z0e-pzL4`4sF~g6=JOu&4Vj%z{8ve#vb?b|Z)RYT=Ps9PC>9!jI{_>Ys!r3NdsL~($ znr6Un1}4YOY(|NA36y>JU!Zy=GRj!?H!ECJF3fWIlhygup6At4C z;o&R4|Jv--hYYEm9mkUgu3yJVyONoOI2WsJj(B(pERLeELZJB6s;*TOUtZk}67a-- z><2tK2Vq$MNIACJ8WSPAGvLNIq&^2GKdKZ}wGk0q-Er;E3~MiGFhTQi_uKCX+}d#o z)(EbkUMAH~$V zf9ACy^{UP0bVDv0|KuD%@Ok0eqU%$BJA{OVg^}tbzr1`GIZ7Js8vc^`^yw5_Ys#R@ zZX=^9IOoKwMAiCF*JPebIvoa-?AFLlya5s8#SfD&gngAb(&da_{?L&VBIp0hp!cv? z^l#IW))asKDbIleHziI@Zi&ZIo3O83>L0Hid6`kBAp_tH7vAg5rt4FB57-ulp<;mU zra%>=GZ<=(E)O*dzsV(!ESb8wKQa6MooWadE+JvLT5g89nOaZlfb2|R*pabDkK9C_ z_~Q&su~RP}pD2!czw7Mha$lJH<4SW!2hDqDW}VZ&!vma`q8@XswL&m8wpmp}V;i~$ zU7-KN@SoH_BnI>FpfO5X@Y}*(-G6k83?LR2w2f$sqBdQI%>}9y8MndP;2jQxD0qxK zSPVWd>W+Ny*P(AA7aSyZe`|d#>9^42ldBXkbuzO8SUwxG)QInnoB|XP;+Qri$JJ&z z%_RdR^eHDw29iUWSCprS<2^CACm+{Eb0s0i0RaRomN~%FNRfd!PqH2|$E!A>B7$rk zKz$^<@F`|i6I6gRj;1ifMF)3?o&3pxDT&+dM<ka+Tfz89%Zemq|>U%5xOKAHDPH*~c#N+2uxuX{#nG^t{3ZIbXxa)e z_<={D9y58!rMn=}pj(AGK3IEs*beWI9jNz#eLMw42|EV|W>n6j?|OQ^KoWxP_f=e~ zCC*dXI>~M5yiCDi5~}2~h*lg@AB0P7n7v{TkUt~R9$k}-2_26>SO%c?k}eqQ1*C91 z_wVR@QToS_?eHr=U4sF8mkcXp4n2bV?@xIg4i`pglCl>QmO2?qK zB6@b$f*ZI`_>?d%UyYk*i7E=ZVZ*1?M7M-H27%MWn5R4N>XijDibWk);C^!#27~tX zm4v`!Rv3)sB9V*Xe%?wT;~7Gfaet^bB$_Ig_f{2^??_!K??YAl4j5)eX?Vs;`Emn9 zxF6_^qspLH%IN7u%We~r7&H~G9v%>tlQXHcalCU61164=rUA7cRAUX; z<&mu<$%0UWP9hIdS>!gm2`)7?4Lx1AzqTy&tV(ruzgRbX=`dV|K}|)rXc(C(VspN9 z$1oQUX0#mNZu&o*C}L@`A?M~u3I34a8Sm{~H}%UnHnsdZ+bkjE^bJQJ&%hX-+WSP| zHw|pclwsA%w6=^F*123+ub;M$$=zk6@{D7N@BR|XWTLpxn@mhl-=hj&yiDmvTtX74{cdtYkyn8sZK>a zvN3VNBdC>%)>T$iyhOoXqh~CbUR(bF<6&b8?rS1JTXVxW+d0+PTyq;2L$f*`w#Ncs z7GR{|W{)BCzP;g~EbEDI3ou=kyEgjww&)c;GPoA;h$mNR_{VbHrfCli4jr$-eVfa2 z73WHXW^9V{^^bsBOxTmfiG;p$7;|FOd1^5L?a;@-OtEdqkyqAj1fV2y0~q2Xx$$=% zb3y4}r5_Jrk@w;C5)!3r27o)PTLOdj@x%9zIYzUlox%Y@%~b zd^jPgC)C%uG0x6E+e2&USM#I!g5Pz$cpYDA?9+DT$wj~ICGs}iwM#DM^Ls>y)$fem+lpnws*4P{qed1+40Y zi3U@AYw8=F0x70Jz9Y-BZ?MOAq|2tw6%MWut*ITpyOgAVp(}D{ik0C_teQ}(BX|XP zhVsmI1Jb@LVMOv^LDKWk-EfWFSe{!6MNZ1Wy=exc#V+l07cV~KXxd&#ib}~=lu^%& zQy-3&rSK`1MbxUeyR@gCDEou&-4=U@VZA$0JC&Z2m;F#h6EMHJkf6oi@xeddB^f-8 ztGNwEBMMD41(!1jYoeVBfHXX7UY8&zWMSyYU0trQy!F)aplvb*QT|1fUo6My18vyW~XlPn%0;;-sy ziHdO=hSHK62hQ2+?992hq1~{Ep{e&k{dT<4dtmfVZ>>;L%_*=`=5>T4OWqQ#11=Ga z*}i^4PmHGZ0Ha_75z7}_XyhRJ?@+{ReQjgSL%d@N+SuGv0ofxUU4x1`aQEUq9Axr9 zL7$D;k_&xxGbHfKXfdu%o!^&c4mvwSUWo9^S3iIAg-=q>Femzf?wYYLzy|0lm~XB; z>yknXNwI`VT;tzltSl-k-rJOfkNg`mN6=(?(t2F_NbQE#Rx(yLHVr_O7$yr^FoSey z%=2LKfN>J8;AvTRwnyW@nnm&|N9DD-kE3sRcC4?8FfB$oT4x`k=cKw;SnkO+Cmpyjf*F|IttY_KLhhshqcZzK6aKh&LKMZ~4|jgEJu$ zJ=tDsdt4pxw}inbpCYTjEIBmRY?9h=wroQ!aeIxeh4a`Mzs>jZz!GG!GSp065$hAJ zXp|X_bz%7|oX_RPn+z`+$c2}O_}*}I1VejiPtkVx6;kcO+cTm z)X6olq-oOqW6&>IE5NDQrys#Jq$~snXkdraN7S}&hCS?;R{zQvK)|5p@GU>TOY7^; z&V@_Znu3tP(DCxqGqz1+zVwIHlDWB8J$6Ec3Jc8G@H9E0B&FTlR!e2O6p8`&>{95u zRFDG+g@Ang8u@Fbx3RF;UjO;JbN{byJNIK4kBj7c8nX)i5m=8#qi7ngS(zTVI_EA4 z@Rlx}d3SFi-`{k^)6za5(Fu3L4NPk$s9%4UeFiTUkQmcy?*-&NW`$3c*_nrVHKoV| z*zSH+OE%tJyhAcD#HSE+oFm88(Fs-$oI7WZdZ4FsP0I=gI65ToU)lQ(XzMCkQcd;* zUQE)g@GG(cHonBh)=`9`hAtafBlU!DVj0t!rdf zgWrTnPLGGr0&-KVOnaypo@!h9DqqF=3l^D1p}2367D7(eSbq(qte3Y(=3`p{#k;|J zRH~%|Wl&c??BHvww^Eey$zuS@CVC?kaljnFrPiMMK?H@nl>~;>V`n7FS`wQsR z2>qoa9h{V9fh8J+A0t&}qV1ao$d{GDK`@J(+p4cCsArMi$m5`qjLZ#XE*?bGMTL?6 z$q)b{w*G9lK8M)YJI=)i1&%>KdRI$iMUTUg8uZeQv-hyEvUa{xS<2$}fxR4sr)QCg zySDd;=s-r8LBd+Wg*=UY)d$v4!}8>{KJBzSyKPfVbOnMU`NCzBMgUR_33;$pdu=)N z-M;MBgRm`4O6A2aGxXn&#EFz=%jS&r@2>PoA~_e53p-Vh@?B*^!sYk=Yn(mM^*h0A z6%;+?p9J#_%ow`8!ksqx3641z~3SLm!&A0~|?o_eRZ*SR@YC^a+ zwKAo$I$4p?ANW}^cGhO)yYJc=;pO%zeq705f5E_|tYHvUNAUg4^Zel%DcY(%iydZi zWVJ(G6To%WJ-_zs8^2juSxIH$?;`!#n$bF-PSofvk-=1Lxv&}7BJ1oj`LT$&jLv1h zmb$keh$FedBqt}U+d&vg?guZ0U2#+0}_wM&W)g^eRd zq8P0BnW1ae0}ecNFc;DK33*eVVF$2oTJP{)yX)CLIc^se%^bpOFxvIS-c^nA1MjN# zt32wf6~m=CXY;vz=L;`zn&E$@G_ugxZzL|RMN9c)(`JT9_rRnrDr-Zn()YLxb&7Xwx_`J)F!osm4nV4; zK^LhzXuVp^UhMj(t&NlLIk+$Al~!Rh}*Y`={{u8K3{G|T>o-Zyt< zO`q{eNh)e(XLmqEM?DJ_mBH$h5r=T7&HHijD*7iRymTAtRvZG61NLL=&{$hKs++l_ zk^c<5ghwq;CN_k%LxbTarQhdzKLK{M zK5p9H^yIg(dn7-ZFJWEpLmQ8jsI+x}JFqflpnGk$OIfA1>+^2yqaz?yqu#2?%Gy68 z)#L`59HDnaQ{vyEeK!h62%7Phcz&HB5_!0r}?rH*A$l!!Qby%Rv+>jc<$U99oSi+CRObiQm_mKEBX`3LmeB0 z@7Vz1LIs>vAU#Q=Sp52hoaXt>eHK_u6vxrpApcw#Oze2Y-S^3!&&%EEPpOMKyb2!= z8%Y*t>?)HrGW+1h>GLwooNEa1+uDud;$)NRGqf$kZb>x%fNr~kJgRscJ`1=mD+JC> z?H>*u;e8f2aPOggVSAiIPiWOxsKfb)72Wd;v^yf7D-ATK0cpk0qu<93i9hMw;0;~R z`sR)7xP(`W*8h&?t^F2Git8f^$z~Iuf{lOV#Wx({jq)v?QaQ?y4=`VEtgjR4%mEZT znXxI+Q2dEPRJ60d+vjgseOxeim%=`i$zV!+F(^d!GmU{Zba|zY$xS_+X1b|qcoZee zM$j6#-+Hm^0&gb5aDjCuFKT4XDbuHKLrXtE%KXUvewx7y=fqsCn$mcW`6!pMtTGD@ zdPdas%c@oJG(E;qcOC3%;%t-G1JaJ;onQ0Mwm{k#B3XskdpjakUMqK!b|L#=mqd}} z{yB%qIy+8JjJbJ6N!-)^&rRHR+e#mEk0x#GHSG2l+0!ZoI(OBQMA*oObvwH z4^>YA!iD%)JP4C-`;uApvyCRWSNdGYXlbcU1NjpJ74&kX=Uus=V1}yjfLyw6yxsv! zKXtp1tyE$qqtNwoZp_If(~=&O%$dL~JTTSk!V-}(jy{nKzm>K z83*4YD5!KTgPzB?w~1n^R`0svfpePu5;x;Q}9UV*#)s&`)*(onlPm3`U6KNA=?te5I1+_7ujZI!Y+zn_5}=ifKr?|ICm{qR z3s+AU*%lXox)|f0QXq}1r7}dYZF}Twhvrfi?=bfFki6>FgC|F{jxfzDr|O)9%h?qo z-x*01Refxa#TE`~k5yTkRIWI{%X~$a{u1eipac3lT}8s-41*@~KIdB2>he>vA1ogV zvc9>am^n*DXO(QBMO()jWKSrbUGAluLm^6LFbwDsk2fEUjYHbc)1NV*EfdTI4e(sE zGi__J@Dv7W9gHy6^Ns_QKYQ0Kx~ClHN5eQ{FFw`eKJ^h96)CW#OO@a3e5=cHl_-SQj%G;fj7(z3F$+C(Vec)CR zEw-zFY^hRt-JB^ZB$G!PX}xD~VG;1cf(m-uxU5RrQ9Y z&#E(;Z#3>{eZ>r-D=1W;uKMY-30wI_ziG58Sqhv>5R1! zGUw_x?U!_xqn(=Yp_sFwdVje4_DE^{*~PLQ?^;ig@dmsN*$oY=&Zd8dSvc0u0#rk>9Jy#s%EMw62ej!!%>Jpu|LWoz;zkU~h>1ZwIne}8v+1UO@3aw~CJwBlkcMIVG#jr%BE5i54l`_y8l2C64Qw%7W) z(`-nfvFR1SDB9X!4{ehocHnH$;(ALz%X zDC@=RHNaM(Cp0_|Sbk75d5h_Z`NzPy^FvGy5G+reUf!kSz|OZa7APQcMocp7qvLeq z)oV_iI<-DC3>1kKW?84-HPpfd<|^|N&Mb{*c++~XMTpd8IH(-{aNpndI{%c2*x1+~ z?d?ZSE=7B{`ONi?qgYijBV2AE?TDdQeLL}tv$F?K7vIupGiR>9F?Ht5^k-w}(9Cul zERo9PKA0oo(SUI<>X!9qd){p3jn0tD8QjdyPX~@cVdJS>kBC0Yl9fq?=k^^vqs?7F zhD3nRY`oYI>QmZap7Z*!rrEiHS5sX$p`~-EPK_50JFxxAaWq-#`qi6?Wed=M)&| zIkL$t(Ed-E%Jl`>jV>Eild3$?FJCL~dSm-?XK({A@CTXo*k?NR#2CTDuJW||=6(IA zHlk0a5T4uZTvwbRwSNS>*V!n~w=xlEmB+n1P+Gxk8vU-(V}HjSv`nqInw2W;g*8}U8UrafBlnn^mYP|D3wh-DV(ielFagrA zL>*dMmi_)6^5<2JY zIEegI1xZc|$%qBk7qO+$YUE7yqT>;{$LW@^Q$(h@8ZiZ=o{7LTW65T8>e}ojpjFH# zA0G&LH^|Cv;@(C*w7Ej+WKgb+rz~K14h1MBQYfMvQI9P_O{pcqeW3QKw9;tjme7so z_)t7QE_wV(0Q@c>hk3xF$We}c@?-%;#U&;W=MnWkrCiAqbgbq~1#>ez2{(X3q+leH z(Dm)hw`y+%b9L46hON+Q$G_z%_-&0wSxiZP+_QBg3YV3o6BLn9s=J>vK_opJ5)V4K zK;nqRu|x+OZUZ;xq($ovUxm1JF<>vt^Oo9T{^Z<*{PoiH>s(~H3bod%#VMFrgch>^ z?A@RQ5vzE-QS*VpA7?HxmnH*@$q7Y=cjIU$8y#1GQ{Z*7g?k(I3UNt;u-EF@BbK0W ztE6eLIpipr;Lu?fhoKKA*N7v;yL7c-&$HY9vaZ}1Fj`<-1wi#wnYcjh{y(q(&_M}b3?t4mj{W1jOapP zw0@#AK!lr<4n@Fzo55nc1r{e7R^%V!V?7BG&6DOApf^l|oYyRxj;0|p4H}O{I?uEfwK3ZPW5zzpKy?w;nw$*(TkoO=<26w_@x<65h9MTzW)u`K_3SKN zUA`Ax00PJ!mMsCa4Z`FFC>m(W8cqR9lD0=~f;OY`}q69$_~t(8{GnKzH`ou~m2nTrf$rLKU3m;Ax0d^n-Zh z4kQSBcQ5wQNsrGbw`F*7_SA-iQx9T%q<(zwss;2bgaO-(QBN72b}CEpulfCt{`02b zoA&%7m$cYES+&XC?RTFKzt?I+?kGy&CYR1a(6ZwaMVxkL|tTo z8vJCnjZ-)U{c3VxVgM^=Rd0%zK#XWa(b4~ZpwL8hKMHQ+NNw$%RDr`aD)by!r)M+?mu z&?LxUlkp=!&~)j&>B7Mm=<2UIx_rfosHhhc(*-8rLa{z>!6nS>**tI1N1=odZFsQN z*_cjLh-ht9J8ndNh5Yh-LPElx^xbs}ZfqxRIM(fRDoye7PMlInJ~Pf=y&AWJLMXU2 zn}vMy53JcN^nD=d!Er~&0_+T9Y5#ve@6|ZBV$%jgXJngl?oFBqES`Y_!UeOlH8fp#^Trx4jY626`4i>l?Arf$6RvMN1 z@7@*r?CRe!Du=5gR1lClfv1}ggRxndyU1E$66X6wytQ$Kum9sr5GnA*i%y3w35c<4 z(TLKi_%@-axy9Bs8b0v@_ToDvwS!G9pM&dJuF%6(=Q6N|+(f}PI9YuJ1Y~L;;=M4j zo7gidz?`t&I^V3XSB}@SK*%#aIWaMuUM!ARdcn3)han}ip}q}^%4%;O^i99}t+efQ zQ`PsumTs2?*GH3u{=q!_m8_4%7Xs#8rleH&zz(JgC#(BG!=v%$8|39J5zvUP`fsZb zh!VN9b)urTz(S4#gI|H|JhaEbYRp$cG0~*i$*&w zboB-*{j~i4{c#=Z#D+jq$y1-w&XMpx&!Jp#zv^&t8y58itg8IIdDtXN=5hQFo{_CP zCmp|F@XNBD6;+XV3yZe{S70UBas1y`EQl`^bd^PlV|Z+&-;Pm6C9v}5y8~-52X@JX zt#ZLU^2YxARUMcw($Hru?CqT>{&sM1%}TS~CohyJ{iCVUj@NDua6aJ&8 zPm$tSf?^8ePmAN1{_ES#cr$BpQEB2&-iY~MbpKd^S&5QJtaJ;Hq`If>XCP7G z?l~;yu)bXSpgY$QP1$Svs(0>dFXMa~AHCC}y{tO$%hH|vy2TkrZ2x?$rW@DHXM)!f zBnfVy(UQPSDM7*Lyldtc__+_5Lgae>)x-~K55-ots^<>nVxix1Su)eh;Zs^l?{=$b zn`P_PCcLdVrt|u)c8SB7$kzY|<7l@-iAw8J8@_GO?>90`<>fTauk&3IQE|TIFLBRG zTtzZk#)WRb+X}UZ|@0DhwkMb7I(&EdGaCBn3kpS zH$>IrUksxf?K3Wp4OJby{h_jbj9EopL*2HlrD?o9A=z%0ERJgYG9x>y7T!u+Eg+ZL zS!8+Spu3fmYGs9gMnLR|^hc)AA-=4MVw0>MC~Vi`R~Y{*TH7s+$AojPvI$%VBaP8# zfan<*-Dc;N{HMFqZmoFWvHnj6uieLwcu#ftXWXjq+-y|>EeuL#`!ptrDM_BbAF$D9 z^@$y2@0SbeUhz{`)JevFyv{jNIwF79T2(cGPt@3%Uo=IlXVNlZCkgq5rHY0+L76>~ z*DL^|W}M5E75y6F6Oyyx-)%ONE(~iiL=toAClb&N?P&Qk82w$+pFZ~QGP}yRf61Yt zU_sX=&0*HPYljZKO6hz1I4$=>n@7ae6AzckFO*ms39GAtIjkF#)=Zb~C$jRVe$;Kd+)Tv$eKc(WWC04D|ja+-vT8ATdn#=NO{Ne1Y zy6d8mtB}5O3ajY{&ZqZzrgRuF8^6Hw0b$;A7s@x@Kql%51Ux-I zYq`o}hkDgA={+3$kyS+#Rw;pvItw!&zgU@WpX05nUFKX^yYlC)Ay`cq3HYj6(KNmwou+&U0E+=eXCAHKuN=MaE;x)^m{%fe8v&o z`&g?ZOAM4ulPj_Wf>oELeAnjrf~a@erFyEVm*(0-yUN~w4=;H)*kZqC^Vr(KZv`=N z?XjZwME+(~uW!o)5Bht1GZ(*o3z>l`59GZ&M4{8o#01zix37#}2N*VYsq5e}hF?L! z)aTP$6dokQetN2dkEAw~b(49VK|%qbk?TN%7^d^{p`?4D_1~0J`1f-Q*n4joFK8$n zThBCx_xf)UU9_s}>zu)`qg*F`8*ypR3o{kUdtp3vX3KM7uh!|WIEpNrcUn~Yzv45P zw|b>*aMur?0B5To#k;zLKJMLWdVAjSZ9DjM#jOGot{d;Psn6*q-0#b0Of&!w(kgXSB@Y}&>O8;b_lj;(2 z=u)Nt@Tmb@S`z00H1h_RK1^~XydDtTSF~|Nx~2c8a@FoqRju0h{O9Uf1s8-}f`9Z~ zJ$r^%%(YLjws=Rkp|)tF%;{%wUt1%@de@zp9BGxI+9PI}5g7YEIJHIMP;m76&s}k< z1_5B~tX?_|hve2AtpA3}C#UVIrygEff9Kz$ z$6o*+sN@!z?I2&@3%MoP1>nYmMC?Il5F8XvH~Aa#`}!x^|A$oh<<4ApM@3I3rVTH9 zLT+nL-#RhZj9Ev@6}|`Ymn@7noqW`gCu`5NaD|u;i#7BM_zdQrc~TtY#A0;z_zgCHftyJr( z;+1#CC+v;2k4!2Ov?%y*B8pg*z}yOHyBAI)D_mHHBu9dh9fWnphYBC8988evQ6E*U zd&V>M+yl4LH`8U-Pj6hb{%M+%OnI?|?9tKYtl8=xWks8g=@fr!iIIGNR-5})Jl3Gi zvYir@3OgmL8@dDJ{Q7DiKMLvkUt-ww{N?|7^K`E5v*#Lwv-tRSx_;`85O!($%XZPL zhh|qwDn-)LKH2JN?VFY5Z)`DC#@Q&yIHa^dw)|yMX-~^>haWxOna`AVEO5B!Fg848 zbTCq3!2fu=f>olVx9uA5%*-uX3Bd{hGcf_|fzJzL!7I?)-Ah~MAyAQe*;GxxIb1nF z{-`caP>z_$vI3S827K|MIdf>FeUHgJj z8>vsR4f7ch|0%YENIddmubyU&Bh~(Uf0&%O*EJ;V^}hE^9tCr{u|as zOuK{Li3n&UtQT}|TH(F({&j1GXHnK=Mv6Kqy$0#^|IcGva4b>G>0X1--(^?07p2g@ zU;b(>9V@=2EHPBC@|yI%;6cFh&iu#GS_?eAN2B#j_B78v?d{z+B_l+rTw{^0grH}W zV&UH7nswLAUp+a|6D7r5Fuk)M?XH*OS?mkOR`gr5-@usve4Hw3xx6p_9C~cimJzr)?Boein^fLq zJO3OsD9C&}^3U-wLe<;9U;gsJa%``X^)Ev|vYpZ3p9#~!xdUzb+$lyBw3 zt9=JE4<4wCoN=V1=H?T{Gly#qu=v1W>QE;fEvDslw55NOJ-v!cR5a$2bnI|nPpGB3 zn(vb)!>$q?)Lfrl$N= z2kye`!e3v!r`B+|>w9&`DTmd&pX+!Rb%z@Aorr7o(mFRCnZK$}j=jR|QJ$Ykn?`QC z3UW*Tz01b`k4a{^_TrM{+FQ@Y-Cp?QaH{D?bL#2e|FotM_vSRl zD_E*>UKgDy=IPWJ_7clA^yi0*%hpbM)yTa-M|YY*m-vT%Yr*OB?gup)3YJi{^o&m$o#=oUcW=xiMMO@z*Bf z{EhWv{zCS_ZcEJ$dsJ7tzC1k*k=ouTH|OxiIWVu`%up9;-tqSw7m-p2+VIa;g#B_P z38%&t#BB{ltp>X@gtUs-Ylj-=)~x&aH7Y5>J6_LwXpm7-nzSe>QJYP_>W}R+Ne_X$ zl})lL+}lpWhQ-oe2AjJ2noWGXcgE93+43L@EUAtat1lMFrWs0?9G$T@TEJMTM<;FU zM&rZXZ$4jWF2AS}u4K>>)n0SuQqwsrDf8Ald-jwLT@yNX=4bp_%M&ZcdkVYS^UWD? zdaI?cJf@Iyhorald`>vl@s91)d+y3P0ryq? zyRo)*20SV1**P!8FVElnLa>NOe}Y9oxlS;aRaE~Gk7h&CB`udFd`lFkVojhNw!IobwJMt%$-@2S{YmVZKx4(L7?_4wUdYxYA z|0{RW#3zYJzIx64pXvmFwd8}To-c2D4hXij9UN?k7ETCC6c-JDtCSHF zX!GXSW6LQQS`ys^4BG?ECf5e9VRsDrp(BDr{qUXdtL;}PI1gN2Cdl*3&uW2=&b>tA z<>j&wnjFE~*(^;K@p~|glTwk9g-y(g)}31M{JIG}q5I=Kj9stU1|$SEtrixY`if&? zjLZI=a~Cb;sXND9Z*85)>boWJ-QD5#J>G}-+ROcV{dYyLpBi?v=$|W(<^sF)->xc7wPt5^<|yB6mb3MH=IysuPq z*yOGGW#7yCK`HhewR%V1#;rM~Ycu{)WbHBC4D0lbPeUu)6YbY8j`V5SqwUG(bH;gd zd#i0FuSVv{IG&Td5$~p0nwXrNv?^EvAJlAS)r_`%rT%+`8g+-bnCtVzU7Pj@K3Juu zx@e`|@)D_h^)(Clgj`tM@|i6M-hzVjryRWsZ?uj>>-ch~7LRZ=%}I%NEDj(qH5 zgLiX_PIFk9aFC!)&`iu1-gm5hFyYE$69`dcRiOQ*aQ@7g&u@ebU)By;gD6H!xWbl%J?1vzMM>;GR|a_}(X&ysl9c(VCV zT{fdtgdr?|&&41jUu)gDG1zQjTJ&VFdz3}PES}%^dkUXhvychPMOKQOGdX@E{&}_i zpOT^r6peq9x9C!3z54dcS?puevpa2XZ9i&vr(KQ5+OPYpWJY`E_BpoTDFf}(PZ-ph znfX)FY9X*t%e=%x@?mxTT8G)G#XC?_f31k^QRuDk*Q!Ma|M$ydPkk^a;5o%f8=RzF zri`kj9iq*`&w6wDS+#cF*tVl{ zZA*KssGZ<`%FTlzx=Jcz`cS{yS=_5CrCN3-z}A3;2fabz|O@ZFfPe_ zIko&fPWjD8C7vIZO03D|vbn-%ng@Pevyovw&!%AXSd;EsoYZ!r9*u!)zEJaw{*Qm|1OM|cTu;8|1*mo_?x$^id}irM$M$LWl)o>w zf|uXU=<}gL`fleC=~DetSrch)i85M`%$IPf=Q_7jz@D@NJ3C{}2vNB7^s> zaP6*2SF|=}OyqX$h-qq={ts4x3uJ-Xf$)i4OZ%durW`J1nyQD^nyO;O<1}B_T823G z%41Qo-sC%^^DCOV@BiFn7}bA62f@jmFs9qG%lm7fySnhjrK(|Y;QgY z*=m)>@?Lo0r+TgR@}AO%>g+iECIKgnyy;8#!=i_hRo?IXRomLG`EOE1fs}Z+-vos_ z8-x&U;0X)ebN;?Gmp)!@!Mdyx>)*OTLwd!Hx;CONhv}h@No7&Q<)7R7rwq&to;Mo& zY<^(xnjt>+e)~M+=+_y#@-0jW?uYVf#26T9zhVCNQeGoB|2oga#YwV+GX}Gdt~AFf zQ#A}}EKe)lomOGW`{J8g*20j*Fu-g(J`>=2z;pgINl|7LRwT8jkLeoMP`^92Jk~%Y=(2Ripu9DPvO`G~%jr3O^E-LO%lbGWhgy?i-t!~Jqg2ZRJ zT`wShIXd6zv>;JTxE-U!9&<$Hi@@7yzUdJhVsT=pDv08w&FnZ^h9Voo3!1~HxE(lN z%?1CA66Usof8%TGSJ{WxA42WuHD_mH!fA@$Gw=B^pPc!Uoa3Edd5nFNS3a?{1&2>P znz(}dfYq<>2lt^H8aHl1^GomjKY5`V^M7{>#R}N<8}>w-!?$U4G#f8Eed0&}=V^@w zLwBe33%W0=*yb439Z2IpZoU?mLF3(NiClf-RyWvO)-m{1&?H8K64@cp&tuT9w9QpJ z)1>(3myqRQ_G8?H=nVBHUgIO7Yn0ySIAl>|D3r@15uNLL^)+MuX;(ArSpI%!>a~CW zFC^qj9mrXA<=)?sRB*dwpryQ|%`v*iv7At?I^!9h|GfCQp;2C-b6{8tb%8EXL0)M` zPEZye_9t5dU=?p8KC|UZ*)mtm!@mv=8RTe%?Uv8`i`m{HorO&Prm`!Cg327h81vW=wLut=D4`R59K zq1aOWV`~Pz1od8n(-xlhWL%g*ca zrw^Y!Z*r)@Ej}WASX5&K1>H8D2fHP)XD?&o$fl0g&DF=5Yxmp}w7z-Qm$i=V_p?Y6 zTOB6)^Hh+xqY(_1px$@VxBXSRF6X(3t8Y5iE`=H2S5_2UP!hF8Sv)7MsZh017QI5$ z4Eou_W*&~@@O$Xz4Jzu?v!_MinCMHp8uXpFF(XY9Dm3NV2L@)#M*DeA<4F@uM8AJu zYFD8`b%jIW_fzI5#{{zz*RNT-3ABIL8OS2Jl!*N&$F{C0xP{_k(5}kzpU%nY7!7^1 zS?07v7%JhhdXPG5$2K33Qsr>NZ1clN6SDwCBzjNdAeW7chww?1aYu%D*xS_01?d(< zefGkH&)y2W4@cs}4E0hM;$o-7lw>Bn@!O&QJZyDKD1Zm69#^thWRGPvE>;r37eMLh znNhl}>MDLx{=r~;CDc3SqAkxF9d-QO^D!$ki_hmir^%3F%loUU&r_uhXuUMxI}VN~ zIzlg5)v^T9QUCclRG1h3Z7ivMfKAl!u%)|$E;zgr=NVniH;o6SybjLSt2ap!=Q_^0 zq~DYo8@kG0TxQtyMxleQAbc-xp!I2bL^^972N#8kT*9?4?6fs+4HXj3zqmr-fcr$v z=BIxDXTd|gT#8CM+(zRbyN$<9){T?bmAO9dS)8T5{O2|l{+r4w`r!;_kF8fwDjZyR z^f5_~NHWMlh(UzOv5ENWySSXOT})l2V)io^Keh9VojFVyo0lR11yxgP9?USt zhK}+bS^rE^vshIsGC-MdJm(qS;IJv@= z_upL+y*##c_t23^w@W|oDej6>X5@@+Oy0$pvJ5p{K}u+k_wny(cj~W{H7KZ$Y-M(; zhVMEFEVE?Q8%P(NCL-_s+;r$<^`$?P*fZ29J61q8x1XdfR1w9{AKvx2!lK_I;+9A$Bb=>tK{^l=R!Uax8r9H zQ$DdNi}P!AWxeJU$Q~BE!uf8I`hwbHt?o?r`Mn=?e#NKK3ahV>?HDj%O*XiBpRXmT zzW8Box4@c&)}-GnVJ9N*%ZkqF_$)Cv*|K%EXX)Lxd+Cdg{NMW1zOzJaOT>pz!>ntm$WJ zk20ApCa;_5m*ZEtjQA*jeiJkJC?(17JcbopRCs0 zhI3-_5{}qo1tI$0t)BvU_1St+%dEVYPD#+sjdCGNR4u3Tfkf`+I4gm6S}M7No~Ppn z+2nKP7thTezTY~k*TrrB^hLEbrrA3)bjy>&g^8z8;vmW1@4v|v3(G{efX5_`LK$>J z6Ns#_!M~w+r?ZJaEtPqxgBjanpYxq%uW(3N#1lM+m+fKgTBY}$2m4DwRv=OlE7 zzM6|>!gb3x`dydH46v|EKiFC=9>5)ApmMYHn%bnFjqg;Q)ErDj8B&Hs=Xw#|cKQ&$ z$C!VN!pOoPmc&h3|72h;3C}Z61G~Ni!}FZ<>X|FF=Z^M`CY!NocaF`3ZQ$&?Uig_% zn)HRJ39`#`wI#JDZmyIKiq2U9YxkLM7ns>nxoH3($I;xJ@1HvB;=uVIuS^htg=hZi<{oV;p{-2;_?;nI6rwv$+d>qYxkA8@F6f{kUW5u z0xp1sU}hijcmkQ-Gf*%mLc0;Pa|JG~{~jp*aY4$pG@{vx1JA_c*6t_lqfjw<6xgZc zNh)UZTG%)&hWmKMv@T?$@%x&Bk{y>w@93!5>5j>UlEpB|By>pJb$k-`fu~)jdVhhQ z|KoTjtWx!j?!?Dbd{oR6vRqAsOtp%`*{sl5m&&Fq*v7-H`-?&~ zb#C^vf7sbrE?US9Xt))ql!m>YX>{-XxsHQac`oZI#r4}6LFe$)BnV-G66WL)sw1$P z8Ucr*yp(iavC(>{m<+^%&p^hH2tZ*tmjRGMg{t-c{z}sa=0vnb^OT#{H+HCSFm@l> zvbtUYg271L^SaarN<(qGys(V1urZ`Jrll8>>QmYx5^36^0w|db;mfnt>bLtql+>h~ zFz>NCJh#F$F+OT#V#%g5B2~i?SF__Ld>-q1bow`U{{}Y1CMqO-6*+tneHGAA!2i}2l zGE%aaX?lZ?DMQ`%Q72|zXzO*?+r=iw-?j{d(MhCX9@rL7JGj135z8|OTaDU!Jeynj zZ1zm5b;uE8PUqPNNwiXE6`$;Tg9!IGnTW!TYe7~()J`${b);&orr*9)sOl%1osp}C z!{-m`qYa$63*yLg*Y&7#ybs7sNFuV|iKP%|Te?)|*^dm;?#tI~Zx!R4-zwk8xH}cI zdkNHq?l1{*LrDoWsB(AW)FBW( z%saeENRf>NxssZi;jQNnmWF0dUwkKT5x~&E3Ve3S)x7eJozCVLTJ6$p77Y5eL`@O&0|H#1FuIsa}e%WF#Szt zCv3!@ipy(1dDImU(L#odh@p4yMLSzCcz*an>>}iuCg2qxP%WxwN#*ht*#KS|XFnig z$T>{v>aov#<92IgqqM#KSgj08uw>}~okSjKHqi^?8a6u&g)qFlJoqQqK$b|C)zljP zVyUA6FhbH8{fmfwGJ0ih-Sx_}A1-j}qsW0IjJ=r1|K~Z&-n)S}KEnB?Fwy(@o|)~` zf@)%4Aq{liJ8-2^Iti+NF5IWfo0es3boxk860EcZ6#=!Y_p{No8=1RPiv&NGVk;y$ zNTeQpp8xsmo283XyxG3|xr24CAu$#%D#E>}zp_*KHLt5y`-I##d*u|$qJqd1h#%q) zf4)}?2_Sy7Fx6XP2DV9S^l0tj%A8 zr1yu8PTBXaEl`sAkIp;#tx2Z_WTYscEj|}SeCbFE*}IH_uQQ=rBnETy-@UJ$3-jgP z(f!7xbCauOTE&yC&?(+M85oPi0tH-eaKtQ@7%}x^~(7rVZBGNxzv}+jWVI`p66KmEISA zFANMIlr~-P1w98sm;pM8P9>=1-ul82`y_}Q_g?-hC|aLd@XDH0mQ70WxRIwgt0{@f z8TANl%`>~X1o{Oj*AtA6k#YjmU3X`J_J;U~))krolDKOkSGgXtQRQT!{Oj8x;WMvujMWho5Exa;yC^_Sj| zbRx_d_2QeOA;LWmq-c?SAy2V`(?{EtsWlRiI5+8=73$C*=4!1f9I}4ZULzP1StI(hlVUl3) zrN?r~4ATo1u*|vdW&-Qt&!x_uYC5-8rICK0vTblm5Ch5kwF)p>#te&LXo(f;7yhw{ zgrIu$Ypgqlq;QVDCJ=0WbeSegQ45a|gG{@NFF&DwtGfs1lO^Bt8UHDK?QMGZ{Vlm{ z>K?WYO+UG7lyVB9aC2dLa|P8^8$K(87%nQW9ff3N8)~QPvq6X~libM!*JuvFJ3GvU z;BlWWYxFf6TumJsT!4E)h|G)s=kehI)8MeX0aBU!SBR{?X`Pvr>|oS-6A=?{UB=M_o_B*l5O`^OP`D_QmP)zxRcE zdb?SPN zeRT_KkP0<7-1;7l}z}f#) zU>HEV7--ebB%oLNsby2&Nv-GdiUziK#V*Y(pL}fsRF7l~@ zmtwz7t@|nln08n@4Z*Zw2n;VlT?HA;kt6(FwL!l60L7wF;?a80tAua2J6$lmv7xxv zZeaHqhj>K%yoJ}tt~<21jJ@@Y*jNr`?kz?bl z@1{{8(z|Pk*>p8LqBWfCJ8Tj@M~HpN*%eF!Q+;ZG5&*Ip2A5{L(@7v1@rAJ}-T16!Gf8*YMVpj5hgI=XK}1i?ZrhR0ugBTVu$~&JwmV@bVBZrJrZATM}`#9`G$u zP;gD%V3RG`$3)4J?LrZ#(dDBy`F$t?scAdeQG9uT`HDKuqv9BqXc`p`%)E@(UrIo< zg+L8q2nRuwz@Q8fjPs~bf-VfE!Q$Q56_A_+A*KLJqQdxJMbGVtjW{hk0WR$>iE<}= zk)P}Gi&_TE3ti}xrNL18iUXzxJJxx2qwkJ!qb5^(YW;uEJ>8}n6y>&N;d<+NX>W?%&J$w?%`ceN`N!i4*qv2L@})Hq+z_t z<1*V)&ylV`i5o)-gSdZjLR9ns5dlHK*#mYXY936~*(#T_RqsyZ1gl=SQ7M}X*Vy?` zsD<@xe0UE1tkwZ*>>E)JmfS^83$ERltA_gd6?fM5@inA)a)$jy^&C}o5s$Z0Y|2ry zi?`mz7m~azNw+gP0u4%E>5HA+bvape%YO8KBsJ&wx-Z8! z4W}q896(%|R;_+=k|3{!G`Mm|Q&0kmQx5jRlFKe56 z3T{DMBd!%5PTfvLFBy;yUQ(Ke_YbcVKZm@u)iZm9g(H10crF*-n3fzr!Pd{($(1qY zU^!TGUccU?$dI$~vyQ}UM~IR%$w{Uz>I84+oVcNFCzHjpHPcNiy5t`EoViwp^1f0< z0@<7JSEs5Bc~z$s`P{?%6l%AIQR;>HgAs+_HI*6|a83(EK^icAh(fHabc=0kKDALl zWs=J7rD6G3+{|@zna$L2z<(p$`EtSUVpvJ*p$x5GHXoN~5q_Z6&+4sa;o>N=5N&x= z=p1nA2|(xI>jQhJ;>&LLUe3?fp3u5WgpqD#k&~y&@ilIk8Ly+rj{nWs)0bn5M7aNy z6+=anF|Hi)d@&PFjMAhK{vgRDI`^%oXxE69g^P7vcV3NBt~)ff@aZwkRXu0KhlTOy zy7~s@R^l|$!@@a_!e`_?7T$*!1vNZ=%7iHK&OgNx*cww$nZ3@h zJrSH$*cCmHd`OBtD_5bdQB~7+L2){+EW$U)B_fDb7QR`okYxjG&w1>{Ly)pZv7=nA zarkWs;HU}s^fhln%hh%Jvh8cd)RYk9Jo$Z{H$8+i$c9XCP4ud)Jt`$ABDPhZj-1zHv4>jC>MggvOZqHa`jNvI zk}dqw!ne12f5!6eSW{vb)E=@1`Z_W=8@zCKb%)if{vu=8+bq=p9|Jpaigqld4jCNG z^=S(c{U$)IBJ6>tuVgM;bO8@;JmD6w~uv-*;#r{>3J zbLL%TbLx?AIrT<}id6D@lI(+@*!um%jtO`fx=OsTKMH<@uouSLCm`p?ImdYBOd$N9 zfl_+p1o(Y%;cV-N)MP)mRvfUxE52{kt8+u#mXx51=UGP2#PQJM0NP80r3X7VCy9rz z`a)ZTH^!hY6L-XO;*q?+ifls$^rwos+9We>cBkI&UD6}ndXlo(|17G=E0iRZTYx?f zS_I4Nrj%wG$YY~(jKeC%!&+iTWv}3I6>-nO8f{^F&bO7(yTBY_C#UfDT_xqRgPs!N zvl{~%glDLs9l z>fR&dGZ`Y0yv9huI=2IRC3XQF9{}VY>&l;tFnqw+zY;O{h@b zqq}ybM7V6+_e|$qJaR9U>${WDp9}`*6}c!(7oC0>c;D&yRV0SEO5{PG^j7}T(2dft zVJ7^(?pyvdJ9KT0P9+{^y?!j8R{MHvy}z{Uc(&|JDAfP{)ZKHVPSJaRrtrzfBNz>9 z@-~*l{M`83{H`o?N-6^l5GXZ*BF-xW!$+~LA9wbwbt&b*og1pI75~i})=ClG<7K^9 z`EFM(;2(QqQxk55S9HidYEmOb!F3rYle~?&>O=V%h5fi)Fw}5 zXjqt%$XW0)_x-l(%s3=27A!qt8Y}Ve=I-73s%OH2*)>TvH4o(g53#4e8!HdgT09IF zu*bR#x+;emF$zkt^fg|HbHmOe=QDDjOs;d275{0!laeHAY0d(_RYuCvL5Ey<|dFNrg^@7P>C z_c@)l`cgHs9fjIlN9Gg0uQ(-hPoBnlclTfi*q<2^akR zp5y*Is@P!DS#>i0Ye`n&z?*|BN&cBs<_4v{{w<+LFBsboy}Jd{(MJ`TJJ)-;^p9Nf9iu<5J(0;OBw*F(RzO34yg6JQrx( zkk)`nay`D`hU*w`?OiAe@H(8RJAO47oeBz=wKH+Jxdp@J#sE5d8Q}NNDpl;53y^g$ zIe42=krZ{&bUQ0{<4Diz{=JF{k#I?YID6+*S+!<)Fdzq(G1tFnPQ@< z6*%Rg{}TQfg0dHA33v@$Y!C?*(%${SwUUypBkr9e@5AAb_4Jne8X1|mKP0aE04hYL zr15;O!Oi};&}hBPrW4IL|C;k}3%+5G1u=(=Pp7oHRq`@KkSP003mLH7X8+1xw)D_k zovsXc7^!y##|l@SgbiBMY#`ALX-iu`s`#yA6^T3$Mu!w}i4MLVi6OG;D3n7x9wr z!yHnqL|ZUwd$GOMG)L-%8IE_pQo0uI=OG7FuMA~ggwg_@TcZA}GLu$1%_HPF=$(Et zHWpRwq6)l29mZZmQvPk)4nw9)cepWDIW>0q>(?g$SP>8$EUTmQc8*b43G55?U+ck* z|GC1REta}6dY!G{v9P(@i8U$GBh@15W;^4xB1NUcP?O+Em@eYf+IU32#`jZKk2S`i zH=Vaph;B0DOJD}&0{s9D1o;i0mUL<_=?0etJgMA5p`OKR)CS+AX>9}%3f_wIXQIfb zqxYa~%w@O3hoW!7bWzo2oTSFz+{eu5so0%>pAi9x2*Nx9V??0O%EUM8qoAIIE&U(i zj~XXsM7ZBhZ+v9FDCYyD$be6<7VuV>2$fiI%vg&@;&ZZVx9Yj~0AKKo(M2PH$&S-= z?j;n{g|Vkn#a7TPff?=}r`3o;j_4*GTP^aC2O3LWwJ$_myT}ujc!~Drk)4GOG8{cT zk4I@%3@c_cBVe+cZx|!FeLtGxIx0@`cg(}h!Xjw4p=FB9d;hqjdLLM$x40tBN6N3t zs`=jjAPW@rTR;^9!igH?_|%ji(o%!M_RD~{e?Ej4Qj~J#RomPXUHRda7UBy@Z?i8X zrKvn)mN>ruT;=(!Dh<GRd5Up{N9%7UJ=(A5S!{>LWfLIAiiDZ%du@qGZm_Qe7 zKmmSH3s6EZI|iU&QPE`x7@mXa9E+IPSs;TU-l_oC@&_JLpSUf&oj(`X+bMY)WY{r$ zJKgGF((g@Vn9~+r7cnC!S)9#Dig@F4rswIPE5b2nLo-DD%!{Yh+!=FyF#m zZM>hT^PKz?M7ig3p~3=Bn-m)Z+6HolhKUUV05$>MGa8rVTrlUIG(#9Lmm_}t&vRa! zf%*5=0eiKs<;jn=hZsc6paiq*k&V1$m6)a7{ey33`+CP>e=(Gx63(5d3yuyp(=!(A z;Xh@%BIL=!bp(>oS)J5O?R!i3Oy=u5H%2yT>RT6oWubx@F-S#cqUG_zFgCjIg(cg7o646|qeH9&;@z8Q& z6q|XOIc*_{X>1g?E0n!-NQC&8;$|sXtpAsL1P$b{YCDbR=84A^(yF)g9*0I;@X<6z z`H3JX0d*F5fFEzY#{`;h>j2P%5_>EG)nW0|nx7W&sRTyp|JA>-Jx4;Vq&^VzH24+D)M%h8hmNY{qJ@)VALAwCal02 z`|-2b&iJ=+R$HSrm~#Nn49s8Ug8r@hz2JtS^OMgaM3bzU>CF!?sfjI4(?qg--}^A2 zu6&gr=X$4m;EfWdrlCTI@g&C_XiljUZ)T0q3yzOBk=dQY{8>c(O2k9Ra&a)o+u}~= zdwKfia-Nx_S8r4K+HXOzxrI*+yFDAL(t`M%-qsg!PRzco`^^n-{Ky#rs21oE7$|@m zUoIdzBZ!@J-Utje5lm)|!;A{|-7hF`j1dH9G5Cs>XaxQ`9^e>hX}w897W0bPg7HL| zNa*9Qn3cnlrs)Zlc?N}pSwKuxXZwg>V@tCCI@-Xv^mH($;PX6F)7vr8S0D1CLgt}k zVy!aWoIa<6RTv+_Rd@xt&Cjphsz2VZPT<3u%jr6v4Dz}694Yaza0$YK1PmgH8fDJI ze!)9~aCEJ~F3Sw~CKzMA2DHpo;D8}c?uZo};7XRxT>R%w`^~@gpZ_F&hSS1^Zo*GM zRYSf-@`Pz>)a!HmsHcOd@KKRM*T~mwvg?;lQoV&3%oDHj5r*!_Ll518m)>r+YskZNHim}@eO6%X=iINuy%P&z zbMAvvKtv1%YVfu9w-6U3@ExrK3>GOR+mYG7Ir@vHiWFsKZ7-Fs<&WW$K{X-Fjg4c= zzc>TVisPy=dg2@VKi2qiV6n5-IUdTkp-0IZJ?P9!nOhzez{X+;FPKk<$*c*DFo&o1 zB(Q&A2bzT8<*zL**T8~KNG^jo^|zE%IUZi8`G618Na4{Gnymr zE4Li;pSNpJ^xa|f4tAKyn+dx9k9typhy8tcnULDEfLS%g5^|_Fgy-87SJpU|NPoez z#4A>4{4x(iLt#Y0*T-fDyRg<3^%9DG|7|Da;E~YQSzfz+ zh}L~khRx*W6eiu4(wt(YI^@%C_$LEn~v&^u_B5&e~+m4+Y zD-*6>s{7e=>1?>(RHmnw<{o?u9F%e~?hxD~1SkRmf{wYA)CmWA33pdDXEb6&`d#AS zGyrboTwSjKo)qD(&BVJA+f-~(t8;0*FRHNUX6M!~r$qoY?(Sml?(Q96_lQ{Tg1<%4 zq}m_JIea}=ucx(puU$};$&Au$*5=P3E7#2%l{|Gke}V0A4KGP@kJ3n!8pUMWewKDv zPr%~adGipR=EKlI_tShFd`;(2WRXf9R+|m|eR-yjVr?Ea?mjV29Il}GVAVU`WZblB z7esc2@<+9hDTud3dOqJ8LTXBpo;&SW^rNVx_PY=ZVvCiLX7m^I)xuFVp_Wmf4Jf1g z0KmNBx@zdtckS9~QKYyW&;nd%IMi{Gm;UppNs8U#d)leGYK=0;7JsNC` z5x#O%ujXglo#CCap?prErGbgOFW+r1jej&bKINn@6+0$O91<7Ml5MR~Iv!!1z|{T9 z!qV!A7};fsC4BAux+7LJkhY2uI7Gh~9LgFdg`Viu+(4brya&V$cP48OYua3iH)-we zl)Q&3P+s9(muR#HhSbLFx7GF=f_lC5a-Srrw10PtBRMaXVG`jg4OVBv0ADKrpwH|X^8tph0jzUSC}c$=;5iam_Budofhb23 zg#vHv7qDt+T+4EDa&{+z_)sW>lny8?a5=$3fncq{mw z>!MN|p*`@%bJ74Eb$|R7JFo>lw^ONer7%(4AyM|j@BF07e8*7#;IZxPEB)f8EKkZe zGdsU8?(dOe2rN$y7=wsY4am?{%QVO1s=_f za5xZ~IZX6|TN%Wf2vHWqEZ5lc9#hEWI$CUBCIaeT07$Ns_7i|}hX4*ytAH<~axDWn z9s;;VjSahu7F?OG*crh>0ieqWj*&uNq>BFg`eh5Jc8kA5anw7`x!Aq}_Iccrd(jUh zyncph7bkccbnQ4t6sZjk+86M%u-p^++(AioiRxyrT#UCDJ-(^2ZwvE{TV*i?dZ8)} z9|Gzp@Szu%@z8Dy4oG_Af%G1Bb}ClFNV$@ZdedG(%pOu!Iz4J<+jlAyxeMik={OL3lhqJhdpIaZ17$0PF-lcQ=L)tAi#A)yI8Cf z8(NsQd5YnWz2X9|m1-KidynjQ?=lB{Z0=!*reocgoDtU~MU~{bPQZ3|Tju_PFXE5@ zkm4p#rFB+$dCAwC^uTJr83h#3&o93XyTBm;)C1$_M}i6DVXJOu2(aL_9(K(+ri~sN0Pbzwi2sGF+ZGyzy(5^}ojzhd&=j;*jYbW} zT)ZLfPHVkVh39mzO)?XNTkiHtFJ=BVdZVd_9`8XKc}|Qw57fNWquXbWL=C)eFftL0d34pmNP@5WFRYfw>EFv!y%PQl1*5p=6D^3(GTTQLYIpEh(MvV) z?a`NuIn^&`u}quIF{K49t!UZ z_dXG;4NKw}Fq5r*R&%$5)aQe#12nuB!8!q&3@}sGSLG$Sy)v4xa@LIv7+1VXUaOB} zmF#z>qF%ij1j0T98$NKL8SSl4-32>eZ7=AYbU?aoh#0Z`w?~#XLVz*j|K36c1{dJ40 zgb?D++I6Mrm4@u9!jadzJt^TF0B0&b_o$L`DS?A!)w*|}f%*c(p>+W)w(~BA2kJ~( zg88yrX5MBrnDLQFUkkE-NIM7MM~kOAh^ z+-?Y@#N6%8p&%{>)&Y)>1$4y2JLr!}*sGmR()&;8g2X5xjAQ@$fyq9xsA6thb`V zu0UKnx^|#IUe(?d*VN;inK7q9@p}V$5uLY{+aE2dDmxL^UceMZ&~OI6n=lXb5aux* z-cX;hM{opQ$D?x{=H}a%#bC>@&$)VUAvsjyDKO?i3;e}?N>M#ivyIG7TOC-mU`z&` zPXw}yM2~+l&cMF}_NBq{gmdCg5`h9teE{JY3Z*Ksm(n))kBBZeN<1ogZE*Zv_;#t! zvTJ(YUKBNzgha;f(&ZbvdAJJn!nHd~MLCl8lA5~u%rORoWx^v557X)<*jZXWzj5LD zvGpM+$d2YuuX5*JImz(R!R(VKqUAd_jR%je*D*z4p_Vz!?vEDGI)0im_^s2R3hBf@ zYjApTpR_}v>g)a6GunwwX5RPwCW0UtYoUz!Ori4b3IN6+c3d~Q2|5q(oXfZ9#zn0H zRr20h))y?G;UPxNcBKaC?RbGFk8XMN(f1kPvM!ZA`g^G+J$<3nR$Jx;HOWjWLH9jf z;|$8Kq9agi^^VoxYafe}hkS_kH2+6bi`KfzX3fH)1TBk*e7T2uLHDvN{1+G!uJ0#h ztMT!kZCj4jI?|z_R?|gNgML-WPokEq3wJ(oc$Dm8JbQyF;|ZJ`GP0@W>3Y88PJFO< z0%xZ>vbA4UK%qi;L&a`vwlg{W7#twR;4DEc#=eV$QyW00C=@POsexv1@X_EuhaL|4 z1q=IU(dUqmD2#&S>Q?g8dzFK`s~yh++5#f;PqODuZVl&^>P@h*aAhoOl*ynks$TJ! zaOCRRS^Az8ur3vKuc@_hvD}r>jynDnoI8r>XT*IcXIAy-nI;0Z?1&Pu8qZH!>8}7nTdlkFVOjIl`~k#0~W|H6eGFDA*OO#l!j;c z=hBAmDYmnIIiJPKt!|RKd(VG~!oIz=%Q3;I;1-)GOsnITfy5uVe$NN6zWYX3B~+!q zXFAinevEc~z#f~maGFKeiv?%zK6972e*3M2EiZK8{<^+!!M!fcR}&=ia@wZ?*=6Qp}ae;X&=3d5t1`mISeJ2m(-Y?dT{YCnqNin@B@+k047Y z0K%GaJN3m8$)8_?{2)e=XJnIFao7FZndV{#tC$bpO%;o~V24r|O zC87!qbg=EKy;sjTm7icfGOb7CYH_M#!J)dj!UT!M1aFc)I^=4-kX??6f<6Qmpowom zc~8jd=V?f?G-qH1f-y{qdDt6=r>~AkxT|9Rd`eegRr^7r#RVGMymV)B6 z)~1%N!nc{*xb7cU`v>gp)7N_*@K~@Ml%go_+U`dk_zD|67QH2zn z_H>sGOP9=F;_M{(wP%bhEG>2unwYpJwzRm(dv5Z~61;u;&Z77<-^YJ`q4CjOsc9Ay zBNg&GHx&Pb(9Edau)dItKVW*LQ{#tveL}^FZ)R7)-Ivb-&htUYoHRIKO?v-WcSot$ zpo2T{49ak84Zpl-`;&GR47D|-J3o!Mt^Y7b1oQhzU_gt{CrYVD6^4W;rg?6|4h*VT zYE@;pb5pqU8}}EkgM)?j0RP`#G}%>6fJd%;lHgI;4=1Rec}8ES5aqYJ!w~~%skgu- zDneD&l!Yz@^D4_@ z_uZTD4^5*pEAMS0#(obpd}X^x&?i8IKZ+N86+2`7N)z$JyT9X~5+u#bt~#FgMu^3K zuCL*tC#3=R?B>F^FDGRAq%rIBn?9)dJb5R;U_Vy7YGt<57SP8RN!ny(6@5*Pt%wX{ zj87VgU%U;-utkmpHqWK<0*Z)x%Kq0=eSn?%9oOfs154M#*7-jpQfke`xkP=3l1yl> zgZ<@D9{}+SgO2dY!>@`WsK3u7mfXdajh!!C%5U>~GQv4Vo`_JQ09mAVf89iVe~;t* z;SmN(C4{clWpkm=!|n1<{d*T;Myrc3^sRQ^8b9M;;qo=R7Vi_=;vV?vv!1A1KQEsmQ9E zTb9s?XKugF&#tN*aV*gep!vA37}fD4pAdAtVKNZ6Evt?VQdjRu@(`hfD+p!1(F|_dUBaCLV_{WW(_@L$h#;3K=lMbR}`g;j=PF|;!Q$F0_Chu|zublh- zvmkmEg#tVvw%0nrYi8!%u@1Z&t%|RMA7}5m;oFfQ)p%Lw?%&65KHloVvZ^-^S?^M2!_(qFISE1PGXXV}x!qY(+zV!tQ9fvJc z>JIizYlH>&UpHf*Tp|SRR-bS~x$8n85a2V!DCEX|2pS#;F-#5&XB(GPN@bv0E zK7|2~OO~yS9a8H!t@6V+%6-1qN<`!zT-h6|_80fuCu^YQfSx`(xix0h7KU)e1M#_U%@nf-;;Xd@h!iWy2t4?D zA>0lEM(kK1M|^dvQq(F05Yk{Qb#^dal@V24^Q_B;=AURF?CR%{0J{sStV%YZ5)=8t zT&{r`9228Cvv68Oc|ejt+aIDK*nOWCMHJ~xqFL-;$7{G<Ym`k(f_Z+zjKm>Q|38A z=65NWG@*h?6JiTY4O6;hmABa4s5oMQ1do>iS_XdRLM(EzPdV|@qZ4N-2Ikf*Q~A*R#5}6R(k%`afsjcInplu3djI(G15;zI6dX&yBJI|P z+o`qn|EP2}??&gWiGi;5#=uE$2|JzoX7y3YMq(8`dd|h7pYW-&|5f=JCa#+UKy14E zW8=dNbW#J(-b4mGG9IHNZ+12?_c+bQxw7!>X^nFJ zQW+zd?-<{=#{mD30nd5%-mB(ZbCs=V z(m0zi|JvZZzx>?RhM1UGRR?Ydl~ox~7lD0Au`{ek8c2k7Jd#33k2qV_x@}JV)wR$N zh7HZn8D9{VgwuD6YeQZJG=<(SR(=R&#}if!&^}EspJhtiZKQcj$q%{A7zRm`=j+kA z`m~gMb$Y1ki|_rQ%Ld-bsJky@@*m#6urlz;CbYR4f^Tca% z*k}e|`#X|t!fkq5XsNmX=x_a2sR4tkr3=6NBwe0>`Yw!q+B{UBE|aA2t)>(ezs_EX z0a;E+!&OQN#)>yKlJlZ{fHYjI2aZ`5ljjc}5gt=5l-Of=mwvn?w21elA$`^sv_D=n zd%1brdCdwvy69cq>di)n{bUZ)2NLk(B5eF=Zx}+N2OZu3n$!Td1Auc}tS=&7!Vt~B z`SOJVuD<)ug#Q;lyCfm7c>k>1d+Dp zB&IW^qNA!kp1F2k;l=2b!Rx)7K#lpKLb})kuFxHZJr;Q$@?%DTKvew=zMx*hI9O;E z0t}CF$dq=*eTkN6*kLG8fFl1e&U{n^H&XYAb3o2p4X@@INuo90A4Achds7(MMibm<=2 zZOg&piQ{_s%`gnLQA|AXX3I1Ts%nJz=eYwG zQ8Hj^oQaWVT8Qd#B!gBv$H314Kl0|GLC`ECZ2?aHlEVSl$4Zq14I;Lu%=l=I=^>i2?<^Xh z+w4LyJN@Eo;-W63Y90Hoh}$*|?X%CvHU}O?jOBQjuG*g!=U2cX7FW8sKu?xcTDo}K zF#IJ5LmcI_W?;2<8u3)3x%DsSbzVv?P$9> zft(A?ZF8|3r;Z{PGzy$>co2$QOn11$$DC-<4yE+;3Ioz4y)yoVC8TtF{IemM9(~a3 zAc^yI6~ov)S#=U*Rx-24o+B11g&9;0qRM&n$Z+iemmVELVa`)>vcU$UR(PIMQ|)x> zi)*24K>}VH;(09e3Jer})mb~bTKSt7&kjXQKMQP@Nml&LlR238%7b(yHXuNWff8># z*G#uz`F)ha0rfEl#pacuy{5u)sAYW0y-SDw5)si1!+cZ=EomTmy%*E_b0*{tzAw~> zD%0iuUKEApL__k`Uz_=1C6yx$RDF&he^g!3NQ-@SIgq_+_t0sQ@!0n(6cWu`Oa#%b zieYF^9uY=#gqXL`S3Pa-07(X_VRsIzba(9Vh0|AAyLB8meKUs2LpsXcQMS)}OrC8z zANh$2Mj>y|H+3c-z6}TniC|3MyIA3Nex374EV&mYz9%;Qm7R2MI23OclE(XBDhMW7 zS5<8+OyHG4-0kGRmqsZ4_eMKRl1$e7gU~k&X4M;LDSy3v?oiY3g}SHF+$@gv5pwJI z7(sLGPV1i*upciTi@Sb{w>igicz4Q}bfiUroH)fK9{YQ7(M0`0*0z8AV0`zY3_>XDL5{S`uXj%^eBLcMFW4sIL;?fvH$RK*f*Um4XaQeHKCX6L)9^W#}1 zU;d+ajP9$?r1xjAqV3aH;1rzqd=tOz1G|^Mpc&6nu9DO(hm-qgLy?i*~L$!y9-uIg01wk^@qp_~U53nfw z9&|hOb=fy4pD8fGzF%&{q*K>B`$jwVYNNqO`S`+x*d$BIk+&YF=;@yY)z0cj+M$S0 z2vGt<%edDjWVE~ejBL?c=<6(I$xJu)5+sO;*Iq_n-Wg*^CiHM9Iklpd^b1r0x{VVF zJ&QHiU}%^LjEofDgvP}`OF_SE13~zF<#B$uM6`C?1DZwE#Z&E(UDF$#KaLxa5c^i) zQ3>lf{5EYZt8T~T{Mzh&va_jFu(OE;*FEs4g3WKby36$8hVgMKGbxO(D;ZnCTex1` zWnHaVA0p26>W+4GR^hdqErlL%sdrT1zyX7RNdyM!4!MTM|C_r9Tvp3vf5+=NY?#6u z0HZpuSx9LZ?d+2X>tMm37?0sbsWePIoODia)Snql3y^g9eXs1!Tw|_$eMBKvj-ZFZEgJ6&RXD&Xvv{(W_%O%dYK)-&gmmk^9au}-FQ-`=jsKo~y~|J0tA zs+d>@4eSP>%pUg0-gl)dC4m-0=}s(QQ~s=Iq}G?h2u+C`Ry4F<)akq;`;}z{A5lWIKaGIPykU`8eZFS+UF- zJHAvF`qkMm?6&u-F08TRnrQXZw-vk2OP;$kS{uRBw7S#Lp$XH8&Y6XH>-FR8#$;XT zHQK&#=&~722=wPd(5dwa5eJukSHM}LnHLq5Txd=)=oXwYFY#TEHl6qE52jsASRw_M$i}JXK zUuyos_?rh7#~}tvvCj`f4D;OwGXnym8)l0+ zP^X&s-OCJJb+$%U`|8Kf{wN#8yzC!}xRs8m(dF+v|2~;mYN1>>YNk=-0}EP8t)3%E z;`;B39iQ|#udORm$ELV`?Xu-_ar%3`3kT6Vxdw-?(d;km)4E7|3*pTfP~g^1`ELxi z^u09|DdeOmr!Z$4zNpQ_^f_3n2Rq)GZGTP;$!mHAN&ppvwfm?S)%u z&hv18ivjrX5rZQF9Pg3#oK7PTwDaNKF-eUP#RdCOI^EAe|FFY4s2xx3n`q#BnpSKD zcbhnc5e!*NTiXkmVZ)-9H=kz*Ba@o|>xCrawxZp?_m=+Ih=|v4LrI;z)M3fWyP|ZN znE1xvdE+Z(X!gbORl!Hr9HD(0qAHc12S&?Y{m%B^4Gx>gt}pcc(0;weMz27EA40c+ zb~Y%hPClfra~JsNcYH{ck4vay-PaK20LE@NRUGbY!_>=H$G(MbH3c{fon|(C!-oR) z<)r3=t#Eu2ZX_NluYZ&i)`iKfFiZV7&n{Umt^>flUBGHvx}*PY>a(ycf!AK_gO(}0 zKWSEQXPb%n+nOCF^s=Uyf_&Z5GbXX7J+Ucgm5DQPs%r z0>n$Gn>IharW&SqA_NPj?+7Ms6Nft1c1U?>(f6|#(GP;3usJaI#5WPHQ;5s9z)e!! z3AZ0AvhGMDw~XNQFwlFw%|vh8u~9TpSUA3{mF4!hd1H{!zcgIb->izmy$?p>Npe45 z90wtK^|g%+%p~pPm?4<03}kHJy-KON!eRKcQgUJsT}k#8Y+2DZ;RRn6Ynu=mae3~S z_H(z?7rmBul=W`y++Sv4=G(|!5uyEPLxI9ex>BNTNpJquI#3cWm0?^ai>p?Ip4 zH2a^yh_~vcn{L%}e8X2Mf5S<0_#h8iIdn*q>(o}zEl_7a9=zrBNh*Y%e(r-|X!|?` zqDvvJ*ut=@ua>{3act!T5IKLpX%$ zQCIEm?Q?V<9z`e8PM_8L{raPI{}W+)s1iTn5%erd+-lFTYOYyKgyM;VB2LvF5j>EuKw&I}-%W?rG?;{1q`~G1=`^C_5Co%a_+z-N zJoiee|289~@`p4|Pq6oW2%E|a65>o?eHZU@Z>L|MI(Fxhn^if%T($2gQie$GaPRWM z?m`$k#K9}bZ-Rh1-HA9fa^)?f^2O=1>!}80FLJC=HSvw#hkD~%Cq&9-Sq48yD?Eyo zbG=N=$K+5(Ho1SiTjjmt^%`QJgT&8+(6=Zl^Ayh0XJxH7CXs@myY<4*Fzg;XGjpIn zohL;tM-c?vovDxiJyoHpIxSYEsyWL6iNaqkG#@#b++S>@`*=8Hg*AO}Bp~2Vczr6$ z>g}vY$*y7;t%j+pm&tc2N-~l3nmsaNKJr(4{%rOM0IfM&3$=ewVO?JVb<2z2gqiEQTZ_4^sQSFkSZ*WiNjvz#L_fGUV(rgA7B1ZQf|Z>C~Zk^cu}ZZ<|5UrS~ea z7xiwe-|mng!HXN4lb1Zo%S(9o|9M2A}{I!o0 zfBO6F1S{fDyY}7eImFHMP-P1#Y_qJbbs=VswkfY}-J`;!K5yMas%y)k_R5(vY~Xo=bSdzy?hL1G*|}kNK34U0<&!XRdXcQW97wu_1 z{52*V!_jkeA+I(?j#q~H>3H0qokUX6zSIVq{l#`hVJxaO&`GtsTsVe#R6kuz@ee>K zaS$+R&6WFmnxCu$y&v@?bHFbSv{;Hck_=Tm`?L25-$vouAM` z&SnTSarzEjy7Y851ekWF6*6|?QT|tvSq|n$*MBYPolRNZD1gJ+@yzHDAqB+nwY9aG zuD{c$BtL(Motmy*Fk?;b-5DSlEh->+>FL;d#qGrn^q4b^oGX9CBjJ$a^?Qt?bC@tLAt;mbcKA@KrfBE*U;Ls~fKQ=z6ILCHoQ9Zbm#G zACBehPmv~WO{(XQh9KTjf3BcAZiZO;Fn7;xqRQ0RCIW@YTKO73FZ_Uv^3Mv5?BJ4# z{gNN+yWkR*?ayWHKl55|*8w&O+-te+`P|k}#XyWichWslA;w97I6=S?(rVk=SjTPQMgT z11yn4`&GKRZnlWZ5{#l+tB6+Y08KOJ9YiW7S}vobtzXuzy~ zMahxT=ouzsgo8lue)qiL5vi?WU>wOsLf_+ayd5v+lPZ=m@@`MOx~JSw<&8qy!?iNJ zBbbSZ$T@&kzjd7H>~g*McwU}&-~WZ4vc^1?KKcU38?uA4eGGQV1Z=z{&~B6m#I zknEvW-^+CDvYq|FUbLsKbeT21V-z|-l9>3Gh^M1ShN&m1BCiZOwX>hc*Gyge!u>{E z;=FW{mfqUY2bl}ZBrf*%&En9v=_ffi8f(9yjX1hqwTZqjp^|oLMyL^bKgfI_cDTNX z(UN?U(kyvr5LuZB^ZY0lbzk&>=(_Kfc>I2ocQlM6-%Wq zY@B{+V%FT|q}rfG(HmUE;YBF8cBYt$CoF5UoupFmK@un8?v_|5(+_gBOiV&doHnsm z%bMSQD{A-p4AIwBmC;GZ!#z_2dbYx3Og_3+EQ-|aWH(wxDoYN?+s-rJyz;1msLvw< zQ>Kt)V0KtFgUbs2*1zl1jvy!1qreRj-n$7A&y3gcJ{@GFxDb-Te}u^u{sYL~P=H~I}-XHbgJ8HYJN*wKrNEdG7>Csv4y?<4EE)0zwMfl=r?gLC( z65wn`+~ebzKd8ExThMuX&((Qu@1E>n{heE!agaqT+e4(&qgWbhy=`mk#XVog1orU> z*hghs+q>1()nek}E_mq` zOKq+)yV`wkrLH6hl#?Ke-LqJO}p#Ew~MCZ<3yLfrJg9MIO_=_7)p-3V`z7Lw@)K~t#L5WF`QKx?c`LID ziNuKSlTF;Zc@<)TouM*b7u#J4Ej!QQ=(^B!ckP1P^a>8Idta7@KR_}wmtrDTzQ<4t z0>OtD1_scI@LWl~$od*qxhp!S&8b-9m7iZ&x#&N}hnSe|3zrwCz=vE`bfrdj(1pmp znwZG9#20laLn_c@4_&v$(VsJOIX#jM4rU8@5#?Ijm@<`^ENh!cQ0o=#*=HBpcptkg z(3zv`i1+E7-+Oa2&)YbYbQLd%+iB2#R88C7FVQ3pO-*@z;q2S=B{sMN2^5eI#CRR; zVwD{q?PEC$(vEr^d*IiOe~@BZi;RknmWFl#2V^g+|MS1QeY+u^^|5=AeMsVHlGZ&U zjowRGl|}a=-Pic?Eh~RoQ*m7V>R_hV@NO#nuyS>QE>5kt@U3b6n80zlUKGfwDPEyP zc?|1U&~d4h(>d*}eGPBtDz+bclIfzI-tTYXiaPzL7^Cp{6aS6_LWWm2k`Y+zMlih| zxOy>wyI;C;r3IpivRO9Jy7J!W5Xf{dfnI_?>z0v3?z+o#t3#NpcWpydpCf&hE_>?0 z!QmT^`D<5XB7D3)GIk#~GAFp@xGeT1vmg&Gj=;T~^D6avy~R#XpWK^osl4dl%kFAE zi0^4?r828o;e-=?adR;2Q+=d|Nrf>OSj?M~#uV6qZOc36&S_Z~1&p*T)1KQU zIud>HBB5`Xy<#W6Kc?#B=P}<>Pb0Wo%o|)kaBi^bhY*;{1oG?FXIxn5iZR@nous4d zd^N^sePcVopaWZz?yB(Z3XHW0{hdw8<5y3#k$TuYy0<2oMH9bqx6BlA=@N4eRN`TU z=0b8nUIli#>H~w6^X^ReE<};7(5h3kQi3d-y?psH;7HZ)ct@Kc7*=I#|67Lplnl|` zjwM1@>M%a+&am@?kBS-GOOA#XarHGG7g+VnCtYnck;u2I5afN|@`P_Jk+Nf38&m|Iv*Z2f@$eTj^H}`> z0Uo7|CwAO6fHWD7EU%_nKaXI9B0h1J@Bh_Souh_=wTE{4wd60F8d_JnV*9ha6*ISs zD2`j5Z8eL-vd@uAU*)-1FziZ79J+ody`Cep(LbJ`;_=a&HXmuMho`AEdaiZkSgO?s zYv(c1E7)wY`gdAkwY98ZcDyA5Q|$S(Y&%@cx*U6#q;k767Y4mgEic+kk6GERrX;Vu zFd{EK8bPCS*j!=~g;_i)4ym@;QI3$HQg52|hXi1uzEf`uyR?mQSI)0-k6cmfJ*?n- zEhG_mynSCRp+z8%R=D^|G~eeDNbv)dyrHNC^O);^65hJp4CzCJpvzrU*@kSfrXuE{ zFe`!U`LfK6_Jurks_`_)Bm!PD$KL0($cDgC_uI67Mv1#)c zbEQScf!swxqm9G;_|6{oOg@^sOJ39R*5tIO7KU@fNBYL9LXQvdys3={Ls(t#3m1e< z%HM_Wm5Bw!akD@d!?VU!GVs)BiQZgR2P!N-U27R-WQMkOhIHP+&^3C6#6+eV3DIi; z1N^h(rwQC1PF?6+QXbi=Kw(Cabp%W)`GUHl5X?iqifL;DK>!9uMnC!QkGg>H1N}*| z&@KE{GyLO6(T5LDL#wVn)Tj=i4+9Id<2C?5wnde1hwcAu10?2)fl8iUr#$iwuNSJ% zhwa_>dpq5V_qJEiwDzTUE}X}5HmGM_Y9BUJ^^({Zn7=%fLazn}z_y#egnLdru5tEL zyB1n?$Y{;F(Ek*Lr>BROYF0h-yu;1wAq+zwii@75%k7gA-};#TV#k;w0hJrv^VVOz z;`J|zxIzq)G=qYTkYX?Y1brc#1}&GhTxLiM%m-?K$J`EI1oxrQG2g6@5fV8_;uI7W zVYKfxKE7gOCk_PHIeh%-5@f~{riN}a)j)cjdUB31-(+)irmhKV>mIJA>DNlGERYl3 z*D2kiu$$AJoh5sRS70T#EP#43G5Xp)f0?;;42&7Ka;gc}8{^5qaM0(PxeAejCt^>2{MOCX0YS!N!iskdQjYJ@h|Y1_ zHT7~w!Lg~?cs@@@UxScsCjPq0hkL)bkLhiD&)e!OTK8|4IZ^Qw5oKk9rah_R($e@AgL&UTYZD@iSn>To zh6lbIb#lqj9@)k)Y~rkLj`L`JWB2w>l7S$TiP~k3h8n#%F_v&n?$9|L^8;|YNFg^$ zF{D3DUzFFC*fu6q`J-1i{P9ndLfBWxC!aiWNQh?Npgy&Y_7&(O5*m$L4eJRP$BFjGK{PiH^j@Qd45lt_ti4 zT3z}f?c;M>y_E>|u~KT;d(US=L?g`m<0d{rG~5wjQq#9xlQhtKr&L5^{~Xui#6AXv zs$;r*@Gn3bdL^Z5Mt6A5tjWoRMIYnr@*iT$7LP7C-t}yDULV;F%bs%+k^A{oZKgL3 zki@X|tUD~8Uz9C${io<6?&<0MK!;bZ&aB#4D>ONs_?(;cwysI%F2s8G(5W}#Qw6!x zMQ(3+V7$&GCkA-_vn1bb?h&aEI9HW0xAw=rD^74=iH(j{%hfibaF>A`90h$ysk~z1%O(^HOdt88F-WIA_0~DnlIl!6Fi; z^0;&-`QT_P>?ZLPpEAq>w zo--4l%PNmNq{<&Vlh7{h=@f4>)6-Wdi{uFlExOQoZHa3OX!nl3p0LK^_#rCfbFidk zcXwTbrQf;9$m4CmryppYd9DUEI*xPvW8?dtn0Gz*zwesidS^2>qE|0+)WN@fyuB-C zt+P}2_F*Y3iv!}Dt- zD(tQp+MpTeFvkL%l%o4vtU++F+vU*P)hpiDj(uXCXfibq?roOkuDfD(IZoj}1jY1| zk)hQKFSU$B&tn5tld-ElqMafk$g1ji*S)rgr7QgSb4Q0*WqC^sgfHl_(ks&tnt4A* z3MQ`r?p?vZzV?WHR-_5WDShJedC2&vEo+| zqU0!n*>utW-Sj&~%cl%8My*nu%r4cB?eQ>84dJ|K+a0{S@FJku( z`C9ibR$gS(x<_2vu7!b&Bxfi`Yg~t?BB!~Rq{q&A>rw>WHtfq}S7w}C`w~_XAektS zvG~)`-^VdS08W;Y8*NClFQWHD=v+>a5N{qU!S@SJrZ!^Y^2c$<38R7RdKcKzdt4SD zQ=_Mx*Tb2bS(@_Fy~-M=xXzkA&1MHna9bv7o+mHJ)Fh>IkSA>(89K-N)uiEpc~bI^ z0uk2mF-K~&cp#KW1@cY>ul)FvG@ZW2)7LGWoo5cMkUQP#< zAGflFt>k&`2*bhkbIL$H1tBY{K9Y>K{f=chVDDj8BtC& zvLT2xTB^&fLEkt7&Ia7vjs-t%`nX*xactAGNU+fMs_^NbT57V(?6F5J;ZMQ1hq2=M1|Q@-htSH5e2#-Ll*Kn&8KXDyGf$0 z*KI<~A}JGIJ9pYR4{LlVYDr6)q7_dVo=K{1teu$WL8@4DiE(eaNWq6x;C`mE$}A)l zqpa=mIXap;lGjR0UhXetW$p(Abi3nynqjWU5?w)iapv=vNbU&r##9BXy6W75l39ND z)b%M!r=ud<WKkwuLgG0Pu~dA7yEW>5p~DxCb>sI2!UZlHtAj`s z-N!v?`YPI|*F4~Y_ZSooP!78PP_~fAqZ+8@iEju_Tu6_I<~3rB5Ms+okBNOh4<|c; z(GC+v=!~Iyv%Sw!2#}aI{^PxKI*c3H5nO?ulztQF9VK9c9TcH^)hR_h-Nh^akRreW z^GN`YYKpVZ?%wsWyIW1(&+~xJS8c>N{^SY0(XOo{A?1_3G8ZjmoaP?O*!b0=E)`J)~j& zLm(}$X?Fm>!{RScd_HdheNunc^b5C;+10isf-qO-ufA{{10rJTC&-!98Z0IzUJ+dp z(=((e30G~bJ*ah_s3FgyG|GDbio)GGwd(lW=RHqB8XG53BJ>bgGZQa8H3lkZIh9e{ zk({j8AcB0P3dg`@4TaH?k*(vb~__p;}+dnT#*h47;@i?jGO!Fm=FVB?)qO=vMbD1L>&N$?_;ChcGclf?5}A z(BD7LdN4Ofb7KiZF0;=-<<~xG!Xf`36h)ZVZDbjzy6MoFly6KyGtF%ox!TiGhXJol-;W(*2Wqi+npfSJ zclWj0#~c;YxjYG>So5{XdIQ(^K(>+}(#U@|Myz*5YU&+bf$Adqy6+c`yq}4dAwda8 zW|^Tb0|u`)7it@>Q@Oac4~4ndT(c4EoytC`AEstPIeXU@8LpN8*C`hLwP1 z%AKI!hgDF}W(!Zt`;WT%@E^umD=^Lou(jw$-mKoQ)a#@4d$99a`NLG3?g7!1O~fcl zj*>!pi8GYrIP1^uByZ&pAq3<31b#a?_ZUG{s%WE8KFf$m`R5mz!+Mc*>$uWYir0AQ zOES$9+nN#!J!Ostd93t3^}zVqUzu!l9VPIfTeA>>%~G(y06hpUDRk@9+9xNdWBggR z!Ok|LhK{t&1>LBnq39Kv?{cL_`JXeh1@sFOJ=OPz<`t5qT=H&$v-}6WQxIA0nOmY2 z2%}NPzv|VppZ9n`rg3^%%Pxs&zDR$`z>O8{h^{Tp^lC+9Xh#=)VJ~pkO^GA- zpmlO6OVa?vJ2Y}3)lhkYql`$_q>?danDm@hKJymKw(CLrp6`okKISqJ;*^F-{%Y2- zht7vn`8=Y!t0I`3jyP1|79vuH>Zt0TG1TIO*6iOK?+ zZZR0mZZw4>S?z<4--&HX?SsM-vq4xlS8@C`^V%mHJw@q#G$YgSouDt zU4^>Z7`jmJ{YRO;$aAbsw_Kx9>#OKNY|_!V042LStP{Of*Gee4Dlb1OyTIdS;Do7a z^hg)uRR#lHT3O5GHIU(Ap%P$Qz7;Fv27@xN0g+~*;xVt{l3Cx_cq}P7{>`5@-l#J+ zAt@qd)<@k?T|hqp*$}j?htNN#g?_S~ zOQyjqKluAVFV#V8(Yjz{<16dc=Hz~l<$UU0)G^11W$WI#N|~z+?DaWE)fq&;S>jbO zAUi#GXRrzwO;sZV_#!RWXtiIS2X zoHp)BC50hs81;%7M;XAU;nfd713P-LX9@;IF$oAjg`OJI?gfGfD&?*)l`QBMIv*P7 znm`1Zpi0weL|jjg9tLwv&(3~$|Gw%qxmG~q)I>EN-3uV^&TK{gYdPwMl3Taer|2Gs z?bZL(qd?@|fn@z-zAgT0^z@YJ(WgGx3X8VWQPB(@-p>DMAu)i z<5gemx^`4%>wtIO5jDG-@K{T~m{~bzJ52qKCYxh2YL<@qNLQ7P^kV%+d0Zn$dZRsr zN0%FQ?Kl#4FJ=xFYMPhXDeFge97d`YjyxH1Y&TXG8P0LU6DGFi zR(XwEY1kh58nJMLho@jTV|T)z?zD9?L%4M{4!KH0X8D#HpjGFoiELw1FnUM7pz@~I z2mzq+Kn?>IG@>Pz6#RRgqGJ)u6G!a5K$}xpRXRdNpV$6+*leh1#meIlYB6CJ4 ze+FrUI~+u4bviE(!UNbcVlV^OXlj$Jn|dNq3OuAe84H8Iax7;T4}oQYX-rUh=CZKr zrCd{nSM|N}0>-iFeR5k(`})WVRXM#ch1f{T$r*t()BWa&zx~sogAPQ4_Xg}up%cjSJK2uQLIUN#5nGtUY2 zd1}?k*=zd42hFM)5kd_0SpkQ1x+?km#4moy9W28moZ*fW9*vmkR|UMvCCB|+UN|^7 zT@Y;1x-H#-<~U&dPemBT-U>J5wHghY>lAZ`X*5rwoy?a2e@{qAh+acus^nOndkah} z75kQ;G(+TV5}B@+|28e7KdWA)nCW#PnMGlDRQLJW<_8vU zX{nBm>R=?tJCVJM5*k7WS9s$)b|h3k=J`4GRhp@eaim`hv3E;&=mua|q=oZ=wbXKy z=Gf(B4ob?UPEs%2PjXzW*RpHeNJ4KnJi|gX)Ql!Jnjw>G$@QyDMm|4>eVR7E_oTE+ zu-5K}ua6G}pY09y^36-Yv`Mr~zXt`un|mdmhkFbi0yaxkFR{fWB;vZzDEK^=NJub9 z07wDxV7|V6`v%C+)2C1G!zdgGp#|kc4!Iml%azKhn1)}v-iBo8R*%C8?3XZW%O4|N zi8R6?PFcvgFg(1!;3~*ej()+bQ{$a9ZI~1qS`} zA2k~GQ;c~E@-6U9zgNs=hf8moo}sIGE)Va+TnBTi0blNyd|k3|?jn3|X0=xvVC=P! zo`WCiW=HS>&Y?w??dJErLa$U!F)o$vpY1>LHE0^%m|CP}!a zBPFPm;4cs6U+3iw2M`p4n5=gQpt5wTPs5BQ_ZH1YS;vx?-@(ggpTn`vbtO3Y7z%cn z8I4#XA|i+BSE0k@0FAfnDw>lxS}cEqh-lHf)>2?*4^oQG!Z%}Ti!`QAIvSle_snP9 z?4W5OIx54=D~4ewe{bH@7TeZ@qW2<#Ygb)~aih@ENVU8`+-}z5$;3Bif#|Il7l8+^ z@GgSUefx%aq#+HbIfZhioLJPgKWT+&nJGF$Sw%&qtB;S5!K&SBvj2^l>|9cmT~v~FIFEp zj#7*)Hf(TY@rXPWgXrr;!YyD4RiPe2r= zD(Tk0B~4e!hSsI6PmK(Lq~zqvn`rP-QqfK)AIJcB9qt=46TNAG8R=v@jn*|Z*iOCU zJ(=*xq20}I?%|G1%wf~Cr=`PY+; zxY9lyygMiHIbZT8z8wb`z63M$X!Z2FWdh&%nRew(@lVDoI^^(AUClqZLTT>RqE*585Ri^B8w?vh>gszeiXG z9P1P+<1J_&Iar)9D7h9490}1b%x?1L5) ziJ<}um;L2JKHKTDNhYp`{H~ifF$NAU%M{>Rtc8Z4aJMj0%6tD!AGgd$Ka|FV6e5R}`u|zB!@ac;Wn9NZQbLR@-r1ar&>auSEHR4gP#VX5R969W*L8zdJe z91|mOq424&Bg<68z2k$8zzWZ~7^k-}A7f(LCFnfP+y`Fs*>mUkrjY+4@si)%!!aD0 zZ&fqdK{B*Gp^_r~$E(-B;NUdk4O<46p~;HPdY_K7=^T7myX<_P1sR7r6qJ7{PdWhE zJ!~R?Gw6HYUOzDW6-R{c`(CSD#io7ao`t8@Muo~ZsDzmY(FBgSs#oWEib&1fv91sN zN<|FUs}2uxM^_l8+uYjv9rP*8N6R8GurT@grVCg^sRKs;4QOk@;`exY>f&(AZ z>%ol?&;~Jg{3-~N+hIoC0F0?iGMTjJ$B-(EAc+lpXn?uD#pR5!C72az*K9UsVX#q+ zFLgMpB{M&C*Y7|k>gG*DiLD=Z^v|vgelC1oSs$i^eCOAfUlDu(sjY&kqz6#j$@>Nd z5C*ZiF4=ctY8-I$_)xmL*c= zEHnuE9;aX!V*xwTKp_%2+PUthy9!weF$sy{WEn_3Y^;=`^dcEWfZD9e?LzNgTMcay zm`Ti_zMTvsM_;{qb@9@r8gA%&c=QMt)Z=EMIRWwm5%AI__6w)08tJ^Gftm?{fVhM- zibZm>H|H~rH zs(`2=crADLPSJo)=3u%O&k=$Fr*aQ{3gYGHUWeMhoqQ=*{duaGlzlme_RD)_odccIskh7-U!l$WUW_a?AzphVw>~sj!>qhlkToW1 zX#cB^-&A~6s1XAwU?3xr(rPN`e3*AS5Y71qE$yyA$_9C3Egnyi=9uFMt|3XjIBQqw z+N~TYiaOioK=mDZ-e^$x^!^{C~^Z7Ihgz?HeV(F6^O}-hY)m)jLsJ7;!{RGh3 z!j+>i*3;a%3^;+nZVg3s;9UiPyx+SI(@#Ab6`*vn)>|sh@2HbZID?c(-|+THYnv^#YXiHj#ZHkg;NC z!Q?}b=B(RXKw!ufFiK`FMC523zOposP}$?(#|Ux$S{{Z%XvwYpy% z{SBd;)V7D`MfEI0n%kc3?;>vn^O1#`enmKZ9sv>80LcYkZE|gQn=Z-^SaI&_=iuvc zw%x|YMlsac18IOg5#GAz2hBQvRyQ?@ZJ;?v^H$j2`(PXVWu>@JAtKfCrDPM>xR*sTHmWypzD8e)i;vx_O>S&h9~+e2OLtX$Zg-sToF5wu2RLvQEQ~fvE2T0k()Dt z&%?>uBf_#mfCAZu+yJ&MtkB=L(efc!NhYyq_K-%!(agVO#MJi$`v@0wDQlRkhS=3k zA`6L?hIiC3pk-^!Nd%9UCPVX6+dvMbHS0^*h@)v_p} zsu(=N_r=D~m)*IFRK~r>v)NeTYfAyCz&&;?z zcNRKwT|Q-dUL(wfUg5DcNrl^`=W?v>8t*Y}3O_I0F2uZmP{sdIQfTRsQ~&T~eaKSk zu*`Kr(Aq^xx{#_b^m^EuckjRKFm3LnGX9Q+oWs}Lx|33T- zBSWdTf2D>|%D#YknTrL}X=+FO(wIG~GTHJsB-uE4*g8A=ipO-B?^l{pk=s$qAs&o#p&;EV-snE`lqB+B_+V(jr);6iaa`|1pzd9j6c^ zmv%7}(Q>)K!D0IcANBfnc+DlJQs`c`H>5A;?&zY3!uZI$OTCeur4?mEz{I`9d*tYq zVd_XqXLVHcBaGCX!){+1o{r#tvR`Ok0D*zJLRjA*0Q>7r-o3)h%3?=(yJlZJl-Kly zP(BqQe;|cW1-M{l)jl}Gjzbn-k{;{mTqW1AJ_Titd(;M_f`8Gd)M6ot+=x~5BPpS~ z{UOcY`So%s88W!33AYTE^nb5E4$BVkKC4xL6#J8;e!J*6rOF~(1se{_Y zdFTd%MoO&%!`!~;%;ya2RR*jc1E~R6ODboyN6#lU^7oqHYaG#2FnAFGL4{s{^b`M2 zG*rl7BQXz^Yt)(KxI9SO^&c^Tgssqjzd}V+_T|uB9uqOQ#aDQ-uH}vO`jvu@f|(od zZrOJxzX8CuPzz4}?I*22!t#FhguRm}_x%Md{mHJC*QK}wB9+4IUi)}#ook9pghtc3 z&gysNg_oaCQx{tCxqD(4T}aoTBY0VDGPirI3!eEB@`TX_^=d zbj)(KicAzvmBPl&OLH1TqkejYU${Hdrq+h)jgi~Fp{SRdOf+z+CDG7m7v4HU&&Bl#Jetyf zVgm~A(nq6OM93|3yQ#$SHrn17OlN4e#;RLZ7khbSBm0;#=S0ko^;7zYes=DxOtqG4 z!|{I0l{Ikx^wKG<{JaA$&%tfQuBkuF7u^kA4NHUqd8n-;hWbG0ve29DC#194IDR%O z7<0Rn3(JXF$p)K%1~Z+nAz3|L9gAAfJxO4MEzXy$)jD5N1ma3JbF3f+hSO#=V| znL*eIG3ab(oHp^{i8a9N8~0jGnaq-Kq<+fr;Rojuu3)H5huf?ST<~UM4#XxBsd-io>04GT4%hGiY2Z{vs%I$>bf7_cP~auwK@-} zS?sDeRzAaYE7Mo}`f14vMEdPi^yoPK@svrCqQYC#aFi+`)6%O5WcOocdQ!u!5$hLe z`Lv^0n79dOs+fYk0xfm&l>;bmz5N+NBM=7lp-7l-U<&{`;=2S9ARV&1V!Wfy`U9nx z;lN^6=7IP?Xc!P$H5x#pVlx<2!7IQ?kEVTB$+-aTbG%M3*JP3@g? z_5L~-WuN$1jImNIx%TaXDOWR*Swx3Sq^BI}pK23Zr((olFhKuod}T(7D-Wt-Q(H~S z^a&(G!{mnrmtB9w{%L_)k)~5cUb_gkkq7^I)~SYIT7$g1JK(orJMI)v4FKT`t9-0h zZozyWQ&)URXv>s?SCr(dsG0S368956_QrEmpIThn$9xw=J2(5*@mqQZ8n^ET@~U%3 zx$xli;6tNGI{AP4KvCb@UkTmjiP#NWLr+bSI8sg*Alt!rSQ|Yk3bVHZ0;L9*JrNbU z9d^v8WT%N&X!j(3_Bmwf2LCOY=STp}4gYeaUIY;rfA;(T!dNP?@Qs&ZNfz{Gwas3~ zWp@T=3mba`vaKS6BbJ1g)v+y$H09w3M^-m6(lP9K$iCUA$JOyY z=fH`(Dh~zF<3zC6x7kAH;iY4!*b0z#k#h`ewqxY#gEh@u8fy1K*?yoY&S}0Ug}-z3 zbyolc;hdmi73aJEta?hNW4vm?>C-l%10AxSaRxP)OlPy7^l*V0g?1J)MSweCKhcJ+ ze#3kB1K|w7&VP8pwINjvas9P;N5sEo)#FU~*p*-_xd_NcfMOBQdlrF2v|r}$_rv(| zKM7mv7_o?zamx+5te4z?=WWQue|g|Tq8&R>s1$iV-}iSo)l)gJC%)wd|wJjHFsRjX`L zeq-Bx%zy=5YB=82*pP2d*?SL1>nT(O@6vY@L`&ksOpKG2;tVEG;k5xNnwK%9b(97u zT8%shnNNn!qhXsn36XN(i^n+lXO};+Za5xO>cc`urv}a|Fr{D!%$L|0;kzcy{O^8p zfxdJNkn<WVn>Zo-N+^bPC!X(@Q0F$icLN?Gn0b3eoss?imVUd*K78gAAxp7GpGEQ zFJG#`j0-|RS5`2Aq+{Lj01PkV05grItzBN#BZkZWx#;Qiph7>xM=zOT&lf5<;KfmgC6L{K`Q02E?5;|RmQR5x))C$u=yF!F|pXkB&u&QeLz)fB@jqXflEZNONJzMui#iL zNc5$rU~nnc{?*VbQ_C~&XMgRd4V1bdtVqYW6-T4z%j|eUwCQ8U;6^SqmdAjW$^#Ji5|G9fw`63U&%E!)EeSJOC56I&^s~M2~ z)=*rxi}k!Pp;9(#>HtI~0-U2`z{n+bV717!ttx(iOAFG)46l<34W9pYS5q4)`=@I4 zC5c*j)Wv&I)F4*4_WoWK7RUioR(~L5#dex;dQYIc=#=h>BX!N2eswIv{*awB&G4ksNZWp<}CE*Z`{z&#PsXR-+XV){PgzFCM=pi!1vuK3@*U$#R0 zzOX~$sn}8l0r;GCwsOmTBy6qEt}N2b!i?!t%uqyOF#bk(_K2+qlew$_RJvkfAFyVQN?F36;>7kEsgw4L+UHS~WHyg9^=q2lN0AyNMI>D6l!&o~cWKU2ogSr3(_a zu%7kk-6(OMfKo5=Aeh}%>hj-WG-c9vd0kU@c|E`DS-wsMiq!1sm$lPsxvPk`)BI0E z!d1qDSP{LedF=Gg*&*-J+G7atJ!ks};?zZmVacoq0R zqk6ZJ*AEUK9e!V~wLiB$b$JzK*m8$y(|f`@pmih_w8C<_?eKn#BM`DDh(vO!yfMYata8If1L;DHI<=qYye}ibSQW!$T77-} zdeG-U{8W5Jettd}W|ph1`yX}cv%IuV_PG10cuL~Ij6^N@TH=(Jq&bu4lX1I#Xmybn;BZi9)oyv5>$4l-Z@t|f@Z46{RgYVCNz(?9Uj)b~h zC3G;scXg`-c6D!Z=zC=FOkf^cN0iol?lPZ&6zHH2J(jr6XcM2`=%V;Dt3u-7gzZ8{lDBzN} zn5WDe?d22WAo*##_k;-R5X-PVns5XoUrQ7U+zmMw?DDbL4wA(}AL`V+z<}p~nidd$ zGOf$Tj?&8?56F-cdfIznxp%-%^s6XUejd0iqy#^Naa*xkJS->F%x4OfK3>JU;Fnxr z!Ok}TCbS$aX0XuzT>a0$9zbxM@zwOn#U0DFN10uTzg*?8qFCIf5xBP6)!{$M?&qJZ z=#u3G`uXI2LYhZT4IvcF7KHvSiw8jP3*spva5=*62HoOYVG6C?4^}9b0~x zdY9BluO~4Uul`d=#yGUL-(3&C*;T?cUUD0iedvYP;e7s*5>2P;_QREkDotT&xZdd2 z3sg?hH`a&-+=zsKGDfq$gX?H&x`>FUvxoHMfoyx9dtk7t_X-z6a-Fl9di1QKvnF$~ z!gk>GnqY3J0MOVLwGP7BKqe;5wa9z@#0JbBVK|NjuuD*!SKn~@=k=yk5(ZGo<@j4I z1|LkthFP45+|u`JlqLJe>d`!2`%s3Po*^&!@;(v@GnT{Hj_DSPyyslGv~ie)(gZ2k zJ6$HhF9r#MIc{Z#yh8)8`ZXA>Ys$IX>OdRi^6^EA_)umSioOgcto8z* z1R9y!XDnz`lggaHOOyAkpuG`r9vml&IIMHcAPBZiECVGRnu6(Hl3Ny6ez&Uhn$PBa z80-&9v%j|Y(k*H`>I-~IqkNZ zHokqr&I4q#e=pUwaK9F8sN97)(*cuqX?E7S08Oweb-nqQ|MSYRUrrr zuK?(D^l|eQ*pY!cA1RS$S{%}rzT9i>{Ormt_WI&_9ncs=Q+NZ7a)QRsaphoyKqbDP zpz6#+sdP7v|HvtPSmw9T>Cn91?YlEz8!fQM@+K{_T@aHDCujn;$ukf+WM{xo%)eJ^ z8Wc`857c$@2Z_4J?7@vUOM5@XIRxd%kbh{7^RiqB?(8_axB=2Hz?T6Q4SFbfv&ZI~ zz;ZoBa0IIbE*`V-p0d)G$QCcn@ZO-Gq)oYq9s3QYxJK8N=t^3F3y8 zUd@IK01U#N9}HzsiXfdt$_Qsf_d<3}4HST2oZ~x^Dd!i4O841<&lVwk=~@}Qw;*(!^_aUG!pa3F#D?%Iv=vi^e9p>i4=}=ZM(R+Za74#eGoVFUo8^3In4xzBmq_v zEaTQ-fxUoxqGqf>FSezxzZAYErT%gO(}-^8_0tvdAlM9SghkClSk2-^>W+D**M@yPE0P*5vte54@H|bxV0q_IuvZ_1-VmPKDA) z(8+~EvVh3oGeD$Sy`E^TPdiM6Sqev>^0}EwF38I+FH7gxb??u6S~9J2rZ}Z_5#@Er zyw-h4??#VwZ#hM>LRLcvH!sdczMO!5BWpvdmdK90Rs%U3uL{ zp3KBKif0VlPy)@BbVRA^&0p(D0QxSXKMa(LLMfwPMBhCQ&98a2nEQYOv_MY1nLHZD1JjuF@#lJozz-3RM~ihdBf^O-9;cICr&U1-s@H@! z?yf;-9vaPhm}m+d77qiEyCY zYU3ljG5a0#4x^Te9TZf2yzd?7G!uG_!u#y!`hs*n_86%Qyh_ z1%wdfy1q+{y>CCD?P;IH4lJPwJLgt=Q*)PlaLXMAOF$y0wEwig6jbK9Y!B6JUbuH+ zvh@`NBr9(Jw>m@z7g)2tN#+kir=9Esxu^1@(s1$}5hsxfTiZe>#%k_qUSdk(3vfKo zSG5Pk)9I?mtHZaMZQayF$sHZF)*R;*ANFs_iY)XO3*RD7b1>njXl#8psM#9a>w|~Z zm)|C_yrAVc6j9TCE6v!Gr1FFac$=xqC~!qC$+v8+d<#gdCThkV4s3ZIe$`ih`L?)s zd4IN~PS9U(LU=ZY5gsR7#E?K;A9Yn2Yrt+$bh;lWgS`i@D=3M742%Af*52;vocKP^ zSAhDym>;W{xD3&+l=*LoUkn)(XX6;hS0%Sg@;_{=Qt2PbS>eJ|MDRyaSdAKPdgFKbP;ylGO^+Vyqf`>N%|Z@$;RKA_ z@cfoTm-@G3h(>V^0Ya#nW(Zg*mY0k>Lbtj|lFs=9#3I&hf(v&y8|e8M=hDOm0RMON zyYza^!}pUB#iu-*>M!qwr#+}x^a7{#caIdJV!xurv)il+KHb)m0N)x`JzXwL|8vyT zzYF@I!4kmpFGhA-GEMw>hpz8KtMvpI4as%569uQ+RWOtZo#LU-2C=0xsiG%EE5b4c zpAszU)ZTGj8i2|VQxBxAR@A9G4~a%{YRikd6ZMBXPHQAB1x*kR)r-E(+-i;1P!$E_ z%zNpdkM{|pS0NiW{@>^KG6<>~fNe1l_Wdh3%QDxl zeNje*0ofDaN;+?Zcjr0uj@ zmnj3_@qREwzp_C%#e$0|nd3v7FbXVZ19&D^^xj{dj11 zOsaAqd*sYQ!6hC9KlrAPI+$4^1?Z!jD9>b#FDMnvFDb8!x^F z9&d{=%-LnP5^dMT3?fsPf;=sYP~cU8WBSJI(OF3x-MuzYVe7Ov(Z9D%Ex?+Q=~-sh zQJ(6f41+&TwVBZ@w6avFxhGd9CuJP8^i){Swvl*f=B04Pj$Tjq{4q^YjeSjU3gjpn zH3!-HZX=8$8pVHu1S{m)&1$)Xiaz^btF>@CSWn`f@4~Npl4F$Axn%nCT8~%AB{PNg z!nt){VC7+Wk!J6yRVV`~DA44;L;mn@yzflv?sc%A`g{*G_vxLlejyju9R6N0^=#nB z5`1a9+D|>}z*4Opzi(*ceoEqoPin{?UtX0Cu0aW^L1P)!9@~x3hxe4UwWNos5iUWEM5XQLpni*8)&lsZ| z4i7RJ$&y?i!wj1B6o~8DzmYL4iaoHO41E+~1hD(H@J(G^mNF$UCm;lJ|C#($ocDfo z^PHeH&jg`VryO4|Ll$Of#8q?Vc!rd^$rGa|UOFDPlFp_(E$OjQdPTTk_i$UddD)ef z{oCb=-|6{_pASxA4-8P8ZTVhMCazqMTZ?g@Zw*Z9{Z4)wO3R{fj7e`@fTlRDO)zRl z5cwr~JhU!c=i*$|%2#w;OuRTaaN&Mbt57CZmMHeCrc%ShyIXl8i-&(tJt~C&OZK1C zCXSA)&L`!@hHn#RIE)nHZKl<(JD9UpVxDa3cxpCIlTf@`@@oXLj|&yZhze}f#&`T3 ziRqI10YPR_x+9XpRXyq!EOQoq+9{khcKOp??2&!v0n4;6$H2P&&ZNKgTy{(pKOP^@ z-pvU;_#s>xNyuG&;S~ZKIR$luI0v&^)iLiQ+DLM5Dr*Q?pHI07>Rd!+LUlVQy7vde z!Bs3`bNq_Dwza7>cm*W%unxYdpz3aDeNB(-qqp^D(Ccp<t8ik=x%tb+2|y&6KEdNfbi5^!{@uY zTYI~kI{Tkhc~hhre0j8b>(*-=x#{|*kx}Y(ypzuPwt=%}M2}-ke1@c^x8~cv6}Iy! zgDT^hwRyfo?#;vViL3jESc{bte62J!b|{yN4=`JCaawntrG@hS&Y9Vqd+(6eM3_gX zboOKNe>$U_Wa;is!`{oDad+Mxb`y?yay1ttqXg7fT&7Amm%B9yvax0wNC_ah3U`7+hYTPJ^%9LE($bPE&hgQ7Iq6>Tl_PBxG&iHp8uZNS0BPF1o#m{d7&rHQK-U zrAYG~(f}0x`?rUihA@GxN~N0w-qUe(tzQZ%Py2if4m==y>m^f*v^^TGjK(Ia5Bc^j zyA&0(D5&E6!W|E$&!?v=YiQJ+fA1JIohQYw@`1CGroeX0r?$ceg2&03zsJs$t=1m> zT(|r5DM9QA8UGkdRNts%&!kE3?#a+89ag21Nn@m*_5yTczF@Xc!1Ih%&-bq=}^$fyoAh%5iN5@YJ^ znF9je=e|$@U^he!nUB0GiAY9r4i$T z{o4cu*ya5*u6rYSxaDKhq)KJR)0#hh=mJ4>0V;(nys%uvqmC$^3?I7gCg_+Rd5U_U ze0ig;|7J^_b!*_Kx=;2uXL}z;!NI7(muM$A%|iC6GbVJD6nVg<2t{o* zIxbnxWaC+>a%`8pKsPSaX7sDZD}=56hhR>GYS6zG(X9Vd5f$!J5TaBn+m`D$==JE~jZQZud}KR*wmZ(OJ-u=E3m2u% zrEE1`%Z^k~{au-q)bQD2{yk1hlbZIE@@;LQRf}ml5b4+HfqL!`p}x%aM>yY^v&2Gog4BcgTsoR7S2KTjzyu zh4k0X*FjOi0PEUe31n&bQPBEy0PvZ%#<%p&=!3VrH_jaj5uy;b_1AgFr0ByuAPS8Y zjZM>19vq0r#R4P=!;mwz8j`R5OOCV!6AI#h%@NI3Cz`?=k92hVi%8}Uu zy$;YMj9)hYr+g#P?*>Nju%28mlXR8N))ErU&tIq14rOaFK`*lKb&no;Ap<3ibp5Js za$iz}U9-G9;M?0T^6z3d zwftzDZTl`|f)zDoOCQ$hc$iS-ej?3>59pT-d!|EoG5l9?PRho zbJu&p;QaMdFzE-|@}khufQT`nn1 z!ltFyJS9Y-qyb3zAFq3(1xL;7*2E8d4-6jQ{kM*ww|=RK}5YUn-@`HTgp;)y4YPkHK{TG$yCiuJDI!3{_5 zqq~!%$0|LPd1}!ZKZN~uASfwfp_E(`O$GkFQ(RG+ir2wo5F?L>Ydr|@VO`vn%S-=3 z)GGi&pTr=NJ&eSU<7EN4Nq|s(0SIUxXXpW?EONMT0N#_*_q_gB!E^LK+s^5Z0bOJo z0rjIx2kYpJX2V9%qITO6WIyES8m|mZEvH)gN=m~*g<=Yb-1^R%SB6iyPc?D?c*U{T zkmh?dDeNC!EO2G7p08%GI0G2?J!t*GLu6!RM2o5Za;NTpK&sEpy8=-lf5mH?Ez}m^ z^og?BRoJXuXe)9W`Ti>V_R>cPKIe1z6M3t`a(M0f^nulAshL7^*1em?yEoy33t*=mHl2)lX240z@$A*hQSUk5VsRm?J8^e-IJHk2w!UgdO z-HO=tTC6!DgIn*^yO2E+p?_)OrG4g+nIXSW0vt&G6jG1Dp~OI-?Wq7=nH>aYmlRE5 zQ$8^7>!4c2n+;Of##mv`uWTuVnGIy_d8sMs=?cK>^-ZUa8#uQ3e@I&L|Eu+;12Kkj zwJ4*KdLN$8(B-1B+Zi5qe<@bLeW?6oJ?hKz@wx<{9u%AIiQTgq%K3KFbE8)c`E|Wi zbNG5D#rYZphkOD$@7uXj9a-ii=La=K+E6|K>@fi3Xe7{;?u{U4JSD0kEIulc&BrBM zD{Fiuu{8Z@wR>^3H3M;8lRaT+THeP*z@S_oqW(1T1MBlf@_tDcWA!fOR^@PT z6ew#I+);bWyVW|!VILLPrk#f#ZM>WyWTt|9K7p~T(|$)`)@@5&dP_aAx0XsHGoztY zPckn0WD#qt?v#w=EAN9*&yOXhTF`uOVF-yGuo$$=UE;2{P*6u z0!fMXuce28m%FTzQR8LucJGIa=ZD{vcN&C$o+Ppwj(YFZSJ$vi#W{@rA}u~6XIgf{6c*H zgyHJ)3t(6+H;P1kn2r-`PJcJ#&{ItHOQ!iQspAJ6c{uG`M;jdZOH}N8yboJM)LhIv zM|ZCzE&A6p09-QTHe%aeGK&QXy!nDSY=c<5Wf>ESD<_FY%lN2m^(3VNMIuOJ0C@$hYY8== zJp1R0acBcm_W-cD!u4ZKC{!hlHFtJ4ZM=ABd*04q&9Xm|Pw;qC*|nF$HDUM3tj;NO zOEMOqqJRg9XNC}}?O9GV8B?3k4p9^!gHbi3Iz)Q|fm02L`6V3ycsKk@TB+o(AlHi} zXD4VA=JRvdhUgjPXfUXFPabj9&3c7t!mMf8hsB@I=c64?W$>F6=M-W{**@0x#!iDa zq&VC4b#=Rx7^uB?Pk?XP~(gY%4k+B4qOp`7jsend)>e=823%%L`|zHUbc zv$B}Y?Oq-Rq{fQXEg(XL$%}t&c+u@SdB)-IQv5K| zN6z<#*}v0ZP9?!q%K6Q?fO$k@@fyegs;D zS^1bh@y@J1l;#F0+WHvv2)A0?vdu1aumXT0u7t~QFHsaOcN|Kt1}gm!H?XmP zQMkhP%+ZEOzKS*wMkIj}_hM^WFn8~)y-h1N>Ee97); znxwa4dGB&cv8+ek&bMSSO8hRV@y*=QM$YI6R-vNfOC43N-uY!wLP7Rj3h%ZnY`zbB z-I?wMmIMlQYqoc;x7=^@h!8Ekcr9!F*?KoYVABQTbjZAp-inDg#1;W?Oa0YBPyNx5 zEGSKXsG3m)kko%3+X(0GRk=Tq$z+e00BDihZEjHj3K^{KptElG)76G0(zu_0D3Mnm zJ@mFI2n#Xxr7bdz+(Y=b@n|mK;_^hn1iXHHTma?qR3w~7tzlgS(43!Eb?C^C!5!?p zx%_!tc9vphA%M;iY9(uLK{YCsL6mwtXdI@K#ZhH1v$+Yq| zud1g{?>cjE&U?HV<$2n?pM4AArNJFycS%f(d&VE=q2b-=ihL7->Ky4wHk5ttU1Ibu z5f?p`7p&P5zv^9VQpnR^4cy@=nVv;;7PWWQmoBXhq8Ok7f|IW|ke&*-ZV#Z@+*ojLBB@nD0-@FC$5nNra>)Lc-F4{J_ zuq7u@zcjry0D0q5!IRT#h>`+Mf|pwooUM-vvTxf5eeW6x_cj|3RIAfgJK`Om>p!5o zz$JRKae4ddwQBTT;L&P;X>|BmwiM2ZZ9he5g}SL3uJ8B2&VAO|>eK;E(0W*BtG)^5 zK~UG}VVOB|yw&Sr^_Z<4g@j)_BS2`~w1Rku4e09oZAdiAQ`A-^WA0sU&XqdEm?wh~ ztnoT+3h2?KBI%_|NJ&2gE0ca?)spiA)-aK??wTEL&t~O*@*#xBMfqjvr!<-PZ*mY> zV8e|AeDXcO$&W=CMp<;N+}MG8x!P_qk=M>ZBmG~}LyjcCB!F&jC_QYO+1c}bU2VM` z!q9c{MRTQU&ONvhNiO#8B6$l`^sb-c{ZMx~ENFpipdvR!Y3ulsXJeoTzabpfu;sD` zbUV^*A2A}A0kx=<*Us+k_RDe?pja93b*dlMC*boA)ZaI!?|vB@p0=-~CgcQFx)!7#-o zH%tv*uLN4nN6~me9SYMCTW7mmTmqH#*2IW=7)9&Z+@oVt9MFiAd($Y@1hvSNYhnS0 zXIjPd-n&?WKN0FPKp>Zq34~;Iw6_mcSgJ`zlE`RlYu9ii>`I7B3Ab7ZVLIzLiVJOv zjN*j}L&LXx7IS#&TbN;~dq0XO4R^Gm?yV|n*dvYC58y6`TSKNvAg3vTlTTZ*4jg-9 z2FtrBAOv1cRKKM_B?d>6$OuKuY<8ay*_zG^7tRV>M|o8N1`0g6`gE{h*U}67d9qt* z7)kM`)V~Hw{Y&8h%s9DJ0m6FJVfv-tp)-Fdul`kY1X-@05sRIdwY02mBWJRTs8^@` z2s2}C_n2(U8bvKvWbRakC=|r>SH!&`pLtqX@1M9l-rCJnaDAUt1ja*vSwGO1?~l%BHtRDB0Cl(a_6zrwbO{( zfegBqzI5-p^T9YvRLT2OKLe-;z$hb()ZF4OZxQzj4WFwz1dyba4jziQI`E^%mva$R zADq#Jq+_PZU`8HkCKS`fk#=ctMm;z-u2ULLr0s3M;(qEN?%Rh|bWK2XeB=fKth>Ol z3!+oiIB|{_fx!ZK8w`3-)z}9n z_54tt-M2bBe|uY@CIdi9ccv<7R(#PuuB_G^xbCf@H8`X2{bJg-Ubs2jg_DkgML8Oh zF%t!p z9-u<5@rQMd_4^Wm2BrVCu+`#Ocd3Oq-vLT~{J!2KcqJbz)MI?JCTb=3DJ;R}9Tr&V zUJIv#S?6a=0o}5=e~5_7;m0qs*=Ce}*WEV;5*GJU`Y0F&qd+D0_x%^cT+`qXa)o=x zum%#;a?l`dK!ef=WM^qNG9R&j>m6A8C^>;v0MO?8&REmEc=4U`;-8cjJ3prMKxcYi z|BkE+8I$wX0)wXADb2pABa_xkB#hfb^1VZbhjz=n4%Ma$LX0Bi_2XQKkx~Mab3lr5 z^>;jaMWx*t568ODGwP8+k631T{*dQ><6rShT%x?Cmd7LUgq}W$jwF3ooau-1^>j9e z&iC=1fFt2Dm&>Y$M*$Kot8go>`q$F7E;9aa#)6jNb_+P@4U>LS98n>EjcCj z6v|}5#caOpE zlFzpl$fw5ZHCctzABKi#qdzSReXpnJ=M{#ZOTuNT<4Z2aD3_9~-{vk%uk{Mkd$dq1 z0r2-ef#E(?Ri|xhV3_frYw6>s)R>;VB+Chsl%>j>j1(e{QsqL*Li2-zZaoo!B(xh> zpTdr++O$i#DN}+WSLv8Bd{F%Ak!l!p=X_+-UWq$JVi+{h>gV;y>|Il1HvFgM_q}$O zy3UFfa%4A3rWDvMp8~W!#(1{uvx8+`a|W~)dhA82iNOTQ%B?byT)}acFq-QY#c!1~ z>D_sXhKCA1HvAf?ksk!%8GVA%Cwrzc6{-aoJEBGB&5?TH_m3)fS5&AzS5`Eesng#N z(;#QWp~ni+CRQn!hYwm>Y6n>7l;Z&^51_J?>bIb-^r!Uj+E)H)AZ9S(XmN%gr`%yH+;uYvV8%e3H*(% zneP6topHg(*mq%k=jo(BL)jbl&2atFhwG|UpZ|DX9x54gRVp-V-ama@0xBiHtj>wN zy}P{%H_HYWd`q{bB#?>;xw*-vIu+5pe^;%|6{pIX3`zc&G9*4#acUf6l~W(yj83t~ zfVpxlCzz~ad5mJ7=6&d6?f4b(R^NM4R5K@aLsP`B78`j>R=RH9azpZ-tcl0`ub^!m>AFd4`xuo$U#HM#-pNWK!0LXDD5yE$Z zxabwXDCRnxj&PC4bUWs|xg{nE=HLktt?V|&;Npt+hQutjw;@Mg^gv0D53>(|QJFnO zXDmcM)OgFrknfXotCLq5{;sn}LeXel{BMMZxAW_hH9YBEqK5PRpHT{nho}86_h7Xn zkLcJtQw-@RBHh#b-rwZ5Ond00v2Uw&XDw0f^*v`TD5|jB@oj|SxqGIDd`_pjEp9<#mU zONs6ne!RP9D>sqhmyY6D0+7*&`}5%5$t$^*xg@M}W$fUHs_Q*H-dk=VBOhwC zzf`J54D+IAz(2Tv=v?tjG7&9vnl!h5X$t#B@`li-3V(YgTjZ1v#SyIZy{Sd`dI&Qg zEFz;hD#}N{huqQ;cbfkZ9_FagF z=B%pkpOy}SbtVoiY;rngu*ieS`hu{>S2&<}>65oxWs_(Iegia{typykfrq3c14 zB&$yFmLqp@xH!wOKQ};LaPRVN&T_4F7x@Zq`{!9@GlH?fue9JjR6{N6>it=JH>ujq z;3iW)dha}s#P!^z)MX1PlWmZC|FOV}5qr6mDq+8Z+mLPpu#nlltJzOpa)Y%G40Sksj)r0X*9i6>Bjy259EPnZ3xk>`n zm9hY1C{e`718+!*;)Vn?t=sx^N%$A$bqar8H-UQjG+^T*Q>zu{qw|=cUXsYnH4XlUC6UsW&&xbTiH%bdou)&PnDM@wWFd6h~C7R zd*!}mD%LEyM5F=-*`FXZ{HoiXAE_8f3}us3%)C3@5)wBft&Hu2eq!;ej$6OH5U-tURD5rr(UlY%yQcPk&RnHCkGM$} zy4T>Ik2W z^f~9Z9SZP7CNF4*8k$4|Ad$gcnYu&zm8u1s1efk!1%7&WG`b#F5n8x3jHHEpwj|_p zvx<~4*j~?s=O$9K+ZQA~{^b(ut2mq+FwLLn-zQM&)X$x_ zIYuV8G#~0{5qPQhv2xiyIu==P`ay~1`@tt>-cw(9q;saF*z0`jbQJstRr{u_U%qCQ;$^$dcG$ z)9AgEDc8|&q7ehaZP^BGV(^RxLvGwLMO&@9+}~WDBb!uy^?n~A2>-c~wVOtLb;(JD zXXB^(?fn)K5dhT_%F(nJy3Z{Qz+)2k= zMBtW@nZU}>35`7=JGy^M9=js>l;7YMNVo9UGnk!KW7#0A0F8Dim>fdDR>@XBk}$E)f!GF z9>G_Z7U?MHaV?`o%Vyp~UA0*SoleK9h4^IT>8(#kgAe;G>%W0%|InKzAo(ozdW)|j z_#*3=ElF;AHaA)J&2Q1&gZu&kv$;OY>nw>#TGJW@PAg6YJ*k9%IL`?NycU7%pIixCb{O6&r6_DSlXPp! zWh_mzO<%o;L2$);VXtYq(wmehM+=a@EuZO?nI#p$&G>UA(6q3+IxR5gvWToz&gR)s zlh4lJ*kFYjZnfsfL=M6UCp=CbbDZdYG3jr`iH3}P0zejZ5nn!m)oLA>*0HtR8b7j4 zm<#C@|+jqm5~+>D;+A)#AnMDRea(S9(AxY{z(qboIjAKH*ueb>%Jq=}2rPkARO{k9eekw3I!{Ph6n6nP#@ zi+p};^3S@KU{+I2X>vHM+8?b|uR8XHzgyUz(wsN9yJ>>FAJC=lPxoiyn=qgLA&JpD zt7}?lvm7@H-P@kEnJJoss*2o9ox@y{g=WTpc6Vum1B5M*b}L>&e($SBEt>D=7-aGY z5lQY%^&2WUtQ1D6>}^`;6BM7hjjyfvqtJ4or=vP~!~u(-F(Vg40>-aA5-A?*P&u~9L`r}UjvnV{ zXY-4T;$~)RZEOF|8eA%kwXE1*nr6Bxrs@B1y>s%m-rYgD*E&+}!2C_dxBQI990t^t z=$C91&w#u7>uq?4QOPcPyk*bw`SQTwVN=FTkx2FId)?|26h@PPEj5cWWKB6ZmjIyk3QYI8CL#`HkYTq zqPn6-`4ho66&J%VO7TpTo9cW)UjI3s5)B6E{Oup{7^G!osVFGs4UYchqT5e&c2^t> z;-{Q>!P%;gOLQ#ucZrGg*=vFt`}aP~rdq_(VGEq-itRS0gg}J06Ei;at*pZ`Ok}oj zs^FNGg5i0Sf+}kWW~#l@!xPM0(}?sd`#t=;{d1310f0XydWU$n9m-{-$J#%d{QB&| zceH=VUB>s+?9dLdAb%fn!$ZD^Qj~Q_Ly58KfU0du;@=wl9h%+Iz?8V~l$BeE6BT}xw@v>`P`5PFv0D{v15(3y_XXwR7xKY3b>s~<&C$qB*rp?YxgBI zPtI8c!@4@N-6mhSw-)9uHW^yw8~?t(K-n#}sz_9y>6F-&N9Crd^k&t)cHi&b#=?G= zrSxD|8tCB9K9XG30x+K&Lj1LoWO^S1!+7nD7ZGrN{<(?-yOL_clPe;@OEW!|u{#&S zrelF`E)PxS0<#3WWEPs{$c+y7`>a=mM_1^BcnEcKj{m%)!;SVH-_O*ZF$>*FTcqeO zR^d6UmCg7sp*&iPj($1nb6-`>;!d8wZr!5tgfn&0%BUFg+g1BtTNe)+A%k9p*VlvF z6QGQc6HsbjX!TLXU}k-$o?9}5Qm8|-prUtnf25dc^{XCSiEVU+K;(2>(goA8P+abi zsJ!}8GU6Zpaq8PK1#J@Pf8T_a^L@t z0z@xPH(W}{ZO8=jHEj!8+|`X>K-RA zx%$Bc6zd-lt$iEBEh{22iB0l#LOa-xk0LeHr$;T+%V?VLh?o3IGr0GTv#!O0zkQ_$ z5uaW=%(I-CP!m{-vB9VBdg(a?+=3V>L&#Sk;)vL5H9VfjY)gIWsvv699axG9Fx@k` z600&e4qyB{`jZa6!r38!HMz~EKG}OdAXVXGaRi;oAhwu$xjp~IYY&8_9L6U^SKzIE zN*nv2#r7ubXsZM;nhgvLrgb4E>A(~G_fpbrr88wSx*!6_teO!}cG#Vr^-F=~|3ALo z0xHTj>>5Rp5KvS^P)a~Ry1P_RKuVGB5RmRJX%GPs=`uh-Vn8~F9y%n3?jB(19A?fv zzVG|}-+#_p=UGcxOV^U&@s8`-dtZBRQuau?e@uS;qboW(=bR-gT^N<{DXwX}-Awe8 zZsBVfKwFv=nI;X@mKpyT7xeJ)`s`^$&mI6S_tR2+;7`qca0%2{t6$H1cG119YblF133#AR`gNsr@lPPKkLStNFTTfhV(fJNNL+lOJ70# z{dNG1PV0YaQeR*Hp?vn6On9C~!5H2E9Z;|FTlRBUPyM?PoKq%~K6OhDkAXsADrXg8 zwGeMow4nz1ZpOhPmbz}z+9mGZ%WLuW?JXCih;?sDv}$XZiVd~A6n0H0bjmv#%(W(+ z)I#N8Y&42{F<>t&Tl4+XUZmnPzu?B7qi$*qes6DZ#`&APCcod|MxOy)!Mv7Nk|&N7 z%{85_;)>1fh5msB*KyVC!jb9QPqL^OJXIK0jNC!;A2ct|Fhi`q`&_j6a^6)k65OR1ge0| z#rWmFw0Y->`9<8@us&@?(^^ESNZ>4OFZD#CkYHqk~A zqeS}6@H<`}`)s)g%^hh4TT}eeRutFsDYXZ_&mKM^`ZI9${^?t=rnx-Nj6PQ`Z+-!7 zXUF=RLg_D^lQNOEG0F_=aCbEE03*QOusy#P`^T}@-zW{k&sGKpd_8t4@KLu^jf7!B z=$YwD-9C1q+<+pq05peS;c3FODo}N?vVe=h8+e>wCnTJYWXY`ldPe8Fdj;hAex#Fb z?Ch(FPfCI*3(vG+SCoDYpUY}5HZ*qJR%o>^4TiPE22R`9P@sjDP>Onc;58f>8_Vt5 z4|;%J;B&KgbhHAzr@1zY{@J2n+ZxLpvcBlmSG3m5ROB2Ph*y=X%C#CbPD*8J%F1JX z6G1B#v=x9ZjSUTX<##UNnctf4bv7>t$JF$srEb)9ohiXAhVYxM@Vl-Y`{dnMkH;1!P|bi}1RC64!Cm!v?lfZ&qE4sFn&+}#g|lL*p20iCor2csWlD2${~|Uv-{3Nkc&iHU8hPRa`LMo-Xb{Hm9EbiLPB(F;isH2}6L0Ak7 zY+3N-3(=M1B?Z;CZOe_AP~W7wL{3;$wNVpKvPjp*KcOQbyz*_M)SO5lu**MhA{Qshf ztWf_sua5+c49-+QCjL>%uvX2sMjKdrL(k# zmQJS~N;G_~G|bY69bHrxB(jmkDs^4DMaKmL=6o}44<6wJ>uH#^d=E%>UyfAlH*Ieu z)7OHU>Qu7l^bHf(XW?4y-O}Gy^I%NM2M}Ae+n<*z=F9iQ9E_els8ZH#M^p(ga*4(t zg{zbvzhVH@w&HX!BsVniHGRxXeQ`c*5Yz)bHD=Yta^`cc_m%^duvvGftH{K(#~?9? zLLfzZwylkm{RUsd!(!zd3j|720nF{gDz@%$Y&C9R^DS(f)+OBoq*0n}}LLB?ZPh6NNQmgtlq(}%` zei|uWA6PS+d?Px35(Gae)SuYcgAfF)U=lCsyuuKBae~3td=&%gY-d_*-b>iaEfj!C zB)^0sgvZ3_6fRoPbe`FxjG&o}B-#Jw6HJI6&zKjm+&S0AgBD-AM;h9g{RC|9YE|KM z;X}6O%#v};#w&Y`CSw=FBi z1tuRp>U+rAbQxnnYeC9a>@W~%?`GxK<5x~Z$m%ozKF&(}^Bevk0wWfPNgoKjd&PzT zDHFtLeQ4HcU@dV^1gXR+C6x&j{=ld5 zT4;Lv(`=seEdWZ(oo<*t$Kk88Z_m{!BbK_iT-PwSx7ba=>*yDR|LsQdA0HxeU-r9o z0k>;LNJ>_N+7Hx(!StmjMi)%OUy3hJ?!O_6)QRK}um4Ev`NCN{SQuJ-A zbxZn{wwj$iSGh5YGm&531Rxb2A9}+6sBw-Yx#z~(*=n(W0igP{zcfI$^i*xl7Xm{$ z#L6Su3T1gn#Li^is-Y#yu4K4B;ILh*e}U*6!xpSkg6FrkXnf<4zarN&ZerXhdx3#4UL@JTNM*J4N zz$QjQLV{At*E4bq>uC+1I7Q3z{a`FPT~z^AbMJ$p7-J}kM{Y4)l-g(8t%c4%;sdFa z=j@U6)Y*N9VxM8VA}>!Dshy+J*-y7vk-vxM6C8IBagh&quZyi^HRlG<=*w>(Carv@ z>wL(d_3c*Hty(Q)|Khu~H?LQbX-GoZ$e;U4%Jcn-%+^Des>iqtozXvQUtse*IxAd&6k-!^IG+L)f zGpRJ_aBCudZwW?`D@*|_nqrJD&d$ie8=IVf-whwNHXke0!8t_EWxY)J;6c(F>d$rx zZsvAfEM^Ki8Ht*jnwWr1AH2BF0GQt>SW2#6ESd72%6m9_Upr~~<6D)t<7<|^nKB*a zRygUWjtnX9=V(0cR-@N0q%6-fH|!Cd?B&a&W022jsXdYEmtOa3sIRT9s-;WT?K_uk zNvrqul?3FpQ`Cho2WG|T70Cp;K_sP(`(JkK-fY_?YFj0bSfB}6Aouj_z*((>ib$^R1?px z-Wk4sng#AB;UexX;en3b*y^@u1+TK5`K~*V1rwp6>d(@g1$cjcTR-e8z`wKXGN#vG zvjKdscARG-8GR-$E|q)}i&ouq9tn&f&#%WB>)Da}QI*~%JA)=~%%FaNRi(GiE!H_WR2<5_6jyup2 zySYCjK{gOl6k4@(f1WT*^s1?4i%Bl5(8R4Pf3#ra{9as-5n^{t{V>0CuW%`*^aa6z ze3}^b)ZX1DH2&+XWH*ky;r2#z(p}YO_;s;4!#rwo*9}#Cg_T+yh6P||VBj^x*N31B zcg>jim;ZI~_J1L1@cFltG;Sax<48=U$=#4Q3Nr>(K z|AhS3{BG-*j1}KKJRCV8VrYL`cE5i2(p`%C;h~$^K{pB+W3yYn40QrSPC}}rd^5em z4s%aKZ!=QQ!KO&4AVpr$kduII_sd4Xi+%4k_+I&4OC$Q*HShkYHs$-Z*N*(SD?Cn= zua3)hci*GFVm>P{)1+THcv_(860?T$Zjf$rR>#qA4!OIu6iTPOA#iU?Vs!;c7#$(M zmvD^0JUt@V>l#y~jwQoYKFzr4m+_i?*w>70L@cBesmRk;i)nimW=~)~g9f4IOwE|HWp5oJGGHv~!^G185XN$rTMNZ^&eys5GL zuDs8pj?z#66Pv~_5r4ns&f6e$ddGb>g{HQ@;b$H&S?&Pi=GU4cZZno>bF>@&C-&;r z%pC0)p0hPa|75gBR8{47fQkcV>VA_QVB-v#e)sfV;42(7I(;M{Z4B+^hHer1txW8n z4s>I7!F@elEDah-^zeeTcJH}YP^Y>199_x0f2R`1?2!RoTmmkUS4 z{+VA}5F0G?qWGVqY8tz)&ITS|+!v!`yX#EU)+mjGTdx}4-5=5^;YUtO#%hj>AaNz~ zrztWv^$>jL+1~~C`wkbygWj`^S;E-A3yIA&Y}I$xYYfZEayCMola$jKR9?#0B&MfC zO>CWe*wvk8NI)%wGQ$$q;JvG($E^k}MxYTOudVDWDcu`?agSENb5SNrp{om6v=-(c zlt$xJEPd-%2(Bh#R!V-SRYP-WbCjzUZFtQ)!S)R)7C%?6nItO%d7{aE7PdlEQ&kS! zjs(TqQ&qUHK)K)2LKHx+mMrSB`Y9&%O8wbp9c=&CLN0zpyzoe-EEa$>^O0 zBRfp!g!%_P+(`MZWs7Dy?&i%{#Y&Y|7 z9J~I}*Uj>xkwmvW#9uDe%2Nb`i~jlbA)jA*th-`=?@Cn$tUCYsp0qZ14p7zjyqd^3 zK6zS&VW0BP))!Ar?~1sO0L6*h18MO?yt|a>2i`>&1~?dlx|21M{AGNXv^x~a9 z9@z!^(+tnGJt-IiAyPcuzvBG%0>$efYzjwBZ9>6(QdSln(8-ui{5D4ccM%ETh{okY z(rOGM0dYXKO0MjHu-~~Ww$fJtwVGl%){7nce2MO*WtuB(D{=ubsSd1L6 zDR{^n70L$kBA&nzgtZ&YFjQ~75-kSdq)Dx_a9ms3*Rk)|6ArQtBEEbJG1%Rv6sc@* z9pg_^4gVB0FOlX>-f0oq;(o1sVt(?1h%{~`+hE)!!vO~QLzHaL>0(Z?t6wQ`!DUNP zOQu(_z%w5lgutLoNzc3p0#zd}c9ZKqrNA_tvq!iGo}ntg8)@a=KlJ%=co7$6@oMI& zN;5>`hVsW#nrBn`ilKTAj9)EPRXE4-*0w&0d0-jeq(6aB<5@zVl4E_&ov=oTO_FSl zu}@46pKpgQV8BG9v#8F+mZo>LK%-S$U^MhPcB(40$jCqBgw7`aZkBSpyolJZ-Kz8? zVcs_^vmKEnim>H;U!8m0Y@Ffx-T?61j zAT9w82rttv&9Xe(Ujasq(l}Um$I;eVqBVnZ+YDWi3H zLt*#*l^B?nsFlRwnvAVTW%D))rYqU8ZXxC@xBMGWLhfv+YkRFtHC<)qe$ypk<2@(nvUUEt6VN*PbZq2Su4{tH<=oAB+B3Ou;d4o_EW$L=VcOMxGhWE!uHX=UPzscqC@iI9+@es%8E9fXpV=gXjr8oRxU)Lt3VHr1& z9H2RKcFJaIIw5VDZBMF=kAm4=nR|7irJ#QzP)@tBI^OLP`a8o112Yb5uwRH zz^ml1_)7Y2Xcf=_m4M4f^Yi3nsD~6T;^wulenmXrTTl@Sy%;}R%H8d-RzTwz>Qwo< ziC5xEQIXVa6jw~nER5Y>Vbo_!3eBqQ*lUC9b(V@@W_%jKe+W4CSrSz((G-YN4qFN@ zAs^2|NSYZHh}7J3gQH&6LBhFb3OsTFI_;YG=Zb2X<;FBtCtm&719DSRLIMO8)2+`o zh=ZZb?*Kakkfcc&pwoV1-hTtiGhYx_*Y1zafLZM;3m_AC+j*ao$!-?~_MSsN1=mCl@l%CTQgAL{8Ohc@j)QKd^I8 z(Oz|-P!~z-BZ{_nhRUSWW!Mq9%r3DjT2b&vQFxx;)vsC=3vX;fiwNWXS)4I?^NCCK z*}xti+yi4v%9ly=n0@`U`yh08`PV;yFhx^GNHq0JDcQdH)Z{*9dxtmC*P~8E`cmqo z62oiZ?Y3iaLFS9g{->_D-ZIsU{i+BSmnI->Fk+&xz&3NWjx1pd60=vjNCeoRWzDurW!v2bT+NkCb#87-(m zu@9F+xT?A1K)Il7R9lZi)4wJ z03t~)&KSTb+=+j%q;|mLvpeSIFjV&f`KONlHfsuWHUw(OUJX(#@KuAm`+ACV*)zh0 zuc=>SyC*dud~N5n*N;;ekv`MWKjF4D{Z%`(HdI>6ipFTeFY~>Q6^(%x{eYV%r){#= z0>TTgcEXOM6Fqlj$ycj~Jlo3i4RFeKyI#UQUiBWPD~;bYKQ`*QRwb!I9|xozI)3wy zXb@pZCRJMb%Zxmh53^}pik=IF`m6@QL$}q~wS?39=8ufrdEF+SCZ#eSn%9+pg-zT+6fMpG6@!A_nn)MO)3`Rl<}lh7oVQar~cVE z>%crsdyNMV@TMLcczkx?iT7;f-mQlEb@fJ%JO#v=RNE@a93mx?Ws+6<0^A z?Hr@=>BkCG!&>!lI#%4!Ab!-)`8>h@s>N|InUB7WNrlM>17XDDi^lBfuKu$9k@_N3 zHB*0%=JX(ZV!xMX;jrIG6}2Fj^B~&FcTL1vpr-PX9af`7&-F-Aoy!CDFQ9RI!c8Fb z+c4abq3_7E1jm8&C#i$X#Uy!phs*aLU5fIKDOtxSbIa6XAs!iGF{P5Z$BxKFAJG)I zKjw|lTxl{NrEKLKp&UmR2DM+-1_T+jq2$Pw*t5{qKDwvD2yPucwT>19*!3+u4_RhJ zAG$Y^ZcY~78frUBUEKNU)<^?=i2!?6gpt`9Ad2LDCd-Y#Z6iXl%8Lx9fet%vl)0+-&4Eohl zf+RoxO#C@LrKegIUUvTEyJC?6N~Hsm5uJ1G8+Qp3bp$zULZ}XT9`nGcuzDhAD= zw^;^&TK~`FHzZl5iC+pOl5``oSjyxoe>^|Bm85H2Wv}japuCvrKe=boC@PYiXZ0rI znbA~?!&2iMl935^q#Pt2WRF&3D`H{%Cfo_?NLh>X?0a`T6Keq29ek!8VrbD-ZU-Na z2!F38)hdN_)1)~L?YD0^zkg!OQ>D^iqi|BXRbWYI4r*DD;~k%n@Z8W4O{%AwA=aL) z5D%bHY_;p@`Pd9APsKh=Rn{1Vj$T7K?-)cH#ln?YU#XvxH`ws*dHU@}UzVC@lS!kc zwOd*vQ>AFK@Q-MYK>hBy9vj}-+2dGK8KjCQwQ+F$NCr_iBD>7DxPTX&+lpoyv@djL z{QTQYXv$+`|9a58bxxi8#l=@cy0v-jr}V$m%Y)#oT`@*GZ<$-ZnICf#kcEteNI{0-mX={;KDHip{x- z>ya=0qYb@0rmC9r7iVQg`ibjM*mH0A$)!H_K{- z@DEiMLz23RSmEV0F$t2yXa4o~4F}Ln&N+EgWb0lgE=69)IO8?nN($fZUJuitGZ@iq ze*cD`dAMrVisd-a;qj@~W8cg#A7V7b)P|}UVEhNME?Xs%Nre@2;(Z9RvN6`viX%t9 zx|pAXdCsii`0d5bM~(x+yDJ&TOJ>MS;sibY&EjloZEype^y?tqDc+KnMZz7dI_&Iz z;kscjE^7bmkOgk=hWvJ=0%z(d0JW7rL!}i@Qf?j9LNFrhL>F{ z7yTgAgpxzQx}+`oU+}(j)NWzy2UaUn^$h}8w%!IvD_&@0KVieAd+s3IVFHS>XXnN8zWE*wl;@&YFCk5)uYYhCzj~GHLb~VW z+xQm50jznd!sHn)C>A*UX@OWquIgDPVh*Op2k>|N5hwH~bQY)n9P_VuYPpPTMisMz zF}lXQ27@DOx4;GIri`qo(KnN7saOg8Cm+B{Hf(0X8d3EeN_0cVuIoKqB|aX`MqVH5 z8KC^WsbGc^@NY#7$kQW`aQ!JYwHu`U_Jh$$yry7MsdabzxkkS$+Ew-j*LDBRm5M8v zsXr=}IM~*-4YMgS$$7H+(OypWYphvsYGMr2e2a|n9P%P_j~!%@>&uwC$!uboix$7u zsL{HJFIUsZHT`h{8i^{09URe@%p*HDUvG2UIWTC&cQOwy+RKk$hgG)j1?

iYFQ2 z5k_!$a;Dw5<316PanAuWiLvyb0V1w;1l z!CKJ=$ibu09mpv%6;oRzSTy*htAPcmdABOL7|b7@@$^DTZ*scO3fu0^XjeOkp01zr z#_=tX5z)|f2iW!nx8Z&&!HwB|Z*_n#CrX z5Z6Oj7c&IWSy3FSjKC2}Yeqn~OgH&Xkac4>Pb`n~+=0Bj2g)p&(Z?;_KI}I3X(OYj z!`p&+x&<#B3hsQ9PbpLdS&Nl_YO$>ps|JvqwYRm+BM^!W-baAdeewjStE=m|{eMA) z%RiVdJbi|w?QOPgLV5jU^{&<*1k^h%#^+N9oEYGImy5|>UCUb1shJmkoa>O*??W}` zqGv-hks2wsaBmP+F;O z5ru#-H+L*(j)H;?6{!AgMQa=Za6a}uAo&pPUv*@EgN<(^bL`Yxhx#BEdLqS=tK`(% z6(#<%xAXPL>DC*GmW)^{0g_-+VZY81yEi~+ns3d5hHnR)dY>l#p^21_dz;(BJ+c>A- zyt)&RTf1OztgEvJ=i%^F8${WqX6N*!JyGrT_u`Y99@qUZ8`ebjx^&cuj-7Pg5S6aj zoSwPz@QUM!N%>k`#5uEw@+UcE6`s&$EKw2QQL@cYk}<6h(DD zclVU^6@Q>Y!)6?GT}<-oOcnv@r6@UTqQ8Uw=FP8rI2T%mF_I`Z+wg<9xcnWD3MYhb zGbGyOq4#sN*&NS>V4SiF9o58Lh6h1O0ed?+>3au}&ih)qp`)ISh*q|;Mn1DeVj&%bMoLyY7x&TmVf!Jn< ztn&Zdi}VB5OB2Y-y}qzkdjz2Gm-?Ff(r3qUb_s7VS$B|p55L*&y`OC+ak`q8e>!d! z1Jg`PpwMdQ1mmXw5&zncW4L*Ghjx_#(+R{a-4>zd$QC0=@av{X+&~TE!M8NqkDZzJ zc7t9jZyQ-Ejv@eSvAw#@o9Gk&ImBPWoOWH4ApV)+n#VkMvT(~0 zvOLE@#?$Ye|3%LF%kH@UU3F#cOf{_x`~VX68PQ;-nfS94xCqTk^3s)>KE#1Smg??F zIe`Igok#8@0dg%IDj|ujKCb<*8x)Jdx6<@KF}Qj zZ^;Tl??vc#D>ZZ@#^GTcQY31COyG<*V z@HD9^(Q#+$p3$5S=eCVjb>3xyPdL1F>q_y{=kLRWTkOi3zmq1uEpi~qdh zeYZG#q74u&OO-ghr4Q3@Q>DF9{_CobB0}(E41jqc!BIw#$w~n2I=K&NTQ4@ zM`Fa?nc2Dd13g8{qT+!kCjm+Qy7pg=5Eu}vy1x_Vk0b|!)3b};Pqgf5rmpZiyf1tN*WKQ6Jb}+I7_8bUob3&diDaI76N_r5DfSKY zv}Ds4ZRkS34_Eh<>`kV^PtUU;866I~W2GPZbY$zgMVtS zV=r=Y3SiziCZ~<8S)=toTcOw-$!`TTJh;o7c}RefW*@M$0+c$ih1mz{mw&?Nv7J}w z%btESfFj_9HD>Niy|*lO@NE9ZF?WwM@3gZu=4_*8L*f(`4E~nZ>Wnb9%3uE*ciiv9 z1d_R!P-Mql>hoWJh<5bwi^8z~Vmp~3={bs`xQ2kO3PD z6SUuyTxeJ$4v6IeqgeT-NXDr;4+pTG=n}aMx`7lZA}XrYx2+G-aif7m?;Ti$w0dwO}LGIM;h5^Y;& zS(7^?`FvAH_NBu4_3?(1_usGAN?f1G3_Yk z*oSAYA3(?_%6HF)I30bfE^S~QXG^nBj|a8il3f5}N9App-1#*MMB+=uY6lU6Uv3MK zacFu6#nyS4h4>!@^s)FRhh7K9WHl23i(g$Kcw$kUkX1>$QtbZPeC}|CrGBQ)ISCYo+BvTE zc|eM# z*!bV(J%jV|ny=Wc&t`v-vUT6xQTvyRhJek?UA0?*aXJAp$&Y8rQuUlBXv3J&`goQ| zufx}?t`&QAaG@%Aw4G35xSkY1gRw8kk?H@~Mc$RuS7Sa=(raxd+u#xe{p|_TRspkm z=HS0dD~T-x0aDf$mz&kGg^+ndPbVAo|MCHV_+%h#42{@M04_4qkM_mhjMf&T6|mNdta85CM((IUmsNw!7Amk4Ld7yWCKhspW?nYv!CJEZ?*Zd zX2+JJR{U!kp(b-OPh=-fm{y)STW~E&84PUVM%p799z5S)FyP!f>c(JHRa^yzKzbL2 zPqM$b@yaV=Id+loI$E0}${%Ubpy>s8V&}@!3-5nR5cw1_SB8u!W{5M@`Tj8vzneBy z?x2>663NgB2|X}KxOigE6;9{%<*>qLrkF16PS37U7--bJQW`v=unTOX&ZWd$z2Qgb zU_7^Cv+IgY-r6_ed+I2dHwU9t=lobVs9KtJBH=Lei1vLe)kD}(h5Bo+RLT3VEZ+~0 z8v*kdsQ(`mYbJ5z3jv4Lg*(y#HA!jLjb^XQ3D=FGT1?ij5)SphTbvzQRgGgnGg;|+dxc@Br1*3RcHA~j_eupUDi&K$~ zlZ;7N19YZmuXl+Hap8T+R|=Dz)UaOzTGpsX2VW$c=E?MTsi8cc#u^La8-SbHv33s% zpa}^XhRJ50y&T7@c~0`q>x{y1?v#}yfP!ZtHtXX%k~n2mY+3uv=H3P-AB-!Sj>YK2 zz=qiNT_nVczI46UXHvcuB@f=TxecYpTAMD>>gX>;If- zB=O{el2w0#2#66=1&QOkLQE(_mDvYVY{>t^n&XPX<0-*3jKcjIu8N0rq!zK(ay@yk zl61lR8q3M93ExWyth5x3Ej)({7h$)OdEP&h zCh4D-vDY{DX)irCX82)I#K<{y)EB*r!!1QKlldYcc-qkBc<^~Pc8#G5n9yJ(?8C``o*LN*O%gFvj!g)27+07uAq8q?R_KdJt^a+V+R12 zzH@@*>?p}*Z>99z(wst%u6LmJ(n&-br@X>>%5oc47xYe3VeWSg2YN!PqmLnFc(Tvl z#)G1M<=(L)<*@;@|E52um5D`z7EwT@Lv)ud^T=G(tTKa6^-7{?q3216RfRmvlm+-9 zkjItVPVB2yF{;@ro=~sNs>0(F5a+8eR*dPe!|Yg5ss0A&IoskaGwJtJEcbx-ByS_1 zJyR_D?6Hm}+)RxD#gIH{$yO0;lLRiZ<(Bku;<=cxAA0GqhN}zcmE$FttZZ8mvG-+G z0*QlcLO!XqaoaxSgk?#z6oOpm6V$$DK{Kvg$XW$;bRb*}2 zvi@rJzJ$zQ;JiTpn;bS?{O@xYj#3WyvYyo}6oPvQ0}tHYiS=WqNt(r6TU*?n6Ukg#OpV_}wMOl8fA+wUPB}!f42rH~2}^Le<0TK&G-7JKi1l z=F)9Tn)@|mzvH=Wi^_tGaH>Y?Lg|h%^8%I3YmemaT}J)#(&T7@%qHlp=|s-FxQiM8 zRn`-+nC2$+v97q%OgN|@ESv%i0!ux;Wk{!cqhW-m%lxtUK=+%2%KfE8k*Xt`U*SAv zvH+TDpf;;KbnQi>3uBE2WCEmQz3ppz+{k^|c0sF}N2iPVeuOovUy+FnG7=RPMIBE$ z6dFM@Lqe|MfIogW{PRYG&q+s54?=?d-*?iPuwFJggjE<;WxaFI6if!?o@hgj6QGv6 zX3N4mDKep7g>S5(`IEG?pdSJeWCBDsZNN8#@aQ_{kYq$B-uMXthbImo*WrD%gO_uI z8@xPv-Q|L}3v1!R~|e)0tvxNV&zp}Q?X`ZRH+#2SL}fw-hC`Ot^FrpKmg zX8TF}DW~*JSr_qYIE4LDFS&bOdzW_cvWSlc_mU!(w2EpZmF`VYGNe6w@<@Nfgr^M8 z-WS;oRb>Utntn&$MAI)^mYFdh!4J!=b-Ig)1k3|gFdCLzdh*#-#%G5V)4tu`k$__7 zEgBH8zBXF@?jMhd*IV;&Y_j!6P7*M{bk!pzCN@Kz?n$6_Tifk{*Z1G2JBu{{xdwRV zx}u7r``w4P?XK_dxqi$@mkJK4n!|6}T2pLi$>I<|34*w$*@@klDczokMC?2j7@$&g zBw%nh1VbUPo|5w)L2DtSfz5BeStEWU6u}UhCEtVx9&!H7ql?wpHS^X;Nh}QFL0B-p zo35ygoeMY-ZbS++K79DzNr5$4Ra%IIr|{U8`+XHF=^c+rX#Xme1!>T817gWc?iH$r zkJt@@9ho#;)pdvE))vjpNPS2m5NK{Sb^mB?Kl5_S@5D@5sV%^nq{v>V?04zJMlE>z zKaW#hCcI~(s0hF|h|(X(Q4jbnNWcJIo5*G9jP>ZR*A>b3tUR`QrK zO~}p^7zgCQOaEO)`iXxX61EqDrB7=9Z-<1v1z8neRH<2im(?)u#RZu^=F%RH@$7*q z&GJobpTjj96Hn@(vA`bz>lkqNu?dB*cn2-@WBCG;!4tRLn~&>2f0k`bJNqg>5obH zcy2mR-{-TaknE&>wWsF6u+o`nmj5xiG_m?Ee5MvBbMs9WK^@%fyhL%+D8OlXkA;Vi zpT>;z@*U#@o%={hI1~sKR>0-=YDg7I2P*kTS!bv>>3(1ctv8t$7x>Kby(I+B0`$zx zVHyRR*fd`-&N^P_!QC2m1K43&3}p5J(UKFdH<Jyv%15GoaNYM$(fnYN!{cc* zE6M3AaL~m$KmH^ASw%xkG&(f|pFnYgzd$fX)wAJ& z?;o1R076aGpkPJ%nae#-^VMDN)5CkoUXqX64JQq=r?p+?9|0$?4WUDG-ZamPu~mtY zADKNjVH@n4V&L?buyvdp5aQ4bW+qSNDMKp#?V#J9$9bU6PnTkWmj%}QtR92DuwMgu z@^FwaeVdxPH=NGD6N`^wW9Sww{_Ucd_suHqO9%&085Ox;!p6TQYIuQfrvgjHIW_X( zB;TO|;KZBsj$r%R$oE_Z{kn7A>;*;(Qw?*q2AD8i;a%%Btvvys+M+0VR(ZwuNV<@Q zD1eCcJQeenm(7--Qg^u36T{&mp>h)Vfmet`&;M0>N$P}a-pz_l+8;q(m#^J# z-i{`Q&`sT}xqqu_2<=w+{^hNSaa!7_ccvK2cKl}t~lth-3 z=J2Ag8^#xNE)00u)klwG0~%7O3;J?a*UT4|d~}D3o`y`>1T^wqJ$OL2eL@*i8e;5u z=}yeaR<-&glkWC>&MT~1s+!Gt*9I|Gg!SQwr3;v?S%Sq{F1AJ5PPu1N3}VYR@ThUE z=-Nr}dJqHt-9-vtD2J$<)q;xgtIIi25KJ#YM6smr_XKZ);r{S(k(#y2Hnifz>O z(9}?BRQvW()fhBkZAvzQkC@# z;nB?%^o`l*l@@yVZa~9rRoD1=<~a9sY4^ zDDNG-Tf?LSh^BsHMN_)Oq`Ri85$*y7_qNpd|A+b5qc&a*g%k=%kPt9#e}ga|i620B zFs<;`Dv}+nk#6@7fNSGCd-g2v)wdR}H4R|52Qo$rHEoK-|e%20n?t|raP3fZO}~j-hoPsgONGaJ*_PxdkD>}r-*8$ z*YClj=h=6Yl?K`v%AKva?TBCV-j5mp>Ni$fb?OUW z!&&xD^*qod_Q2RJmF-xFjUxKe;0=*L8vlJZ7_6`_4$skqhQatZvj!PA|Mtsa{Ccl@k+??De6o+vzC3Q?4KHQFL$l%i0wO?6TFx&8LUU0<0fk0SlJ z-v@;f=SQpHe1xl$E=*mB*WU2cvC}lvKl{!IhTm!-k@;}F_0e}Y4jz7b*i!GdCljXd}*wr5Q zycH5(&0Wss*{O0W6s_&po(A?G?U(Rr$Y7GZ4&pfARSwgs#{0U)AgVb$Zn!YtbJ=WE z0flbtPxAEGcj=B%p>(tNpDaT&JL>EW@GA;Gz(iqQEZG(ZCz(qWXjuzli3U>Ishb`H zp_xnHHi}-A-cFA4e5d?&sRiMbHa|6lM`z$GMKs^3qxReRKChWUpxo?<_1&%{GYC_! zH0DHE#g+q9e$GWT5@lQWu>)Pllj0Ain+r0G+JEE);Dg1!{7ZdViv-98=>jm9!rb+{ zS3w0RakdBQWLUv!AY3v(f%aW1*;`AsH02@$&SG@gU*Gg%j9w%#0!J z>Dqg6Q|C>(<^mW2@?wgtqk}Lz>es#H*bwGFx?=U%Mp&a~Dv)EGSSQ-6%6^uEV8Kl%A*i*;^K4N(>#7)wn$S2CyrqYZ< zR6K>LRB>Hzn0BzX!A=lZ3(?ix5s6Dw_I!W9`|0`W!i|K~31~HAfCS?ZEsmlH4WW4D zFdET$a19!>{a8nuq3ORuXPKkwDEEA14KR-xL3;TCm+f(|tftQ1M=lf24}K<})@4ns zy!!*N%HDAcv%f-#I0jx|OnFybAWO<(VVZTPgUIhr{PluST@)AOs-Q}sb4=KcvaMLi1;wG~&gaTU49*p` z)J7N(?}AwZ?VD({+po#Ut_Eh@PnH!!d(qMg@_p{92fmmWA2-zMde7S|a7iwP;_y=b zkjL0_pqRD$0Z`PABYLPB5I1YsSF8MmW%(K-w_p2)McLXvIw>_w7q-p!TvRv&g z`R81_08y4wIlz;>_bF<)8hfLL}IObm*9|izKZyvF(sLN%w0k zLD`GS9JgqmT&MVOf`ks}lPw_lcXpv!T5Wpr3|RxS+9{5&!{YlY=2^}<(oOvDV}1OT zdLnwvV8n6kVSPmG((k2>-29fNN}5qV3ymxDWWHACWZ)aUNPB6Q_u$p2m6j%MF!IE2 zp!(htPW3$FS?W-OeL4%dgH$(OXBT2hybX<+?_pNx zMH*xLs&altQXoq|-CpkVnVEaDjvaA(SqQ-9e^2@4&2nGl^^SfWRf}SqQ>_2Sz>BI} z!?(uYY_1)zKTQyxdjuklzyDNj2UnP!-OJ0Ha)HF;dC=Xx?+^L3ruMn35(u zd|y8Hs$;v$)J^(c%ArJ_0Pd=z|KnGSyKAn+ z|NqeSmSItDVcReUqEaeSQi=j14bm|N0!k?gNT(p(4KpeN(o)il2#9n^4lvSPA~|#; zoio(Tw{CSm&-Wh3``&-{kFCNTYpv@#uQM*(IEN4xnx0>4X_1ym_nF_P&4ZfOO3&9V zPyqm44%@(J4VXf8+brd1f8*O}c9F7MS_zqAIC{Pqiv$c?L=(Qr7Y$ny*;N<*e5d57 zO9Q-~yRS0q-;o$ie0cZJASR)%0-d_{8tQ2pAJ4yO2Lg#Yi(JTy?1E71SoMg2tKOKI zw^IpYcz^K@?$r@cjy_NSm2Nr@$gfqPUV+9Q^TSr4Tw0T=i~=vn9%LKv$Kz(N3Bhac zE3FBs|0!2Eh5goX18MSA!b#nNxAnPNts9sob!% zGUb&TaOpsXSAKd@C-X?Ve|2RAx5r0RqQLuxpV`bz~<)SisD?rRkNK<@NmNoY*}eSCK9!z zRJWJ*W5?I68K-IMe#tz~V$wbOvIi+QG&3Lg8W0Y85JZ58;70#ny~)+`{S^u1rdVF8 z0aX5sBVoWFzUAf2>IkFP2|0jKf8m@dCiZ?)SK%`yVe^2eRGBASFq{Rl4#>Mnx~-gtWbXU}Yhki{++S#0pF zYlmxGrHj7IEK`finit(}vaTL6Erh;by^3{FMJ!?2zcZd5%`gCajbkMmKbc&Sh=Eu0 zAb#-ERo73gKco8QwzdcQ>95B%$RWhYnzLTUbruaN%$Z;<#7679#sAC;L}C+=v3}tj z7LUr$T~?V5H1Mt~iZyVEM@zRiTDirw?GzH=1ys5Ls)ybW-W$B{&;@HUVyE&WcSPln z{}5%}J^*_GY!Cuz25!Hjzd0)Lzut84F<9yQJz!Z{(syE)<=W=Vug!z;t2bt(6U&TYvQr?pR3BiQqDDqG z71y+M6st#&ih#3?`Oq;9oHFijXbmPYb!@xDm+Z$~UUo|eERzwe&rC8N6)^Y-8!zk$ z{jKJLpVa-^B_nb^cA0ICy<77x-hG2P4DI3)vQV({mf}&Fy)QcU#(eKL?~YG_#asE; zuknZep#b=8?_?4h&qw%ltsEF&3DY)PymflO^ty1=z5%mHmhibFir-V<#W$zNqrgL# zs{vT_>o&tBHmNEHb%|I{`z)=q-x9JrnGT*vRCqT0ABH~~h$6oWOl^0@Pm;la;>2NB ze;~VPKAxd%x19x}-QO6DeW|b~)F=3b4IHB!zc_xa?Lj40Mn-F0K~fSUfY^X@i(6Jk zM#k2!=c+(PK&oA4-A2k+eb3CyEbHza=nsP{<0hc91y&{=7%PWh4w-DSckszPTi+}2 z!#{)s-+hZ0+WHXU96~%P5qHN99op94Ut7x)zW52&GA8W=%rAk!dtzz(T#8wbS_P;@ z`+%#F<6-E*XG?zL8@r5@G$-+3yNRq!9VB;;uSsgYQgTJh(>-f;Yt1IRQn%dQTq4KHpa$N{hP@2LS!ewG*bm>?ydpF<2$wz^qQf(Qi-)Mw$C(C@WRNQSnN4 zR1h`_YFer#YIgxUNV0VHw%n!_Xjgt^s|OCFQ4s)x58{dN9~J{@(z_L${6igw-Ll(3}UZ{VdickwdtFYfu4 z!aF;;b_Bdn*2!<4r?j6K5FWBDH0+NEo2c;A1oRqg045>bP3^{=lm;kHNq0 z{T@}^_g7blbSx7+;BkI|qVYxAO3non$$`=wzs0=3qT>_-L@(g2aQ+QyqZVUv3~)q4 zpoa!)9FP~80SxV%naKg9TC@GCFrH-F|5~+SLC0-6Aus7*H)batM7q45enrt`_UCm# zky33n^mXvvfvnU~t_j>spPiC$z>jD=p59M@cx~uWJZVW*D=h3%1Yk~-b^CTc_v_gq zFJ(DcVo+nZ0R{MgiDAuKUXzY+ZjKu_PZ6mG$Jga$S9JoFpauhRD;qU3sc9`rv*`Da z$A1n@0S^T(@0u}$LX&05_Rs9txDuzZe%#x3cK-MZ@Yz)NuQ4&1LubgV>{gh?c&QaF z5wMT?dFu17PwqiShr)5=GGS1sZ7Jl0IFAN1R;S zmPMYUy$VWHA=Y{Um@r4o5gZi`6i?&bKGl1(Qe)fGta*H_i>*?`lOgXnnt|Ug=Aj|< z^K*SN_FwE5tll*fvOX>(khIL|=%}YI2cD=PG}ppZfxld|;L7tkYaqII>}cT{WbXXvuD4NR z5*BO@ZCbp-!6S{Ba*@SU=Dj4DbVthxDGC_wk~|WT@*Hte7ln{+1Nj-QDEww~13hBR z{qL)cl?7(Q%gf7gUEQ%v63!n0u=~8m(*?{^=IO9lEXS=|?;9H%Bi;Vnq#k|x7}>=1 zT%%}Bp;)oYSY&H`jfc+BhWw9aR-nNj8cHZ=7fw~Lu=I#^w?4W#;^CMH_*8;B+KWRK zY12$>RU{EP+WHj7c-GF^3u<9vLvZ-DE0nsCWnhv176Iuvu|>S36!xDr?I6Qy4+r^E zOAIxG7Twsy@Q*Q8Ji#=Nh#(*vU}J1R*9`&LMu(dsV}oC-Ogj0=!c>nx;AoqD2QF%G z34}{M*d2w9=M(M)leiamMU9R?ND^;60N{W%lmc+#<>YJ=*k45M(@HW)x;n=rV`Js- z-MhCX`Jej4I8eEBs*E#dIPv3x+Rpbl+|s%R@PzLAD^q>wA6etZLIm~5oWV4-6>wc` z#+z`ay4rLfh$=<0;p`KT_8GSY|$ls5_ zNQO4~qs|#rF^`0}>6gtqJkekdc5O7+#mf?zal=|48ra@yv2@DvBXQr_t6^Fiyk2i1 zb%5Fs6L9#_AEy}>YQCr)C^moWxtb-S>Oc_2ask>IP};lzFBdr5oi;q&T_ys~f?rjU zVQr>gys+ykPT?#t6>aTnF?ZDO_-Xt%<#@0LOKDzvu5sB%upwh%*1~LbvrKpLF5zfz z4ChWUF7$pZt|V->XhdxL*Yk89z2C*nsdEp>2rgm*SpvMmEWwnJJQA33K$I0thK4uf zB|wH`?_6>uK2gW-F5&D^%q>um80xiQw z8go-ao6`+|8fuR8`2;^X?911a)X39*1cV^v#P%~6!fu9^s#BN=1a&_Ksq^HMX71@ZO+gdq4)34`dwTay| zS1WkzoIRTL-tC5C%oErJ1Io87EPlE}Q|v^-5_KnBpGWduVG==eq2(#JcdJ*ZcS{VI zRJO~RuX&q_L=ENNY&m|AoUc=~ajPhh_?U@9?ZJqL2J?xh5R%&k!MG3N3UK@QFj@6Y zCmm4_kLi;IQpUV!q_OU3Bj6+_w(W`-323?3bgYbN5CDY_>{%b8OTnkTUNK7{7ee4+syS6#Z3OXR+F|V5Xw6`*ns&^{HFi zm4xmTIc;Av-XVC&w7pg-rzWsX*Olvh*#dg+u|JdYj-sI}3mfxg6!v0!{5NA^t->4vB;(@Tqv&&*-e-j&v~|F08twapYdl1NH^*R7XV1g{Wn_S{v# zmJ*ovfHy|X`bJ;Tla`-$`rx}yCxReB*G8$({wZ zoNF^3kWZk{7~$Pu@z)F^X_t!yzYioY@&Jds7J_FIfPOoU+wn2+fI1}n&hjeZoNT_v z1=KV~z}{M?&Pkixx3<97?cT^9*ot zkLyoOwf_anp;-&MR>zg}7~6*iz$mla&ZhgJ^lx#$Y87D1(nmNF_3y%;^e?>c3empX z4q&=Q$@j}Xng4cc9^(HXu=XM_aPYu&gw7?S{7&~4s$Dk-U%3}^$)M%sVYWJ*hX-vn z9(z=~xSXLHW$si1x!IZN!GhaQ#RLykw+?V~kZX7od91sA{j8x`fndAK{Im=pDj%*s z1B+yf-%Bhsxl^KTyP<+ z^_l5UocuX3I`g$b^M5nr_dCb*)S#YXe1G`sg#4=mtf)B_O5Vw)o zF2d^GCXnt+LSNCBsw5`TQ855&5@^#yo+{W(f>L&Hm51YR;@Jo*1f&MSJ z*sy((KGii#EajzERsjMw?vN4@y$0KAdl7~{;w1uOqVSQ)@K##=+nZkd7MQPF_-5?* z;HYewk#SET&PyM+6~Q_q7aG<(|M=x$*@=U@51kbGo$`wBf8KN7U*VA=CEvX4!UZ6Q z-P-0C`u0lwE1x8hc?~`ZGyrvcoL7J`MW}%rh)X$Ry8;f{c5#69%Xy2P^9UiUb)2rh~1nfC2j+ zJK^-cwp(nkDHc*>#*U-N8>UV|Hz43bw^WD)d-xKAPmaxyRoL9ApjGO=j6`Pn>9|KKV3`=-jn|Mb?K^%oW>ewr7Eqj&OKQC72;Y zN9;l^1thk3!<&2oF=rIJMXju~n*WJK&ic~m8?B)pwLkcDdnlw?27b$tz|R*AKfwXV zU(oVY1KJX_oL_@r2Q+4q9t=rUi$hQSZ%-z-kl;OpXGP@WVjh z$w(un`d?ke2jGllyD}QO!>UpjVVR14VF3NOV#i;2Z(qnPn!4}%?Xdf-QszV81s$`+ z)eK(owWsDpc6!)PU<;1R+BaTt7hUS28mLJ5_0U3IMCSmw-Zl zdFx%5d~0`!{_1`@W3X!X%zj(WMKYi90p6gthuXB)4nh7-7UnB(3(L#RvorQ&5i)Qs z5qn^;cBgbUN(KqIXy7HytAILLKtKQ(#(@uZ{%%TC)T|)3j)bMrR8YWbFxOc0_TN|O zSlZE6vb7nTh;yAaD&L!CvfLb;T5x}P5ibJsUVV8dgn{JVk~H?|0g!BL7uuaovFe&P zP$V>AkH$BKeaw9&Ju>>a+~b0)9V%J}EU*u76=zlnR5K#=20+INPzqVzs&AifgH`Tu zP3LF*-kt@vA#qcJ>uPdzL6}%etp}_37-L$X?l9PVUi24 zYjCuo7hpyQioBXU$#B#v=(-Y%l1>+OMq7(z32Wxe*=Tx;H>pFh=(`OFm zb$L{nOJ`CAS<}<2>{$DU67moDgvxqaS){DslL^uN?ZT*g0Qq z2E?1;8k7m#(^i7=c#8;PXGs$JG{U!EuC(yF`+z=n%q;FFwMeJ)^Sg$4pO3M6gL=@N zW$8bwNpe+g-xety-VBN)k&`OZ?oW~Qce+JH>%jg$^EU#|Px0>Z)Q zniE;Jn)S_gD)nut;8Y_4J1WdqD9}pP{zS=tb?PCf(NW>}un! z7}PU7;|ls28-TF*^$=fCV&%+rN_bI3l2oE1kJOLgg7unD>*TDACUW}Bpfq!LhD*dH-s?^XrSO4fd8vaWoMg`C#aXn@iF??C|48EsQSQ@zDm=5W6RTtk^JVEPkvno*UP<^yM3V3J=n z5+b<#j8_%1J%DY@WU#myR~+O${W~|6-LbrJFl1_KkE;t%NWg}ga69do!C6L-Fv^pK zs`LD1BQH}djzYpp0AD16>p_G;-+L|8`EFDV{&_KEQzOt-|(TcW&SOJ4pTYs@Ld zkM^k~J;?@N3|aL+Ij?gCH%FW-LDFr*T%FEPO%uS;$8Jhb2(d;@DM^P-N^n|PdLEoJk{V;z;r`iLE zxOVeK%AIZH=AOhPpPPi8cM7<#5qTsIO=mNDUCDSiEJ0tcJFUbXg`vE1T)089ZS6ty zf_y7OQF_R5w|Y-veakW$oXM^6D>x&?q`zmf_`dd?vlYyVYkDY=0hxpk0PDB#t}A8# zB;o{T+Sff=q#L((UKWO^9FFw22c(s}fMou=MgPn1uP-|-cE1Am?e6X#z3?3BhQXy9 z0f^T2m^t-7LA3{7T5-a~Oydl4Vxc@R(Pf<#h<^N{HC zW2xgf5IH)ej>?BH4nN82Q(v9mDkG_-GG`n37bw3Ce_jPA^X1A;U`5VC8MSlt@Iw5B z+z_2*cWb2XPTEvVVWpcrrz?z?KDZnINbgcihR{166-PdBFq z7EcwP>i0@K_<|yybtakwfZ=2}VZPhn?UN?~I(%ajC<G0m;d?~N+fB5nS{@dZdvv{lUD&Ex$Ac3>FCGmZjPE>iE)-QHM zZNP}{1w9V_&)sj-*7U$XJ?HrB%T#QanF$C~@!shY1*3J~X#LNmtkN=99C%lL&AY+b z;9~9zWKgwLSZ!eaLE~Oyure5yrp!%!=jm*}IIisV^fx=PvbTM~@?eI%>SzeE?gT9ftUvzLA6Ehp%<8qlAo(b>a z*@vD(OM1Wm1ik3;2Ki9TLR{*cg4*^#sskhqvxs%v>`uuPa&stpH&xc~jcF!k5M6j~w=AnVh;uLhJh}4{I*)1k`&>M1v!*aulF-f%cR3 znR@0s)*iwfY)Z zI|EbNlG!4l4CM<4V-Ion4nwf(I)>Ou)`mu)!@&Q(xP1|_-TD*2)~cQgrkj8;5d)$p zE?>IzIdkMq0saIF$`E^*^_BI;+dX=)rIaXpxE)$!&_ZP=YUAH0XwO>|MpC7ut5tL&nUD#cF@WMn4OtEEb8mDGT+Sj zpAA$7lo$!XkOTl1u?Fz?M7!F`-byepAu=zVD}n^<&bZ@~@tawm@zgc-$QLBS6-ZjAAgV+LN$oohlKulBg;D!#UB%Y8TXoLmMd%qO5P%$E%$ znOeE=G2$goYVy*zz9J=U=oC={wJacK2g$KJO0A+rJiMsxPO;-FjvjA!(m%+S zJbQNbk#pYpcjVNU&)&c9xgqUGbfZSO5)C{3)`&gk(s^(Fo`!}_@x>JJSP3Dl6PklN z%cf#^b)Rh@KE@s5*8Cf+0&r0_Y zNy)?$x9~59EfmNdbmgJ4zTQE0&jDX3D%$O^=HZ7LZ{++VUON9%Rx)qhJnI+irV>ef z^bNtL7SWFSGEIH<<(VMl;z44ZOMLu8O0u*OyWp%HB%ycq@Dv?;kD>-?`Bz7NA#_zN zm-=nCn@EOB?p*wMau@RJ!4(M^4X5X^P^xa8;E zYR-NV6%O9v7MU;5?JQPaBtbuT_^_?BvvJx!T3m#OCp094>`Pi2SE<;)sxqN_O~id6 zQMtPEhh!nK{iDQDC8oZSS>u2Rr0QV3pT3FS@^ZS*E-YLGunKyR1f%V$HkrkZ=LM8@ z`#w?kbUydMBSohha%o{cJy>*@pKHcO-WVaLE*@K z?v@}jXS*xP$(uys0=8$&^3?_k5-gAExc!y2>pw+vBTvG2?Yeg!VvI|py|aE^qkeS4 zxiNE|G`;t90fo?xiFwbKP1@*oT@1@5CcyNA-kDLh{bmkm8R`6=RnMDP$eipxCu{f& zs;e07Bxp+j0H^YTBZHKT6%z~h_-qN{Wq?Gri=zB-Zi-t);+fOmU@mv|*_EY6ACl-P zu2NB{Buq95p}TzAqe zPU)-Q;^FNb^Atw4A0?tXFDL4Eip?@dlj8&pFB-v()XQAKTZ%~dT3=TH+*6o=tqqvk zD~{JY0Bd%oIQ~;nZH--`liu3F|JuYXi;Ky&&4{uDJ`G&Hn!Q!|VYSrpmn$3g<#KXr zgcEGx08bSW4ZJ_J5xseuJ%^eo08=+fLkI<4GKW%d_nm2}$ypC;Gac%r&{mInt@AJ# zd4GXlr|hK3>6@w7OSEGF2IE+Zsu*m#OJ=h)AYLbN z?6le7xhXWcJV5)poZm&(+;&N~pwk1G%+F*8R@eS^acxc<*6tCyHCUtKHnENssoS5r zql5bMqFzi-4FS6&i}5O9QX1~h2m&6$q$_lJiOTFIyQ@8R=wNWESUp#IaOpa>u>q%^ z>UP@ysM7iEZWVMp=os&E-Vt*`P0h8Pf*UphiHhikaSfEn9xP5S4=tsz`drmWkk%)J z&z^d7^IbqZ$~?bkR_;4<9aXH<+mRR3;U}z%1r3<#8vG{4>fVKzhoh5cu{>{a6a!Uv zp_iQuvIrFVZ|JW%xpvph^Hw##;1m3~uAElg>|JC`h}Tbf5n-X|DzqP^epRA|R9=<@ zbFiACsPOvo^D#5AWK(Z!vZlId`Sd~)%)qq$kWZagRoG z!p)P!oqNcy^Y~or0-e~UqjfHL%1Z02<9A!!iuwyq$0xLtJw9t2{pJo4vt(y9lml4s z=fZR+O^)lQ5;RlC_XbcR;7&zajdf!Ouv`8CQmCTmua|-{^b^`kC{k{*B*HPQLx1YMEPz#Pv3q5)Qe- z(rq$c0ft`_|LIAOj-uYj>fMAjDPqRbar< zz%K=FY-!oq?N=>vUREp`z`cv zch04FV58{{J#%+VLMmT}fQqYSp7Cvuov$4;xA?9)L6B}I@lP3M{xa`}_HJ%}f6hq` zlL0lX_KPyNOKZcV$Bd6-N<)*|N@Vt{P&Ef)6!uwy5yAKZ@H>U_*v^ft_xyD~a=a4dV0rTEiNcd7tel(|VmlsL zTXTb({T5WJ_cRQ_X+Vr-`g2!~DKBe0g~4|YZ0P$^eao06hnTu~r^2wGN;t*)b9zkQ z(fXIpHuLTUW1t^z_v>|ier3WpkTmFo|9*UTJ=u*<=VzIFPk0~)pHJjUB3A{gn)M>z z<^Wx*%GS7D*@F%A{qTSv)>#v!01Cs|->4=a(RMNjtxei+k;IYa<5ZZ@iIX&aiCKgI z;*?z}Lnk(rmy5Z-2rgE#zMH>3m2~`;D2uMRkcC&Gw<#Bwkt8$0yl<>ZkAdO%mA7)x2+E7 zi(OI4ioBKS6;L!7C}{fi3ZOk4?Bb!(Nf9kJ&tr>ro`z{62drmG8Tb}PgH^n3T^XfX zYf*A)6;`xaiJ~SHWVdD(5*3pOhTZq&dKkLaL&o)eXrlZUy`*~7LkqGKqAq7GClT`# zkef$|Y~g{{WDQ>*jH?Wk^MdU*RdZZl_E0|Z6H;Sr5N>tq=#a7zewX3)Xtvpp37GUd zgPi@0lAFT8Uj=Rq)R=PKe>Is*WTDx2YGCLbZtgJt;84wmiNyDW{$fs-bNRL(+)3dNc}p~JLJIg6Wh;!b0p0M z99QPvK58-;N;3iLpOIG{8vHwqJ^gAwf3tkF*22Ou*UkmLHw~%+GU1()@ZO0-v10D3-AJ=QLRB=H0c@!=y?3EGRLORP$+P zNFDL%E-y3NpnkeJERIP&?o`>m#$rf5l~-#Q@f0?_tMXvZ`^u$L#1ojj(^D)IqTc6y zC(oj%EcKvdk|h4`BTvH@y{r<4saK?Jf7H%1_n7BDX`FUMwq-mP|DSXB=44I}pgVq?pw8oc>uZL+G}v{vYxYIY-B7tZ;S!`C?3F7M`+;nn}TpjLafg-XQzqkN+H zt@oZAwZUvs)fF~75nW>k4aw;X8J<0Mq|@X04M&UPj7;EAxkyELN+&|Bl<7S`@QpS^ z>tY@WGT8;JNQENyY30`jIW_W_cl(*|;a{vGE?@fH=PHJJN;ge zgl}W51P_kZ*vkfY$O5&LHS|~azGx~uh5DarEyb19zBCd8_yHSelEgvk1-n?ql5Oki zsx=-E&NY_&QUEuwy#N~0gKLhl*jV+NoAOaq?&%Fu=7{^QGK?cJWiIvJ%vOSlfnMw4 zW0jSrT}SkavS;@u6N3;dgH0xb>G{)m8xw%6qm?VB~aNTb{~*nwv) zNZ|%!xdqqn)(_S3cL6^V87<}1jc`dJe$jR-S0i>b+RqgpK6LJnYzq|HZ2eKr6 zp-?CtcZ!^3bFaG~7;|xzy>=9@?y#AoS)$$Oe;5_%!O95Q0PoT}5ng;lp_c|ye=N@w z(XZe+ z$>(|sZpTAsod$1hZ_-*hRMx=Cg|ny|The%Mh^)%azYWsp#KVWM$0y|=Ag!7!5QL^S zM&qAw8RXhclUI}qS016eurk_hgRFzbR!){~p@V+cW>n>rdNU{m|>;g=ye z_v5)FEWnAErMa;>uD9weQMo^A1p;~~9UL4wL0l-{;sYz$gVmCr^A|1{fk5RkSBy;kg$jRI4eMp%#3fw%w)#}kYFM@m3abFI; z$+3pSM>Hdj!uC}Ot<(j4Gj%hgQ1BM50<~ zVe3M>1A@hD{rPd=Ak`fP_H?r=Q!VJXIhP`be$+4a>}Nwe<6YsyJ@DooWv+ktFfY|Wn%M(-4 z3HO^(9IEH0z`(>De|$ilAPEdhy*6rS%Q0PC1=aO)ZBd|=a&bT?D4Yl5rnv!41Gd|@ zzeUIY+1F*$zm9m^PUO-zXl$j_*zEZ9LU1R!g1oiUtyq|?V^M%9aWQ}q8|7IV>?IU^#wkVNt^H_N=v%jMt6#KfwmA}@#Nr8i?TUjX4ktY ziP*9xtho%=;BvV^c)P3$60}aQ`%|ITsk6@Js!iSmzr+XIP}Z8}&X+}mkb;c|r6-0o z4eUHdiWiV?nU^P%76ZzfEVrr~Ul(JtPQuZ?BPHX7S{@u59NT&aN!hmcdS4C1lbek~ zbJov7_r%LjO#zRY3y0yML^E|^y+FXErMUOt!D&E!$kHT$4NDe;DH7o}8z{N%Vu4wY zxaaO|fDcXquA#(9)8Dd1A+&JksmKtzCYmX0!tHZxx>x^gnJN^6(Wyt&setFwt78{& zSRdSEQVh5ha~D(%E5`L(fb;-kHMx*M&3BN5*0C0ySR?SJ?)P3y0loAWe%e}}^D-#< z?VsLc{Dy6a93L&P5vsZ5+c@%=MJ;AzH?GR!l@3a92js~$Zm9=mHdiYBGA0dgP?vkV zT^p*YiB2cm`1t96u>&^S68fK+ltrpCTE}D5j6Z}#<8d{W*u_+KBjq?5dkL3^)j3tr zf>St9*V?pN>P>~x-Lk*#aDt7UP4N>Ezx~ubqzSOBhZ;;G7(h+@7GTU*QEO$G@1oUz z*AQ|2yhsH$ErzknCHg4tI?B(@ZbRuKva})(Y>X6W;WjJl=GBq(9^B8gE`BnAgx!f6 zbPp$4(kmCW+w%&(r(;+-bjwlyvG$oLueuZgY2R%OaX4YTI$Ari6u*dvxgmhc{QSm? z%cA27zv7N==TbMDh)NE>7(x|0i~K?{_B*RAJ==B7l-}Jw zuNt^Kgs0Vyx#2QLkGidF-JVewWSva4x=xzBGoBjcGnxuEB>gtA2NirVJEh|wtp#q{ z(||DY>d{X>S<%vF8GY2#Sjuc-mMivDPf=<^S*JL$2$}n(cUceYqQl*bSp5eNg0};{ z*KSa?`;?nHT^~giJX9i?K+QPIA<4l z6hSO}TE*|XlY#Uu6B!v1@fMAS1qKFsZ#B_EnYw_UU3Mi0I^;m*+{B(I)jI8Z-=NLI z8y=H{58&_|6EU203>WCYrWdU|ZL*!Zq-tAUhel6!3QbzqYuW_(KYQEB=tRE;5}vGN z090nxrBhw}TfO*9joV=5t@SGt%UFsV`ycytE1{2@(VWMtweltxR}Rj>(69}omo(IfKS}X{_5y zD_~2;da3+*LgHv~#;_UWo>r>kPba2Tv80auhaN5P?pH(8QrkXbZTimuZFlAfJKxTA zULrc^Xz7{tY_r4M@M6~k%^N-W@K2Kz_ywoh6$ZVYldko}*Q3vn0E`kk3S&91zv{+CrEWnU# z?yDy+6GisufJ1kbi0VwoxWQpU*m@~yXYP?|WMTy9&(*yIq3?>e5P0}WM1|SixbVnM z#T*<>En!T-80~Uf2$*~bvh;v8%yhhYO4id&s7r)iZqkFy#BmD`Yglwg_e1cXq>iwM zm5kk-sg_U0{jM&2d_}oE605VcJwkEG{b&uqy53YzQ-s1fG-cX?8o0c}&^60zV*|(e zj+1w9IZoW}yy3o8g0tJ#Bc}X8Kc0|#s$e?ET6=~LypXJs{A{-v`AsV~?$yAz{#hxr zRq59I=ixr6txH(fg5ZH%Y#()m-qc)?IOf2~;A;;)CA7fOth~z!lE|IC(2b5+TIsNX+>S`-aj56DPZ-hHTK>=| z`uLG5!@$W&v6f-#&sy6U^EujwY7JW8Ts2%|;xQA*0RmK*NGO@p0S`!j)|+!>Xb~5C z)Ym5?BK2p%MUegkYP?gRZa&Pl4>6*<^>wjGm@Sy61#!!~c*UNA*KGjKyOvNKh|b5w z(MnFXZx0XZpWKHJI-b;;Qk!kH9?QRUagXW22S`eH>e*~atnCH;5+l*5&wci>EN4z2 zHnd@M@d9fu{15UGQ8M=cPGDhTQchI18Fn`{dFI>J_-@~9{ct3djXR|GzVB3f&Vi>H zYtn8l;c1z#vW4y8ckap3Yj2}Nyw{H=9m~>j69S71`FrlJ_By#VP7XQ~hkL6mPO$v% zcZtRTrTAnB{c|%a+#%qs;B@%~Z#2ORxuZ9E?8)w&_@>EvYXoAevSwzszcZlXP`Sm& zR{$MXGBUL|2V-Ma5DSYH=M@m>TNKCu^hK4eSum1~5zBX3p)%Ot7)Oq}j`xqmhx1j1S{cmA4xwpL7gg(r)%Yz1hd& z6?~*Z;d82Kx?o6>Y|(#>^z}!VkN#SmFYQv)ep@lNOE)ed|K*1|(LLACfd|Sv zxVMXX z&d=-f^_SnDyaO|cxW@M7l$cAdcK&EQMduv6 zbT(&ZYG}U8>KRvK_(Lt?rW2L&I?Ws^`hgOM()@CEAR&z>ff+nZ?LTtTeA88*j**D$ zW&PSXYvc0!>Be~5YR#JKn7H`M8RK|o`Kvo!wFQ~dqX*CAzSBQ@wF2!z5s~T=xOrF9 z2rwU%4`V!a{mz}BbL7_yMG`y2ws%A51l~0T)0P<4odyaVxpLcasYCP5xVn73N0_q^ z(TzIcC?BY&Y(ropWYDQZCvHOZDruZ zRcH1O3R%#D+*gMO+KTeOLKk0u>vA{g{0#Ad`S9MQbcd63E9m{qrt_3vg zBS9@jCqn%tF=dS!{1UBNcLAhWFg5fuJtKWC{Sw2~-~r=>sxTLiMa`bEt#+k4P{cgl^vN1q}KNQusXWXV}odTQ%E)d}P8e*GGQ~H@^?$ z;PrVo-4&o&*AAiI=~7Dm(<^!UZ42GW9f0=MG}9E^*4@3oL`p&N)x0MK6dPrYHgu}7 z{mbbfs}}NH1$7%gh-NpnStp~$Ti`h&Q7Is6pXFdNiVnV%2TPE9$8>dP}Isu1;Uec8teD6Q3MTG1oq(F`H{MDk%af1Re zLJ~Tn0WMwUxo0UTD0rTfH0@ykg}IlwOy&5wqo0l=k&$fx>xMSfJwqC=usK|H;Zy3< z)-(w%{_bb1!4`7qvT4C?k`^hAz2TUW7Al-Ay(+{bi0l)m{o6syo+M zOi5<|&fAyen)#&Qd3rNvC$X2|vdP8by5A%;I(xaGXPjeHrXQ)JyVN1tj~@ ze7pm;LTq8z-XsSUFCVoR=(lv!W0He^`)RfaTAeM&00w^P}OHAPOGi zDDz%*_L#KZ>D;~sauJ*7;87M-;|t%1xYTY|Bo!{Pz1$HPL!Cc;Q;~UvR@Vob%SSjR zjF_6)_tz*|(=LJR@!OymRJJ7Uue^VjYoB?KZs#EA1jNNZUvl0Vj;%b>^HsXfLRkn8>dZH$W*NV6fZT-aZ4s=v4V|=41TPhU}w9 z>dRjkM66puc8ELhMU5A>q5-gCvspEr#I!J za4!ua&v;~gzRN1$k|Cc@VspPb);toPUK`tgw3NU}$!#nLwQ(4z2Ze{)#rk3H=?Ba~u~ht@e!e z1o?6;UBM8)!Ti=KB-Ey4Y^?>gc$csTa^&0kG{YVytqATgg&a&i`LrrFQ-ZeZpaULa zXWs5p#hxd{u9*;)ygpQW-MTC~Hs#WpAEFn6x44^CnWgV@YLE!p-8yFYa;l;rtEmV_ zt|5(Z5ZWgUzThU?av|Wx1U-&-@4RE-T|i$6;Gp@+#gv9=d;2D8O?R8h+Y{dhARr!` z*T0qQ_KfMy1ulE&pDNG`aI&xL=`^Q8H7S*h)-j(d_~dZo*TsD8x>(g(KDR0|*kj&1!n~9S!yaN(4kwO~M*VUyWedw2k%p*|V;vO0>+sf_7+j$(>8a1x7kI2@Z zmRH4kb&JKt_k6&{masLUE(-hr;M&&eRM|_Og_*MIy_FHfhEK)MpFga>uIt!ciTwC< z49#Xk3ZZO^uKRUmhU|NZvH@|VM?E>UK&{xZJUensE zA?O>!<$E0_I(N%Hkf*Bao+nRxC&erHf~{e#*#pPlJ!i_;*7h<${Mk`9lUl0DBwz8mk@&%XWKs$WLPei~ybF1k?m2S6La$t~ zW_gJitrCaw;w6+X5rUA3W9OnKT&ZG~J^nwozB(-G?%N(kKtVvdLFrD3A;m&M5Rj5? z>5?uLDJcPw8bky|siC`Thyj#_p}U)5=-(OM_kQoazvn&=${&L`b3W&sz4zK{txdR? zKqS8B7R2AyC*VNemVIQP;jk{9XfK!4mNsy~Dl~~%x|Z`)kpeZ%7&=D`x`!R;eUoLo zRfs(>kXGgL?b`@`v*&BHTj_xfeVZFC=gk-}H$U^0&Uq(hm)v|&4^~cJCIEyUEa{y_ z>_w0`-lmuMgvaRh7PvH}smYzc{5LO;wVioWHRdm|tcwFvDgS9v1cBtX3S8zs3~LzT znHj3}xOwlQTm+Sp-G$zgietK@Ygg143o&ww>YGX2Ur~=^^e$BbUO3v~(&)0(B6AfB zJWBsNPLzKmJx?2y-#mDeP#e+#ap-drTsP{Fo~fC(2T1xTdw3m@g3a^IImy7c^bW-A;(=Pndm#r2iQ8lRs-W!=i3>9Q$s^& z`Fsff)OMS2;7KRT0glPSa{R#v93p+F*nWQU4J?Zt&Y)S>Tdw=JoHX1yFhU3B1P!Yl zyWP1qjHO83(G_L*hWu~ z_pOTNUx)?w4vbb8k^^Zfd)fQfBy>vM=GS19MH#K3$2zNGQQmaoaTR7b(OKWS45Jq; zm>uhrcT68C8|mv-0^jGxSm=gZo+$Hsw2skIBN}h^vpIK*O~@sYFjZ*%_>+qH%>XB| zB`S|`w+QHf0g*1+N%xHVMYCZjFrpoO`D;4=RU#(U6irQUivcmRCtF{Z&++s}UGhb) zA>;fwqWp-IJ@RawkAr`Nxt7K7k9pbQlX4O~X=;65)WAfNsaE`1HYR zo|W+*AO1eE-*qu^h+fDdg@>}TzVMt2ga|HO2x}WvYgTw~kRpS{Pio~q@vW3DQ9za% z#%_shhhz=0#n=Rr85}b0=-k9KOSv5Zy3N=GOLHj{pj~6)_i=QZNCHmXs#f*Z?l2l2 z;vJKQC0&G;r(0LmS`b@}_Y$UkdCc+O?Mu=a@v+UpWQo{1&k5D0pFQX@`_A55;=6&V zMvXRJI>ZAWD9l1Ee-#cg4h{|mo+nx-IA>>PoB-B8*u}hLtqInsjlvXU37@LTE^~Jh z!=OR{DyuJ@n0u>JE0Rhbh;~z>74TDc4O@M4W0d!P4B_rgud+zOd(0h1Z2L@SM-83O zQ|#VHp%MY2-?vho?)1Zq!{MWIwT}#Xg&jHfyb(<=Z)cOm4;?8?fBJBPd7EB-Je28O z&74-hDxp6ZuMq_32O$kZ4b5)hgMbQ|EBC%C5J#b4TPVQPxhMC}*2$V7a9aq2>b!bV z_TAD#xjf5(%N97=6~!m>;HUKOPi5b#M4LS<#_#Z-P5fSC8Limcv0T9rT2HpxKBgKm7VV#PI-qug4OlzrH=0{q$~^iy>xRR`3qdRImkeBV*H<) zCP&k12h`86=p~%r0!CzWGq*>m)zhcAfNe7foO1dJFoZgYKBf%D4-9{nAeo}su>_z5 z#@v~*MH5N@E_Ap-N2?ch3ocsGIt9(czg5^@Q&T+d*1T?X$bv1DxUY(?EwkgAs9;ji zAdOyNrB_tMaPEdXI!I$&mY3#Ko*%w;V@)8L+taGcjWkDGyQa&`EQn7~)CMuM;Yd$hQr#o0~`zl~oGTJ2*`Yb9w6?Zx^v}gLW^v?@R zL3`J~lvi5%Y#YE&t1UbWo?G6ROwn6S@9m>1YXc%VvNg4b5wFf_$M!|wR6MBkiY6EU z-GXf zOVlRPZKm;B{mB~7R_%T-{&QKoVO}i$>Xq+Gd>ESbf_*32@>=*j+}!o6#_-gv$z|m( zW@8u1VwQ9aAIk6XbE$Aqx5Ew-J28ovq+3EPOVCE!HJEl$MkxrI-kW!>Z`41sDA=Hf zhHC8E4gMPRj^|MMW^zgtBc@cK$zvMEsxnlhpd zDf>m_%-wdy&N;qG+J9_7x5qM|l@fA<@Tb=pPL$!7V<2+E*14sRNa6joBfhdLn3T(2 zh#}Oux@YEkTek5?OmIwPXD?VWO0rNY&>zf?1spO2$T4^;2N+yqBU`}*x&6eSm!9GA39g8NDMy;9Ol|KN` zF6RI8NI1?v5FIgS0lGhUyqaw1H~FA2-bI4*cm{~;hwQID_<-gKUUwTO00McE-~34H zKDNv47*lu9_D_YXqGSY#E|-nJk9~SKR4YjW(E-9yji<8uaS`P<6LD7%mxZ$dmBkW3*qS8d|_LgDfkIU-*0X*fLzti!oUUh?9=RC0DRHnc%z}pPUb7a1ZD1Gt9AUpncfYL zb)_^NqurBQYie9rzE6}5-Opp_{$6$BzS6eNPRZv6A7J4FI!-WfX?lWDT9+py4PZb= zmeX`S7`G4vTtvS3Ai#d6k@b_nBclJNnkGq6K#mE4uH~O8P9%HRJYp}8;G;xf=Nh~% z-eY*8fB&Zih0Y!ZLDQ};*$CqK?f4jrFUH~TD6`lHZI^rZBk`N(Jk%>MAN)l{*y+rj z-AiWiko&$}Q_)(bB1TumqeF>PAMGaL%qSU|@1YV7ADiQ5oRd7JR(^B89PPD1Nc-6* za;z_=GEi1?ElDIJZ)fCp226ePsZf0jTH}!i^Ku<<<`qXXVkeblMc5U{x<>@gT)L8 z2*6BbFz`Pd%G0li##{;Ad=-NBqk@ueRvx^3N&e#HtQ~N|26b?JpX_Pm*_j_Goww_b z7y|;T$sT~6(Cbgrz{{xvkem5gSmcX9hf_}1ude?=D4|fTxx`Jad#%&2=yz<%V{x8d zOQU=2JY5(q3#ZzcY$?bsK)o($hbPdUz5l*RPM``Vc=j%2^#>W(1LVKR!TiCg$Nghd z{^3dqgy>9(>9dE|U0@Ae7rXO?Uk|36z?$Mo$#(qm$n5Rse$so5oxtEywv@)TTnBSB zm#Z9ICc_9nvaxaesO+(YW%+wtFkWen;dx#U^Eo}zOa`b9jd2~XEqDY4BoUL=0a@$ZD-Wv^Quj|MPme%Q~VGRj)0(kke z^}Hxaxq#C$BI?lAjZvk|SyD((`J%%*A|lgdT^n4@8l=~#j1A_5F7owBCSU?!N87vX zj?T^P*!`^?+`o@^!^CN{a8SrvIV1N|5Cl)LSCO5IBYXjB5myqg7he_eaLLBmlk~`YI;^yn8bJ6Q8=k{o!z&zZ%QaN)SF;YA^bD+@i1N1u7nIL`A?;;w(6Q0>M9o&`bNmS#Lo>(QoS zsWf0t3)tAaAZaKl4#uztuAYRm$yrbmGy(a&$DX?N0!4O)BE!nH6`g0Al;PP#>-Bs< zBX1s7kXnfgt<{J-F{!i>P-d`;G4%%wNuc z;Le$H?-z@*fdp%vJ`9mBWQiHCQm}a{md{s(uCRS_U^)4khE6Fy=`QTW>)!sfja1R~ zEJznoS~50v(TC7^B{jLfE3UI3^U%v!mxz(grO>zM4MB1qmc92yu~u6536H$t7rY&W zXhwX0q2rU-3jQr-#Pd0|a&(3qS z(Ma1VLhpZp)aIRCA-?}4!p`aN5MrA5`zG9>M!xx#J+LPPwskVG4QIO&%5h)N(pz6q zM>~ye#IOI!NNBS_`{ylypoQI*R5Ua+bSs_j(TO`Qbi}e?0VwHjwYwcDDQVcD3mnt^ z?WkJuS|JF{>{>z;;dqr%NKBGjP&or)I4FPg*BbHr_`ycv$q!76+^HNCE@S98H&*@JFFx%+VR zr;D=E6-0VASIhlRqMe~-fAbbkL-qx9mGM0GaQnLld|gR;ik3>}_edFtu(%Ca3a+ld zn_yu`N4A4;yFyGp7~O91TaW4?y(3YFJEMb zQjd8o*uac2p*=?@_`NZ_t-alK;JJ7Lgdgaoq_!%tK@J0SvpoA6n2Xgq5)|~LOrW5D zWQ-NO)ZaoTPqcmaVRTOe!nIrnZ}7?~T2vuaTJEc=~~aTqm^c+byzlTP1f;=W??&&8rm+I0n7 z%1Z`C@oi@BFU!)=Sy>v!aBYp9+{mnyz#5OF>(@aVL~$M#)1qDB_#2KEbu_TO9Q$IJ z^jxF2Xw6`Z1E4!bW$Q1d?oBNI3iB9Wzj<&b?$lMyb=yYxMrxZ!yRrU4q+89sl7qlR z%<}b8JW3@zycuAL`*2~HTw>K+{*y@hmg4&HkAUpCavC1gNYac>#z2p`ae-wo?ez9v z{v4{_Qx-`w3ImTrEzq>u3_Qd3&1_E!F=&!$h&G>(jgp0{u86&()1UHsWp~xlDaq7m zg6(m0a+;tv#<*|sfG4sH&GW-!f+?F@Rv>9AoVmp^ZJY05tp?otfx?6A)s1(No#m*Q zYz&UzJ5xHBHnyP#H%MbvE4I}b^$x2PkQMGyJL$a@*S%g+N-9dE&>`=FmChV-(l`vWU0pw+&Zc~c}J3TK%q6J?7J zO2(l`f!kMxBl@zm9VLal6^=_hcwCjyr{pvi*3Mh}k`Bfze=i;{JAZdvU)ZN^v@YWV zr)G71?+8vx5xlNzxi}vwT>d@EcWG7Qv`>`3BBI4R6a*Rm3zt30$j-4&3rKM(rzbk%nxrO*ZA|QoM-!kp#SP^@FtNUlVHo99d`I+PBM}u6s zrS_iJ3{*~oOJc+Bh(w=z(FAfZQW$ydxW z(ae(Y6C75M_-uF=w%rx{LtHXuu;$&dD&@^Hh&|7gwbgp5tqeOZug$$c9xNHkP9Mtw-&_g?^NNs>_3-z@kTd^H%Xmpw37 z4x5f}A;Q6!yn^qw)6eZ!ecNvK9&}=fWT8J+NRvz(69GH}aPeICItmv-k#v%60c+)t z+jJ(5Td!iQ@haMEA-jgprcMZTn;#a9z>Mc=6~`1Y;QMJY%KK2SN84!ako27PKk1V4-r?pIaIaFjT=Y^hYA4aOOk+DLyi}Gl406b**do~B zR@s4Yi%ZzPD*`t3>HVy`GV#p_UTAVMIqnBng;QMkdov6@2mT%S#E_5-A44VAZ*+Sx;y6K0>oB7SspNUMZ+-{hi~Z&fFLA0#AJ6hkezW zSh5VjY%iaO{pcjr@$w4~5Y}g48mj3UFE8@g{Ju$LH_{HU4pCVm-!w_U^s&`iD9nY) z?phqKX&BGjgKUK5Sm9Z(D(cs$``-S+`-74?6)eD|YXtz;`ORn+G0rdOZ|J%5arv?$ z{S7R{s92Drk3MsXL;x%QG%HkeSt^wo6yL9(!c`X7oZf z*^49{gHE1`$dASeIlN0eCfZMR5f+uV-uURyb{_7Yx~K*SK2M5PQ{ErhF;}~ zw;p5#TDTu4Z z+jZ&w>Z;JNJs5@u7qRkFv+qX60eqscx|=8v(nEQc9{5}1TGB`1)SeGdLr7?8w2QE} zYx2+;-LtIguW<)@Jc{Ytx#Ww?wc7U>9wU)OlQ1l0N21{~1N-AD36{)T#rhpw2j}Z^ zcD^0O*R8JgUPpL2jwM{-A2xDBNNE>z=&1=HHtR|?hZ8xH<^HG% zjZ31W{{>{p!htJA#>Vw>dIB#8)lLgLB}$}b3D=uye@3#{^iIt{6&ChwP7<6kEPDK# z^U*wCIo<0*uQLN|F3^VQp}|paX(R!gEMl4m?N({TpgM_w<*gZ_o<@b$Uj_h}Uw3{E zWNqdV5qyW#Gf)D5fSCEnSNCxn!kJF11)arod$i54&BKXNx7`^6>8Z#=10m&|d@*BE z*6~oyli{>%4G?35Pyzxud+uJye*l;c8Oj7f?qb* ze0^o8mXe@?aZdsjL2=smduC+fuu~d+Oq-q_k8MxMJ~xrdYD(~TBo#TS2kwoZ>_g$YFJmI%v2g{v+t#-CQ?kH=$@{n&p)4a5no+TNs1 zZJ9GiL^;FUtO_2V>Mcsc3iEr@qSfTtV;?60l||LPvOEFvth$*_(zMj3?ZJ33$gbM-XCdzAsRx^vXNoe`0Zib+%YYA|KpZ)Rr2Xu#315yxcf1H8EO7@O>2DNrHrRG~Zjw z)l->q=6&I!O2bOkTiC-yhnY|8IrGgP{i>+7;U@$3owa#KEl65xZk1lLQOXW7Fvk+`PJNNfg?*- zOr*A_$AVGrEp;dF`D@7Cbg8!@Z^w8jxC$5K@oH0T%)C1${W6nGap;lj^Bkfey7E*{go|3S=vwQGza^bEQC1G)SKV?{h+>0_3LChlcZrwHIe_v z+!zKJH|3lTIduf0kJao_^^E1=t;m}!62j0=Nu15!Svrg=3nzxQzHy!<996)ng?zC= zKi>DCOs+JQDzSF{Z#mm{+4Ka`Gw#ySXBBuPMqb}36LEsRlHeBgtb5xJQIl8*Fvb_> z2r20BA0~m+)MxSMr+R;zdNZ6aGv9>!_6_3olJsgP%|Na9LI)KN2lct>XY!q2Pl4Jv zsWEAek4kM4jELp@au->(yWSw_E((-vFE}byf5dFlj+A}WVRKFK;QIiBKRTI;gSVE3 z$U8Nk(0nScpM#Mg3;Mf(AvXzxA@(Acbx_%4YRa2&(|8k|BY_^O(7H|XPArCdI#^ZQ z81{StrA(*2>?%_^i7RkX7~C1*alGL~P0?`>YiM(4@pM1VIG%kl^?(w{j>6S--;lkU z`H=`ZoR7a)<;`XU6RXc~?@024xA`0UXaC(&`JuE(utmb4WaJnHS}bk%s_XvbiN%yr zWjufijm}RW+3x!q&{cCdVckVCK2vIFlHTv|e>@bvOB+~RD0_?dc-?vJ{8SSp{}Y}m zrk&@LuZIvSJ532Ukne95h2AkEzSlwdl5fbyRx-6sy6LyP*)F_ymkx4-t>kn=&v8PFmX~LZKPi7>>C5#Q`R(V(r5FBI?>Hw( zYpzT+=h@OWcdH!iQz=4vctcHKH)$(;J!xk-w5`wWUAj4P$GpW+MOTa=RlOeQp8uN! z%s2~s^lsM3+}t(Br;Nc`Sia@gUHtE6cfBhVrn3TeZ+O@k&lSBhtTB;b7@b=u*V@h@ zi>Xu3)EIFKrziI$5dj^>cOOPgC|dFT+f}lt8 zpElmARvosKJM+b2;xoS;fe%J*gu8v`5#EXf$Ln(qJ}>2^3Id+_|0GIMGH^CXfMU#` z!=Uy+tNpe)s)m6XUzW z`IsPwrW?qhTSW|Z6?nb}p`(v(mg1Um|B(DlzDsbBt!+YIwwAE3+>NXguvO!G6K6Oh zNkYmqRb9Md-5v6Nsw%Z zqY~cro+$wq?moqZqz(pMF#- zB5z^RMA$Scyor)Z$F}VL+dtQ^Z!!pe)3Ad%Rz(|No^y~P7ley3hIomw)xB%G1wB50 zYoC~mW=--!x|Q;KVD97ICsG0YG6xjX*8n}b4Eb{UstMNn7ErZV zfG~KlmTagbEhK*1!yg_d#scg$B0P8O{LOGvh(kn#3O2uGr9IC7_+eFT^RY2vEfle~LUI($W}UoH@y zmfspH$)^2sWyV>KD7{VDaKcYA?XP%VtFmS`w?C@N8iFpE8oJBv?(M!m~me6DjXPzp6Fczcgw>pLH3 z3|_lN%%}c)pPM69pn7_h#l;pbdgyrAgK zJD)XVyq4c3A$qo0sH=EHd{0kSp2!zKSv86^Ri$Mqce*$`01xLz+0hetp+T%BQ* zBEZb{pDW|3j3*-(B)J&nG{L$ZhTz*_s6$5C6%??7gmv#&? zEUz0?ss-Z-aq;h4HZf1=ZkJqFjmC,|SN9_cUmSqe9a!kV&_zs-#Lrr=JR{-L`w zw6JldBFp+n=pZ@BgHA}P4T^GVcs3Ig_@dCAC-IsBErl?bpYCqbST32jpNaBW-WQSj z_D#H!O1JTJ)}OwCjClXl{JXXC+`fOAC%lh87^(ZXum4`p-ka9CJRBSO?ZGj%vCj^i zi@F3bMjnrCKY(^9P>JRK^N_q0J3Fkq<_!I+{g|qP`9p@+5i&H>UXiOcfjF0Tkhd-G zX5Tp8X&U|T^T1I(*B~PVbitDAZ6u8yPvfMsw*_XRbEE(0Xt~hv3*!r`=2m|ivOoMK zxm4z0sfW}Xe*}8kFBFwOf*}X*&mC@v=MujZJUB=+!m5Pw7Y~SZO2+l_T->vuTbgrD zQb;KB!z=pvbK^~xSx9h@C6s|NPk0?MZX zK{Ta9OH!IaEyH|g%)s6NfJ(0c(~$ZK6KRx$7^imN(PREMrtaS^l={FWE53#rVL!>VuStdo{8>K!T~Tvi%7NNDA8%rQUv*n&LELqfk-bp zPq@XLineCVi2k}1<4sJeuf?!GTE4enYf*m3L0pHhn38v}c26VU0I0;h>BTz2rLj9e)=08cJ?bKKK@^zxPWx54onlrXt5y7U zdS)u#mbaVx-}ElOuNnP_>XyBEuV||o+{VIiio?p&Uz$$}UY3caO_ zQCxBgdL0o`1!R+31Z3>zpS?P2g{=Zbh>*XH)m&5Re-;ba3d)?jBcT_(s8B1xzYNb9d}9 z`+UKkOYO@q#77J~!8~L+AI-|nr)Ii!Vlg2*)v%y{j;?m`bVN!Li?e%_Zz;{Yn4^^+ zR=DFr0Mbb?9Nnoi~#Dk>|0f49SB6!C`grqt}M%{TQ+$FXJpHev3cuVH7^8mn@3afiZ$?XR`XBD1UNGbG`(^v#;U;Ee6&C*ohEhb{h@Fu;Ekvna=weevtO z)rx|BeXZMk&ku~Yk7)|~K>gGNdx9j6{rpwI&ud7h?Zl+h@$0b%S$#pz6Pz)OM#SKy zz(l6Z`Mv<4?qh8sR8Ud%p&{vkA^9E-298u}eEuIHQp;$+N%`lvy9mp^72hi`nLs6> z?v#g@cL6i2Zhbf(@cuAD8lQs|b*%v)VCe9+UlecnbcV99|IOt$eWV_#ku!#H7>?)9 zz;(EEWeSokXJLO>ue~={lEch| zT9BrCO<-Y?`3^B5TTZI+@2hgMcVP+KMtN$5QG(N0 zg_S^a)&=wmz%cjvOgo)ZYhPcV9pWPePvutix`gx0TLoVrY)S%(g`%SB?w8=bO4Eci z^~W^aTR-Qz|7=`7-h4BW-O0knPYSjR&LqY+YQ-ojjDXIl;FyYof2rl!nh8h|iuD==M6=UWld5l(g=@b1>C&Tkck-2FxDN+qT+7eQ% zYDa)@dk!SC($y?-ZsW$B z-@i*8kYB(g6t7J#8{y=Y!QW)vi2_HGs?rJ8#z4m%yANdUJLe}B7;J<=hdjm|>lSK=08IxWaI(r7Ctoe0@A%%VD6I{qOPOwT zaC2gTa8xxflT(qyJs&l+SQTlZN8MsgsmUwxYs-I0m-UZ!-Ybp|vV_B{BKLHnC8v`G zDMFi6i9Q_X-PdmR`0{SJ(e{|s^K_%iGNd6ov_6H9Mt98sq_7zij~L|(7B=D;ms#Vr zlC*}hUvL2h$XpNxh9X(xCi238Yw{FQin$$LGRV~rnd|CO1D^}dQ!EmQTqQjKywQhp zGu%JomRE3IWSf`qVdUApymg?+Y!Rduw!T>-EY0H(bfN{?4TAkj_UWdkcVt21E&(Ddc{fGs)pXO&Qp>G!yb@_| z$bUgJy-x0Hw$<(K`GZBvwDFxE8itra6A{FRgp?LgvOvArw8?ph!qYU|H@pgYO9Vj> z>xHPV{^q8pe$s#&&D-&Ng5D~AtY&SLivk|)q)uZ-uHpI7Os~?K>U_KRj0i z9h3Nhn!Qxe3=Eum+2^z1Wp-RdHfiqfjxCkE2Ks03vv>))r@KLJ~ zxwql8k=POsP5xzeb$n(-;MV@I>m&~TVJOJX=pR3}58tLUJ*t=cd!!1Wel}5*6juW&Quq3HBc)>!>FR7=wX@b3in6Fm?&#Z=i`) z2~2$eYweZGDX^k>2qqW~WUE;N-6B4w*=_L66(5Spjx{N_=S-~LDNLul>KUapN?cmy zD36dOr*G?UbYGDZz80AJ199xeP=)F30mm8Lg8Z|!8zZI`$k^MClGJkD;d>{B^VMI) zoATY>by_h0p?vm@?-Rh@lzPBCqpIGCxyI4m2cw-@aHa9`J@B01u|ZA8>gq!dm7Tl@ z))K*zemC9c+A&JzY}yRDbconffyI7tDsvew-{t!)))nv)nPmqJS>KPDI10Hl2vyMH z>+}4@^NVv(OX`=Hhw>N&_LIlG+zPU1C(OLZ0-v=-OS)gL9Fe1KeDy%6h@CI6aV3Po zd?qtbynq5xIsQw3iKE)T0Pi8VWH9!bFD^gocufTeMo49!E1LOy1&aL>QER{HpA_|- z9keu^v$ZV~Qrkx543kSkPGZU4C;izpl}IMJsoYC&ax8$vGta25+EyDw_7HvgYVTfss!+O4KZ|0YJDL$piTB>GmoD;? zRE*tCa-@;(`zvm`&m;9MR{c{GN2=WHJReiM`X}`zG;rFc8FD1oxnFxdw`v35%oT;H zLG0ykjJ?16%hj!tP#jqC$Oq zBKA(+=|RiZKSqtQw9Me>8oB!xnOt6vLBTxyw7_(c>1r|4YhuECs(xw-+s^x&;}(vq z+6@n37#p3;pO=bO^_~fKK-B{*hF#66`XB?=y{v)r?6z`38H0U=0Mo&A^nHboybgh9 z-o@nlw87XP&mvs#C`qbzK6?-C&ol*n0F)Ln-^2{VdOrvju&ZR0@$ovWa6nQ5`i+Lh zp%FC(G%exqUY7lGxy-l5cl@qAAI9`5K$rJh;WnvXf*3@}dZXeINu zYsKc9-m24{mtWWae?%tdQp!RU^p7Li_$7LIM%`;a>bw>G)@7{ocmew26JdiiK{xv& zMGEfcR+aYj2N!gs2iqCoDt)H-FNwG?@S4JY4CyS>yM~_TPZ9IU7pF#7*M)J$e>ztn zSi|bEQiByMJXkXp!`GI<*g3C$R-%SocC6u~SC6MX)BCE9thcNO)iR%T?ba7qi6g!F zW<`C5Hnq0OFujP5V;Ev4v3rqOWZ=j{m`A7J#y21kAKjcvfB=>-SaA!B&xXO6G1a!^ zbqcQNeVE#S->N}MjoRlSrrLe+i;*>a*lu0Q$Q(!^R)&B1CHSDMx0b;aG7)PyJ^dCb zcUam&F}j-??ASCrPi?ll5I!)4ORzz(`K*~+CbVVqZnhvi_d6y8lOkhqx`C$@$XpjI za*~s;1ZLV>+v>k$23b1X*XCE=z77_Cvd~=WcFj+OvHvJRn*Ar??Etd1H+l6KyT-7= z7cy%ofKPk`l!cE-s3XVp=1Da2)iiG7PQ<*flr^KBSqfE*UWr_q1768%qneHUQaRn? zRO|Z553?*KOcWN&=TbG!>UBGyG+Sj|Ob(2bHGk@Y%ow?hDD&W*eH7|N-!Q*Iw3X~h z9J)v3-NW$TPJYRD7`;j?fijR~0M9+t_H-XG^s(vB3Rjp0-u{?LvS5${8)JG}+VdAL z*p8MkKkYWPw4j1Xae&PdbpljAH69csK_2Jrh@Xk^*86aa_wO9cY23Uu$A|cg!>mBV zdJ4J2D7SS?QZeh#vDm4Sv@-II$+My+RZQK1+h3JgtH-LGWQ$3)GSKVCvU4RpN6yEz zm+Bxe4&<)sme1>+av}97s|k1x$Jv7GfdU(tG)*-X#9ud(7xSmQL$bTK9%$0!#&(qI z@Mm3UW%rJ5_tbTLOe*(L)|c2;G7PZ#y-WMXg(B7+COeW2DFADIX<8xLIk359gD66P zQN+A0;KIdyopjlwhS2EDg#c9lz=zoM<=p`Vk00`%m#t?uf(Za01w$`^ISO`RUOLkLi)m z2u(^x!%82Mt}^MZLIBIbXs9hZv>h~pJ2^XM;(O9R%_F$n3X&SB>lya=u8n)BJkFY< zJqp3;n|JetOQxCpNqteXKyW1snnQm2`3UsLZ>8It9;L`LI$)#ttzA#=we?x&D=if9 zyfUls(aGX}Q+?bZFe2<5b%ct*q8cJtJ|{^F$4Bk25v-2HbkuAyJESb9Jj5en{r3Ft z^N3uc0<(>?A^`u^MaLY(EC6fMe&*ufZ8A)y6iZ+Pwj>zBkPO`Y_Cfka!obi2^ev!9 z!itgJyBT772e{#sjTo)(k3Na`$Ug`Ssmg72Ftj*ppqe?W-nY{eXlB06c209SO1)ObEA`2e8W#jsVas!JZAJsVC9l8;xt0{d`Cc(kQYw0# zT{62E{c=upt@N^PEHHg9`EEBdDUe-dLhW5)>Z4Rt6Uaoq9w5!U0V>eN@na*>_(RyO?_ zjAx7T;QBFw4-w_{e7>gnntb?|boQ08In^%^-+{^0GH?wr1Fm+`7wCD+pEiFm*DEd~*A=K*UUYrFhL;&}6%#{SY6u7?to$k(w$Cl38ll_{wwQ@@JuL-PexeH8KK1s4l-|W z9NgPOA+eHxF?G!O2{lng%X(6ElY6pH+e?rIa`kax%o{pz@(?prjlgT6@*IUcMj@Rd zkig6U`25Xva-<)4W28ODv!P5_m1{Yn`8|l6qEK$G z|JXl$J~QaN(%(ewm8=}JAVc1=ciS0)_86(CZJ6~(S!{8+`2n=fv$Q^l?$yMDuu0Dy zgU8&~fJDl=%&N1J^iPQncty<2_0+gYDr`r{Ad^3I&R$Y)stFK3j0LYQ>b)v32?FTz za6l1(wQ!P(&I}GWqKn80NVXT-t3JFtL^|s-amM%i{sq$W#;KkS1yxt{pndcL`H5fD zznNi@l5c}2Y00E&-EU1H2H(D;%>t+79pnvzkZwEs_(JwtsTo7F>*ihlz2MV=!4}-W zhXI(_f}zqWV1Rm|RSzvz^_$g)LAZoU(!f(e$hwyv=-#8~66?w^W~xvrSAsNpu(c}LZKw<4iVRK0*DQ4eXo3;gaRl6(sh7_(Mq z>r^S4{x>%-v@h|~BEmlB3lG&tYYqvJ#A;IG=A{sU;TR;-Z--zDMNvK!=T`_9H<<;2 z*@K{Kh%rcZr9;yb_E$gg`nP%;?)dZ!B{erI23}Zqpl*9qUD;FGd#3bN@mZ1e5j%88 z8A>~Sd_zTUg5p{9XCe=+uOB_ZxjdOSrp~sPyY+{=m?HjEjlEf}IrnPdWox1^XNC-*?zUh=9t`;ewsD!7A7F_&>4pJ}vH2bbZ~5Bxq>rlZw*Q5T#|SItF*q?Y z3;@?ozvN6{&d0Ohm;bU5vj-7=Q>PB%m#5|ekYElagut(nSY-LxV-=%nIC^$W-$n>! zds3DYHD=(UcnWl6=+^@Ze)WfL|Kz4rlhHte_P55!3Bm5*%j){mJ!(B?<_qwYa;`BM zlM=ktlPLdj2dkkDm{>ouNB(YaP_k<8$ZUHSZCps!;YN=z&XPutq zKk}m_^OMe@W1xfnYPFO|%aYz#>$gT=-E}$hTC3Y9Vy~H8_i7PLD)k`U&hrOO=ZWox zUF)}CWUmQ$kFGkYj~`&Dwy^}`Q(5=f1(X-M6Yx0S_ymqE-q%kaJ8*5!6R3XVsuLtO zSbfCUPP7^dKHXhAFsVa!bmv-$C*tQHL)|4b(0Wl%I^H)luQ3x&VBox>UaV_WG>#BA zzLTpvols2@p?und7g7Hn1U4YU1efo<=b<~8e$eamI&-m}^N>Oo(VB<*Rku4j+{ooT zm%-439Tp1#1Ko))B?q2y>?(IJ+NfmeM=(QXv};=xQf2obp6NdRUaNbFh5EJF=9r_( zZ27k$8hoSr)}yRbbY50*bFwO##{cqT=4XOfIkA=YMf!}fI!_n(hVCxWOH?M(+o{F( ztnd`iAS4&D3yh&GU)HFXMrn)Q#=OgW21MvhcijUf=LAw`*d#9W*o4a;$#nGzUInGx zo{JiCnh{f-*hqrLBaqzz*KM`D>SHcO^=o^~XY2hJc=eY=+4t9-Zq}c>VkIKm zN&=<3!Nef{UT^pLT<`MJ65!~F4^zNoYB#P3BURt>E=+~Oo$8;*GvwNMQ)Dt~F)f&NA?>=5&1qtK<5Su^2qx3Fy>P{w}csY9vGB2Khjp<@G zcO%~fY;z{8k?>STmTgAGmBx2ifW>vOivHEAC%cm#1ag(u9PcTtXIec zbQw%4*+sReys8YUPp^}YVeKRM+)^Q6Vid$*8ROr%U@hvVndlR_HI!6M@@_HZ@+fOM zx!ROF@F#k7ZiPI6QVs*Vq&aM7sXC{1rG3T(sIi%jpkRTj^QaCfN00_-HD)%*b~T>^qd3ujjpE>6K4Eqdb3z7&=hflewJC;nF~`TGQGii^ z6`FcFAo9!E;)G9uT$5LqoKro)Su{?MkIGp(Od zdzMyZ?r~u5dS+Mrt10rmWDuIMGxTvxtsg5iEAosK?mv}BoTgFXBEcC4%@F^)-V|#< zI#=`Iyc1*#7=8`0Sil1%Ri)(l`S){`B|T}k9mGCknujwRNjMSij7gLnGST=CY&>oX{2 z+RrDFHpey-;@h%f*$${R5!1O&<@AMBu}0c>0}V0tjk!O?uNS7`MA_NF6bO?%w^iHIN-g^r*3IB;#-+SMm8Rw2OTsi0D zl)cwpdo6@A=^c#TX8dJyU4U?S8C3othD?IRahH!I4T^73?^pS|bP*}+Us=Z!OqOb| z0;N6Z4zxbN05b$K`aKsH*0-oJ9AbZgi&nzRA5HGpnEk@9G# zv&ea|3fl=7YTDVR^*fXIsZ;}A0-YLOr_lfl;r*`KM1c0z{~#X?P8x8&%=}DJ`|C1E zpXQO0uHEKJUT*sKu&KWO&rxTzs-GXw(s?A@-Rq0VvQh>d}J`7-jjnPbn~U#Ed>YDU>uk*dSFYE%KHw(b%}?5W{TP zGJw-EC=R-xWX{KN0rV|d0K8iy+(tKZ=XYn?f(6s_a%|5R>sIp)tEwgQ%?mt!zp-G^ zKHx;AhT)yZvqm9fbAk@pzt;N!5v<1K0nxFO9Rg^6tNRgh93l)zT=-XF;+|M)0?7J324F2bPfD71 z7kgz}Kyv&zf+3`T$zt_aNgb$7NFObJ%B|N2+j}RieQCOKsqFR#qAU)&-*av8>aRE* zl+ghb>#Fr!XuA~jgcaSx{(1F645+*X6E1wHapGo-bhzyv@Sg8AqtV#UwL)i3$&Yd2 znWURtRDOj2z)yPBz<9Ur(g)y$O*pppBy^?(qcb!EYSAhx6=-m z-5Xwl-uuSmuoGD+VXp)U*>3{D0IvL@%IAD@fW@!`v=&5jm;L``Ab}{J^pb)bowR33Kq8Cs+btc9wrj3b}k)}P( zaos|{%nWc8xZ$Rx-?|Vt&?#um%T~ne+mUs0FZ@N0Q_1fpOwp+SmCO1%b$9JB)#_m5 zy1VNlLjKVOj{OW3?Rx{Z-M@bnstG&>3<`Do`ui1uhzJl1$%#}|QJDq4b-=1i$L|H{ z_k02x#H@sdMBoJHez&%`3U1(FzZW|2`To%TsI7-og)>F|##8eLV7ab=dNxL+;ao}%sI!p9E*e~RDbSyiuKw&glz-c^it-#@ITPeP zVE<~9C@`GF{qiQ5ns8k}pIFqIJ<1w7q4?@>N%FV$Nh&iKwCDO*5-|M2rd8`Mv;L{7W3g&a32Sq-jfn?JJCkjY5vFKzB0P@ z!?X>Nv&>ULDJLz~MzXg?qfO=ybfXKeVgp6s%eb%4`27P(V`8}FO55QJfdb|XM}h3c zBLR^mtv``SS=L>=sFU>FdUPUBJCNY`A4#0XeA9f=RrI7<=k5V098(Ji5d#_yrZ0eU zR+6CEX&JxWMk}6ts%Z~NgF2r(zypMdq+$xnSKw#$4!?opZx;hk@hUx<6MtifilaUv zvQ~?oTy7y6?msK)7?!U$3hnnk07{yrJOS*MoFvTR0}4(@RunAM;{9gr+n0(@RU*W` z>H194P11ECQktZVc69FQ`sH&;A6I06UD4)+qM;tGGOv3FeqTs3{u1I9?SY#54Zn55vNCsr4;;4d51=^)j_^WOxFLSDikV8e(2 z=xEkC=$H{)Mn9%p-lwLn-uVnlHq!fJ?ia5KWpfX4fnfPl0XCy6J4DK6Yb)jASXsv> z=UXD&^k9hgMfE*4$1%_7l&vbg-$OpYZXyMIwd~8xF5t&50CE=`bGNR9biP z7Vo^3thfG(^y+q5k{N97^lvT<#DLhmhj70NkJM1Y>5vvz;sjNK%ucZtGQrU_Zyi2WuY7}5@E+Xi`>eua8Ff^5O282cI^|;&V+)LHZ`AOIZ zIX9mm4rB>+Gu=u5YF_V=?D^4oKV3rpDD64)d6Z#l{qI8CPYE8tR(o}H9=}M8P7Kpz zB1($x;{K>m7?_eOFt1-b=Yo!jqZQ;K;3ANut9^ZaNt(V@0-T&-D0>XhiUGb>%If~4 z2Ycb|SOF8AYU<-2y$nsc4CnD=MW?NtGdjy7 z`k!>D@wx}7JiX{$Q60OIY@+b7aFy@Y?@L1}@T<#O(A1~Qrx$oBf<5cWK@Fi^h%_UK zutAF3`3Y!HUQ7)8`Vv-qxbg8yCKAD5lJlPyaB4p{Eq(+YBnJpf=0G+Hay4{HXMD)1 z(jC~&lf>2%_)U}mvrH`r1%_>{3Cau^uq36`ceWxwySbfM8yrswcsDfL2h1#9U$3n@ z|4?$^tCMJ=M39%67|h`~81N^4M&oaRD)Xh0=Rh&XO?Ae;*l3kj>KYhbbnMwdjE zS-U>+qJUG$u=CLo`Oc2u7Xi0B31h1H4Jpl@r#iyI<}Dw7Qc1hk+XH$G6F<3_BjbN* z(3eu=gN&PCeSyAjF`(%z59Bogql5Dl6x$5ez(?$Oz4p$zJT6rAA!HhpgB6dm{t`!BEcDK=s5k*cO@aR!~ej|X%UL-=+QF_WVoWIYSGwn zs;*)iCG-#mOr^H_<^=9oZ!(37G6I_>(uSQ7qH1huGV5)(jH)dO)g{(qBLr8!=_23@eR2ASAU7KUybwBy4q*)I{&v?#7~Jl;0v52Yz_9Z4{lHo zt_xB!pnY7T&+9N+QTSWKFwaQ}TIRceN$W68d)@EovrDVL7BE;&@9mguG*(g8D!GKm+PYccf8J0bwS61iR zb3n?dU5Ia(`z;sztL{4CRdf88M?h92t{m643MCjuJ= z61gsL{8;$h<2-X9prI#H6YlD z`r$9u!?I|W--+~u>tRZbIxen{1RKsQ^r7yr2`z|Ud`U~LCkO_Xz`@7oUf1bht`U3x zor6@&Q6`SZPm;9nL@{4gNzrB@x37u|?;}w}gXr(b!8iBjpLz%wyM!JHc64gJ96{M$ z9i)>}Q&TyS*ecgs;L0j@!q$|ofD~Nf7ND-eDzZygh8LEqk2oKwE!Ucj3g8VFo(U$s z9X-89=d`Aqp((?tB~`w9uMCt<{U-H5UrO7^L;H3n;HUW%xI2&a9#Pqm|HZbQ?_QXy z)B^`aepDhMyJ<~u5_aj5spEeM|BkX{KAbt1RL24*m7ip>4TQM^fXr_Is56cMi?N(Y zLU$1HC7@j0xRLqaW>Zp2OxYOX6t?}UR>9Jd<*0GRFJ0)ftCf%mmD7Cpuc24uBP7n` zyQ=aV|K8s#UGSDVbnT9+tDKUlV8xow9{^*6N>oimiqW_V;1JBI{jMc`Rx&>C&hmUf z6#&4_ke3_}X2u6AenUeNa)oLps%evw1LuphmS$=JQ*Zjukv0VZ%l<@=Tt>k5c~abh zt@X|Ym1R?NiVGKfm-BKD7C%J+^9*prKmWdqpazFj^spl7#@cYN5&^!-D)>WfbFs%_ z*{UNfQClys=(#jwF|NRZ{p?{pVZ!nS;<<#>yr5%~2$cmZNCH*vT?D}1OCawLY7zyT zhrf@l4G11Lnearq|ANO+c+dX%Vsf38ZBj@xiysgzLIfNAUS88Ym(7*Zc7F4*D%Uz- zH$^hGHujy1x3IQ8v$>BxNdvyHq`cc5M2Zj&l@A{aS>zwh7PPjtF2vHlQ6V^r1MT2C z@((Hj<8{<0_!&3^`p{m}g-~dG(#U*3elAWTDDqa=H6v?5c)thL0yDpR08ob_ZB;-r zH>->@xvxSRQeBz#`2YeWdkySX4|f3U4s7WD7qOF0GdRy?z>)5>4*`ccPf0l!LT~zX zsK5XM(rQyc*o0hcYP3Ubf4`!>3I1qf?Z8{uwzT=;C7#&Fc9Wj)Qz}vJr{+lyh{X+R zX?apSbGmfSEJ`dtE~qFy@mJ|qUjQ7loz{7E5^3K(VCVHb3DE&yjk&1aG8zVceRMLP z+?BvU2HiTn?W?h5et0rqBKixDu(u>F;(s4eXikxIiV8XU_Gr>&RcK_TrwUN(Dq4Ur!RYuf4%|kU#Pcpr<&d761P_H@x%ARX<$qae;VO)~D!d7OJh( zktWxrZ1T7%)S&nzDr*5(^Yx>6rtVJJZ)2R_8wXs^U%;IkeM{lA3wWs9v7VTC2_i7u zy9s<70knjiMu01B>#N{JlIshJH&e6rUSUG>L?h6yC&b?C#B^+4&|p!!y$c?Nm-F{# z(X&iC3BtpvnC_h&rg`LY|2uQ1aT{-7%TRBrkG8f!g`|-s zHHS-(=Z4KzWwuyro?XAkCXTW$rDHX($7Yht@Gx#Me;$-Vkf-nVzw#)Dd5^#*!ScOL z!%nr-!dLIk=Hfi-R&1V(U5c<1SE5`I^H|V@KwK0bJO~Ehul@JG1_R~Y8)AkaTyav%r;`JQccVHmevJ(#?$Lf z-q4@=UE*kM)>&=k4E!{%Rt(*LrjS+RHSD#~elTLJFE_jV(SD2QyTo%1#)kw5WRgpb zIj$l3=9&?|XO()*0&@%QRl}dPoGjJ3eL%~?QUqL6)c}3y^?Y{*GA@n=zBg)JV;4iA+99fZp0iWR%ON9hfHhE4Q54)e;iq-J5{_Mt&tYFCC{YOcIL z-O@{D*4ww_nrz`7_sV~ivwo5?8smZqOuWU?Kp^*2W2@B-g$$GkOMUJz)o2a^@8KX~ zuj0?xK305j*;I~ks-B}G9Lz-30FF$2TwFG=qQSS>fOvzmxdp#DNN*sDU7kjw$6GGcgH;7#XD#CM#`QX4h7`mm>E3= z4i2`KuVMRh4o^WKb`j-keK86-x6aRD&OTQ1#w*x~79Sm*{%=gr4ZlDW1kO`aV}Sd! zsN3ulKGQn=SPs=bXEfn62FOtgeI|ztXeb(a+pp{z9QpgG&q*G{?80M<{3a{s+TJ?k zzdlcmIQb|ml#+(rj89I>_pU@E*cFwv9eBJ9Cq8LXM?|n9`2RKeCtzw@|3Xf@z({4~ zHf0PxxT_B7y`)gKv!-|oQZ-#5-#4O{gF-5b*&ljhB4eCOILmJ#zZ zZ3Uu9*+h(p=H|n25Azfb{?3!pHxnY9u8|Egn}rUcoo0oK960+Cb@Q2)d=q`C$_+Bn zcT=d}>e|TtZX&%{yv@=HX@h(;p0_r zXEIUMA5OOYOiPnL+LDlhqqvbQreo!>c3IY;Z?2mw?Oc6(>g_O{Rfp%RV8q#5c|Un=WPY)!XIZ92#C%ZJQAHxp>ho<}^%sttC)s|?Bl+>FlI`8k zh6vbX8@y`g;O(kN9bTk-PUG(#zGAA;6i1_6i%t;hN`MaO zu8)x!#EU1wWLg$>)RjUE_VnZh?6+fXjwoFT`W_i8)uDTgl9aS z=ut5{)xh5Y%i7RFV`|@Q*kR6-k5Z-mYjb{p;JA+27{RM$24(X=XWw+m zT!y&6vN-Yn@*!vI=;6NOe@6oHJYSQYCopX=Oa21NrZ-DzzN@vh*B)8_8JLvuNJto% znwr`Tm*#|0vlg|+a<;d$w5&o4DrYagNAr5>+ZSX{(jliI z#tl_OXuS>BaZk>|O(~_96LB}Cw1Z%Omjs$Ga-gF_#dP;0m>&;2yL`;-#s>1+w+D_R z<%+9X5mHsnJZwN0E}xk>hdej4wiba2Jn60l+tA_5K&&7Y=P18_a$a%W7Ej(+*|V^YOlt zqN1jSMYin08(C7B$<9h?|`Ak88u#IWs~Gy@$R3V z>~TJi7CG{^(e~B4uy4HZ^5a{b6QZzO3U!G>=YPi@fa})oUfHQ_v7LvC#@SBrJgiJ# z1zBP`Vxq6XZtx9Z5EC`5oPXt6kj}vOjMXV*8RE4`RhE9LHNvb; zWOJ@&{W~1TQqHK7#@eK7MV1oC8HrWBdpjWEqhYX1M z%fa%4(L_bi&aL-Y$e>UO?ok?O6Hs`>@=Jetj4zK-6g|i%skwk3a>a;H4c&qpesNxL zN$TXt#mFUitFW%(?_CHo`XX(g-hV`eld>_!-^JODDO<(v+`DFX-9ED%IFMAO(Daq# zYl>#N$k8gM&=jfFmSikjy|)7H9Gd%OqWm^B3eDYS$Ts|z8FN-Vy6Bo2wPid|T2PE$ z!MIlxT38nH>-MhDEDXbP`;E~;L`R-nP&NmBsvW|c=YhN3e;}UK& zAC&kZ&ri0M&`D0P><=YoFBU-;?!+W*ApdN}NxK=r1Iw$A^P=d{-a7I>;uZ+|6NzlN zrTzuu#gzwu=#3HXmi{z5YXRTy9tD=)59P7DE`4863qQ}-g?gA=Hni7exN{H7zc zaNtggirtKJe~g0vmxmt3$~&tP)e7#H7i6Z7lXeifAMKJ&6hg;%;~bf%a11p?d^@n6<{m{OTva z>5Go4$qjyMBPBZH9|8FC$V@1OcZZ6oBo^KFGJCx`ZGOl( zw(;+CYYE?bM|N)s_Fve*lf2tgx(qYB^k+wCa50@jCyUj4q-2*y>Li-fdbL`Wojk3C zA8!Zs1nn6c8YZiy-p%aM*sqWsDKQTlsdPqyew20xF&=JiB^m#{fo#Rt0bq*zb^-!1 zo^#VJ?3-#~ahKTjOh~+Vt(3eWMiQqzDkqzD!^Fhsl&$>_T~gM_N=29~9~zx498I41 zT(Z`Z@9W}T>Gy~-+WT9BQtJE zi&e7MAJ~n2@_1b!nCtPrN2f~{pI^SlgFRn%&3ZlW@);+HbZn;;wXy)nnikUK0&QNF znhyu`kQboj+D{p&OWU zlvziC=Fv}RA`a;X+yk-BERZ5M354!ak9kzP1xnYZPBA92GSP`hfKzx}J`%e4a3!h$ zQ}EqjB(9-#X{^Dtp@wsv1vE4pMx17lC{&>I@Chf74~^0*51n4XFj)K&*4!e3 zPIm~JXF6;IMlWk8#>{}{ZyuLCyAHRZ50k2oJ+Ir&UdF6lC zvdXX^Am_Ou6QA{O zce;St{8GH0X>XeMcpkV~4S4BjX=!b7%Y#QNu#^1G6!Dde6MXK22M=}uv{p4(+iKyK z$1g10U+8=4qYzDzQSPdBdOY{lBgq!K!OYgEym2{LoO2XQVrs=Tu|icN#0fh=-M@72 zZdy^1w+VK5SB)~_TK=VpOL_O&cqcbn4Vw(FPt?O&1za0=uZO z+oH6I&Fb6RvG}H(ZSc`R8Sr11#HL9-J3G4^ zlP`*~9Pwg$I)~VeK=0N&tgNiKjR_by5ZYkFpgd=JJz7vc6@X-f6H7x-ppu5#r{lF3+B5!-br*&3_H{ZW6*0}OJS>Xdj=y$+LmnZLOM``&eZ z&AmfGea*;%uO(Dr*Tw_6x2P-0+X@a7dp(*zTuiA`=V(2)hzZz-yRTPuLiO0q__|+x zHO#mWaxE}Gj#8Tp*Cl>m0isAf7gp|qfoyE}d2v2e-J>-Jwc+@Q3IaUM2UGf&y8`y*fF>~^+xI-8rah9x45rIG@S%4c}}N1F~D zv9ql6bzK7b%OVe!x8Kvu``~VU)ZHn4Ld@Jt!^A@ z<|lFg?DJCQ5%3(m_Hgh%6uoh$Z&Ez_LrOjpq|(1X_u^fz;+53Yii%{7`+(_bQm56h zgP7w)gUK0kD&8`uTwEyB+*(QB>+egB*&_u7F{?{O-VVr+}!?8 z(6#Y#3O^iKXHlgNYtVMNTey2#q%G@^>5Sw%t&AUV3$Y|@1BczmqM2;fjRZLVHbdU1 z4+uSDqJmHrWQ{NUdZCY`xZq)N@SlXOf7h5YPSy>UDxFs|yI*_l>G>FOW!2M+Ki;-f zemdYL@7vc#Ps*XW7lLYpl5$o!72)Yq_P77cY2Z55OBm?9DFkzx25AWBowivUK^ay! zBoTLWKe@M^SP>w8I8we2+Gg*&I*2A|5*(D~XG( z7v$W*3gjVd4%+SZ`nscbdph1N=)~#HmJ@42F}B8Gt~Etdo(mN<@_ZU`JUQ%2szm~5 zj;yqS*>hYt&SPYKK+fL7)fdH!U4#9cko{L8v4NF6MA?(;E3k1@tsBL@fdIr=s4xovF`r2~ zM(|?T)Ju?}$?S9p$7BNw+W&7tO2DHw`efaR@#>3D>5De>0G{7y(hRc1no<*?AnZ}5? zOg1WiGXM_OgFp;~v$Y|@63`yI(u~TiB20az$Vlr6Kibvw@F-`?tE$qsA1dhWSC=aU zRUV}nB5@zhbN6o5vNIio+eUd=u~R=jd#<*RHr@ZhM$tiK9sjQ*&VuVk-s><#2;XsA zyF*_ZMx^h%Y02C0yH|eLN^AJD=Yj<^@WZjhR+QgH(+k8VI)zuOQlo;N@I7Fn$+F@q z8L6}yy{oKbs&zD05w*tS6Q7I4?yWI(I-Hyz_=h8wrf+~~2OmA0X+G>*y4Yea?4`4s zJal?#QTKSIz=>M9vL0G;_#83ld4&{hg^4A|viL@LEEqwX_Vlv_R8=IrduXHfcxv=q44(E1qEtAvMMG8XIc`LH#(xR%3j4xZ zra)Q6_ju;6tvkny>s7U)p9xpx$sU=^W51fuW-F#lB#NP}VJHMI)$+LiQl-CD=Vd%-n06g#&z%K+Zd?bw#)XCz004LZqFd??d__Q?hAi**W_o4lf+5!i2yC2J)#GN#`GJ;wzAn@%Y5;c zgwum=`JtYwM*K#_?QHt2+qEbUtw5D<{6?(yQ_z^DFb8DWSHtiOEa zN_m(9PdI5Rl2 z=;eEDVgFeHhRsmjl#v51^Z;f}h84f{WFNv zLQjjI>o&XE_req8D0`EyI$JULcWKb&qdudrr>eHkH3Z~HRcUvq_BF{pskprKowzgl z$O_z6m5JzP6N6MTbcH|#GSwqN{W(1R=j#gSqvM^8l~>w}y3e_&%UGU=%Yg_Y`WtIT0qE_v*mkQc?W<{85%Ha~Xg<;X!YPMp9n}q7`+1!bJhoT!%9?-4)ISLk8V=GR?&2{Mpt^Nv? zeszVw7&gZ@!YhOR*)55`L>yg~di0uVC`Cb&3HFM2)UppIua2>KkOv+Hul?c3sw|&t z!gz&_)03IglZk)ud{N&XWZ-7Q3+PPI9>yT_#vGFjT2lXrD(`wbpe|S4lWgcl*;}zz zp!Wj3i64e<4m#UXziP(b1RC(q_RpkRUWI-O>#OVrU^3L^-9Q%X5a+P+&!5ACiW%hz zq`WG4ZzAZV9(W{v2DD-A7z?qK!D6$AAWfS!#xC{e4kGjf)K~rK zQ!X4lX#Jf)*^k~qDnncSAvOGtxsNySk2~G^EJc%W#|=S4z0Y@ycz%d|L7cCD=D8rM zxxMA(hzkRF>8{nc6hjapcI=$;4OKf%=^{w}6n`>--ATj4+a~xgi~j6|<>aU<=E+s- zI&a4qPFPU~H1Ec!a4eWZCYA1%>iFgA&25i>iPypFFC#AW#Ot(;L)8T14FA8Ht&yQA+T)v^%e_i#j{#gx9ct3k)&kv~^? zwS`w5nXuj66xET_J`YZiGysAfydOMe7SFPebLAuRWnRVyj!O&^Ll)I-WTg1i)kIV9oCHR zKF(Xw&%-gt0RQPnrzZ0+U%lbU7(~Q~rNqT^=WIw{IV*MLAE^d)A&0r{TfM;cL_vOaD0g*kF9IzgJ78J~ksPJ=xWm*Y7k(}DdD=8Z@O69r%APdTondS>TagPz7q zlZooj2VAQNK0Yb=8~;M`Uk>wzh?JoXC%lM0t_>cm;>T~FTI-R6pmNSC75T}*lAk-C0cKbbSap>%rZ`&)9;{w>D z%bEX^AaFU6qqDNI$}QXJND1DlQ>S9G#T-X|N7N|0cwim5?v665aL_b0O$Yb^H_{yP zC9{G>I_DZctINzJ}cHgKfZ`-{_&RS{cW7Yf6Tekqh8W$mUmzA0GdPVmz zA!@8;ND8lKXziA-Xt>hb9P`?CIT)Q$G{Jn&%(k;^Z=a)V;K*yK!FRng1kv8>*he4r z#CEg|nMcn~nat6Qg~H)~OK;Ml;Ohis99=M%-X9T+gSqOgihTeKECMnti$LT-8^nG^ z)$1}5wtA<+5ld3SSc*7vPxUDXWRxU}Ds|WT)fHRV6^YGc%bvIIE2o$Un3P+7 zimEhdl2llkN-wcv`lTMWtwkg4@Zt2)u2)t4ClD{C*BJL#s^}?sy=_Vls1MLtv}t}L zRJ@!>&dvYcD}ci>m5+JcgLv>+0=loThQP)(ud+uFOeq|ET6fMvP`dZdaDA*CsH>c* ziRFqZJlIBB|6*h1%p5If0s-?L->sFiFruiY@7oL6q9^Kx`!yD22{4Rh!GLrAtRAvP zl7x4VK0a3~U&1OU0&79#F-Ja}F6}~~YU*80QdHySR!S1Xb4P`m{d`ukm}H%vR~LEe zl*k&G(HVR#iq-$S*}(fFC{cIHQ>yTyz(aF{N3ft-2(zK zmu-i&_V#gEYetVAm&bk^9hMu4iakg+Ay7FLBe&zLbMeVN^Q3gxs;hG2+axwd!)8Nc!&BrIs%2Ndadhm4$;Kt_c~bvo79M4R|)cMhqr zySL1?)_E7c?o(Nwmb|-S{r0Dv@A|LBv5Z{e~n-o zGgTEh<4_TI`g#5|<89G(yP!1i#CG)x-!ERX{`ORgDeUFV5y=L%qsX0&GU{QNKUylN z!y%fXycGElUGF-cM=*w#8Sy}~sC=4iDKUwAf2FvU*E8tRGC{Ak1-<$gH-+#|dTq9k zjvCeb;c8}mjEz%=3XO6fhEXj6butPH3Mi=RF*(VcgP`Q^yS|}wlKgT)EQW=t&m|hT zn=LBGB+q5r+m7~AY%neip`c}2xX0fZ!IRrz>o~yy$LZ&`3`kd1v*np#L(f*)#f;5> zAYqTTL5v2ocb#Jsi2Qrr zi`=CS=2XC$&ieF;$KMdL`DB=^IeCBQRQ<7esgOE-yiw(itO&1HVp_FVxN=&$0~IJk z$M|B0S9c9twj*ulk;1s8adk`40xJ7jvDJJojGAWHIcDP;YM)C@@2MP1kt3)X;k@h~ zlq=;-PSH%xL1v6PHh=A3$RvX^YwNTCmokMd2e$5Kp!7+$HfJ{86-3y-*58_xO~!dl zDjX}vFqB8L99BPbpQ>YtN@T_hfm*Z-7k2j*VTbb1Zhg+kEptEyc@IWw2Z9B7G4Z(@ z1yfitTb_yb`v;%#_m7Te_Res`2A`vb5?W)^LK{lMOGUkpjXOsr)XCV%sO*ixDu&|h za~DG&ix=NCS1rc{^>sTd+m0F|_P3))u#VIL^4?}40+3Cb$Oo_(Z4W~`Uj%#*Idja` zq`PHLI%u+dy+qW)s&4euTK4mEv}jY+!EIfxzz=a%i};eqX@C*nM)beSLZ=| zMg}Q909o`eGZS!{_(8zqGAI?m4J#X3#RNy>ao?6bc{sE=cBO5v5OLah0*b>lzp{g7 z&@ErngK@+vlI!gyu!$ZtvCd-hbDHdd4R!g`{nMRFsX41z4q>Q>GmQ3bxve#*;X=Vn-YwCZ zAH zNCRNFyd9uTP8T$3HE5+erYJsZqjG$Qq1Wmao{zXbakWTo=w|C^E`H~ZOiuCLrGxRE zmgQH!^!Ap89#c4);$%)kq#yoy755L#!NGb|8rML;Q*Fn4%r$sj=(@Kr}j1Qs}qb|??%&mC`OslTA5s2DatXr z&8WW9q7=D^S`~n5(;pvuc@OLPpMo@AD@%m4=O6K{!sJr#Ts(ejmiXzmg5*JXT*}hj zT}z&lu$H3Fv?`PjN48(=Vkfe{@gA3`x@SAhARYcEgf(nhD<1;r1Uo2w0+OscxZVrj zrb8z4Ey}JLmPkDm*4@y0u*?;PQdk+n?044MKkci5wx|;~f+*lF*}R&X{3h-TTWQG^ zmdSyV+p$fOWvvBL<{JawBd^e2yR_6dR@Fv3|727akc+tLliSvj%qN1xr^76#P(iIuuUOdO+y>?_wRmN$2L-TPXxd*xS}_s^f*iEK#(0%`)fS#~stq!pcP15db5Zb= z-N9M4SXxG=m2x_k>*k3vtGJcZ$K1E+NAHfhZng`&?xBZ#^!resU1bDd%L4g5zOa7p zgmEFEgw{dfXtX34Nam=@BBkQ zYtwr+&1U3Q+{ed3FDW$q?zxYBb zi_*tIVJxh&`!K#rWEH=KkKNnU{Q>54MCJdhdQs(z>{#nt80fW=({pJf;&!HynKwEC z@O?q=p9_vo)CDHIe7%R5r+NdFiIPzqyMWdF`}2hk94w$|&QC(XMygyt)p)HL(#o7_ zq>b0oC^$rC+jT{?N+xLev>sX+D2VtiKxzGbS2D1V^9(+qIrbt4P13SvS4@t`-qok9D)yhe4`W0SM$Fg%J!jvfcB-eoWM_Z!e8jWz(*3_L z96_G^5$T0vxFmefDt0(Rs!69=FngAD)ktie_*h8Em=Tj*JK&i*U{wqkrG-GG(?FW| z1F*YNWp#n!Ff{_-qo-lM6p6{H3XGTfo9IE*09OxCocp)u+HrtL^95j!ByQ|rjxs&j zxpR_Cm@?~b3i0z0ou!|70iA}mFjmJWI`>NoI~dB$`rB8m^3BFup)E#%TQ54i_g#;Z z|L_XPzGq9HZ8nGWxy=9+_0g#py!w2`s zA&r9$ed>FgdOxU+6QY--RyTyb6m?q=ISH40i`a>}jV}&ArC(tD%{+c|0OAW;HQoG0a2f^iPWbbUDIC_yLhFC~>|;hu1x$ zd{v-mP|U^=hD9{NV_^09YXB2g|F`EJ#Uh{R2Auez@R4N61j||F39|_?A$To3yruL} zdEr?41EWiSdB70kJ7Bv2H+hel2=Qgm-XF7i_wgXSNUHN|1Of8uorljDgK5*I9g@jQe(0wmkjQV*RAM{@b{ za@%Ce2qR)z5kNDo#%Sv6iVy=WH1^3 z=Ur%XoU(gXrl`o16H&It6XC&R8t8a@qtCnh37G*@Mcw~W!JuPIV$R428RX*-XwJBD z(scRHihi7TkvagvE?k3oS_+_k;I6ZK1`_`bJXn}F3OG_Wsn*U_S%lrPh#1~Njifz^ zW}|dS`Se97|Ak zCj?Jl<{Ni=azZdeL)B*`3Ye?ylUNQMee=wHH#*`Rf0_s#A6CQHK;YepmDStv>e9Ne z_@ZfKD6K?D$8A&^i2c!pAS@^5P@@LP`ObR2{Jo?ZwD1;7Eu z)bEcY=uLg@1Epq*K>>(fME~z@hBL|aVg}bFLey4LLJtu;_1;ee;($O#f4`?2@q|Wf z??xihF$a^&|$bNVONV(5(It?dMYX^I^dz=mq;?ou9nMLBxFG2;$f@h4W3Q+RG0f7RHB@3 zgEj>!!kD_m1gv}+^I4Uj3|%0GNMPgZeOD+Pm!7y@EY=~Qaz#?D=Y7?UQXhlVt_Z~U zEw*FbNbtP`YZXOERNXI)e`1B?{+~tE8R@@4LdYp7pG}`!4~AN`x8%vxta}_~WCB(3pMd%$aXiJziuYcLy2v z4cs$UKNyzpP$AOwBBk`riNux2@WQ&K#kn;hX2W z-t^CDX1J(_5>K2-dR%X5&F2XHD}rUa8N(&q0MVoi<}~O=cIvl9W;>ePtA6Tx^ogAS zN5)0JYUvtNtyJ??sYwAH6HI`9@|}Eu5y~>YA}$trYZdvFa38Xr2^qouwK*=C1jcM4saGRL5CM#>rr{p2@+xeOGrJ zyD|ArjrY_^Q!BfmgM>SN>6E?M*QRvS0p=E$7s|X^we^hdLUEzstlecK2wz#6e}YDE zHK*a0@g6K%&%jo$wr!kI=brf>WORt&d`WY9c!$vQZTPm8;| zIdj%{&Jh827Dd$hM3lxAo5Ki~Yv}UNormhxr#{xudVCk>o+WBPVjN;(EDnbQtQrZV z^A(_(E83htEmV|_Y19cgzK0%^Q#tBAqbbaIx_^f?zQNm{GB%LbJq~;FGM&`EGMD7{ zw1fffWtG3}jq8?~0l6W6+Vf*i+8|j#oj3IS!m$6ETKaVC-vK7S#7s4y>Zp?J?W>Uz z-Cp;!X|UJBR3)Hw=I2ZK2^e1rHjVqxUC+$Q8kCu7sj~cAOvq=pyh-RsLXLQIE=got zLAoCPl|L!>x7Dc;qXp*NDwddxN%>bX)A2IoyB}#s`{Vs8`P2WC0Xo+?-)F|$2cmAuTZy071P=BAT-9Kuk13pOq;Pr5|C~C6=1GHcqvSvO zxCDaLz#xi0DcFrDQ(vjL%UV%pKH_TR=(BR6!d2|TB?-PBkESa}(kPFo8S5j;}G-ETKf*FH)qmzdufMO1fvV4WSB0?65Ex}T;6jnYFv=s1&Hnz zA_iI*bhVIJ)x4lrmE3(_(gRo=Zr2%)dIG(Q^G$lWDM5*Y-)gpJh_}qjrRAc6E-+zQ z3`aB6V?A}sYN?GDnO#v?xjg53V?cB{g8HKG?8vn?y-b5--m}tZIT08r=MzE_SnV~s z3#8Dpv)t<0@3YzKpPSfC#^gC?e}D!rhns?U>Ehq zpHx5~^0eB!*Y(X29@s&*Pkd5Jx#5T@ZF7{=2}zmx0j$C$d9u8Na0Ht3Qe#Moq8hFo z-N_n%g#@;bM@~W)6{{iXT-S|Z0SQKd6ptc%2sBBf)?<|Vc=29WMVLdE(4*z6er9zcwOWaC zMS7DCQwbfGxSB+%lRmbm|IQuzL^NT{+A-7E18-18E&8NSD9`8COy^l@=^An*Pl-1# zCgY+=ERB#{rVMNOXMVwtd={?ZU9i|ZiJ12(b@NT3!7Rhw7jZkGMRHS9hu7~HKKT`lXy2fd#A}Y!`V?qN6J>b#gw4J1!X+Da31WD)*)d1{R1JVf8Td2@4jYcAio zg~32yUlRMFvJwy4N=Q$Qh=@pVGo5Fugw9J9T64anPT(otSAEd3`R#>(OQDf1=~VMw z4+;+IjXHGD$|tN6s@kYt>2q1-2>H2bRgTBH_is)T%gVpX7)jIj$=?@3gb=web^8ZZ zeCki1it&w*4B;tbq5l%NC@*PUAPoe55BJ{9X8V(L6)sANhvVNpnVf=j`^U>O^)^UE z%#|+r@2(b9FmZ9^ep!O6Z1>|RPg)qJ&Kx@Mj!S=(d}?QfDXfHD{qmA;WLvfO;rr13 zN;=sgNs+mv$@EzERZLoDd5E-h19k4`$hYo%L zvo9`)FMlB6NCFZ~yotxw4DZg2C8p(awA)3%Gl4WRl@M-`Qkr3UKCp)7h4oU)KIKrL zcZL-)NJX28#9d6SH+B!8(1j}}u@4{>Tob}2@(GJe?QGSJjK+Vxs&`7jFt2p~{&Q=A zktw_<`&yfW;^kb6iWcf!rn{<7eFp_qK3|CwKMT1Z;vVR4HdMDB36APubKX*Qht$T& z^Z&%y0f^j(rF%{-s0!>4=3GSV4Dq#90koO?wqpoAgl_$y)01A~P8U~uZ29)e6EpUa zKP=5pzm_V~Owt?vbJV;qtyNK;anXBxv?b5DNRRdSpzmfQMmbUMu(OXhe}pKP=p8Oh zWb!9hin@QHeGO+N@JzMACq%Okmp8vH;;E`rgJ>4Oqmj`&z>xZ}mmwoLP*^sk2IaEm z=4QK*&&R!ed<4^E<>l321i~pw%4C3!wN+zL)GQu@(%o1WC8Q?2s7)znTK0vZ*ID+G z>@<05;S(|P?Y9!uyGyi@I%OJI<+-;HfB{~?ks2NG=GI>DdW@ld) z?#V&B{Huk3PTWz_c(XP9@T`o-`{p)+l(#y&zwt$TN)~%6CcOQ*{l5{#@C*^w55onU zkd+gVpH{D#)fZT}xb9rLc1=zkjRozs@4X&|hQ+RwMEG|Qlk|oI_i$}ueCKjR+m=hX zJ-WS$3UkL6=!Q7D(KM}O@cL`{Oj0gTLs^qW7dv0DMEeDBnu7jH{ruu6MR4tcok?wbw ziA;6fSy?0|xl?mLd(eC)_*kZ}vDjD-R zEUW*4$~~9&4h5Izu~yP=476sX{xUK(m71+r9iq6)U@N>^WI0PQ%>6U-&uI?wPhy0HzcsM1y*sYA+BDx*iDLJNxW0WoV%3W!&Z5vxpnkWn#Q&72@}z*O zN$W)I>JK;@u2+myA2e01j6CZN?*PqsrIJkavpy%2fPe*mi|JklBF$T68N^Zh9NAoVg!C0ZH9bPk3Ew>l;%jD$lGh$veUiS9d%kHDL8VU zVNV!fR)R~_0p*;6ucCl>pqO#AyTy(lQ<-Ko@W-Hck3vHyKyACXnji4!M+f`j8@V$R!7-gt%N zc1bnsyR`80owruaiK4P0T&GzZ2*!fEii;#dY&}LJltd=FouypgNpg&m^M>u%mkY}W zm2>s@y~CFDV19&~SRx51xd`4kYz#5)WIxY!!(6Gyx{OA6X6mgqjB=r4rrWRv6r@{9vHIXUU1K>TX5Io=QH&(6>;Q8mdW zIw$CKQGPD1Gc$mfTKOe@EN}O5#}(Ph-s{2|k0MF#hv*wk4Lo{U+n+P4r9If^UaPSf z|8TB6^}4&kn2`w(0shLCtFPt#lPzaoZ+{pLJoY?A&_aO_CkW^4ymb60>I-WbqX82$ z>{S8L0afRM+R1}r2WO-jo4(+sHiz0TF=zAlvoD9dJs#F7s2UljRZJeueZUXx9W= zBHLU-WvP0yUotHFs-!-CFxm#$f;VA-Oklp`IGo~ubNC5b4~4Sbok*S3tT+V{2ad1M z_EyXM#pdo=>OazJA{k7@z7U1V8YbEfbL!Iu12gz=x&N-quV0LNFmQT67pl9(?$s#0 z6_K4FyB^~*U`MSvJZ-BK*2D9)_Q#v9JIw}f%NkSmDUDBrf;yGC5AFskqeo_yRGKN;BR;Xi0haOH! zH{%iSpScb7+O>&xaeuWYY#b5V@_Au-GEH%I=!XM44b&X`k*N&dad8^=TI%po&_77?NL5fR`Q;^<$pRd7zuHhA{wWVPO zGU(5jnt#iH8cg+~J|v=rG&jTBOpnF9WoFMdu4%jEvgJcJQ^z-xCz6*DKr`8N`%($- z9fimx7UND|Uptfc`why4@7z*+c@Kx-n0xWUW_-aEuXGjdHoM$NsLFfp;=VN16;f2M z8yV0kQK?DRUbt>eyud;zPTF5@frXqbv87hTl2ZoC=j*yk;n$ia$RrPYr1jD4*(IKu zT3wS(S30PUCRrT(M6Xde5wp8h=|L?LJeOCl_#(OrZ0VqOc#?mBTC%M=+C5#5uo2)( zMl0etz5a#3JT#Gbu3>_4Ie(8C;`0F@ZJ^_QSgSSm;(?2ieTRF3-fF|?Xq{@r7Fc$o z3^|xQlJa)zo2xHnz7*(T$)~+|kSA^N2Ynn_Jh4%0Dx_h_!L=DW!zVx@>??$>H}4;{ zy5n#yTSN;ZX4_8BvSjzNWpj*l(&T#P-CldyAh*(bgSHXNPA%qTda28VHBoO7J|&1s z3pQn-;2VC#(!{t{DKVdyw_H6yd+XIbFTLL4LlJ!T3esBo7=!6NT2s5{?)e#uAIWkL zS6mO1T3GAC^u`!;=93Pyw5i>!-m`8*1+ddl=3T9$HH(Xj%P^2^0)~&>k(2X3TB!it z#w*P-9NMaZV&*TLixuW}U!{??LnjpsRafkmz_%dH`=%w27dZv{{Ig+{wZKZ}IOk$x z<~hSt0Mlu5wSJ4o6t|&05)jbeZ;dqsZ`FgwJz+|pq-T2>ZYrv@2k9+^SUD1(5FwPNNnbPm9<`>w)o_9VeSy= zDS^pr)X^L0t$l$~D~EbtL&ceoI{Z#h?05cdhI4I!K%Y&HiX@BdgwRc>#ab2d@+m}c z%%~fZ0l7^sha8748P`lcFSAI(yj5C5cT-yv<-R9j@icMle4lxcgyI`$1c!s1AhUE#3&c(z{~O>(a~sw<=WLM_3W`BX2KBWZ-gYdYx1F(_v}inU!|`Us0ELDz8C|3`7a!5D9G1r(~|YqMRs4 zEgL1IE-$?NR1QTvjwbH5a<{(YT=dEJ4j%1wDJW&q*c!-W1;wvq8NKirRmbZ0vPVi zxM2jw6`ifK)0i7JX#2|Egk;tz(p1-)4Qxx1z(Sv^zybt}qoYz-PMxiSuu|UkK?*;h zk3oKIFe#1X*E{N4)T^ieJo5FMLo6z1Msh=QJF<`LoTLW5`_ko!b)B^;?@vRllWCxUp@I#;c3?kGMzQNSQHW`kxI90i~&kmTdK!?nt_5^aB znq*Nn8tC?FnTnF~^?6)UI|kh9aqkqkiOfqRsnkFI4g{ z-OWQcin)2UTCeB;Jx$Xk?}*}guL zE2GxbQHwrH)!RGdUrP#(rm8T|Ru5)-bsk=+YU9o0c1~_F?d@4NMA9O*>l%eUv~t;y1dUIa{zkKZXbKQNK(3@x{+_0 z`LN&P&gZhVl--X<7BkTOgZ5lYTqtO&&?q&^PJ4{kH6MS{5N0!TO8cb8^Omjml#ee^ zcQFD=JhOhgwl0s>vP*nYC}rDvp)xL*Lt54{tHRPXafDVwSolX#3c168k>(TPk98%n zAUe-tcU)qSe840a*a?RYg!O9ClvuKX6f9gY^w9>&jgbqc$uxXXu`$eO*(gSl;1T8`KIdAI zZps^RpWuzEv$YyIH#m=yPxmAv&0DKWERH%AZRylH1^pXpeaO{$p8efp-)W|7N|*J@ddT!M3XALu8CF2% z!qC59hDA=MW%t1WgHzEJ2`FLh%j1n|9Ua*g=ysFb$+|d>mA1Bmzsj`GI);>KzZSlR zGHp+}YP!-3wwcXlnpJ6?OO!mP4^x47lBH9gbobsTneQW`DZD?whRl_dy|b^0+Az@0 z)#p}N2aBrtS-h?BGbm&oT%L`On8V=&qBa^>50af_$aKl5hZbbjgpAhZM-yGq`G9w5 z%VXN|_d=3NeSGF~jXvP{X(L#6%Fo~M#zkKBIf znyra=v|YNn4IA7?~NgUjmTKk3Ckykl{cQ|hYljBX~67hOe6%H**< zU)y@`w+c@`GiiRs*Vg+y`Zg=9X3^}mV%0EnpJeZv{y)mPdP|SEv!k;_(-m-@9fE_iB$XsPFuZf z@ouweJV-IhT%9uT1Lavf}Q<5YxzfjETWO5BDgh>X`WX24iAhs)~%1di&+jEy< zGq>hR3MD-=3xKXgim{jJPOdNu%f#4kMDlQ(fzK-z(ZQXo>(04JnYp965zZ?^;kVy% z;&OD=^SZLP^qfkhG~v@(6<@Y**KgY=d@o2SdPvG*1tjdg6R*SOP0zLA1)ML(L5I=P zXi3M$poevYh@{G_=Jy}M4I6aer*qYL}YdRA^P3TVr~+AZ7*fKKXP40 z7-uv^?2X3V#d4iei&pxxTIW$EiAHJxMy^#7tVu?&oj zsj8&21{YuHt06HNgSjT%_n@phz>;vIc!6v*z|<9&=azD+y}vw1&5X9DPKla&?{NtF z{+y1t^D>cd7hhSXPwts-|2!2Q5>}mPJb{)`TE(H6An4l2HyO1N^``gM$Z~yPNZ)`o zsa19Xk|x|Wp-B3Sk`YQ1izpLTB-FYEh6$w1n7@?g8Fqth;vgtY)VN~{s5Ulk)TZ;`S^6{FsZn@OvvXs zd5le$58e=ulR{6Xr&}@!bM)$(C!v!Np4w|`af9TUR7J|ic6O=4U;hfAtlfb;?e24` zDPx-oMp^&!8ekTOeewj?8AtG+PQ!?7`6mP&-xQcKTQ}8c%yt=jy0FNJ>r-m_yIi}B z)4|ehROKspiEdxp=f25TS`O5t&|;;QlL^SwNS}OrHhWY;@0sUjf=a+R$Sp~3EFqtu~vx?t=E@%D-OTDgJW#t?8=5;+|>&1qCvzG1^h>#O#r?zY*XWMWg zxOO*%v=2`qFAqrZ0N!=wwr^>Ibl2Xep2Uop0z}UblYSC$>g;bpzB2v@phNY6j7kHeT)qnQtAXc3-PBU)=M#?}D$-lPJ(X zDsQkkv0J;>3olX2;s5c%vA0LPqfOp?DQkY~tu%CTqMR)L@xo|(6&VZA6)a6fDMXcl z>bz36_jzkw_^m==N?ksx_2uQLQbcVA1mYGy2Ks7eYxP~YfQ|crz0THACYF1`h$;0- z-QfH$$4iS}SbN5%Ic~;p=0RqxW+UzQJTY~Y$hCuImV#+lCs}uk*>X=c3ls~X2$&xQ z3Y1i3t0tpq>-16-e(=YXQt|8h`RXeM%{Uj+DNUp?fsgeUU&>w)kg9zR$ObY99#wja z(+t;UhGt!HmE**n^vqqC3T#s|kKVIvL~0ivqCMl@iMWG2O4q1Q1uMl(`RD||gz|^p zeXr0V8N{(<)8teTNxuIVfULIN74=SoZf2{m9NQjcfX#nr?Y^oDA{tC%3BNw=cj8Xd zA~yW|{9(|oIW#;>{>L9N2-rE(r2-SZ!hC&wnO8JUGTeSqqmQl8S3;^X_7+;g4Ua4I zlLt&Rjjp3hx_`U>vH5sdj*D;QSKKGFCNFc^)@Ij`$w7az_MWe}dWQJ4z;ZV&7nGdF zyl38$?f7dBDQxm`TfpACd@=Hx#eZh;zZD9#R`UVT;P%*dLRhvkIQHPL!a@(aSuoD7Q(HcQ6MZK$@Av9fzs^Lw>?bdZ zDQ_B%*6FY9<)h8rz47*$B{ZAF7R)Q#fymm!LxyvZL z5N@|+suVc&`_GnCQIy z=E~8sr3S&prKQxnmGe#y+&90yYgwytX^KAeU04K>%Gi-$WIiv=(Z8HSaZDpA{Ea0x znc-2G8GZ9EzIi-AGP6pGNmL+X&T4LBje#Tirs%}1W8CF6Y4LGMHp}2rtsc3P|Lhzh zqIjoP)qoO*8Em~_ARXl7bYP);= zTuas~-IQT%3H8gl84A_Gl@Y4UL!NZ5ulH^;DetLVazMq%SlUeXFbmjAnK)Eg4v4!52X&T`(YcUVq9;TFhk~XLn zW$!QR|LoX!Iwy{vFP-;vWQcq%GkN|Q1=4>Qh;RV|*J;SNQ%pE2wsQcAN)enb6o+OX z%lHQKGmk5mjMRQhMOS#l>zpPpxZij&emoH;;9w0MoLv%c)y!f$CQVXp$VWlkBHOb2 zRi09gx;E|9$t0Hqp{GVbK8&e7C*-7n)z7}WUVwNfh028;QRKNeIxEk!T)8sevjrV1 zAVnbLICw_FVYVk3VfR84dDvCGTc5&jJXe7Ipx$j*q%3IK&Du-uieuFYVV+IF2HlO{ zNsTiniSI?rd@9NaKxsmgO&p-rsSmL6a=*rYo;L7hsen?A_tA>D-39g|W9}?Sj2QS^ zkGzziQB+iX8Ra_557eV?%M&f}4Vvkz!7q+h@;NrNx2sY3`8;wT2x$xyX>aO@WyjOh z3#TA|$qK1VDNL2NnLlm3tFTtZU33jX9}`y`B=hvl!XQq%f zNG^;k^c6_amo1iAc@x==x(}wuVe1^@dCylAYrqfCk#Nvg?Xrc{;@t0ZLYZDHFdO zZ1OS1_Z(7GD3c&Mq|~}OibJ#WSAbs2On(I$yFB1URxD%*&f;h#o)re|O<`TXL73t?vsUtoy_y&*&?eYiRRhp2Wc zxNohpp&S6Fc9|_2r|T?mFoS_1_5}f{TGwwqe6#<}De^y51{ZoSmZ;dMjRIqCs~q>1 zEohZBFzs~OV!rvy*4f#`Y92EVZBZ!`7q3(8=*89aO^>~gSHW0_eSLIhCY@d;BuPG6 z1kv$@`K6$5%n@Fn@NmEDd^~ishKXW!$!AYRC28sxOp;irJc1A)0|%B%Py|OT7pz^d zCdf4R2=>md)`8tjW`Vt>ATiW@A+)Z5%)o76f3~D%mZt!0tXxx5lXI8V1ZJ|$RuDW8 z=ot7-YQ?fKi*a*V-x1v)`>)Pr>36#}yju-!eyPLq@9ra1>Hzr8agjl|$q@;qXY6u9^v;yGE(u*Yep3EQr)-4&4 z4*7gZZF|2-+Cx_3t;E%L?GNl{pRXXElkHPK3u=}ja$|3D3cUnm?>Pocu=aOBk92B< z0+Wn4SMqUrBX2#?r0RR0Bi@puqH2Wx(`t6+G=9^ehBsz&b8n*Y zP6Y~x3ff1-rMnpp@&-@-hC`ngO#{g)=!>k?nVEgLpGsLh{KrN@eim**WS`RYRG(?# zBociVNfhs(lBWt6r-pr`>k_3oze?VS^Su89R(trIYbM z-RfFVXtRGKVwS|uO1IN{4Dqd#pJF53#+~X_F=>_J4DFfQ3tJGOsNm-A89!})K-)`e z6eQVA(W$m7t-;ipX3!4%LoQzWy=-mtRIyKQg~;CTHX(h87Bk>q_Y-a%_5FCWSyo=z z{JUBG@usBgFKWg6Vydo~W#Ca0?B4lFT{=JhAM#TnHhbu_0yXrI16Doc!{0ZwH|is@ z+V`F#CZ*L6i-6fZEI0Gu7_#*K@)ncdkxN)4!FKgAArE=xLTIP#-h_ZCH~=1oI#pf> zjm{jIi#sb8ii0e0iw1(*X zGyyp^rLd3>VrNkPLAn}%!`1{*G^lu>YaI6iQba_vS1BLsYE3Zo*uxyrmDyfRCMG5< z9jHXYETE*UECnnC7nc#H+l31kEb3jWsU4A_2O?`zAU-OMwXwBrr1pFIv?<4+t`+{P zEA?vzjU~F81ESQ3?M{*WFQca5UzIQX&Gx*YmUDxqUc+;P6&1S|lB$ju5^mqId+o2|*DN zX^>%S@`R@o&+e^!5D`Tzl{b?XP9Zv?_wK!+GhP__G}%{HLRVwPGX-=>kR)$8KqeGp zS-x=#1d$bh1gIV53IVbK4H-18KqYe&n2SEN9(M&DXInu@6CW!PP*+a-i-)qp{b%rD zGC#8dB$?*Os0EahZTCh=M4qWa%08u`axZ^Op9)fbJA zhp&*9kqM-OhDT6mHU)8amzn(b<&E*^Dd1JN&S40y1!LStW0^H>q6A8)&PL0$s%}$? znO7|ZbUhoV&8KV>HQ|cFv>FN#lIopy6~h;{V|j)*B>sLwh2eY>c?BS3jb)8>bu0+# zaN0llcVM4?`*y!?&#%8!SP6&_%vE{WX=dpR`plzkk;3gDmk5xHstFmaH8u{iB9+O~ zU%m<+?0-raK?WJwegUfMN3q&lXur6z}GmF3O zD}_)Y&UJEHTA=H2>wo?OF#=ea*pUZGMLB}>Ei;KqWs(&|ioiG609j=#3&>>C0VLA!W zX825L!(4Ub`gzP=A;t-MXc6fG6~j<$OwLDBAm)zXB6L_P*X#LMdtQKM;{Yb?cXZ5IXwb`w)~rpse##aO?x= z;^X3iocrqPWM031jhIhFu))C4Fr>2*=Eki6r)LrYZ(y9}=-eE};#dsx-orO7ElL1J z5j;~D7|n+?oFYa6Tolq+2J4TAGsBTrTbd&_A=z_+yR|1o4~+6VFUBF~&LRW0n`*r! zkqPRb_Y6K+3$`FiG6TrafJW4PGY>oybsPAuajYIpOb~P|O-hoy4273QQ9>H>j4jJn zLdthHP3 zA|f^^sT{t#5V76oH+7G4d-)s6(rZx!GqPx0);x@hlO_OYxy?u#;qw&ho%FlIm zTofESX57ZjA7ALNudkm0rG62yNAwN(tfp@kr+i4Cyk39&v2kB?!ieRpwfN z%xcijxLH?4Tu5MD2a4t2{!3>F{8 zLA=Px&0U#H%~Y-7Co;JC{acYQcckB-EX-P7wi90dD!#b3)(*d*rf#rb?YF-^nQ~*z zf)y&c_eVaz%%}!Hh$W}Kx;inZZXYxGk9YV)I@c0_RhkMRPQVt*xcP>#%W|Q<=)3S2 zKd(^RTJ*qa9^+=Z9T-m!17QjYexQ(4Rub1mYgoP4_dC{QUCX;1Q~IDAOQEGzCHom$6VI=2Y$h-%Ern zu0=AzON73s-e@sOSe(x*d@yrml6YSV(a&Fx3`P;4_=Z%u=oXGA&1?L8C=QNArUPJ- zT|l6oUJ4|yE1)!|pdP4{Z2xH>`=Oz>S_sHU0q~e9aOkt3A=LyRlowt)qdCF|8a%1 z4ywmD0s;cmKnUJ`aYQQL1TWq1v6X&ZTs+fmLagZLM|Wr&XFh;o@h0%bVqigX-~Squ z!bq8P<;N%WYuts}eHAk3tSMGP&7M*$`23G8la;$TCJHJSu#pu&L>-1TBzJV&#Y1bv z5bPTdhme1LLxY2PCe#3K2JKlAOSXm-_P-n#3*)+c zBT&YZLE6;!`EwO`7%9c9F-rfO7+Hwr!PMK&_VDLh^lE@t!nu(<4SJu}YK#PC^rY`0 z=HGv8rOaKp7371(_&-%wBPGFVf9gARA#5OYp9m4Xn!UPM{Lh<1-l(wh^EXayCmIa~ z@D7Oz>z@9xHna}pCr@6|yPb_b2pTM7F6E(@nN1ml$bK*HA zSJlj0ApMeW*g}E@o%tYM}U#QG^&R53O`_XzU8lSIxX%pbP+HNr>{Hd6T&2P9&oruY}`htnPW z^*v6M|M%WfS_NtDs1=_U8e-4|gSCnN3d~D)n2eyT?+OEdw7ZeS>di2(MNh}@D>(Sy znj~mdU>CsjV0OrP@tJ6JyS8H_(r1K>6I)nVpjqoLu#85gMWlkb7S#1$LO2R7=85I* zTa%#Zq5x5+K_g{f&Ey6QKJe1P?ARl7Cm=ix%*@O@cg*MBy?gAgQQpu$KLL5U5}Y%Y z0t;=(-;mvo#kU$7>Hf4A-)QnhIS#N`F{u&z~ zk^-56vycT$df4EG#Ut$9*R2Yd1Jp!Ls7JfM_zZ1LB;?Wy5Ke$~&{{1e*`BW?106o=NIfkrt*2|=j@gwf*9ewq z)kfYOQD42$?H0nYO@y|V=HS7)KgvYii&MxUgj($MekwkWBOi;pLF?VFcVZFC&GN>4f0 z03Bhsf$-t0bqQu?R}ESfMr}d3CM`spLdG&q2$UCL;;hoP*xBCxqG?$+y4nFjJ4X&Y zMFKsBx5uiIbnN5Plb^~Blgi=d_Cp2wDl(thG_7TV8~MBmyJE;T^4ood=k|{`M)QJo zW4D>^=;$~xLo%ngGUc_~-{fH+Qk9!%d9-WUmlld*~k!*ROl06e9;uN_bCBNgfxUg-}=-CZxYC*TOeFQIWOu;^v^ zgvkFJ`1#G1b~IGISmCHt+#Y{2`QCTKvLowt!tKyEfNf}02hgekGJ}Yt!SAF)uO70P zHa0fg7Cn5(m}%T}bYNgCgf){;p=^VsAsE^g^YXZ%l?Cou3K4>Y{|RpBJf&W(oqrNd z+AEBd00i`i!t>5@i+u5Fm+1=pE;FD$fK}Dj)+)KVmFl?8pz^_I-$2#Bx+;!%T{vtS z3APU%bB16S2kck$oG!DBcc0M%kL9;rG;mI4@vJB` z5RZjuT3p3`0ip;;C`n)A+r=t9FSifbYS9Wy%k*~b&3JvhtD%M+ZJ#tb3#*{%v8y}Pj)L+}Y+GCv9@ z*=S|^w9hWBYNP0mXl-R}`{5nYG$0kimK91Jui#rIWVyf^rIO9AAI#)7{BlMLfsJ}B z`a$Lqh*$VL<2$_$-R8K+Oxy7WDj}yVx(UD>!FzZlPeRU|NS6%Eghe>c4rVZ3jlK~c ztP~ces!UiPoO{mO$2c)GE?n7_7GjA!YbN|vlC80D%jMw!(OO_qkx;NQwreHrroAEz z-MDoNCLyU}M^+1jz=)1m@d?cmK;*EH#W{JV*f_MQB zT?kYWt#?@D{6F9C2OY!^XLhH{soJ#51ja8W8zFF~HmYAR*6{Nceo-Fwfqp)+54_dc z`cLeLN;mj!a2(qaCC=4A*5NdT$9op!j^Q3ck)~NVSLmD_z!7nDpruvZQE`PN9Az5f z=I)+v+RlzZ1_mjqg_>LxiN-AnSoXeS3(z@iQEk>+tkV36*ZOWaeoQJXk1MwMTV1^9y-EgOOD#|+82!GvhB0=J2P8V_tKBR@rExsWXoGuxS z{(isk4m=4-#XvAPs#^gxY*L%igpKHn-3^g~i9k@O0pnK{bOXEzLLiXFBH(jcTH1*@ z&G~LV2%K7w(I1oK`H+PvOG-YWn*hoNqQkF^c@Pa)Nu$VCn#lINI51?U;&D36_?~$Y z)jCIK^PWOQtliE^`xJQVFh1)yVJOoCa7xb?)C4-I7kj{xvfd8_TGPoIitKqj!fC;zpHIvc^ezk-X43#C_eG~c{i6%+(~pFL9s zSQLD$C6l0FNr$t;Tp#q48YVl;bh(TMt4XUINUJnK4gfss@;blWex2zT;L4m}V}T{O zs6GS7MnV_4^Pv{ zfGXR{h4aY5mK1PX&*}rEixYFaz!uur+++r#9^&0Wm;r`1rQpyW2&5zcIZ*aid#ai| z+G8UgCvDPe+Fh^UMVO<=A!Q>FIoVV6oj?cvh6RutV`F0^FGN)D zEP9Ip0sJ&`4#;l^ZU)&j%Cb)gwWHvyC9vNAcUu}?E*faeLE0=AV1FsgV_ z7`-FaJl&ZKeO#(5QP&z>CxgfFFa{I@xMI72GnW~UZPotJ{b!cw4nk&%r}795U%$95sy z>t!jBzRg4?Zz?;HCb_|if=B@5l(CTBw?k_ps`)JN_QAj5HX|5rzxsYwkNJo%jW^<; z3X?o8wxSomeY(_#LnBXeYi+I_K#a`RR%P(Aj3|@}mr+9y>?}xig?A}hdWvlG?Wa@$ zYgJq8$6%Nm&#CIgPl>-FNGQVIRENF!U)E9*j()Al20P0p#x=(TS>Y3^wW1K6>HUb# zAP)Py(u1X!c}CX)a6%@-+m zCsRi_7m(wnYFf-IKvHtuw;Z_4I<8!*0r=?^gg&@gZ!fhhmvVwxYm(wE>;w!Y&mQ)t z5!TS(ZP#(FpDBbnzh|VX8XFtWiF%aYK*elkY3@G1%D~1}&x$VZ$uq&%N^%0U$TcxB zLElG!3WGJZBRmlLJ~Dhj@Y3A)_;{Cy?uXwBYy)7j6lx53j_f&XA|idi9XuHN;>91f zFc0!#H*qw^%q*?CYp^%X&Haox!O~H+aV@%u5PPL$wExT|W}B8}9CqfY!RS;a{Iw(( zF8CRk(oQx)73^OE1b=^K&9`49IRW8@j852MoRZ%r4xZ7) zi(dc)1^bmE3IRnVc6R_55;7LX%g5KDZ)8+LT>t_K?S+W)n)0}3ic!J#>5#B66FA=Y z=|Z2Lyc}{rnzzrX(!R8s#x+g3T6_hf`_sGou0j|(BZP1;pF6{sYd6B0g~OS4-j)#c zn9n`Tf(93wWmDe!8myv29Yz=V8?v`_RLkLH>yEmWh=Y>>R|YwPE|M^)ak#Z!D%B7Z z^PGB%Avw+73f-UdQsA?g+Dnn9G&@*+5XPIZO~X@s zNrwz?h1!X=p!|C_v&IPiSUC2n%($q4M3S@^2I_r?lw-*4mf;fWtV-T?nE2ZlT#twwx1isHjSva5Wo?WlH%D4dX(B)JaO3{keiWDd+|MX_;E8A;ARSy z(yYL2SDr+!1IY*3Ap`OeRQ(K=X63^N4yQ5*BvK(?t!-!sLFAI&ULS)AVR4AyUPQz> z;KvnRA4^k=eH`a!S?bLnd79(XkjI_D^@ZV#>VqtLy6&o<9i4lf$0Mn zdG!z@^fC@AZOY#k6z!+61eij$gLU4WM(%DgTm&8thj{Qj1Ki~MEk}Sez=TXX^h1Qg zC@idfVy>T&pG6C{@%%^P4R(%A%vO#J?BGFvawHIB`9zF7Q=#nZ-=l#a&(!;e7oz_H z4*29{xmL&+QTIMRAr5Vsurpbm?!?B)i)`Ew-B}K-&hCdLf7#Dxg)D3n5O^<2ffc&* z()=I>68`*Z+TA|qaq4zG)x0jhg7K14m@9SxLK-NdOhU?HBWMWaLu?!jxbB6Rf!xJS zna#2hgXHpC43{q>0Vm#}q083i0S_>iuP^HR!{xR7D=X+m zTr*2ecP0{0R(CIH>nsyi$7bN^N;o^Y!h)qz9&maJ@hzoP0^oP?T^#VqlAPGvV7XqJ zgZJaaeu3egLbLrK^VYr_<&Vxalx@i|kl}*f_+0R1fx*F}T_Hh1`eO}|rWwVP%-{z$ zH^bm0P;_Eo+E~VhX1<95QlPPgJQAD}NHrRMcyt8P`!iArfRe=vU|{K|!Yq>_v9hhi_p4QQ$;y>A$^N1Qb=C(Tet|5Fq&EAiR0qm2IZn-t@+N40Rmb+vJTtJ5znjp;n9gQ_IY;I zhKj2MoNi)^F@)>}4N=0T&#sO^C^A+w9>pb>*doaZBmwrN4Jg`&n4q1=ei+c_U_cjs zq9&G*mDN`Nl8AkGGB~ixXT=HV$gFu1o!FGVGXPsTb@EmL^MZnM*iJweBAO$=|IWDR z2H+3C1oAyJg}p zPi-N#G-Pa490ivC^g00d_BHkc!^6Wa84!{)Akul`jbt8e2LC;XWS-pf9@OS;Ld;sA znyxWweH*fT;0ebjy z6UhLz&oo_rP0E=*}y1@p~{J?dXd2NS)S3|1+BN>@WWejBoSkNGF>R=}>Lr$py zJ2|kW=JMCr?`eAzp|Sz|snIgQ6aLHM+=P134%9Ms`#k`Z6-4?K{&~f8=^1~g-u(Qh z|C@O0&o4p#@c)Z1nkRHaA_IH-4P*;7&?^8zH{$E*nAq9n;Lpc=Vom(g#$XeSxI`kr zFn;u}o46Zd`4h~Zs;R5{K2rg3CKKfGSPwk{JO^a+UqeQbqFG=;CE`{Hn?nsBUra|w zmjn=mw7fiy4l*5tkO0u03b~r9`)%MVVYywF@1rbw3Z+f|BFkh6{=mOuuABFjszYRr zAXroU;01sWG6`nsM>iShbtyvIM!cb`Rjh zWJrdc1aSCEU?U-WE8MbN9c0)P0Y#|ZAXx_7{IjWYVS*@zqle#hH4P$zWVuKICMX2} zxM9-t#+$}@wdEtCAsQx(~1f&DvkxxLGFv)BC;w zJZ%JFNf1?Lsb*-p!2@*lm9%M+l}WI&nz^D~wb#7)4SiB8vubBZ9cAHi$+YXg66{xr%nwC^67g z#2CCpL1aNNiWk@j21JZw)GE3VLGX&)LBmCmrN85@I{Dr&j>Bcy{lD*f&U2pUIpmiedgI)1CK<-b22AtIg!{O!d#k_Z@}ls9 zvQ7^Vk03}D-;Y1O&QTQah4hxdpBOc!6RXt#SEO?jaus17 znar8;o+FebxKz7`?l&Xb<(Cs--wNjj1tDpL#PJEx6==6)*;8IcY>j9a0ft{;(8o`i zQq>vI9;=NX=+|!@K5+(24WD`2iGSASvIPK=t5BcBr66*-`)SvV@6i$BEKll%^Aa)x zaJ1VqW&tg%ky{Ixa+9xXP(iXaP>4yV%<|<2<^CJq>q=@)@oXht?1L)d4;P%*4k`BL zD36~PN1e@miYB?B9p_vel^hgS8tP#_AgU(z@z0!k*<78>(pi&MKYE~N6AD!)D_c-h z&cKW+37Dlv&mwoQ=D|`o9ZdXh6B;sBxNa9w617kaP^U!tVCd6za5LjOF@P zh%K6L;hbi={m{O?Jc{RVfQrfpHEmF!8u1`!!zm{#!YYsa7;!Rdi#t;%c|ww@#Xgo> zY>8Gnv@O5EZN)Hy-o!ck=s9}S=}Ay%a+M}ZI{`5&2kf3Zk#~HR0k)#anZMu+KAXZz zHR?QO%!k~5Q6a{++nuNTyKegW_$2S$yH^0L)Kn3(j*2mj13r%pA(97(H72c`{KVF6 z{O^HHY;@uWHk(0lkA8a3kPboE1XYsL-fIno#$hkAMm4_Imgtf8_5qFkTPLErf1_xC zF?BsUF6%>Dg4O!{y=)WDOX6T;W{~Dlte9Cx&pidztTI2`JkgtbGExsHx4cyu$Wl-u}X29m206W>?7yQmFHx>^Zr~qXFHK_ z8)TRikm@jIo+%^WFqInJ6jD`un_gYr9lcGWdmNBHnbeSN))pVlA2 z>#DjQm6b6jPhI`;%MG@Upm8Tbw-R3s=}FUjebCgM zZkdORS$3R@i?1O+yyk>M2GwQB86I7ufZyi|wB za?)9-bKaNhpX8ZZKR9_4Cv`fE_?qhCrT+<9*gEJzTU#D^-p8e?yRt!Nq{}a3y444s zprMSiJ`wvnn27_akaX}lV*V<-b=z0wIss0`NMr;jif>g!r86y>OpfkBaF>sPrasFS zuXXO9dyFgumpLhAW*EQc_V!~zI#owQ9qS)*ZWJpCk>L(&TWU;!n90ZK4>Q1&6A}NU ztfh;wcgD*8(vlK|5+>F8TF0h8jd)kt3whc(V5uE2EsG&p$PEQb6%~TI)0%DSR4>8X zXIOv4=Y}s}HBnL&(aF5jJNo1c;8xzPED_@j^Ur^JaK)Y|Z%mOA*Ew%Y^zb@$LJA@o za)$56&fY$v>NVe#mWIr|f*QUa?Ug&V?T1gMMm8twIxYnP5qDz^J`5j`)K?o})-YIZ zqsX}!f}eGkZb!|o_lRJf1|nwqa=CZ#7P{wgV?x>~JNVNkI&6Y&B)9Y)dg3x3fIG~_rr{$ zH-CJ2g1Hj2Xul6&qifY0%*SV6m(ZMDxI#D6R^#zB`Xzc;~kCkWq=Ct;NMsuE< zlJ{4U@Vi&@sT@2}|I1pvK0P})4l%Uxm*d%tWTZ<)QYF(TzCSxTA1)OK!wo8^2pk(1 z$gbQiDY~s=e9u2xZk=8Pi|A^OvYx2Qb)~~YjD>@(59dtdyL0#CR2^F(_gDmz-~&=^ z{TXiMqipdrVSF=vATsD(kwsT=fZFzma}uULH$Bg|zpa#9^m_sFDnqKlRG&~VG zNsO5+-AXpKQ9R?Rwq%IaeYwGZ@8Rr_%+b+PVZ1F`!sq#P)ApXj*{g wH(z;WpYHi7d;ays8=r^t7r*uY`4NlP(5q*?|4`ArR9;v))9<6)8B5py4}((uFaQ7m literal 0 HcmV?d00001 diff --git a/runs/frontier-qwen30-vllm020-profile-v1/plot_mock_ablation.py b/runs/frontier-qwen30-vllm020-profile-v1/plot_mock_ablation.py new file mode 100644 index 0000000..c61deac --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/plot_mock_ablation.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Render the preregistered profile-ablation figure with schematic data.""" + +from pathlib import Path + +import matplotlib.pyplot as plt + + +CONFIGS = [f"TP{tp}\nMNS{mns}" for tp in (1, 2, 4) for mns in (8, 16, 32, 64)] + +# Schematic only. These values are deliberately not derived from experiment data. +MOCK = { + "Real (mock)": [0.42, 0.50, 0.58, 0.57, 0.60, 0.76, 1.00, 0.99, 0.55, 0.70, 0.74, 0.73], + "Old profile-only (mock)": [0.38, 0.44, 0.51, 0.50, 0.47, 0.58, 0.70, 0.69, 0.66, 0.84, 0.95, 0.96], + "New P-020 profile-only (H1 mock)": [0.40, 0.49, 0.57, 0.56, 0.58, 0.74, 0.98, 0.97, 0.53, 0.69, 0.75, 0.74], + "Per-TP calibrated upper bound (mock)": [0.41, 0.50, 0.58, 0.57, 0.59, 0.75, 0.99, 0.98, 0.54, 0.69, 0.75, 0.74], +} + + +def main() -> None: + output = Path(__file__).with_name("mock-profile-ablation.png") + fig, ax = plt.subplots(figsize=(13.5, 5.8), constrained_layout=True) + x = list(range(len(CONFIGS))) + styles = ["o-", "s--", "^-", "D:"] + for (label, values), style in zip(MOCK.items(), styles, strict=True): + ax.plot(x, values, style, linewidth=2, markersize=5, label=label) + ax.axvline(3.5, color="0.75", linewidth=1) + ax.axvline(7.5, color="0.75", linewidth=1) + ax.set_xticks(x, CONFIGS) + ax.set_ylabel("Normalized SLO-feasible throughput (schematic)") + ax.set_xlabel("Configuration") + ax.set_ylim(0.3, 1.08) + ax.grid(axis="y", alpha=0.25) + ax.legend(ncol=2, frameon=False, loc="upper left") + ax.set_title("MOCK / SCHEMATIC — expected discriminative result, not experiment data") + fig.savefig(output, dpi=180) + print(output) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_flashattn.py b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_flashattn.py new file mode 100644 index 0000000..bfba530 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_flashattn.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +"""Profile the exact vLLM 0.20 FlashAttention backend at TP-local shapes. + +This deliberately uses vLLM's own v0.20.0 attention benchmark runner instead +of Frontier's FlashInfer-only attention wrapper. The output is raw evidence; +projection into Frontier's split attention CSV schema is a separate step. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import types +from pathlib import Path + +import torch +import vllm + + +VLLM_VERSION = "0.20.0" +VLLM_COMMIT = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--vllm-source", type=Path, required=True) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--tp", type=int, choices=(1, 2, 4), nargs="+", default=[1, 2, 4]) + parser.add_argument( + "--batch-specs", + nargs="+", + default=["q128", "4q1s128", "q128_4q1s128"], + ) + parser.add_argument("--warmup-iters", type=int, default=3) + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--device", default="cuda:0") + return parser.parse_args() + + +def git_head(repo: Path) -> str: + return subprocess.check_output( + ["git", "-C", str(repo), "rev-parse", "HEAD"], text=True + ).strip() + + +def main() -> None: + args = parse_args() + if vllm.__version__ != VLLM_VERSION: + raise SystemExit(f"expected vLLM {VLLM_VERSION}, got {vllm.__version__}") + source_head = git_head(args.vllm_source) + if source_head != VLLM_COMMIT: + raise SystemExit(f"expected vLLM source {VLLM_COMMIT}, got {source_head}") + if not args.model.joinpath("config.json").is_file(): + raise SystemExit(f"missing model config: {args.model / 'config.json'}") + + bench_dir = args.vllm_source / "benchmarks" / "attention_benchmarks" + sys.path.insert(0, str(bench_dir)) + import runner # type: ignore[import-not-found] # noqa: PLC0415 + from common import BenchmarkConfig # type: ignore[import-not-found] # noqa: PLC0415 + from vllm.config import ( # noqa: PLC0415 + CacheConfig, + CompilationConfig, + DeviceConfig, + LoadConfig, + ModelConfig, + ParallelConfig, + SchedulerConfig, + VllmConfig, + ) + from vllm.v1.worker.workspace import init_workspace_manager # noqa: PLC0415 + + def create_vllm_config(config: BenchmarkConfig, max_num_blocks: int) -> VllmConfig: + model_config = ModelConfig( + model=str(args.model), + tokenizer=str(args.model), + trust_remote_code=False, + dtype="bfloat16", + seed=0, + max_model_len=40960, + ) + cache_config = CacheConfig(block_size=config.block_size, cache_dtype="auto") + cache_config.num_gpu_blocks = max_num_blocks + cache_config.num_cpu_blocks = 0 + parallel_config = ParallelConfig(tensor_parallel_size=1) + scheduler_config = SchedulerConfig( + max_num_seqs=256, + max_num_batched_tokens=8192, + max_model_len=40960, + is_encoder_decoder=False, + enable_chunked_prefill=True, + ) + model_config.get_num_layers = types.MethodType( + lambda self: config.num_layers, model_config + ) + model_config.get_sliding_window_for_layer = types.MethodType( + lambda self, i: None, model_config + ) + model_config.get_logits_soft_cap_for_layer = types.MethodType( + lambda self, i: 0.0, model_config + ) + model_config.get_sm_scale_for_layer = types.MethodType( + lambda self, i: 1.0 / config.head_dim**0.5, model_config + ) + model_config.get_num_attention_heads = types.MethodType( + lambda self, parallel_config=None: config.num_q_heads, model_config + ) + model_config.get_num_kv_heads = types.MethodType( + lambda self, parallel_config=None: config.num_kv_heads, model_config + ) + model_config.get_head_size = types.MethodType( + lambda self: config.head_dim, model_config + ) + model_config.get_sliding_window = types.MethodType( + lambda self: None, model_config + ) + return VllmConfig( + model_config=model_config, + cache_config=cache_config, + parallel_config=parallel_config, + scheduler_config=scheduler_config, + device_config=DeviceConfig(), + load_config=LoadConfig(), + compilation_config=CompilationConfig(), + ) + + runner._create_vllm_config = create_vllm_config + init_workspace_manager(args.device) + + rows: list[dict[str, object]] = [] + for tp in args.tp: + for batch_spec in args.batch_specs: + config = BenchmarkConfig( + backend="FLASH_ATTN", + batch_spec=batch_spec, + num_layers=1, + head_dim=128, + num_q_heads=32 // tp, + num_kv_heads=4 // tp, + block_size=16, + device=args.device, + dtype=torch.bfloat16, + repeats=args.repeats, + warmup_iters=args.warmup_iters, + profile_memory=True, + kv_cache_dtype="auto", + use_cuda_graphs=False, + ) + result = runner.run_attention_benchmark(config) + row = result.to_dict() + row["tensor_parallel_size"] = tp + rows.append(row) + print( + json.dumps( + { + "tp": tp, + "batch_spec": batch_spec, + "mean_time_s": result.mean_time, + "error": result.error, + }, + sort_keys=True, + ), + flush=True, + ) + if not result.success: + raise SystemExit(f"attention profile failed: {row}") + + payload = { + "schema_version": "qwen30_vllm020_flashattn_raw.v1", + "environment": { + "vllm_version": vllm.__version__, + "vllm_source_commit": source_head, + "torch_version": torch.__version__, + "torch_cuda": torch.version.cuda, + "gpu": torch.cuda.get_device_name(torch.device(args.device)), + "model": str(args.model), + "dtype": "bfloat16", + "attention_backend": "FLASH_ATTN", + "block_size": 16, + }, + "rows": rows, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-qwen30-vllm020-profile-v1/run_flashattn_smoke.sh b/runs/frontier-qwen30-vllm020-profile-v1/run_flashattn_smoke.sh new file mode 100644 index 0000000..04318bb --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/run_flashattn_smoke.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +set -euo pipefail + +OUTPUT_ROOT="${OUTPUT_ROOT:-$(pwd)/artifacts/flashattn-smoke}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +MODEL="${MODEL:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +LOG_DIR="${OUTPUT_ROOT}/logs" +PROVENANCE_DIR="${OUTPUT_ROOT}/provenance" + +mkdir -p "${LOG_DIR}" "${PROVENANCE_DIR}" +exec > >(tee -a "${LOG_DIR}/smoke.log") 2>&1 + +if [[ -z "${CUDA_VISIBLE_DEVICES:-}" ]]; then + echo "ERROR: CUDA_VISIBLE_DEVICES must contain the fleet-allocated GPU" >&2 + exit 1 +fi +IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES}" +if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then + echo "ERROR: expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2 + exit 1 +fi + +echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=${MODEL} runtime=vLLM-0.20.0+cu129 operator=FlashAttention3 tp_local_shapes=1,2,4 specs=q128,4q1s128,q128_4q1s128 dtype=BF16 block=16 output=${OUTPUT_ROOT} expected_wall=3-8m hard_wall=900s hard_gpu_cap=0.15_H20h" +date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" +nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader + +test -x "${VENV_ROOT}/bin/python" +test -f "${VLLM_SOURCE}/benchmarks/attention_benchmarks/runner.py" +test -f "${MODEL}/config.json" +test -f profile_vllm020_flashattn.py + +git rev-parse HEAD > "${PROVENANCE_DIR}/aituner.commit" +git -C "${VLLM_SOURCE}" rev-parse HEAD > "${PROVENANCE_DIR}/vllm-source.commit" +sha256sum profile_vllm020_flashattn.py run_flashattn_smoke.sh \ + > "${PROVENANCE_DIR}/source.sha256" +uv pip freeze --python "${VENV_ROOT}/bin/python" \ + > "${PROVENANCE_DIR}/pip-freeze.txt" +nvidia-smi --query-gpu=index,uuid,name,driver_version,memory.total \ + --format=csv,noheader > "${PROVENANCE_DIR}/gpus.csv" + +timeout --signal=TERM --kill-after=30s 780 \ + "${VENV_ROOT}/bin/python" profile_vllm020_flashattn.py \ + --vllm-source "${VLLM_SOURCE}" \ + --model "${MODEL}" \ + --output "${OUTPUT_ROOT}/raw/flashattn-smoke.json" \ + --tp 1 2 4 \ + --batch-specs q128 4q1s128 q128_4q1s128 \ + --warmup-iters 3 \ + --repeats 5 + +test -s "${OUTPUT_ROOT}/raw/flashattn-smoke.json" +sha256sum "${OUTPUT_ROOT}/raw/flashattn-smoke.json" \ + "${PROVENANCE_DIR}"/* > "${OUTPUT_ROOT}/artifacts.sha256" +nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader +date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ" +echo "FLASHATTN_SMOKE_COMPLETE" From fb12e3b502e2a4acc38c22d65d5d09eb86b30334 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 21:07:24 +0800 Subject: [PATCH 069/148] Use local cache for vLLM profile smoke --- runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml index eee4f5a..e9410f2 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml @@ -9,6 +9,9 @@ command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/run artifacts = ["artifacts/flashattn-smoke"] [jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/flashattn-smoke" VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" From 2f04971c90c283e80e841f49514d8a421ee2dfdc Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 21:09:51 +0800 Subject: [PATCH 070/148] Freeze Qwen30 trace profile support --- .../extract_trace_profile_support.py | 215 ++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/extract_trace_profile_support.py diff --git a/runs/frontier-qwen30-vllm020-profile-v1/extract_trace_profile_support.py b/runs/frontier-qwen30-vllm020-profile-v1/extract_trace_profile_support.py new file mode 100644 index 0000000..e3e37c6 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/extract_trace_profile_support.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""Freeze the exact request cohort and its operator-profile support. + +This calls AITuner's production trace loader, including its input-length +filter, uniform max-request downsampling, output override, and sampling-u +threshold semantics. Prefix reuse below is a no-eviction upper bound; the +actual cache state remains scheduler/config dependent. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from pathlib import Path +from typing import Any, Iterable + +from aituner.spec import load_study_spec +from aituner.trace import load_trace_requests, select_requests_for_threshold + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--study", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--cohort-output", type=Path, required=True) + parser.add_argument( + "--thresholds", + type=float, + nargs="+", + default=[0.125, 0.25, 0.5, 1.0], + ) + return parser.parse_args() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def nearest_rank(values: Iterable[float], percentile: float) -> float: + ordered = sorted(values) + if not ordered: + return 0.0 + index = min( + len(ordered) - 1, + max(0, math.ceil(percentile / 100.0 * len(ordered)) - 1), + ) + return float(ordered[index]) + + +def distribution(values: list[float]) -> dict[str, float]: + return { + "min": min(values, default=0.0), + "p10": nearest_rank(values, 10), + "p25": nearest_rank(values, 25), + "p50": nearest_rank(values, 50), + "p75": nearest_rank(values, 75), + "p90": nearest_rank(values, 90), + "p95": nearest_rank(values, 95), + "p99": nearest_rank(values, 99), + "max": max(values, default=0.0), + } + + +def prefix_upper_bound(requests: list[Any], block_size: int) -> dict[str, Any]: + seen: set[Any] = set() + total_blocks = 0 + reusable_blocks = 0 + leading_reusable_blocks: list[float] = [] + reusable_tokens: list[float] = [] + rows_with_hashes = 0 + for request in requests: + hashes = request.metadata.get("hash_ids") + if not isinstance(hashes, list): + continue + rows_with_hashes += 1 + total_blocks += len(hashes) + reusable = sum(hash_id in seen for hash_id in hashes) + reusable_blocks += reusable + leading = 0 + for hash_id in hashes: + if hash_id not in seen: + break + leading += 1 + leading_reusable_blocks.append(float(leading)) + reusable_tokens.append( + float(min(request.prompt_tokens_hint or 0, leading * block_size)) + ) + seen.update(hashes) + return { + "semantics": ( + "arrival-ordered, infinite-capacity/no-eviction upper bound; " + "not an observed KV-cache hit rate" + ), + "rows_with_hash_ids": rows_with_hashes, + "total_blocks": total_blocks, + "unique_blocks": len(seen), + "any_position_reusable_block_ratio": ( + reusable_blocks / total_blocks if total_blocks else 0.0 + ), + "leading_reusable_blocks_per_request": distribution( + leading_reusable_blocks + ), + "leading_reusable_tokens_per_request": distribution(reusable_tokens), + } + + +def summarize(requests: list[Any], block_size: int) -> dict[str, Any]: + input_lengths = [float(request.prompt_tokens_hint or 0) for request in requests] + output_lengths = [ + float(request.completion_tokens_hint or 0) for request in requests + ] + hash_counts = [ + float(len(request.metadata.get("hash_ids") or [])) for request in requests + ] + arrivals = [request.arrival_s for request in requests] + interarrivals = [ + max(0.0, arrivals[index] - arrivals[index - 1]) + for index in range(1, len(arrivals)) + ] + return { + "request_count": len(requests), + "input_tokens": distribution(input_lengths), + "output_tokens": distribution(output_lengths), + "hash_blocks_per_request": distribution(hash_counts), + "interarrival_s": distribution(interarrivals), + "sampling_u": distribution([request.sampling_u for request in requests]), + "multi_turn_fraction": ( + sum( + isinstance(request.metadata.get("turn"), (int, float)) + and request.metadata["turn"] > 1 + for request in requests + ) + / len(requests) + if requests + else 0.0 + ), + "prefix_reuse_upper_bound": prefix_upper_bound(requests, block_size), + } + + +def cohort_row(request: Any) -> dict[str, Any]: + return { + "row_id": request.row_id, + "arrival_s": request.arrival_s, + "sampling_u": request.sampling_u, + "input_tokens": request.prompt_tokens_hint, + "output_tokens": request.completion_tokens_hint, + "hash_ids": request.metadata.get("hash_ids"), + "turn": request.metadata.get("turn"), + "parent_chat_id": request.metadata.get("parent_chat_id"), + "type": request.metadata.get("type"), + } + + +def main() -> None: + args = parse_args() + study = load_study_spec(args.study) + window, cohort = load_trace_requests(study, study_spec_path=args.study) + block_size = int(window.source_payload.get("block_size") or 1) + + args.cohort_output.parent.mkdir(parents=True, exist_ok=True) + with args.cohort_output.open("w", encoding="utf-8") as handle: + for request in cohort: + handle.write(json.dumps(cohort_row(request), sort_keys=True) + "\n") + + payload = { + "schema_version": "qwen30_trace_profile_support.v1", + "study": str(args.study.resolve()), + "study_sha256": sha256_file(args.study), + "trace": str(window.trace_path), + "trace_sha256": sha256_file(window.trace_path), + "window_id": window.window_id, + "block_size": block_size, + "loader_contract": { + "input_length_filter": { + "min": study.trace.input_length_filter.min_input_tokens, + "max": study.trace.input_length_filter.max_input_tokens, + } + if study.trace.input_length_filter is not None + else None, + "completion_tokens_override": study.trace.completion_tokens_override, + "max_requests_per_probe": study.trace.max_requests_per_probe, + "replay_time_scale": study.trace.replay_time_scale, + "ordering": "arrival_s", + "downsampling": "AITuner _downsample_requests before sampling_u threshold", + }, + "full_downsampled_cohort": summarize(cohort, block_size), + "threshold_cohorts": { + str(threshold): summarize( + select_requests_for_threshold(cohort, threshold=threshold), + block_size, + ) + for threshold in args.thresholds + }, + "limits": [ + "Trace length/hash support does not determine dynamic decode or mixed batch shapes.", + "Those shapes depend jointly on arrival history, SLO pressure, TP execution time, MNS, chunking, and KV eviction.", + "MoE expert routing is not present in the trace and must be measured from model execution.", + ], + } + payload["cohort_output"] = str(args.cohort_output.resolve()) + payload["cohort_sha256"] = sha256_file(args.cohort_output) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + print(json.dumps(payload, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() From b3e437acb03dce14ce68c4d8e62463d8fec5eecd Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 21:17:27 +0800 Subject: [PATCH 071/148] Serialize vLLM attention profile results --- .../profile_vllm020_flashattn.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_flashattn.py b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_flashattn.py index bfba530..9a37c12 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_flashattn.py +++ b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_flashattn.py @@ -46,6 +46,15 @@ def git_head(repo: Path) -> str: ).strip() +def json_default(value: object) -> object: + if isinstance(value, (Path, torch.dtype, torch.device)): + return str(value) + item = getattr(value, "item", None) + if callable(item): + return item() + raise TypeError(f"cannot serialize {type(value).__name__}") + + def main() -> None: args = parse_args() if vllm.__version__ != VLLM_VERSION: @@ -161,6 +170,7 @@ def main() -> None: "error": result.error, }, sort_keys=True, + default=json_default, ), flush=True, ) @@ -183,7 +193,9 @@ def main() -> None: "rows": rows, } args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + args.output.write_text( + json.dumps(payload, indent=2, sort_keys=True, default=json_default) + "\n" + ) if __name__ == "__main__": From a70a1fac7186276a04b2ca7c8cc209804d9fb331 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 21:19:22 +0800 Subject: [PATCH 072/148] Retry profile smoke with cu129 vLLM wheel --- runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml index e9410f2..16be440 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml @@ -1,7 +1,7 @@ version = 1 [[jobs]] -name = "qwen30-vllm020-flashattn-smoke-20260716-v1" +name = "qwen30-vllm020-flashattn-smoke-20260716-v2-cu129" gpus = 1 gpu_model = "H20" hosts = ["dash0"] From adaf68badced9099524aebe3e61be0d49660964e Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 21:24:11 +0800 Subject: [PATCH 073/148] Add full per-TP FlashAttention profile sweep --- .../jobs_flashattn_full.toml | 55 ++++++++++++ .../run_flashattn_full.sh | 88 +++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/jobs_flashattn_full.toml create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/run_flashattn_full.sh diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_flashattn_full.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_flashattn_full.toml new file mode 100644 index 0000000..c4bc708 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_flashattn_full.toml @@ -0,0 +1,55 @@ +version = 1 + +[[jobs]] +name = "qwen30-vllm020-flashattn-full-tp1-20260716-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1320 bash run_flashattn_full.sh" +artifacts = ["artifacts/flashattn-full-tp1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "1" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/flashattn-full-tp1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" +MODEL = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-vllm020-flashattn-full-tp2-20260716-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1320 bash run_flashattn_full.sh" +artifacts = ["artifacts/flashattn-full-tp2"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "2" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/flashattn-full-tp2" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" +MODEL = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-vllm020-flashattn-full-tp4-20260716-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1320 bash run_flashattn_full.sh" +artifacts = ["artifacts/flashattn-full-tp4"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "4" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/flashattn-full-tp4" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" +MODEL = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/run_flashattn_full.sh b/runs/frontier-qwen30-vllm020-profile-v1/run_flashattn_full.sh new file mode 100644 index 0000000..7188969 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/run_flashattn_full.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +set -euo pipefail + +TP="${TP:?TP must be set to 1, 2, or 4}" +case "${TP}" in + 1|2|4) ;; + *) echo "ERROR: invalid TP=${TP}" >&2; exit 1 ;; +esac + +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +MODEL="${MODEL:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +LOG_DIR="${OUTPUT_ROOT}/logs" +PROVENANCE_DIR="${OUTPUT_ROOT}/provenance" + +PREFILL_SPECS=(q64 q128 q256 q512 q1k q2k q4k q8k) +PREFIX_EXTEND_SPECS=(q64s512 q128s1k q256s2k q512s4k q1ks8k) +DECODE_SPECS=( + q1s128 8q1s128 16q1s128 32q1s128 64q1s128 + 8q1s1k 16q1s1k 32q1s1k 64q1s1k + 8q1s2k 16q1s2k 32q1s2k 64q1s2k + 8q1s4k 16q1s4k 32q1s4k 64q1s4k + 8q1s8k 16q1s8k 32q1s8k 64q1s8k +) +MIXED_SPECS=( + q64_8q1s512 + q128_8q1s1k q128_16q1s1k + q256_16q1s2k q256_32q1s2k + q512_32q1s4k q512_64q1s4k + q1k_64q1s8k q2k_32q1s4k q4k_16q1s4k +) +BATCH_SPECS=( + "${PREFILL_SPECS[@]}" + "${PREFIX_EXTEND_SPECS[@]}" + "${DECODE_SPECS[@]}" + "${MIXED_SPECS[@]}" +) + +mkdir -p "${LOG_DIR}" "${PROVENANCE_DIR}" "${OUTPUT_ROOT}/raw" +exec > >(tee -a "${LOG_DIR}/full.log") 2>&1 + +if [[ -z "${CUDA_VISIBLE_DEVICES:-}" ]]; then + echo "ERROR: CUDA_VISIBLE_DEVICES must contain the fleet-allocated GPU" >&2 + exit 1 +fi +IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES}" +if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then + echo "ERROR: expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2 + exit 1 +fi + +echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=${MODEL} runtime=vLLM-0.20.0+cu129 operator=FlashAttention3 tp=${TP} cases=${#BATCH_SPECS[@]} regimes=prefill,prefix_extend,decode,true_mixed dtype=BF16 block=16 output=${OUTPUT_ROOT} expected_wall=4-10m hard_wall=1200s hard_gpu_cap=0.33_H20h" +date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" +nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader + +test -x "${VENV_ROOT}/bin/python" +test -f "${VLLM_SOURCE}/benchmarks/attention_benchmarks/runner.py" +test -f "${MODEL}/config.json" +test -f profile_vllm020_flashattn.py + +git rev-parse HEAD > "${PROVENANCE_DIR}/aituner.commit" +git -C "${VLLM_SOURCE}" rev-parse HEAD > "${PROVENANCE_DIR}/vllm-source.commit" +sha256sum profile_vllm020_flashattn.py run_flashattn_full.sh \ + > "${PROVENANCE_DIR}/source.sha256" +uv pip freeze --python "${VENV_ROOT}/bin/python" \ + > "${PROVENANCE_DIR}/pip-freeze.txt" +nvidia-smi --query-gpu=index,uuid,name,driver_version,memory.total \ + --format=csv,noheader > "${PROVENANCE_DIR}/gpus.csv" +printf '%s\n' "${BATCH_SPECS[@]}" > "${PROVENANCE_DIR}/batch-specs.txt" + +timeout --signal=TERM --kill-after=30s 1080 \ + "${VENV_ROOT}/bin/python" profile_vllm020_flashattn.py \ + --vllm-source "${VLLM_SOURCE}" \ + --model "${MODEL}" \ + --output "${OUTPUT_ROOT}/raw/flashattn-tp${TP}.json" \ + --tp "${TP}" \ + --batch-specs "${BATCH_SPECS[@]}" \ + --warmup-iters 5 \ + --repeats 10 + +test -s "${OUTPUT_ROOT}/raw/flashattn-tp${TP}.json" +sha256sum "${OUTPUT_ROOT}/raw/flashattn-tp${TP}.json" \ + "${PROVENANCE_DIR}"/* > "${OUTPUT_ROOT}/artifacts.sha256" +nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader +date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ" +echo "FLASHATTN_FULL_COMPLETE tp=${TP} cases=${#BATCH_SPECS[@]}" From 5a958077a79b63b562daa3fed9f2c01f30318341 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 21:27:35 +0800 Subject: [PATCH 074/148] Add exact vLLM 0.20 MoE profile smoke --- .../jobs_moe_smoke.toml | 18 + .../profile_vllm020_moe.py | 361 ++++++++++++++++++ .../run_moe_smoke.sh | 47 +++ 3 files changed, 426 insertions(+) create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/jobs_moe_smoke.toml create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_moe.py create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/run_moe_smoke.sh diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_moe_smoke.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_moe_smoke.toml new file mode 100644 index 0000000..48cd8db --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_moe_smoke.toml @@ -0,0 +1,18 @@ +version = 1 + +[[jobs]] +name = "qwen30-vllm020-moe-smoke-20260716-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1020 bash run_moe_smoke.sh" +artifacts = ["artifacts/moe-smoke"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/moe-smoke" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" +MODEL = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_moe.py b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_moe.py new file mode 100644 index 0000000..927b316 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_moe.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 +"""Profile vLLM 0.20's Qwen3-30B unquantized MoE kernel at TP-local shapes.""" + +from __future__ import annotations + +import argparse +import json +import math +import statistics +import subprocess +from pathlib import Path +from typing import Any + +import torch +import vllm + + +VLLM_VERSION = "0.20.0" +VLLM_COMMIT = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1" +HIDDEN_DIM = 2048 +INTERMEDIATE_DIM = 768 +NUM_EXPERTS = 128 +TOP_K = 8 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--vllm-source", type=Path, required=True) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--tp", type=int, choices=(1, 2, 4), nargs="+", default=[1, 2, 4]) + parser.add_argument("--num-tokens", type=int, nargs="+", default=[8]) + parser.add_argument( + "--routing-modes", + choices=("uniform_random_logits", "hotset8"), + nargs="+", + default=["uniform_random_logits"], + ) + parser.add_argument("--warmup-iters", type=int, default=3) + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--check-reference", action="store_true") + return parser.parse_args() + + +def git_head(repo: Path) -> str: + return subprocess.check_output( + ["git", "-C", str(repo), "rev-parse", "HEAD"], text=True + ).strip() + + +def stats_ms(samples: list[float]) -> dict[str, float]: + return { + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + "median": statistics.median(samples), + "std": statistics.pstdev(samples), + } + + +def routing_inputs( + mode: str, num_tokens: int, device: torch.device +) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]: + from vllm.model_executor.layers.fused_moe import fused_topk + + if mode == "uniform_random_logits": + logits = torch.randn( + (num_tokens, NUM_EXPERTS), device=device, dtype=torch.bfloat16 + ) + hidden_for_topk = torch.empty( + (num_tokens, HIDDEN_DIM), device=device, dtype=torch.bfloat16 + ) + weights, ids, _ = fused_topk( + hidden_for_topk, + logits, + TOP_K, + renormalize=True, + ) + elif mode == "hotset8": + ids = torch.arange(TOP_K, device=device, dtype=torch.int32).repeat( + num_tokens, 1 + ) + weights = torch.full( + (num_tokens, TOP_K), + 1.0 / TOP_K, + device=device, + dtype=torch.float32, + ) + else: + raise ValueError(mode) + + counts = torch.bincount(ids.flatten().to(torch.int64), minlength=NUM_EXPERTS) + counts_cpu = counts.cpu().tolist() + mean_load = num_tokens * TOP_K / NUM_EXPERTS + variance = sum((count - mean_load) ** 2 for count in counts_cpu) / NUM_EXPERTS + return weights, ids, { + "active_experts": sum(count > 0 for count in counts_cpu), + "min_tokens_per_expert": min(counts_cpu), + "max_tokens_per_expert": max(counts_cpu), + "load_cv": math.sqrt(variance) / mean_load if mean_load else 0.0, + "counts": counts_cpu, + } + + +def reference_partial_output( + hidden: torch.Tensor, + w13_original: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, +) -> torch.Tensor: + output = torch.zeros_like(hidden) + for token in range(hidden.shape[0]): + for route in range(TOP_K): + expert = int(topk_ids[token, route]) + gate_up = torch.mv(w13_original[expert], hidden[token]) + gate, up = gate_up.chunk(2) + activated = torch.nn.functional.silu(gate) * up + expert_output = torch.mv(w2[expert], activated) + output[token].add_( + expert_output * topk_weights[token, route].to(expert_output.dtype) + ) + return output + + +def main() -> None: + args = parse_args() + if vllm.__version__ != VLLM_VERSION: + raise SystemExit(f"expected vLLM {VLLM_VERSION}, got {vllm.__version__}") + source_head = git_head(args.vllm_source) + if source_head != VLLM_COMMIT: + raise SystemExit(f"expected vLLM source {VLLM_COMMIT}, got {source_head}") + model_config = json.loads(args.model.joinpath("config.json").read_text()) + expected_model = { + "hidden_size": HIDDEN_DIM, + "moe_intermediate_size": INTERMEDIATE_DIM, + "num_experts": NUM_EXPERTS, + "num_experts_per_tok": TOP_K, + "norm_topk_prob": True, + "torch_dtype": "bfloat16", + } + observed_model = {key: model_config.get(key) for key in expected_model} + if observed_model != expected_model: + raise SystemExit( + f"model contract mismatch: expected {expected_model}, got {observed_model}" + ) + + from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + from vllm.model_executor.layers.fused_moe.config import ( + FUSED_MOE_UNQUANTIZED_CONFIG, + FusedMoEConfig, + FusedMoEParallelConfig, + RoutingMethodType, + ) + from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( + UnquantizedMoeBackend, + convert_to_unquantized_kernel_format, + make_unquantized_moe_kernel, + select_unquantized_moe_backend, + ) + from vllm.utils.math_utils import next_power_of_2 + from vllm.v1.worker.workspace import init_workspace_manager + + device = torch.device(args.device) + torch.accelerator.set_device_index(device) + torch.manual_seed(20260716) + init_workspace_manager(args.device) + max_num_tokens = next_power_of_2(max(args.num_tokens)) + + rows: list[dict[str, Any]] = [] + for tp in args.tp: + parallel = FusedMoEParallelConfig( + tp_size=tp, + tp_rank=0, + pcp_size=1, + pcp_rank=0, + dp_size=1, + dp_rank=0, + ep_size=1, + ep_rank=0, + sp_size=1, + use_ep=False, + all2all_backend="allgather_reducescatter", + enable_eplb=False, + ) + moe_config = FusedMoEConfig( + num_experts=NUM_EXPERTS, + experts_per_token=TOP_K, + hidden_dim=HIDDEN_DIM, + intermediate_size_per_partition=INTERMEDIATE_DIM // tp, + num_local_experts=NUM_EXPERTS, + num_logical_experts=NUM_EXPERTS, + activation=MoEActivation.SILU, + device=device, + routing_method=RoutingMethodType.Renormalize, + moe_parallel_config=parallel, + in_dtype=torch.bfloat16, + max_num_tokens=max_num_tokens, + ) + vllm_config = VllmConfig( + parallel_config=ParallelConfig(tensor_parallel_size=tp) + ) + with set_current_vllm_config(vllm_config): + backend, experts_cls = select_unquantized_moe_backend(moe_config) + if backend != UnquantizedMoeBackend.FLASHINFER_CUTLASS: + raise SystemExit( + "runtime backend mismatch: expected FlashInfer CUTLASS, " + f"got {backend.value} at TP={tp}" + ) + if experts_cls is None: + raise SystemExit(f"missing experts class for {backend.value}") + + w13_original = torch.empty( + (NUM_EXPERTS, 2 * (INTERMEDIATE_DIM // tp), HIDDEN_DIM), + device=device, + dtype=torch.bfloat16, + ).uniform_(-0.01, 0.01) + w2 = torch.empty( + (NUM_EXPERTS, HIDDEN_DIM, INTERMEDIATE_DIM // tp), + device=device, + dtype=torch.bfloat16, + ).uniform_(-0.01, 0.01) + + class Layer: + pass + + layer = Layer() + layer.moe_config = moe_config + w13_kernel, w2_kernel = convert_to_unquantized_kernel_format( + backend, + layer=layer, + w13_weight=w13_original, + w2_weight=w2, + ) + kernel = make_unquantized_moe_kernel( + quant_config=FUSED_MOE_UNQUANTIZED_CONFIG, + moe_config=moe_config, + backend=backend, + experts_cls=experts_cls, + ) + + reference_checked = False + for routing_mode in args.routing_modes: + for num_tokens in args.num_tokens: + hidden = torch.empty( + (num_tokens, HIDDEN_DIM), + device=device, + dtype=torch.bfloat16, + ).uniform_(-0.1, 0.1) + topk_weights, topk_ids, load = routing_inputs( + routing_mode, num_tokens, device + ) + + for _ in range(args.warmup_iters): + output = kernel.apply( + hidden_states=hidden, + w1=w13_kernel, + w2=w2_kernel, + topk_weights=topk_weights, + topk_ids=topk_ids, + activation=MoEActivation.SILU, + global_num_experts=NUM_EXPERTS, + expert_map=None, + apply_router_weight_on_input=False, + ) + torch.accelerator.synchronize() + + samples: list[float] = [] + for _ in range(args.repeats): + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + output = kernel.apply( + hidden_states=hidden, + w1=w13_kernel, + w2=w2_kernel, + topk_weights=topk_weights, + topk_ids=topk_ids, + activation=MoEActivation.SILU, + global_num_experts=NUM_EXPERTS, + expert_map=None, + apply_router_weight_on_input=False, + ) + end.record() + torch.accelerator.synchronize() + samples.append(float(start.elapsed_time(end))) + + if output.shape != hidden.shape or not torch.isfinite(output).all(): + raise SystemExit( + f"invalid MoE output TP={tp} M={num_tokens} mode={routing_mode}" + ) + if args.check_reference and not reference_checked: + check_tokens = min(2, num_tokens) + reference = reference_partial_output( + hidden[:check_tokens], + w13_original, + w2, + topk_weights[:check_tokens], + topk_ids[:check_tokens], + ) + torch.testing.assert_close( + output[:check_tokens], reference, atol=0.03, rtol=0.03 + ) + reference_checked = True + + row = { + "tensor_parallel_size": tp, + "num_tokens": num_tokens, + "routing_mode": routing_mode, + "backend": backend.value, + "intermediate_size_per_partition": INTERMEDIATE_DIM // tp, + "output_is_reduced": kernel.output_is_reduced(), + "time_ms": stats_ms(samples), + "routing_load": load, + } + rows.append(row) + print( + json.dumps( + { + "tp": tp, + "num_tokens": num_tokens, + "routing_mode": routing_mode, + "backend": backend.value, + "median_ms": row["time_ms"]["median"], + }, + sort_keys=True, + ), + flush=True, + ) + + del kernel, w13_kernel, w2_kernel, w13_original, w2 + torch.accelerator.empty_cache() + + payload = { + "schema_version": "qwen30_vllm020_moe_raw.v1", + "environment": { + "vllm_version": vllm.__version__, + "vllm_source_commit": source_head, + "torch_version": torch.__version__, + "torch_cuda": torch.version.cuda, + "gpu": torch.cuda.get_device_name(device), + "model": str(args.model), + "dtype": "bfloat16", + "weight_quantization": "none", + "top_k": TOP_K, + "norm_topk_prob": True, + }, + "measurement_scope": ( + "vLLM modular MoE prepare+FlashInfer CUTLASS experts+finalize; " + "router linear/top-k and TP all-reduce excluded" + ), + "rows": rows, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-qwen30-vllm020-profile-v1/run_moe_smoke.sh b/runs/frontier-qwen30-vllm020-profile-v1/run_moe_smoke.sh new file mode 100644 index 0000000..af7b784 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/run_moe_smoke.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -euo pipefail + +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +MODEL="${MODEL:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/raw" +exec > >(tee -a "${OUTPUT_ROOT}/logs/smoke.log") 2>&1 + +IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?fleet GPU is required}" +if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then + echo "ERROR: expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2 + exit 1 +fi + +echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=${MODEL} runtime=vLLM-0.20.0+cu129 operator=FlashInfer-CUTLASS-unquantized-MoE tp_local_shapes=1,2,4 tokens=8 routing=uniform_random_logits reference_check=on dtype=BF16 output=${OUTPUT_ROOT} expected_wall=3-8m hard_wall=900s hard_gpu_cap=0.25_H20h" +date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" +nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader + +git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit" +git -C "${VLLM_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/vllm-source.commit" +sha256sum profile_vllm020_moe.py run_moe_smoke.sh \ + > "${OUTPUT_ROOT}/provenance/source.sha256" +uv pip freeze --python "${VENV_ROOT}/bin/python" \ + > "${OUTPUT_ROOT}/provenance/pip-freeze.txt" +nvidia-smi --query-gpu=index,uuid,name,driver_version,memory.total \ + --format=csv,noheader > "${OUTPUT_ROOT}/provenance/gpus.csv" + +timeout --signal=TERM --kill-after=30s 780 \ + "${VENV_ROOT}/bin/python" profile_vllm020_moe.py \ + --vllm-source "${VLLM_SOURCE}" \ + --model "${MODEL}" \ + --output "${OUTPUT_ROOT}/raw/moe-smoke.json" \ + --tp 1 2 4 \ + --num-tokens 8 \ + --routing-modes uniform_random_logits \ + --warmup-iters 3 \ + --repeats 5 \ + --check-reference + +test -s "${OUTPUT_ROOT}/raw/moe-smoke.json" +sha256sum "${OUTPUT_ROOT}/raw/moe-smoke.json" "${OUTPUT_ROOT}/provenance"/* \ + > "${OUTPUT_ROOT}/artifacts.sha256" +date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ" +echo "MOE_SMOKE_COMPLETE" From a9aed87518f303f25f1489bda8cdea317ac57a7b Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 21:34:20 +0800 Subject: [PATCH 075/148] Add FlashInfer TRTLLM all-reduce profile smoke --- .../jobs_allreduce_smoke_tp2.toml | 19 ++ .../profile_vllm020_allreduce.py | 173 ++++++++++++++++++ .../run_allreduce_profile.sh | 53 ++++++ 3 files changed, 245 insertions(+) create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/jobs_allreduce_smoke_tp2.toml create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_allreduce.py create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/run_allreduce_profile.sh diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_allreduce_smoke_tp2.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_allreduce_smoke_tp2.toml new file mode 100644 index 0000000..802066e --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_allreduce_smoke_tp2.toml @@ -0,0 +1,19 @@ +version = 1 + +[[jobs]] +name = "qwen30-vllm020-flashinfer-allreduce-smoke-tp2-20260716-v1" +gpus = 2 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 840 bash run_allreduce_profile.sh" +artifacts = ["artifacts/allreduce-smoke-tp2"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "2" +NUM_TOKENS = "8" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/allreduce-smoke-tp2" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_allreduce.py b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_allreduce.py new file mode 100644 index 0000000..8665b9c --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_allreduce.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Profile vLLM 0.20 TP all-reduce and assert FlashInfer TRTLLM dispatch.""" + +from __future__ import annotations + +import argparse +import json +import os +import statistics +import subprocess +from pathlib import Path +from typing import Any + +import torch +import torch.distributed as dist +import vllm + + +VLLM_VERSION = "0.20.0" +VLLM_COMMIT = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--vllm-source", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--num-tokens", type=int, nargs="+", default=[8]) + parser.add_argument("--hidden-dim", type=int, default=2048) + parser.add_argument("--warmup-iters", type=int, default=3) + parser.add_argument("--repeats", type=int, default=10) + return parser.parse_args() + + +def git_head(repo: Path) -> str: + return subprocess.check_output( + ["git", "-C", str(repo), "rev-parse", "HEAD"], text=True + ).strip() + + +def stats_ms(samples: list[float]) -> dict[str, float]: + return { + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + "median": statistics.median(samples), + "std": statistics.pstdev(samples), + } + + +def main() -> None: + args = parse_args() + if vllm.__version__ != VLLM_VERSION: + raise SystemExit(f"expected vLLM {VLLM_VERSION}, got {vllm.__version__}") + source_head = git_head(args.vllm_source) + if source_head != VLLM_COMMIT: + raise SystemExit(f"expected vLLM source {VLLM_COMMIT}, got {source_head}") + if os.getenv("VLLM_ALLREDUCE_USE_FLASHINFER") != "1": + raise SystemExit("VLLM_ALLREDUCE_USE_FLASHINFER must equal 1") + if os.getenv("VLLM_FLASHINFER_ALLREDUCE_BACKEND") != "trtllm": + raise SystemExit("VLLM_FLASHINFER_ALLREDUCE_BACKEND must equal trtllm") + if "RANK" not in os.environ or "WORLD_SIZE" not in os.environ: + raise SystemExit("launch with torchrun") + + from vllm.distributed import ( + destroy_distributed_environment, + destroy_model_parallel, + init_distributed_environment, + initialize_model_parallel, + tensor_model_parallel_all_reduce, + ) + from vllm.distributed.parallel_state import get_tp_group + + rank = int(os.environ["RANK"]) + local_rank = int(os.environ["LOCAL_RANK"]) + world_size = int(os.environ["WORLD_SIZE"]) + if world_size not in (2, 4): + raise SystemExit(f"expected TP world size 2 or 4, got {world_size}") + device = torch.device(f"cuda:{local_rank}") + torch.accelerator.set_device_index(device) + torch.set_default_device(device) + init_distributed_environment() + initialize_model_parallel(tensor_model_parallel_size=world_size) + + rows: list[dict[str, Any]] = [] + expected_sum = world_size * (world_size + 1) / 2 + try: + for num_tokens in args.num_tokens: + input_tensor = torch.full( + (num_tokens, args.hidden_dim), + float(rank + 1), + dtype=torch.bfloat16, + device=device, + ) + for _ in range(args.warmup_iters): + output = tensor_model_parallel_all_reduce(input_tensor) + torch.accelerator.synchronize() + torch.testing.assert_close( + output, + torch.full_like(output, expected_sum), + atol=0.0, + rtol=0.0, + ) + + communicator = get_tp_group().device_communicator + fi_comm = communicator.fi_ar_comm + if fi_comm is None or fi_comm.disabled: + raise SystemExit( + f"FlashInfer all-reduce was not selected at TP={world_size}, " + f"tokens={num_tokens}" + ) + if not fi_comm.should_use_fi_ar(input_tensor): + raise SystemExit( + f"FlashInfer rejected serving tensor shape {tuple(input_tensor.shape)}" + ) + + samples: list[float] = [] + for _ in range(args.repeats): + dist.barrier() + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + output = tensor_model_parallel_all_reduce(input_tensor) + end.record() + torch.accelerator.synchronize() + samples.append(float(start.elapsed_time(end))) + + gathered: list[list[float] | None] = [None] * world_size + dist.all_gather_object(gathered, samples) + if rank == 0: + per_rank = [stats_ms(item) for item in gathered if item is not None] + row = { + "tensor_parallel_size": world_size, + "num_tokens": num_tokens, + "hidden_dim": args.hidden_dim, + "payload_bytes": num_tokens + * args.hidden_dim + * torch.tensor([], dtype=torch.bfloat16).element_size(), + "dtype": "bfloat16", + "communicator": "vllm.tensor_model_parallel_all_reduce", + "selected_backend": "flashinfer_trtllm", + "per_rank_time_ms": per_rank, + "critical_path_median_ms": max( + rank_stats["median"] for rank_stats in per_rank + ), + } + rows.append(row) + print(json.dumps(row, sort_keys=True), flush=True) + finally: + destroy_model_parallel() + destroy_distributed_environment() + + if rank == 0: + payload = { + "schema_version": "qwen30_vllm020_allreduce_raw.v1", + "environment": { + "vllm_version": vllm.__version__, + "vllm_source_commit": source_head, + "torch_version": torch.__version__, + "torch_cuda": torch.version.cuda, + "gpu": torch.cuda.get_device_name(device), + "backend_env": { + "VLLM_ALLREDUCE_USE_FLASHINFER": "1", + "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm", + }, + }, + "rows": rows, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-qwen30-vllm020-profile-v1/run_allreduce_profile.sh b/runs/frontier-qwen30-vllm020-profile-v1/run_allreduce_profile.sh new file mode 100644 index 0000000..6321a8d --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/run_allreduce_profile.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -euo pipefail + +TP="${TP:?TP must be set to 2 or 4}" +case "${TP}" in + 2) HARD_GPU_CAP="0.40_H20h" ;; + 4) HARD_GPU_CAP="0.80_H20h" ;; + *) echo "ERROR: invalid TP=${TP}" >&2; exit 1 ;; +esac +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +NUM_TOKENS="${NUM_TOKENS:-8}" +mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/raw" +exec > >(tee -a "${OUTPUT_ROOT}/logs/profile.log") 2>&1 + +IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?fleet GPUs are required}" +if [[ "${#GPU_IDS[@]}" -ne "${TP}" ]]; then + echo "ERROR: TP=${TP} requires ${TP} GPUs, got ${CUDA_VISIBLE_DEVICES}" >&2 + exit 1 +fi + +export VLLM_ALLREDUCE_USE_FLASHINFER=1 +export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm +echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} runtime=vLLM-0.20.0+cu129 operator=tensor_model_parallel_all_reduce backend=FlashInfer-TRTLLM tp=${TP} tokens=${NUM_TOKENS} hidden=2048 dtype=BF16 output=${OUTPUT_ROOT} expected_wall=2-6m hard_wall=720s hard_gpu_cap=${HARD_GPU_CAP}" +date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" +nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader + +git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit" +git -C "${VLLM_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/vllm-source.commit" +sha256sum profile_vllm020_allreduce.py run_allreduce_profile.sh \ + > "${OUTPUT_ROOT}/provenance/source.sha256" +uv pip freeze --python "${VENV_ROOT}/bin/python" \ + > "${OUTPUT_ROOT}/provenance/pip-freeze.txt" +nvidia-smi --query-gpu=index,uuid,name,driver_version,memory.total \ + --format=csv,noheader > "${OUTPUT_ROOT}/provenance/gpus.csv" + +read -r -a TOKEN_ARGS <<< "${NUM_TOKENS}" +timeout --signal=TERM --kill-after=30s 600 \ + "${VENV_ROOT}/bin/torchrun" --standalone --nproc_per_node="${TP}" \ + profile_vllm020_allreduce.py \ + --vllm-source "${VLLM_SOURCE}" \ + --output "${OUTPUT_ROOT}/raw/allreduce-tp${TP}.json" \ + --num-tokens "${TOKEN_ARGS[@]}" \ + --warmup-iters 3 \ + --repeats 10 + +test -s "${OUTPUT_ROOT}/raw/allreduce-tp${TP}.json" +sha256sum "${OUTPUT_ROOT}/raw/allreduce-tp${TP}.json" \ + "${OUTPUT_ROOT}/provenance"/* > "${OUTPUT_ROOT}/artifacts.sha256" +date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ" +echo "ALLREDUCE_PROFILE_COMPLETE tp=${TP} tokens=${NUM_TOKENS}" From 9bc38d8851de2f29725641665066b20ab7a93656 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 21:37:01 +0800 Subject: [PATCH 076/148] Profile FA3 KV-cache updates separately --- .../jobs_flashattn_full.toml | 18 ++-- .../jobs_smoke.toml | 2 +- .../profile_vllm020_flashattn.py | 84 +++++++++++++++++++ .../run_flashattn_full.sh | 5 +- .../run_flashattn_smoke.sh | 5 +- 5 files changed, 100 insertions(+), 14 deletions(-) diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_flashattn_full.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_flashattn_full.toml index c4bc708..5af0e6c 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/jobs_flashattn_full.toml +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_flashattn_full.toml @@ -1,55 +1,55 @@ version = 1 [[jobs]] -name = "qwen30-vllm020-flashattn-full-tp1-20260716-v1" +name = "qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2" gpus = 1 gpu_model = "H20" hosts = ["dash0"] command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1320 bash run_flashattn_full.sh" -artifacts = ["artifacts/flashattn-full-tp1"] +artifacts = ["artifacts/flashattn-kv-full-v2-tp1"] [jobs.env] HOME = "/tmp/wjh" XDG_CACHE_HOME = "/tmp/wjh/.cache" VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" TP = "1" -OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/flashattn-full-tp1" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/flashattn-kv-full-v2-tp1" VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" MODEL = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" [[jobs]] -name = "qwen30-vllm020-flashattn-full-tp2-20260716-v1" +name = "qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2" gpus = 1 gpu_model = "H20" hosts = ["dash0"] command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1320 bash run_flashattn_full.sh" -artifacts = ["artifacts/flashattn-full-tp2"] +artifacts = ["artifacts/flashattn-kv-full-v2-tp2"] [jobs.env] HOME = "/tmp/wjh" XDG_CACHE_HOME = "/tmp/wjh/.cache" VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" TP = "2" -OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/flashattn-full-tp2" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/flashattn-kv-full-v2-tp2" VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" MODEL = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" [[jobs]] -name = "qwen30-vllm020-flashattn-full-tp4-20260716-v1" +name = "qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2" gpus = 1 gpu_model = "H20" hosts = ["dash0"] command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1320 bash run_flashattn_full.sh" -artifacts = ["artifacts/flashattn-full-tp4"] +artifacts = ["artifacts/flashattn-kv-full-v2-tp4"] [jobs.env] HOME = "/tmp/wjh" XDG_CACHE_HOME = "/tmp/wjh/.cache" VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" TP = "4" -OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/flashattn-full-tp4" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/flashattn-kv-full-v2-tp4" VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" MODEL = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml index 16be440..c811b28 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml @@ -1,7 +1,7 @@ version = 1 [[jobs]] -name = "qwen30-vllm020-flashattn-smoke-20260716-v2-cu129" +name = "qwen30-vllm020-flashattn-kv-smoke-20260716-v3-cu129" gpus = 1 gpu_model = "H20" hosts = ["dash0"] diff --git a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_flashattn.py b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_flashattn.py index 9a37c12..0e25d35 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_flashattn.py +++ b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_flashattn.py @@ -10,6 +10,7 @@ from __future__ import annotations import argparse import json +import statistics import subprocess import sys import types @@ -37,6 +38,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--warmup-iters", type=int, default=3) parser.add_argument("--repeats", type=int, default=5) parser.add_argument("--device", default="cuda:0") + parser.add_argument("--profile-kv-update", action="store_true") return parser.parse_args() @@ -68,6 +70,7 @@ def main() -> None: bench_dir = args.vllm_source / "benchmarks" / "attention_benchmarks" sys.path.insert(0, str(bench_dir)) import runner # type: ignore[import-not-found] # noqa: PLC0415 + from batch_spec import parse_batch_spec # type: ignore[import-not-found] # noqa: PLC0415 from common import BenchmarkConfig # type: ignore[import-not-found] # noqa: PLC0415 from vllm.config import ( # noqa: PLC0415 CacheConfig, @@ -78,7 +81,13 @@ def main() -> None: ParallelConfig, SchedulerConfig, VllmConfig, + set_current_vllm_config, ) + from vllm.v1.attention.backends.utils import ( # noqa: PLC0415 + get_kv_cache_layout, + set_kv_cache_layout, + ) + from vllm.v1.kv_cache_interface import FullAttentionSpec # noqa: PLC0415 from vllm.v1.worker.workspace import init_workspace_manager # noqa: PLC0415 def create_vllm_config(config: BenchmarkConfig, max_num_blocks: int) -> VllmConfig: @@ -138,6 +147,77 @@ def main() -> None: runner._create_vllm_config = create_vllm_config init_workspace_manager(args.device) + def profile_kv_cache_update(config: BenchmarkConfig) -> dict[str, float]: + device = torch.device(config.device) + requests = parse_batch_spec(config.batch_spec) + total_q = sum(request.q_len for request in requests) + max_kv = max(request.kv_len for request in requests) + max_blocks_per_request = (max_kv + config.block_size - 1) // config.block_size + max_num_blocks = len(requests) * max_blocks_per_request + vllm_config = create_vllm_config(config, max_num_blocks) + dtype = vllm_config.model_config.dtype + with set_current_vllm_config(vllm_config): + backend_config = runner._get_backend_config(config.backend) + backend_class, impl, layer = runner._create_backend_impl( + backend_config, config, device, dtype + ) + required_layout = backend_class.get_required_kv_cache_layout() + if required_layout is not None: + set_kv_cache_layout(required_layout) + get_kv_cache_layout.cache_clear() + common_metadata = runner._build_common_attn_metadata( + [request.q_len for request in requests], + [request.kv_len for request in requests], + config.block_size, + device, + ) + kv_cache_spec = FullAttentionSpec( + block_size=config.block_size, + num_kv_heads=config.num_kv_heads, + head_size=config.head_dim, + dtype=dtype, + ) + layer._kv_cache_spec = kv_cache_spec + _, key_list, value_list = runner._create_input_tensors( + config, total_q, device, dtype + ) + cache_list = runner._create_kv_cache( + config, max_num_blocks, backend_class, device, dtype + ) + for _ in range(config.warmup_iters): + for layer_index in range(config.num_layers): + impl.do_kv_cache_update( + layer, + key_list[layer_index], + value_list[layer_index], + cache_list[layer_index], + common_metadata.slot_mapping, + ) + torch.accelerator.synchronize() + samples: list[float] = [] + for _ in range(config.repeats): + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + for layer_index in range(config.num_layers): + impl.do_kv_cache_update( + layer, + key_list[layer_index], + value_list[layer_index], + cache_list[layer_index], + common_metadata.slot_mapping, + ) + end.record() + torch.accelerator.synchronize() + samples.append(float(start.elapsed_time(end)) / config.num_layers) + return { + "min_ms": min(samples), + "max_ms": max(samples), + "mean_ms": statistics.fmean(samples), + "median_ms": statistics.median(samples), + "std_ms": statistics.pstdev(samples), + } + rows: list[dict[str, object]] = [] for tp in args.tp: for batch_spec in args.batch_specs: @@ -160,6 +240,9 @@ def main() -> None: result = runner.run_attention_benchmark(config) row = result.to_dict() row["tensor_parallel_size"] = tp + row["attention_core_excludes_kv_cache_update"] = True + if args.profile_kv_update: + row["kv_cache_update_time"] = profile_kv_cache_update(config) rows.append(row) print( json.dumps( @@ -189,6 +272,7 @@ def main() -> None: "dtype": "bfloat16", "attention_backend": "FLASH_ATTN", "block_size": 16, + "profile_kv_update": args.profile_kv_update, }, "rows": rows, } diff --git a/runs/frontier-qwen30-vllm020-profile-v1/run_flashattn_full.sh b/runs/frontier-qwen30-vllm020-profile-v1/run_flashattn_full.sh index 7188969..5d84abc 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/run_flashattn_full.sh +++ b/runs/frontier-qwen30-vllm020-profile-v1/run_flashattn_full.sh @@ -51,7 +51,7 @@ if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then exit 1 fi -echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=${MODEL} runtime=vLLM-0.20.0+cu129 operator=FlashAttention3 tp=${TP} cases=${#BATCH_SPECS[@]} regimes=prefill,prefix_extend,decode,true_mixed dtype=BF16 block=16 output=${OUTPUT_ROOT} expected_wall=4-10m hard_wall=1200s hard_gpu_cap=0.33_H20h" +echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=${MODEL} runtime=vLLM-0.20.0+cu129 operators=FlashAttention3,reshape_and_cache_flash tp=${TP} cases=${#BATCH_SPECS[@]} regimes=prefill,prefix_extend,decode,true_mixed dtype=BF16 block=16 output=${OUTPUT_ROOT} expected_wall=4-10m hard_wall=1200s hard_gpu_cap=0.33_H20h" date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader @@ -78,7 +78,8 @@ timeout --signal=TERM --kill-after=30s 1080 \ --tp "${TP}" \ --batch-specs "${BATCH_SPECS[@]}" \ --warmup-iters 5 \ - --repeats 10 + --repeats 10 \ + --profile-kv-update test -s "${OUTPUT_ROOT}/raw/flashattn-tp${TP}.json" sha256sum "${OUTPUT_ROOT}/raw/flashattn-tp${TP}.json" \ diff --git a/runs/frontier-qwen30-vllm020-profile-v1/run_flashattn_smoke.sh b/runs/frontier-qwen30-vllm020-profile-v1/run_flashattn_smoke.sh index 04318bb..327a261 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/run_flashattn_smoke.sh +++ b/runs/frontier-qwen30-vllm020-profile-v1/run_flashattn_smoke.sh @@ -22,7 +22,7 @@ if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then exit 1 fi -echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=${MODEL} runtime=vLLM-0.20.0+cu129 operator=FlashAttention3 tp_local_shapes=1,2,4 specs=q128,4q1s128,q128_4q1s128 dtype=BF16 block=16 output=${OUTPUT_ROOT} expected_wall=3-8m hard_wall=900s hard_gpu_cap=0.15_H20h" +echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=${MODEL} runtime=vLLM-0.20.0+cu129 operators=FlashAttention3,reshape_and_cache_flash tp_local_shapes=1,2,4 specs=q128,4q1s128,q128_4q1s128 dtype=BF16 block=16 output=${OUTPUT_ROOT} expected_wall=3-8m hard_wall=900s hard_gpu_cap=0.15_H20h" date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader @@ -48,7 +48,8 @@ timeout --signal=TERM --kill-after=30s 780 \ --tp 1 2 4 \ --batch-specs q128 4q1s128 q128_4q1s128 \ --warmup-iters 3 \ - --repeats 5 + --repeats 5 \ + --profile-kv-update test -s "${OUTPUT_ROOT}/raw/flashattn-smoke.json" sha256sum "${OUTPUT_ROOT}/raw/flashattn-smoke.json" \ From 9bbfb87a85993103504d41aea0415a0e44b04140 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 21:40:07 +0800 Subject: [PATCH 077/148] Keep KV smoke artifacts separate --- runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml index c811b28..650fc76 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_smoke.toml @@ -6,13 +6,13 @@ gpus = 1 gpu_model = "H20" hosts = ["dash0"] command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 900 bash run_flashattn_smoke.sh" -artifacts = ["artifacts/flashattn-smoke"] +artifacts = ["artifacts/flashattn-kv-smoke-v3"] [jobs.env] HOME = "/tmp/wjh" XDG_CACHE_HOME = "/tmp/wjh/.cache" VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" -OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/flashattn-smoke" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/flashattn-kv-smoke-v3" VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" MODEL = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" From 4fed4329cfe9383e294719886600cbaf5f3a1212 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 21:41:38 +0800 Subject: [PATCH 078/148] Profile MoE TP-local shards without collectives --- .../jobs_moe_smoke.toml | 6 +++--- .../profile_vllm020_moe.py | 10 +++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_moe_smoke.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_moe_smoke.toml index 48cd8db..f8403b3 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/jobs_moe_smoke.toml +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_moe_smoke.toml @@ -1,18 +1,18 @@ version = 1 [[jobs]] -name = "qwen30-vllm020-moe-smoke-20260716-v1" +name = "qwen30-vllm020-moe-smoke-20260716-v2-local-shard" gpus = 1 gpu_model = "H20" hosts = ["dash0"] command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1020 bash run_moe_smoke.sh" -artifacts = ["artifacts/moe-smoke"] +artifacts = ["artifacts/moe-smoke-v2"] [jobs.env] HOME = "/tmp/wjh" XDG_CACHE_HOME = "/tmp/wjh/.cache" VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" -OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/moe-smoke" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/moe-smoke-v2" VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" MODEL = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_moe.py b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_moe.py index 927b316..74da2d1 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_moe.py +++ b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_moe.py @@ -199,8 +199,12 @@ def main() -> None: in_dtype=torch.bfloat16, max_num_tokens=max_num_tokens, ) + # This process profiles one TP-local weight shard. Keep the global + # runtime context single-rank so vLLM does not initialize a collective; + # the action-conditioned shard size remains explicit in moe_config and + # the real TP2/TP4 all-reduce is profiled in a separate multi-GPU run. vllm_config = VllmConfig( - parallel_config=ParallelConfig(tensor_parallel_size=tp) + parallel_config=ParallelConfig(tensor_parallel_size=1) ) with set_current_vllm_config(vllm_config): backend, experts_cls = select_unquantized_moe_backend(moe_config) @@ -348,8 +352,8 @@ def main() -> None: "norm_topk_prob": True, }, "measurement_scope": ( - "vLLM modular MoE prepare+FlashInfer CUTLASS experts+finalize; " - "router linear/top-k and TP all-reduce excluded" + "one TP-local weight shard: vLLM modular MoE prepare+FlashInfer " + "CUTLASS experts+finalize; router linear/top-k and TP all-reduce excluded" ), "rows": rows, } From 1fb777fc113001bffe39e26967881834ee122795 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 21:48:52 +0800 Subject: [PATCH 079/148] Set vLLM config for collective profiling --- .../profile_vllm020_allreduce.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_allreduce.py b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_allreduce.py index 8665b9c..33fc9e8 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_allreduce.py +++ b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_allreduce.py @@ -69,6 +69,7 @@ def main() -> None: tensor_model_parallel_all_reduce, ) from vllm.distributed.parallel_state import get_tp_group + from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config rank = int(os.environ["RANK"]) local_rank = int(os.environ["LOCAL_RANK"]) @@ -79,7 +80,11 @@ def main() -> None: torch.accelerator.set_device_index(device) torch.set_default_device(device) init_distributed_environment() - initialize_model_parallel(tensor_model_parallel_size=world_size) + vllm_config = VllmConfig( + parallel_config=ParallelConfig(tensor_parallel_size=world_size) + ) + with set_current_vllm_config(vllm_config): + initialize_model_parallel(tensor_model_parallel_size=world_size) rows: list[dict[str, Any]] = [] expected_sum = world_size * (world_size + 1) / 2 From 4bf9bdf28fc61f9bc588be9d6329b8e04e6d02bf Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 21:52:43 +0800 Subject: [PATCH 080/148] Add full Qwen30 MoE profiling matrix --- .../jobs_allreduce_smoke_tp2.toml | 6 +-- .../jobs_moe_full.toml | 18 +++++++ .../run_moe_full.sh | 51 +++++++++++++++++++ 3 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/jobs_moe_full.toml create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/run_moe_full.sh diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_allreduce_smoke_tp2.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_allreduce_smoke_tp2.toml index 802066e..7a9d10f 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/jobs_allreduce_smoke_tp2.toml +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_allreduce_smoke_tp2.toml @@ -1,12 +1,12 @@ version = 1 [[jobs]] -name = "qwen30-vllm020-flashinfer-allreduce-smoke-tp2-20260716-v1" +name = "qwen30-vllm020-flashinfer-allreduce-smoke-tp2-20260716-v2-config-context" gpus = 2 gpu_model = "H20" hosts = ["dash0"] command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 840 bash run_allreduce_profile.sh" -artifacts = ["artifacts/allreduce-smoke-tp2"] +artifacts = ["artifacts/allreduce-smoke-tp2-v2"] [jobs.env] HOME = "/tmp/wjh" @@ -14,6 +14,6 @@ XDG_CACHE_HOME = "/tmp/wjh/.cache" VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" TP = "2" NUM_TOKENS = "8" -OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/allreduce-smoke-tp2" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/allreduce-smoke-tp2-v2" VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_moe_full.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_moe_full.toml new file mode 100644 index 0000000..aaeae66 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_moe_full.toml @@ -0,0 +1,18 @@ +version = 1 + +[[jobs]] +name = "qwen30-vllm020-moe-full-20260716-v1-local-shard" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1620 bash run_moe_full.sh" +artifacts = ["artifacts/moe-full-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/moe-full-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" +MODEL = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/run_moe_full.sh b/runs/frontier-qwen30-vllm020-profile-v1/run_moe_full.sh new file mode 100644 index 0000000..374a78b --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/run_moe_full.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -euo pipefail + +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +MODEL="${MODEL:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +TOKENS=(1 8 16 32 64 128 256 512 1024 2048 4096 8192) +mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/raw" +exec > >(tee -a "${OUTPUT_ROOT}/logs/full.log") 2>&1 + +IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?fleet GPU is required}" +if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then + echo "ERROR: expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2 + exit 1 +fi + +echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=${MODEL} runtime=vLLM-0.20.0+cu129 operator=FlashInfer-CUTLASS-unquantized-MoE tp_local_shapes=1,2,4 tokens=${TOKENS[*]} routing=uniform_random_logits,hotset8 reference_check=on dtype=BF16 output=${OUTPUT_ROOT} expected_wall=4-12m hard_wall=1500s hard_gpu_cap=0.42_H20h" +date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" +nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader + +test -x "${VENV_ROOT}/bin/python" +test -f "${MODEL}/config.json" +git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit" +git -C "${VLLM_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/vllm-source.commit" +sha256sum profile_vllm020_moe.py run_moe_full.sh \ + > "${OUTPUT_ROOT}/provenance/source.sha256" +uv pip freeze --python "${VENV_ROOT}/bin/python" \ + > "${OUTPUT_ROOT}/provenance/pip-freeze.txt" +nvidia-smi --query-gpu=index,uuid,name,driver_version,memory.total \ + --format=csv,noheader > "${OUTPUT_ROOT}/provenance/gpus.csv" +printf '%s\n' "${TOKENS[@]}" > "${OUTPUT_ROOT}/provenance/tokens.txt" + +timeout --signal=TERM --kill-after=30s 1380 \ + "${VENV_ROOT}/bin/python" profile_vllm020_moe.py \ + --vllm-source "${VLLM_SOURCE}" \ + --model "${MODEL}" \ + --output "${OUTPUT_ROOT}/raw/moe-full.json" \ + --tp 1 2 4 \ + --num-tokens "${TOKENS[@]}" \ + --routing-modes uniform_random_logits hotset8 \ + --warmup-iters 5 \ + --repeats 10 \ + --check-reference + +test -s "${OUTPUT_ROOT}/raw/moe-full.json" +sha256sum "${OUTPUT_ROOT}/raw/moe-full.json" "${OUTPUT_ROOT}/provenance"/* \ + > "${OUTPUT_ROOT}/artifacts.sha256" +date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ" +echo "MOE_FULL_COMPLETE rows=72" From 3121e35b0ed15ba5017bd611e8466995e0eaf91e Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 21:57:44 +0800 Subject: [PATCH 081/148] Profile collective dispatch across tensor sizes --- .../jobs_allreduce_full_tp2.toml | 19 +++++++++++++++++++ .../jobs_allreduce_full_tp4.toml | 19 +++++++++++++++++++ .../profile_vllm020_allreduce.py | 9 ++++----- 3 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/jobs_allreduce_full_tp2.toml create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/jobs_allreduce_full_tp4.toml diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_allreduce_full_tp2.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_allreduce_full_tp2.toml new file mode 100644 index 0000000..9debff3 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_allreduce_full_tp2.toml @@ -0,0 +1,19 @@ +version = 1 + +[[jobs]] +name = "qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware" +gpus = 2 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1020 bash run_allreduce_profile.sh" +artifacts = ["artifacts/allreduce-full-tp2-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "2" +NUM_TOKENS = "1 8 16 32 64 128 256 512 1024 2048 4096 8192" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/allreduce-full-tp2-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_allreduce_full_tp4.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_allreduce_full_tp4.toml new file mode 100644 index 0000000..28194da --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_allreduce_full_tp4.toml @@ -0,0 +1,19 @@ +version = 1 + +[[jobs]] +name = "qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware" +gpus = 4 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1020 bash run_allreduce_profile.sh" +artifacts = ["artifacts/allreduce-full-tp4-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "4" +NUM_TOKENS = "1 8 16 32 64 128 256 512 1024 2048 4096 8192" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/allreduce-full-tp4-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_allreduce.py b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_allreduce.py index 33fc9e8..c37ef26 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_allreduce.py +++ b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_allreduce.py @@ -113,10 +113,7 @@ def main() -> None: f"FlashInfer all-reduce was not selected at TP={world_size}, " f"tokens={num_tokens}" ) - if not fi_comm.should_use_fi_ar(input_tensor): - raise SystemExit( - f"FlashInfer rejected serving tensor shape {tuple(input_tensor.shape)}" - ) + uses_flashinfer = fi_comm.should_use_fi_ar(input_tensor) samples: list[float] = [] for _ in range(args.repeats): @@ -142,7 +139,9 @@ def main() -> None: * torch.tensor([], dtype=torch.bfloat16).element_size(), "dtype": "bfloat16", "communicator": "vllm.tensor_model_parallel_all_reduce", - "selected_backend": "flashinfer_trtllm", + "selected_backend": ( + "flashinfer_trtllm" if uses_flashinfer else "nccl_fallback" + ), "per_rank_time_ms": per_rank, "critical_path_median_ms": max( rank_stats["median"] for rank_stats in per_rank From 32ab58478996b7b143039a390be3246b5640bcc0 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 21:59:58 +0800 Subject: [PATCH 082/148] Initialize collective profiler with model config --- .../jobs_allreduce_smoke_tp2.toml | 6 +++--- .../profile_vllm020_allreduce.py | 17 ++++++++++++++++- .../run_allreduce_profile.sh | 5 ++++- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_allreduce_smoke_tp2.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_allreduce_smoke_tp2.toml index 7a9d10f..72721af 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/jobs_allreduce_smoke_tp2.toml +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_allreduce_smoke_tp2.toml @@ -1,12 +1,12 @@ version = 1 [[jobs]] -name = "qwen30-vllm020-flashinfer-allreduce-smoke-tp2-20260716-v2-config-context" +name = "qwen30-vllm020-flashinfer-allreduce-smoke-tp2-20260716-v3-model-context" gpus = 2 gpu_model = "H20" hosts = ["dash0"] command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 840 bash run_allreduce_profile.sh" -artifacts = ["artifacts/allreduce-smoke-tp2-v2"] +artifacts = ["artifacts/allreduce-smoke-tp2-v3"] [jobs.env] HOME = "/tmp/wjh" @@ -14,6 +14,6 @@ XDG_CACHE_HOME = "/tmp/wjh/.cache" VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" TP = "2" NUM_TOKENS = "8" -OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/allreduce-smoke-tp2-v2" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/allreduce-smoke-tp2-v3" VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_allreduce.py b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_allreduce.py index c37ef26..09b3752 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_allreduce.py +++ b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_allreduce.py @@ -23,6 +23,7 @@ VLLM_COMMIT = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1" def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--vllm-source", type=Path, required=True) + parser.add_argument("--model", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--num-tokens", type=int, nargs="+", default=[8]) parser.add_argument("--hidden-dim", type=int, default=2048) @@ -69,7 +70,12 @@ def main() -> None: tensor_model_parallel_all_reduce, ) from vllm.distributed.parallel_state import get_tp_group - from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config + from vllm.config import ( + ModelConfig, + ParallelConfig, + VllmConfig, + set_current_vllm_config, + ) rank = int(os.environ["RANK"]) local_rank = int(os.environ["LOCAL_RANK"]) @@ -80,7 +86,15 @@ def main() -> None: torch.accelerator.set_device_index(device) torch.set_default_device(device) init_distributed_environment() + model_config = ModelConfig( + model=str(args.model), + dtype="bfloat16", + max_model_len=8192, + skip_tokenizer_init=True, + generation_config="vllm", + ) vllm_config = VllmConfig( + model_config=model_config, parallel_config=ParallelConfig(tensor_parallel_size=world_size) ) with set_current_vllm_config(vllm_config): @@ -162,6 +176,7 @@ def main() -> None: "torch_version": torch.__version__, "torch_cuda": torch.version.cuda, "gpu": torch.cuda.get_device_name(device), + "model": str(args.model), "backend_env": { "VLLM_ALLREDUCE_USE_FLASHINFER": "1", "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm", diff --git a/runs/frontier-qwen30-vllm020-profile-v1/run_allreduce_profile.sh b/runs/frontier-qwen30-vllm020-profile-v1/run_allreduce_profile.sh index 6321a8d..0cf4565 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/run_allreduce_profile.sh +++ b/runs/frontier-qwen30-vllm020-profile-v1/run_allreduce_profile.sh @@ -11,6 +11,7 @@ esac OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +MODEL="${MODEL:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" NUM_TOKENS="${NUM_TOKENS:-8}" mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/raw" exec > >(tee -a "${OUTPUT_ROOT}/logs/profile.log") 2>&1 @@ -23,12 +24,13 @@ fi export VLLM_ALLREDUCE_USE_FLASHINFER=1 export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm -echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} runtime=vLLM-0.20.0+cu129 operator=tensor_model_parallel_all_reduce backend=FlashInfer-TRTLLM tp=${TP} tokens=${NUM_TOKENS} hidden=2048 dtype=BF16 output=${OUTPUT_ROOT} expected_wall=2-6m hard_wall=720s hard_gpu_cap=${HARD_GPU_CAP}" +echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL} runtime=vLLM-0.20.0+cu129 operator=tensor_model_parallel_all_reduce backend=FlashInfer-TRTLLM tp=${TP} tokens=${NUM_TOKENS} hidden=2048 dtype=BF16 output=${OUTPUT_ROOT} expected_wall=2-6m hard_wall=720s hard_gpu_cap=${HARD_GPU_CAP}" date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit" git -C "${VLLM_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/vllm-source.commit" +sha256sum "${MODEL}/config.json" > "${OUTPUT_ROOT}/provenance/model-config.sha256" sha256sum profile_vllm020_allreduce.py run_allreduce_profile.sh \ > "${OUTPUT_ROOT}/provenance/source.sha256" uv pip freeze --python "${VENV_ROOT}/bin/python" \ @@ -41,6 +43,7 @@ timeout --signal=TERM --kill-after=30s 600 \ "${VENV_ROOT}/bin/torchrun" --standalone --nproc_per_node="${TP}" \ profile_vllm020_allreduce.py \ --vllm-source "${VLLM_SOURCE}" \ + --model "${MODEL}" \ --output "${OUTPUT_ROOT}/raw/allreduce-tp${TP}.json" \ --num-tokens "${TOKEN_ARGS[@]}" \ --warmup-iters 3 \ From 48d2c2fb80bb655f2279d1efc133067d82edb620 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 22:11:04 +0800 Subject: [PATCH 083/148] Add Frontier linear profiler smoke --- .../jobs_frontier_linear_smoke.toml | 18 ++++++ .../run_frontier_linear_smoke.sh | 55 +++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_smoke.toml create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_smoke.sh diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_smoke.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_smoke.toml new file mode 100644 index 0000000..9f1eed8 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_smoke.toml @@ -0,0 +1,18 @@ +version = 1 + +[[jobs]] +name = "qwen30-vllm020-frontier-linear-smoke-20260716-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1020 bash run_frontier_linear_smoke.sh" +artifacts = ["artifacts/frontier-linear-smoke-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/frontier-linear-smoke-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +FRONTIER_ROOT = "/home/admin/cpfs/wjh/frontier-qwen30-vllm020-profile-v1/Frontier" +VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_smoke.sh b/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_smoke.sh new file mode 100644 index 0000000..1d9232e --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_smoke.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +set -euo pipefail + +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +FRONTIER_ROOT="${FRONTIER_ROOT:-/home/admin/cpfs/wjh/frontier-qwen30-vllm020-profile-v1/Frontier}" +VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/profiles" +exec > >(tee -a "${OUTPUT_ROOT}/logs/smoke.log") 2>&1 + +IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?fleet GPU is required}" +if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then + echo "ERROR: expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2 + exit 1 +fi + +echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=qwen3-a3b-30b-moe runtime=vLLM-0.20.0+cu129 profiler=Frontier@d9cfeb6 operator=linear,norm,rope,embedding tp=1,2,4 tokens=8 precision=BF16 method=cuda_event output=${OUTPUT_ROOT} expected_wall=2-6m hard_wall=900s hard_gpu_cap=0.25_H20h" +date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" +nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader + +test "$(git -C "${FRONTIER_ROOT}" rev-parse HEAD)" = "d9cfeb6d8791fbf2f295dd9744c56a666171776e" +test "$(git -C "${VLLM_SOURCE}" rev-parse HEAD)" = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1" +git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit" +git -C "${FRONTIER_ROOT}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit" +git -C "${VLLM_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/vllm-source.commit" +sha256sum run_frontier_linear_smoke.sh \ + > "${OUTPUT_ROOT}/provenance/source.sha256" +uv pip freeze --python "${VENV_ROOT}/bin/python" \ + > "${OUTPUT_ROOT}/provenance/pip-freeze.txt" +nvidia-smi --query-gpu=index,uuid,name,driver_version,memory.total \ + --format=csv,noheader > "${OUTPUT_ROOT}/provenance/gpus.csv" + +cd "${FRONTIER_ROOT}" +timeout --signal=TERM --kill-after=30s 780 \ + "${VENV_ROOT}/bin/python" -m frontier.profiling.linear_op.main \ + --disable_ray \ + --num_gpus 1 \ + --output_dir "${OUTPUT_ROOT}/profiles" \ + --device h20 \ + --models qwen3-a3b-30b-moe \ + --num_tensor_parallel_workers 1 2 4 \ + --num_tokens_list 8 \ + --profile_method cuda_event \ + --precision BF16 \ + --is_moe \ + --yes + +find "${OUTPUT_ROOT}/profiles" -type f -size +0c -print \ + > "${OUTPUT_ROOT}/provenance/profile-files.txt" +test -s "${OUTPUT_ROOT}/provenance/profile-files.txt" +find "${OUTPUT_ROOT}/profiles" -type f -print0 | sort -z | xargs -0 sha256sum \ + > "${OUTPUT_ROOT}/profiles.sha256" +date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ" +echo "FRONTIER_LINEAR_SMOKE_COMPLETE" From 0e9b64893f0f0cfbf4e1523478394dd5fc138fc4 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 22:18:30 +0800 Subject: [PATCH 084/148] Add full Frontier linear profile sweep --- .../jobs_frontier_linear_full.toml | 18 ++++++ .../run_frontier_linear_full.sh | 57 +++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_full.toml create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_full.sh diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_full.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_full.toml new file mode 100644 index 0000000..59feeda --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_full.toml @@ -0,0 +1,18 @@ +version = 1 + +[[jobs]] +name = "qwen30-vllm020-frontier-linear-full-20260716-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1620 bash run_frontier_linear_full.sh" +artifacts = ["artifacts/frontier-linear-full-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/frontier-linear-full-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +FRONTIER_ROOT = "/home/admin/cpfs/wjh/frontier-qwen30-vllm020-profile-v1/Frontier" +VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_full.sh b/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_full.sh new file mode 100644 index 0000000..aa8b75f --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_full.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +set -euo pipefail + +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +FRONTIER_ROOT="${FRONTIER_ROOT:-/home/admin/cpfs/wjh/frontier-qwen30-vllm020-profile-v1/Frontier}" +VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +TOKENS=(1 8 16 32 64 128 256 512 1024 2048 4096 8192) +mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/profiles" +exec > >(tee -a "${OUTPUT_ROOT}/logs/full.log") 2>&1 + +IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?fleet GPU is required}" +if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then + echo "ERROR: expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2 + exit 1 +fi + +echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=qwen3-a3b-30b-moe runtime=vLLM-0.20.0+cu129 profiler=Frontier@d9cfeb6 operator=linear,norm,rope,embedding tp=1,2,4 tokens=${TOKENS[*]} precision=BF16 method=cuda_event output=${OUTPUT_ROOT} expected_wall=3-10m hard_wall=1500s hard_gpu_cap=0.42_H20h" +date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" +nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader + +test "$(git -C "${FRONTIER_ROOT}" rev-parse HEAD)" = "d9cfeb6d8791fbf2f295dd9744c56a666171776e" +test "$(git -C "${VLLM_SOURCE}" rev-parse HEAD)" = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1" +git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit" +git -C "${FRONTIER_ROOT}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit" +git -C "${VLLM_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/vllm-source.commit" +sha256sum run_frontier_linear_full.sh \ + > "${OUTPUT_ROOT}/provenance/source.sha256" +uv pip freeze --python "${VENV_ROOT}/bin/python" \ + > "${OUTPUT_ROOT}/provenance/pip-freeze.txt" +nvidia-smi --query-gpu=index,uuid,name,driver_version,memory.total \ + --format=csv,noheader > "${OUTPUT_ROOT}/provenance/gpus.csv" +printf '%s\n' "${TOKENS[@]}" > "${OUTPUT_ROOT}/provenance/tokens.txt" + +cd "${FRONTIER_ROOT}" +timeout --signal=TERM --kill-after=30s 1380 \ + "${VENV_ROOT}/bin/python" -m frontier.profiling.linear_op.main \ + --disable_ray \ + --num_gpus 1 \ + --output_dir "${OUTPUT_ROOT}/profiles" \ + --device h20 \ + --models qwen3-a3b-30b-moe \ + --num_tensor_parallel_workers 1 2 4 \ + --num_tokens_list "${TOKENS[@]}" \ + --profile_method cuda_event \ + --precision BF16 \ + --is_moe \ + --yes + +find "${OUTPUT_ROOT}/profiles" -type f -size +0c -print \ + > "${OUTPUT_ROOT}/provenance/profile-files.txt" +test -s "${OUTPUT_ROOT}/provenance/profile-files.txt" +find "${OUTPUT_ROOT}/profiles" -type f -print0 | sort -z | xargs -0 sha256sum \ + > "${OUTPUT_ROOT}/profiles.sha256" +date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ" +echo "FRONTIER_LINEAR_FULL_COMPLETE" From 9b54075e75624ff9f7dc13ecf10a544a6bdab04d Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 22:23:56 +0800 Subject: [PATCH 085/148] Adapt Frontier RoPE profiling to vLLM 0.20 --- .../frontier_vllm020_compat.py | 59 +++++++++++++++++++ .../jobs_frontier_linear_smoke.toml | 6 +- .../run_frontier_linear_full.sh | 5 +- .../run_frontier_linear_smoke.sh | 5 +- 4 files changed, 68 insertions(+), 7 deletions(-) create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/frontier_vllm020_compat.py diff --git a/runs/frontier-qwen30-vllm020-profile-v1/frontier_vllm020_compat.py b/runs/frontier-qwen30-vllm020-profile-v1/frontier_vllm020_compat.py new file mode 100644 index 0000000..b1d8403 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/frontier_vllm020_compat.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Run Frontier's linear profiler against the vLLM 0.20 RoPE API. + +Frontier passes the pre-v0.20 ``get_rope`` arguments separately, while vLLM +0.20 carries the same values in ``rope_parameters``. Keep both repositories +unchanged and adapt only that call boundary for the profiling experiment. +""" + +from __future__ import annotations + +import inspect +from typing import Any + +import torch +from vllm.model_executor.layers.rotary_embedding import get_rope as vllm_get_rope + +from frontier.profiling.common.layers import rotary_embedding as frontier_rope + + +def _vllm020_get_rope_adapter( + *, + head_size: int, + rotary_dim: int, + max_position: int, + base: int | float, + is_neox_style: bool, + rope_scaling: dict[str, Any] | None, + dtype: torch.dtype | None = None, +) -> Any: + rope_parameters = dict(rope_scaling or {}) + rope_parameters["rope_theta"] = base + rope_parameters["rope_dim"] = rotary_dim + return vllm_get_rope( + head_size=head_size, + max_position=max_position, + is_neox_style=is_neox_style, + rope_parameters=rope_parameters, + dtype=dtype, + ) + + +def main() -> None: + parameters = inspect.signature(vllm_get_rope).parameters + if "rope_parameters" not in parameters or "rotary_dim" in parameters: + raise RuntimeError( + "Expected the vLLM 0.20 get_rope API with rope_parameters; " + f"found {inspect.signature(vllm_get_rope)}" + ) + + frontier_rope._VLLM_GET_ROPE = _vllm020_get_rope_adapter + frontier_rope._VLLM_GET_ROPE_IMPORT_ERROR = None + + from frontier.profiling.linear_op.main import main as frontier_main + + frontier_main() + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_smoke.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_smoke.toml index 9f1eed8..6b40ea3 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_smoke.toml +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_smoke.toml @@ -1,18 +1,18 @@ version = 1 [[jobs]] -name = "qwen30-vllm020-frontier-linear-smoke-20260716-v1" +name = "qwen30-vllm020-frontier-linear-smoke-20260716-v2-rope-api" gpus = 1 gpu_model = "H20" hosts = ["dash0"] command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1020 bash run_frontier_linear_smoke.sh" -artifacts = ["artifacts/frontier-linear-smoke-v1"] +artifacts = ["artifacts/frontier-linear-smoke-v2"] [jobs.env] HOME = "/tmp/wjh" XDG_CACHE_HOME = "/tmp/wjh/.cache" VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" -OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/frontier-linear-smoke-v1" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/frontier-linear-smoke-v2" VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" FRONTIER_ROOT = "/home/admin/cpfs/wjh/frontier-qwen30-vllm020-profile-v1/Frontier" VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_full.sh b/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_full.sh index aa8b75f..60ab82a 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_full.sh +++ b/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_full.sh @@ -6,6 +6,7 @@ OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" FRONTIER_ROOT="${FRONTIER_ROOT:-/home/admin/cpfs/wjh/frontier-qwen30-vllm020-profile-v1/Frontier}" VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +RUN_DIR="$(pwd -P)" TOKENS=(1 8 16 32 64 128 256 512 1024 2048 4096 8192) mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/profiles" exec > >(tee -a "${OUTPUT_ROOT}/logs/full.log") 2>&1 @@ -25,7 +26,7 @@ test "$(git -C "${VLLM_SOURCE}" rev-parse HEAD)" = "88d34c6409e9fb3c7b8ca0c04756 git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit" git -C "${FRONTIER_ROOT}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit" git -C "${VLLM_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/vllm-source.commit" -sha256sum run_frontier_linear_full.sh \ +sha256sum run_frontier_linear_full.sh frontier_vllm020_compat.py \ > "${OUTPUT_ROOT}/provenance/source.sha256" uv pip freeze --python "${VENV_ROOT}/bin/python" \ > "${OUTPUT_ROOT}/provenance/pip-freeze.txt" @@ -35,7 +36,7 @@ printf '%s\n' "${TOKENS[@]}" > "${OUTPUT_ROOT}/provenance/tokens.txt" cd "${FRONTIER_ROOT}" timeout --signal=TERM --kill-after=30s 1380 \ - "${VENV_ROOT}/bin/python" -m frontier.profiling.linear_op.main \ + "${VENV_ROOT}/bin/python" "${RUN_DIR}/frontier_vllm020_compat.py" \ --disable_ray \ --num_gpus 1 \ --output_dir "${OUTPUT_ROOT}/profiles" \ diff --git a/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_smoke.sh b/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_smoke.sh index 1d9232e..59a481f 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_smoke.sh +++ b/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_smoke.sh @@ -6,6 +6,7 @@ OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" FRONTIER_ROOT="${FRONTIER_ROOT:-/home/admin/cpfs/wjh/frontier-qwen30-vllm020-profile-v1/Frontier}" VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +RUN_DIR="$(pwd -P)" mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/profiles" exec > >(tee -a "${OUTPUT_ROOT}/logs/smoke.log") 2>&1 @@ -24,7 +25,7 @@ test "$(git -C "${VLLM_SOURCE}" rev-parse HEAD)" = "88d34c6409e9fb3c7b8ca0c04756 git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit" git -C "${FRONTIER_ROOT}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit" git -C "${VLLM_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/vllm-source.commit" -sha256sum run_frontier_linear_smoke.sh \ +sha256sum run_frontier_linear_smoke.sh frontier_vllm020_compat.py \ > "${OUTPUT_ROOT}/provenance/source.sha256" uv pip freeze --python "${VENV_ROOT}/bin/python" \ > "${OUTPUT_ROOT}/provenance/pip-freeze.txt" @@ -33,7 +34,7 @@ nvidia-smi --query-gpu=index,uuid,name,driver_version,memory.total \ cd "${FRONTIER_ROOT}" timeout --signal=TERM --kill-after=30s 780 \ - "${VENV_ROOT}/bin/python" -m frontier.profiling.linear_op.main \ + "${VENV_ROOT}/bin/python" "${RUN_DIR}/frontier_vllm020_compat.py" \ --disable_ray \ --num_gpus 1 \ --output_dir "${OUTPUT_ROOT}/profiles" \ From 885db8527a3b3f8632bd408c3b981f3944117c36 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 22:29:10 +0800 Subject: [PATCH 086/148] Initialize vLLM config for Frontier profiling --- .../frontier_vllm020_compat.py | 15 ++++++++++++++- .../jobs_frontier_linear_full.toml | 1 + .../jobs_frontier_linear_smoke.toml | 7 ++++--- .../run_frontier_linear_full.sh | 4 ++++ .../run_frontier_linear_smoke.sh | 4 ++++ 5 files changed, 27 insertions(+), 4 deletions(-) diff --git a/runs/frontier-qwen30-vllm020-profile-v1/frontier_vllm020_compat.py b/runs/frontier-qwen30-vllm020-profile-v1/frontier_vllm020_compat.py index b1d8403..9730897 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/frontier_vllm020_compat.py +++ b/runs/frontier-qwen30-vllm020-profile-v1/frontier_vllm020_compat.py @@ -9,6 +9,7 @@ unchanged and adapt only that call boundary for the profiling experiment. from __future__ import annotations import inspect +import os from typing import Any import torch @@ -51,8 +52,20 @@ def main() -> None: frontier_rope._VLLM_GET_ROPE_IMPORT_ERROR = None from frontier.profiling.linear_op.main import main as frontier_main + from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config - frontier_main() + model_root = os.environ.get("MODEL_ROOT") + if not model_root: + raise RuntimeError("MODEL_ROOT must point to the profiled Qwen checkpoint") + model_config = ModelConfig( + model=model_root, + dtype="bfloat16", + max_model_len=8192, + skip_tokenizer_init=True, + generation_config="vllm", + ) + with set_current_vllm_config(VllmConfig(model_config=model_config)): + frontier_main() if __name__ == "__main__": diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_full.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_full.toml index 59feeda..374470e 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_full.toml +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_full.toml @@ -16,3 +16,4 @@ OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" FRONTIER_ROOT = "/home/admin/cpfs/wjh/frontier-qwen30-vllm020-profile-v1/Frontier" VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_smoke.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_smoke.toml index 6b40ea3..4baf066 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_smoke.toml +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_smoke.toml @@ -1,18 +1,19 @@ version = 1 [[jobs]] -name = "qwen30-vllm020-frontier-linear-smoke-20260716-v2-rope-api" +name = "qwen30-vllm020-frontier-linear-smoke-20260716-v3-model-context" gpus = 1 gpu_model = "H20" hosts = ["dash0"] command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1020 bash run_frontier_linear_smoke.sh" -artifacts = ["artifacts/frontier-linear-smoke-v2"] +artifacts = ["artifacts/frontier-linear-smoke-v3"] [jobs.env] HOME = "/tmp/wjh" XDG_CACHE_HOME = "/tmp/wjh/.cache" VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" -OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/frontier-linear-smoke-v2" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/frontier-linear-smoke-v3" VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" FRONTIER_ROOT = "/home/admin/cpfs/wjh/frontier-qwen30-vllm020-profile-v1/Frontier" VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_full.sh b/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_full.sh index 60ab82a..7371f3a 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_full.sh +++ b/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_full.sh @@ -6,6 +6,8 @@ OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" FRONTIER_ROOT="${FRONTIER_ROOT:-/home/admin/cpfs/wjh/frontier-qwen30-vllm020-profile-v1/Frontier}" VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +export MODEL_ROOT RUN_DIR="$(pwd -P)" TOKENS=(1 8 16 32 64 128 256 512 1024 2048 4096 8192) mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/profiles" @@ -23,9 +25,11 @@ nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --f test "$(git -C "${FRONTIER_ROOT}" rev-parse HEAD)" = "d9cfeb6d8791fbf2f295dd9744c56a666171776e" test "$(git -C "${VLLM_SOURCE}" rev-parse HEAD)" = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1" +test -s "${MODEL_ROOT}/config.json" git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit" git -C "${FRONTIER_ROOT}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit" git -C "${VLLM_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/vllm-source.commit" +sha256sum "${MODEL_ROOT}/config.json" > "${OUTPUT_ROOT}/provenance/model-config.sha256" sha256sum run_frontier_linear_full.sh frontier_vllm020_compat.py \ > "${OUTPUT_ROOT}/provenance/source.sha256" uv pip freeze --python "${VENV_ROOT}/bin/python" \ diff --git a/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_smoke.sh b/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_smoke.sh index 59a481f..eaa2f7e 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_smoke.sh +++ b/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_smoke.sh @@ -6,6 +6,8 @@ OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" FRONTIER_ROOT="${FRONTIER_ROOT:-/home/admin/cpfs/wjh/frontier-qwen30-vllm020-profile-v1/Frontier}" VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +export MODEL_ROOT RUN_DIR="$(pwd -P)" mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/profiles" exec > >(tee -a "${OUTPUT_ROOT}/logs/smoke.log") 2>&1 @@ -22,9 +24,11 @@ nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --f test "$(git -C "${FRONTIER_ROOT}" rev-parse HEAD)" = "d9cfeb6d8791fbf2f295dd9744c56a666171776e" test "$(git -C "${VLLM_SOURCE}" rev-parse HEAD)" = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1" +test -s "${MODEL_ROOT}/config.json" git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit" git -C "${FRONTIER_ROOT}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit" git -C "${VLLM_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/vllm-source.commit" +sha256sum "${MODEL_ROOT}/config.json" > "${OUTPUT_ROOT}/provenance/model-config.sha256" sha256sum run_frontier_linear_smoke.sh frontier_vllm020_compat.py \ > "${OUTPUT_ROOT}/provenance/source.sha256" uv pip freeze --python "${VENV_ROOT}/bin/python" \ From b8f5f1dcf3109b5d1bf9ad20697599481b1e1f3c Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 22:34:23 +0800 Subject: [PATCH 087/148] Adapt Frontier RMSNorm profiling to vLLM 0.20 --- .../frontier_vllm020_compat.py | 20 +++++++++++++++++++ .../jobs_frontier_linear_smoke.toml | 6 +++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/runs/frontier-qwen30-vllm020-profile-v1/frontier_vllm020_compat.py b/runs/frontier-qwen30-vllm020-profile-v1/frontier_vllm020_compat.py index 9730897..4366c9c 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/frontier_vllm020_compat.py +++ b/runs/frontier-qwen30-vllm020-profile-v1/frontier_vllm020_compat.py @@ -13,8 +13,14 @@ import os from typing import Any import torch +from vllm import _custom_ops as vllm_ops +from vllm.model_executor.layers.layernorm import ( + GemmaRMSNorm as VllmGemmaRMSNorm, + fused_add_rms_norm as vllm_fused_add_rms_norm, +) from vllm.model_executor.layers.rotary_embedding import get_rope as vllm_get_rope +from frontier.profiling.common.layers import layernorm as frontier_layernorm from frontier.profiling.common.layers import rotary_embedding as frontier_rope @@ -40,6 +46,16 @@ def _vllm020_get_rope_adapter( ) +def _vllm020_rms_norm_adapter( + x: torch.Tensor, + weight: torch.Tensor, + variance_epsilon: float, +) -> torch.Tensor: + output = torch.empty_like(x) + vllm_ops.rms_norm(output, x, weight, variance_epsilon) + return output + + def main() -> None: parameters = inspect.signature(vllm_get_rope).parameters if "rope_parameters" not in parameters or "rotary_dim" in parameters: @@ -50,6 +66,10 @@ def main() -> None: frontier_rope._VLLM_GET_ROPE = _vllm020_get_rope_adapter frontier_rope._VLLM_GET_ROPE_IMPORT_ERROR = None + frontier_layernorm.HAS_VLLM_RMSNORM = True + frontier_layernorm.VllmGemmaRMSNorm = VllmGemmaRMSNorm + frontier_layernorm.vllm_rms_norm = _vllm020_rms_norm_adapter + frontier_layernorm.vllm_fused_add_rms_norm = vllm_fused_add_rms_norm from frontier.profiling.linear_op.main import main as frontier_main from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_smoke.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_smoke.toml index 4baf066..2208aa7 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_smoke.toml +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_smoke.toml @@ -1,18 +1,18 @@ version = 1 [[jobs]] -name = "qwen30-vllm020-frontier-linear-smoke-20260716-v3-model-context" +name = "qwen30-vllm020-frontier-linear-smoke-20260716-v4-rmsnorm-api" gpus = 1 gpu_model = "H20" hosts = ["dash0"] command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1020 bash run_frontier_linear_smoke.sh" -artifacts = ["artifacts/frontier-linear-smoke-v3"] +artifacts = ["artifacts/frontier-linear-smoke-v4"] [jobs.env] HOME = "/tmp/wjh" XDG_CACHE_HOME = "/tmp/wjh/.cache" VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" -OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/frontier-linear-smoke-v3" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/frontier-linear-smoke-v4" VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" FRONTIER_ROOT = "/home/admin/cpfs/wjh/frontier-qwen30-vllm020-profile-v1/Frontier" VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" From fcca259475c309f149ffc75eae859da8db851f81 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 22:43:55 +0800 Subject: [PATCH 088/148] Profile Qwen MoE router on vLLM 0.20 --- .../jobs_frontier_linear_full.toml | 6 +- .../jobs_router_full.toml | 18 ++ .../profile_vllm020_router.py | 195 ++++++++++++++++++ .../run_frontier_linear_full.sh | 1 + .../run_router_full.sh | 49 +++++ 5 files changed, 266 insertions(+), 3 deletions(-) create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/jobs_router_full.toml create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_router.py create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/run_router_full.sh diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_full.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_full.toml index 374470e..c24a55c 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_full.toml +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_frontier_linear_full.toml @@ -1,18 +1,18 @@ version = 1 [[jobs]] -name = "qwen30-vllm020-frontier-linear-full-20260716-v1" +name = "qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens" gpus = 1 gpu_model = "H20" hosts = ["dash0"] command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1620 bash run_frontier_linear_full.sh" -artifacts = ["artifacts/frontier-linear-full-v1"] +artifacts = ["artifacts/frontier-linear-full-v2"] [jobs.env] HOME = "/tmp/wjh" XDG_CACHE_HOME = "/tmp/wjh/.cache" VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" -OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/frontier-linear-full-v1" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/frontier-linear-full-v2" VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" FRONTIER_ROOT = "/home/admin/cpfs/wjh/frontier-qwen30-vllm020-profile-v1/Frontier" VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_router_full.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_router_full.toml new file mode 100644 index 0000000..95cbc71 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_router_full.toml @@ -0,0 +1,18 @@ +version = 1 + +[[jobs]] +name = "qwen30-vllm020-router-full-20260716-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1020 bash run_router_full.sh" +artifacts = ["artifacts/router-full-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/router-full-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" +MODEL = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_router.py b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_router.py new file mode 100644 index 0000000..b1feca8 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_router.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Profile Qwen3's replicated MoE gate and fused top-k in vLLM 0.20.""" + +from __future__ import annotations + +import argparse +import json +import statistics +import subprocess +from pathlib import Path +from typing import Any, Callable + +import torch +import vllm + + +VLLM_VERSION = "0.20.0" +VLLM_COMMIT = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1" +HIDDEN_DIM = 2048 +NUM_EXPERTS = 128 +TOP_K = 8 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--vllm-source", type=Path, required=True) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--num-tokens", type=int, nargs="+", required=True) + parser.add_argument("--warmup-iters", type=int, default=5) + parser.add_argument("--repeats", type=int, default=20) + parser.add_argument("--device", default="cuda:0") + return parser.parse_args() + + +def git_head(repo: Path) -> str: + return subprocess.check_output( + ["git", "-C", str(repo), "rev-parse", "HEAD"], text=True + ).strip() + + +def stats_ms(samples: list[float]) -> dict[str, float]: + return { + "min": min(samples), + "max": max(samples), + "mean": statistics.fmean(samples), + "median": statistics.median(samples), + "std": statistics.pstdev(samples), + } + + +def measure_ms( + fn: Callable[[], Any], warmup_iters: int, repeats: int +) -> tuple[Any, dict[str, float]]: + result = None + for _ in range(warmup_iters): + result = fn() + torch.accelerator.synchronize() + samples: list[float] = [] + for _ in range(repeats): + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + result = fn() + end.record() + torch.accelerator.synchronize() + samples.append(float(start.elapsed_time(end))) + return result, stats_ms(samples) + + +def main() -> None: + args = parse_args() + if vllm.__version__ != VLLM_VERSION: + raise SystemExit(f"expected vLLM {VLLM_VERSION}, got {vllm.__version__}") + source_head = git_head(args.vllm_source) + if source_head != VLLM_COMMIT: + raise SystemExit(f"expected vLLM source {VLLM_COMMIT}, got {source_head}") + + raw_model_config = json.loads(args.model.joinpath("config.json").read_text()) + observed = { + "hidden_size": raw_model_config.get("hidden_size"), + "num_experts": raw_model_config.get("num_experts"), + "num_experts_per_tok": raw_model_config.get("num_experts_per_tok"), + "norm_topk_prob": raw_model_config.get("norm_topk_prob"), + } + expected = { + "hidden_size": HIDDEN_DIM, + "num_experts": NUM_EXPERTS, + "num_experts_per_tok": TOP_K, + "norm_topk_prob": True, + } + if observed != expected: + raise SystemExit(f"model contract mismatch: expected {expected}, got {observed}") + + from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config + from vllm.model_executor.layers.fused_moe import fused_topk + from vllm.model_executor.layers.linear import ReplicatedLinear + + device = torch.device(args.device) + torch.accelerator.set_device_index(device) + torch.manual_seed(20260716) + model_config = ModelConfig( + model=str(args.model), + dtype="bfloat16", + max_model_len=8192, + skip_tokenizer_init=True, + generation_config="vllm", + ) + + rows: list[dict[str, Any]] = [] + with set_current_vllm_config(VllmConfig(model_config=model_config)): + gate = ReplicatedLinear( + HIDDEN_DIM, + NUM_EXPERTS, + bias=False, + quant_config=None, + prefix="model.layers.0.mlp.gate", + ).to(device=device, dtype=torch.bfloat16) + gate.weight.data.uniform_(-0.01, 0.01) + + for num_tokens in args.num_tokens: + hidden = torch.empty( + (num_tokens, HIDDEN_DIM), device=device, dtype=torch.bfloat16 + ).uniform_(-0.1, 0.1) + logits, gate_time = measure_ms( + lambda: gate(hidden)[0], args.warmup_iters, args.repeats + ) + topk_result, topk_time = measure_ms( + lambda: fused_topk(hidden, logits, TOP_K, renormalize=True), + args.warmup_iters, + args.repeats, + ) + + def gate_and_topk() -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + current_logits, _ = gate(hidden) + return fused_topk(hidden, current_logits, TOP_K, renormalize=True) + + combined_result, combined_time = measure_ms( + gate_and_topk, args.warmup_iters, args.repeats + ) + topk_weights, topk_ids, _ = topk_result + combined_weights, combined_ids, _ = combined_result + if logits.shape != (num_tokens, NUM_EXPERTS): + raise SystemExit(f"invalid gate output shape: {tuple(logits.shape)}") + if topk_ids.shape != (num_tokens, TOP_K): + raise SystemExit(f"invalid top-k shape: {tuple(topk_ids.shape)}") + torch.testing.assert_close( + topk_weights.sum(dim=-1), + torch.ones(num_tokens, device=device), + atol=1e-5, + rtol=1e-5, + ) + torch.testing.assert_close(combined_weights, topk_weights) + torch.testing.assert_close(combined_ids, topk_ids) + additive_median = gate_time["median"] + topk_time["median"] + row = { + "num_tokens": num_tokens, + "gate_linear_time_ms": gate_time, + "routing_topk_time_ms": topk_time, + "gate_plus_topk_time_ms": combined_time, + "median_nonadditivity_ratio": ( + combined_time["median"] / additive_median + if additive_median > 0 + else 1.0 + ), + } + rows.append(row) + print(json.dumps(row, sort_keys=True), flush=True) + + payload = { + "schema_version": "qwen30_vllm020_router_raw.v1", + "environment": { + "vllm_version": vllm.__version__, + "vllm_source_commit": source_head, + "torch_version": torch.__version__, + "torch_cuda": torch.version.cuda, + "gpu": torch.cuda.get_device_name(device), + "model": str(args.model), + "dtype": "bfloat16", + "gate_replication": "replicated_across_tp", + "top_k": TOP_K, + "norm_topk_prob": True, + }, + "measurement_scope": ( + "vLLM ReplicatedLinear gate and fused_topk; measured separately and " + "as the actual sequential router path" + ), + "rows": rows, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_full.sh b/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_full.sh index 7371f3a..e68cf88 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_full.sh +++ b/runs/frontier-qwen30-vllm020-profile-v1/run_frontier_linear_full.sh @@ -47,6 +47,7 @@ timeout --signal=TERM --kill-after=30s 1380 \ --device h20 \ --models qwen3-a3b-30b-moe \ --num_tensor_parallel_workers 1 2 4 \ + --max_tokens 8192 \ --num_tokens_list "${TOKENS[@]}" \ --profile_method cuda_event \ --precision BF16 \ diff --git a/runs/frontier-qwen30-vllm020-profile-v1/run_router_full.sh b/runs/frontier-qwen30-vllm020-profile-v1/run_router_full.sh new file mode 100644 index 0000000..b4fd7a5 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/run_router_full.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -euo pipefail + +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +MODEL="${MODEL:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +TOKENS=(1 8 16 32 64 128 256 512 1024 2048 4096 8192) +mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/raw" +exec > >(tee -a "${OUTPUT_ROOT}/logs/full.log") 2>&1 + +IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?fleet GPU is required}" +if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then + echo "ERROR: expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2 + exit 1 +fi + +echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=${MODEL} runtime=vLLM-0.20.0+cu129 operators=ReplicatedLinear,fused_topk tokens=${TOKENS[*]} dtype=BF16 output=${OUTPUT_ROOT} expected_wall=2-6m hard_wall=900s hard_gpu_cap=0.25_H20h" +date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" +nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader + +test "$(git -C "${VLLM_SOURCE}" rev-parse HEAD)" = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1" +test -s "${MODEL}/config.json" +git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit" +git -C "${VLLM_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/vllm-source.commit" +sha256sum profile_vllm020_router.py run_router_full.sh \ + > "${OUTPUT_ROOT}/provenance/source.sha256" +sha256sum "${MODEL}/config.json" > "${OUTPUT_ROOT}/provenance/model-config.sha256" +uv pip freeze --python "${VENV_ROOT}/bin/python" \ + > "${OUTPUT_ROOT}/provenance/pip-freeze.txt" +nvidia-smi --query-gpu=index,uuid,name,driver_version,memory.total \ + --format=csv,noheader > "${OUTPUT_ROOT}/provenance/gpus.csv" +printf '%s\n' "${TOKENS[@]}" > "${OUTPUT_ROOT}/provenance/tokens.txt" + +timeout --signal=TERM --kill-after=30s 780 \ + "${VENV_ROOT}/bin/python" profile_vllm020_router.py \ + --vllm-source "${VLLM_SOURCE}" \ + --model "${MODEL}" \ + --output "${OUTPUT_ROOT}/raw/router.json" \ + --num-tokens "${TOKENS[@]}" \ + --warmup-iters 5 \ + --repeats 20 + +test -s "${OUTPUT_ROOT}/raw/router.json" +sha256sum "${OUTPUT_ROOT}/raw/router.json" "${OUTPUT_ROOT}/provenance"/* \ + > "${OUTPUT_ROOT}/artifacts.sha256" +date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ" +echo "ROUTER_FULL_COMPLETE cases=${#TOKENS[@]}" From 414838a7998bfc3a2912e39412b2aaf59c0ff589 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 22:49:31 +0800 Subject: [PATCH 089/148] Profile replicated router without TP group --- .../jobs_router_full.toml | 6 +++--- .../profile_vllm020_router.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_router_full.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_router_full.toml index 95cbc71..15719fc 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/jobs_router_full.toml +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_router_full.toml @@ -1,18 +1,18 @@ version = 1 [[jobs]] -name = "qwen30-vllm020-router-full-20260716-v1" +name = "qwen30-vllm020-router-full-20260716-v2-disable-tp-init" gpus = 1 gpu_model = "H20" hosts = ["dash0"] command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1020 bash run_router_full.sh" -artifacts = ["artifacts/router-full-v1"] +artifacts = ["artifacts/router-full-v2"] [jobs.env] HOME = "/tmp/wjh" XDG_CACHE_HOME = "/tmp/wjh/.cache" VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" -OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/router-full-v1" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/router-full-v2" VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" MODEL = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_router.py b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_router.py index b1feca8..7e23862 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_router.py +++ b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_router.py @@ -115,6 +115,7 @@ def main() -> None: bias=False, quant_config=None, prefix="model.layers.0.mlp.gate", + disable_tp=True, ).to(device=device, dtype=torch.bfloat16) gate.weight.data.uniform_(-0.01, 0.01) From 07b1eb4b75ae73f6156ba9ee696c1b0083098317 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 22:54:17 +0800 Subject: [PATCH 090/148] Initialize TP context for router profiling --- .../jobs_router_full.toml | 6 +- .../profile_vllm020_router.py | 138 +++++++++++------- 2 files changed, 85 insertions(+), 59 deletions(-) diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_router_full.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_router_full.toml index 15719fc..4dd0908 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/jobs_router_full.toml +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_router_full.toml @@ -1,18 +1,18 @@ version = 1 [[jobs]] -name = "qwen30-vllm020-router-full-20260716-v2-disable-tp-init" +name = "qwen30-vllm020-router-full-20260716-v3-tp-context" gpus = 1 gpu_model = "H20" hosts = ["dash0"] command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 1020 bash run_router_full.sh" -artifacts = ["artifacts/router-full-v2"] +artifacts = ["artifacts/router-full-v3"] [jobs.env] HOME = "/tmp/wjh" XDG_CACHE_HOME = "/tmp/wjh/.cache" VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" -OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/router-full-v2" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/router-full-v3" VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" MODEL = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_router.py b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_router.py index 7e23862..8da2070 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_router.py +++ b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_router.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse import json +import socket import statistics import subprocess from pathlib import Path @@ -93,6 +94,12 @@ def main() -> None: raise SystemExit(f"model contract mismatch: expected {expected}, got {observed}") from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config + from vllm.distributed import ( + destroy_distributed_environment, + destroy_model_parallel, + init_distributed_environment, + initialize_model_parallel, + ) from vllm.model_executor.layers.fused_moe import fused_topk from vllm.model_executor.layers.linear import ReplicatedLinear @@ -108,65 +115,84 @@ def main() -> None: ) rows: list[dict[str, Any]] = [] - with set_current_vllm_config(VllmConfig(model_config=model_config)): - gate = ReplicatedLinear( - HIDDEN_DIM, - NUM_EXPERTS, - bias=False, - quant_config=None, - prefix="model.layers.0.mlp.gate", - disable_tp=True, - ).to(device=device, dtype=torch.bfloat16) - gate.weight.data.uniform_(-0.01, 0.01) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + listener.bind(("127.0.0.1", 0)) + distributed_init_method = f"tcp://127.0.0.1:{listener.getsockname()[1]}" + init_distributed_environment( + world_size=1, + rank=0, + local_rank=0, + distributed_init_method=distributed_init_method, + ) + try: + with set_current_vllm_config(VllmConfig(model_config=model_config)): + initialize_model_parallel(tensor_model_parallel_size=1) + gate = ReplicatedLinear( + HIDDEN_DIM, + NUM_EXPERTS, + bias=False, + quant_config=None, + prefix="model.layers.0.mlp.gate", + ).to(device=device, dtype=torch.bfloat16) + gate.weight.data.uniform_(-0.01, 0.01) - for num_tokens in args.num_tokens: - hidden = torch.empty( - (num_tokens, HIDDEN_DIM), device=device, dtype=torch.bfloat16 - ).uniform_(-0.1, 0.1) - logits, gate_time = measure_ms( - lambda: gate(hidden)[0], args.warmup_iters, args.repeats - ) - topk_result, topk_time = measure_ms( - lambda: fused_topk(hidden, logits, TOP_K, renormalize=True), - args.warmup_iters, - args.repeats, - ) + for num_tokens in args.num_tokens: + hidden = torch.empty( + (num_tokens, HIDDEN_DIM), device=device, dtype=torch.bfloat16 + ).uniform_(-0.1, 0.1) + logits, gate_time = measure_ms( + lambda: gate(hidden)[0], args.warmup_iters, args.repeats + ) + topk_result, topk_time = measure_ms( + lambda: fused_topk(hidden, logits, TOP_K, renormalize=True), + args.warmup_iters, + args.repeats, + ) - def gate_and_topk() -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - current_logits, _ = gate(hidden) - return fused_topk(hidden, current_logits, TOP_K, renormalize=True) + def gate_and_topk() -> tuple[ + torch.Tensor, torch.Tensor, torch.Tensor + ]: + current_logits, _ = gate(hidden) + return fused_topk( + hidden, current_logits, TOP_K, renormalize=True + ) - combined_result, combined_time = measure_ms( - gate_and_topk, args.warmup_iters, args.repeats - ) - topk_weights, topk_ids, _ = topk_result - combined_weights, combined_ids, _ = combined_result - if logits.shape != (num_tokens, NUM_EXPERTS): - raise SystemExit(f"invalid gate output shape: {tuple(logits.shape)}") - if topk_ids.shape != (num_tokens, TOP_K): - raise SystemExit(f"invalid top-k shape: {tuple(topk_ids.shape)}") - torch.testing.assert_close( - topk_weights.sum(dim=-1), - torch.ones(num_tokens, device=device), - atol=1e-5, - rtol=1e-5, - ) - torch.testing.assert_close(combined_weights, topk_weights) - torch.testing.assert_close(combined_ids, topk_ids) - additive_median = gate_time["median"] + topk_time["median"] - row = { - "num_tokens": num_tokens, - "gate_linear_time_ms": gate_time, - "routing_topk_time_ms": topk_time, - "gate_plus_topk_time_ms": combined_time, - "median_nonadditivity_ratio": ( - combined_time["median"] / additive_median - if additive_median > 0 - else 1.0 - ), - } - rows.append(row) - print(json.dumps(row, sort_keys=True), flush=True) + combined_result, combined_time = measure_ms( + gate_and_topk, args.warmup_iters, args.repeats + ) + topk_weights, topk_ids, _ = topk_result + combined_weights, combined_ids, _ = combined_result + if logits.shape != (num_tokens, NUM_EXPERTS): + raise SystemExit( + f"invalid gate output shape: {tuple(logits.shape)}" + ) + if topk_ids.shape != (num_tokens, TOP_K): + raise SystemExit(f"invalid top-k shape: {tuple(topk_ids.shape)}") + torch.testing.assert_close( + topk_weights.sum(dim=-1), + torch.ones(num_tokens, device=device), + atol=1e-5, + rtol=1e-5, + ) + torch.testing.assert_close(combined_weights, topk_weights) + torch.testing.assert_close(combined_ids, topk_ids) + additive_median = gate_time["median"] + topk_time["median"] + row = { + "num_tokens": num_tokens, + "gate_linear_time_ms": gate_time, + "routing_topk_time_ms": topk_time, + "gate_plus_topk_time_ms": combined_time, + "median_nonadditivity_ratio": ( + combined_time["median"] / additive_median + if additive_median > 0 + else 1.0 + ), + } + rows.append(row) + print(json.dumps(row, sort_keys=True), flush=True) + finally: + destroy_model_parallel() + destroy_distributed_environment() payload = { "schema_version": "qwen30_vllm020_router_raw.v1", From 630de9f573575a5deb999e9f6dcf3eb993042bf8 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 23:14:27 +0800 Subject: [PATCH 091/148] Freeze vLLM 0.20 profiles and capture trace routing --- .../trace-routing-fixture-manifest.json | 108 +++ .../capture_trace_routing.py | 272 +++++++ .../extract_routing_fixture.py | 162 ++++ .../freeze_frontier_profiles.py | 695 ++++++++++++++++++ .../jobs_trace_routing.toml | 19 + .../prepare_profile_comparison.py | 104 +++ .../run_trace_routing.sh | 50 ++ 7 files changed, 1410 insertions(+) create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/artifacts/trace-routing-fixture-manifest.json create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/capture_trace_routing.py create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/extract_routing_fixture.py create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/jobs_trace_routing.toml create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/prepare_profile_comparison.py create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/run_trace_routing.sh diff --git a/runs/frontier-qwen30-vllm020-profile-v1/artifacts/trace-routing-fixture-manifest.json b/runs/frontier-qwen30-vllm020-profile-v1/artifacts/trace-routing-fixture-manifest.json new file mode 100644 index 0000000..ff78c14 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/artifacts/trace-routing-fixture-manifest.json @@ -0,0 +1,108 @@ +{ + "contains_prompt_text": false, + "fixture_path": "/tmp/qwen30-routing-fixture-20260716.jsonl", + "fixture_sha256": "e9e7f5b4e0d3a788dcd99d432f939d9e36bff2a64e412649b407b0609f0e39bb", + "prefix_pairs": [ + { + "child_row_id": "17366", + "child_turn": 13, + "parent_row_id": "16914", + "parent_turn": 12, + "trace_hash_common_prefix_blocks": 59 + } + ], + "request_count": 8, + "rows": [ + { + "chat_id": "184516", + "fixture_index": 0, + "input_length": 3791, + "output_length": 73, + "parent_chat_id": "183921", + "prompt_sha256": "6b59c3ecb9a7d8bd3178f10b847a7c16c221fef113df3e80ede55543bc66ea6b", + "row_id": "16914", + "trace_hash_blocks": 60, + "turn": 12 + }, + { + "chat_id": "183512", + "fixture_index": 1, + "input_length": 264, + "output_length": 975, + "parent_chat_id": "-1", + "prompt_sha256": "0af4fe018ad7afc152951995e5cb18f389bacf4076239f5a72165a9c84dd566d", + "row_id": "15910", + "trace_hash_blocks": 5, + "turn": 1 + }, + { + "chat_id": "189951", + "fixture_index": 2, + "input_length": 488, + "output_length": 863, + "parent_chat_id": "-1", + "prompt_sha256": "b84c0580e9cd80e8dd388f176bd51119f06ee4686838d957b4018b8a1feccb15", + "row_id": "22349", + "trace_hash_blocks": 8, + "turn": 1 + }, + { + "chat_id": "177472", + "fixture_index": 3, + "input_length": 1037, + "output_length": 895, + "parent_chat_id": "-1", + "prompt_sha256": "146dc187af0eeb342dd7bd6ebd9453973074209ab4da5b122718b5d9e06d46d1", + "row_id": "9870", + "trace_hash_blocks": 17, + "turn": 1 + }, + { + "chat_id": "177528", + "fixture_index": 4, + "input_length": 1993, + "output_length": 654, + "parent_chat_id": "-1", + "prompt_sha256": "c92ba8acd0d637b796d60b5b01e3ac54bde70a986e83809b434c46f50e5242cf", + "row_id": "9926", + "trace_hash_blocks": 32, + "turn": 1 + }, + { + "chat_id": "193539", + "fixture_index": 5, + "input_length": 4088, + "output_length": 1842, + "parent_chat_id": "-1", + "prompt_sha256": "38eecbf8766bd8432ac41d6a061a74244c4b3f8b20e8dfee7237b5e6c0e9e13e", + "row_id": "25937", + "trace_hash_blocks": 64, + "turn": 1 + }, + { + "chat_id": "177590", + "fixture_index": 6, + "input_length": 7995, + "output_length": 1128, + "parent_chat_id": "-1", + "prompt_sha256": "4bee5b9d1aceec7010a80b8585fb5071b6d468c751e5903abbcd656a8285fcd8", + "row_id": "9988", + "trace_hash_blocks": 125, + "turn": 1 + }, + { + "chat_id": "184968", + "fixture_index": 7, + "input_length": 4017, + "output_length": 72, + "parent_chat_id": "184516", + "prompt_sha256": "6885aff07780bff5906669057d46935d78148fb7f4edd0bf58d44c2ffec76952", + "row_id": "17366", + "trace_hash_blocks": 63, + "turn": 13 + } + ], + "schema_version": "qwen30_trace_routing_fixture.v1", + "source_trace": "/home/gahow/phd/aituner/trace_windows/traces/chat_w20260311_1000.jsonl", + "source_trace_sha256": "f539f38eb0ee0f750e3c23ff47df6eed3faf723a25f1444d55665a85871750b9" +} diff --git a/runs/frontier-qwen30-vllm020-profile-v1/capture_trace_routing.py b/runs/frontier-qwen30-vllm020-profile-v1/capture_trace_routing.py new file mode 100644 index 0000000..e1f4b99 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/capture_trace_routing.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +"""Capture exact Qwen3 routed-expert IDs from vLLM 0.20 on trace prompts.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import subprocess +from pathlib import Path +from typing import Any + +import numpy as np +import torch +import vllm + + +VLLM_VERSION = "0.20.0" +VLLM_COMMIT = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1" +NUM_EXPERTS = 128 +TOP_K = 8 +NUM_LAYERS = 48 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--vllm-source", type=Path, required=True) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--fixture", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--routes", type=Path, required=True) + parser.add_argument("--decode-override", type=int) + return parser.parse_args() + + +def git_head(repo: Path) -> str: + return subprocess.check_output( + ["git", "-C", str(repo), "rev-parse", "HEAD"], text=True + ).strip() + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def common_prefix(left: list[int], right: list[int]) -> int: + count = 0 + for lhs, rhs in zip(left, right): + if lhs != rhs: + break + count += 1 + return count + + +def distribution(counts: np.ndarray) -> dict[str, Any]: + values = counts.astype(np.float64) + total = float(values.sum()) + mean = float(values.mean()) + probabilities = values[values > 0] / total + entropy = float(-(probabilities * np.log2(probabilities)).sum()) + variance = float(((values - mean) ** 2).mean()) + ordered = np.sort(values) + gini = float( + 2.0 * np.dot(np.arange(1, len(values) + 1), ordered) + / (len(values) * total) + - (len(values) + 1) / len(values) + ) + hottest = np.argsort(values)[-8:][::-1] + return { + "total_routed_tokens": int(total), + "tokens_per_expert_mean": mean, + "load_cv": math.sqrt(variance) / mean, + "load_gini": gini, + "load_entropy_bits": entropy, + "min_load_ratio": float(values.min() / mean), + "max_load_ratio": float(values.max() / mean), + "expert_utilization": float(np.count_nonzero(values) / len(values)), + "hottest_experts": [int(value) for value in hottest], + "hottest_counts": [int(values[value]) for value in hottest], + "counts": counts.astype(int).tolist(), + } + + +def phase_summary(routes: list[np.ndarray]) -> dict[str, Any]: + counts = np.zeros(NUM_EXPERTS, dtype=np.int64) + per_layer = np.zeros((NUM_LAYERS, NUM_EXPERTS), dtype=np.int64) + token_count = 0 + for route in routes: + token_count += route.shape[0] + counts += np.bincount(route.reshape(-1), minlength=NUM_EXPERTS) + for layer in range(NUM_LAYERS): + per_layer[layer] += np.bincount( + route[:, layer, :].reshape(-1), minlength=NUM_EXPERTS + ) + return { + "token_count": token_count, + "all_layers": distribution(counts), + "per_layer": [distribution(row) for row in per_layer], + } + + +def main() -> None: + args = parse_args() + if vllm.__version__ != VLLM_VERSION: + raise SystemExit(f"expected vLLM {VLLM_VERSION}, got {vllm.__version__}") + source_head = git_head(args.vllm_source) + if source_head != VLLM_COMMIT: + raise SystemExit(f"expected vLLM source {VLLM_COMMIT}, got {source_head}") + rows = [json.loads(line) for line in args.fixture.read_text().splitlines() if line] + if not rows: + raise SystemExit("empty routing fixture") + requested_decode = [ + args.decode_override + if args.decode_override is not None + else int(row["output_length"]) + for row in rows + ] + if any(value <= 0 for value in requested_decode): + raise SystemExit("all requested decode lengths must be positive") + + from vllm import LLM, SamplingParams + + llm = LLM( + model=str(args.model), + dtype="bfloat16", + tensor_parallel_size=1, + max_model_len=16384, + max_num_batched_tokens=8192, + max_num_seqs=64, + gpu_memory_utilization=0.90, + enable_chunked_prefill=True, + enable_prefix_caching=True, + enable_return_routed_experts=True, + attention_backend="FLASH_ATTN", + disable_log_stats=False, + ) + sampling = [ + SamplingParams(temperature=0, min_tokens=value, max_tokens=value) + for value in requested_decode + ] + conversations = [ + [{"role": "user", "content": row["prompt"]}] for row in rows + ] + outputs = llm.chat(conversations, sampling_params=sampling, use_tqdm=False) + if len(outputs) != len(rows): + raise SystemExit(f"expected {len(rows)} outputs, got {len(outputs)}") + + prompt_tokens_by_chat: dict[str, list[int]] = {} + prefill_routes: list[np.ndarray] = [] + decode_routes: list[np.ndarray] = [] + raw_routes: dict[str, np.ndarray] = {} + request_summaries = [] + for row, output, decode_tokens in zip(rows, outputs, requested_decode): + completion = output.outputs[0] + routed = completion.routed_experts + if routed is None: + raise SystemExit(f"row {row['row_id']} returned no routed experts") + routed = np.asarray(routed) + prompt_tokens = list(output.prompt_token_ids) + generated_tokens = list(completion.token_ids) + expected = len(prompt_tokens) + len(generated_tokens) - 1 + if routed.shape != (expected, NUM_LAYERS, TOP_K): + raise SystemExit( + f"row {row['row_id']} routes {routed.shape}, expected " + f"{(expected, NUM_LAYERS, TOP_K)}" + ) + if routed.min() < 0 or routed.max() >= NUM_EXPERTS: + raise SystemExit(f"row {row['row_id']} returned invalid expert IDs") + prefill = routed[: len(prompt_tokens)] + decode = routed[len(prompt_tokens) :] + if decode.shape[0] != decode_tokens - 1: + raise SystemExit(f"row {row['row_id']} decode route length mismatch") + prefill_routes.append(prefill) + decode_routes.append(decode) + raw_routes[f"row_{row['row_id']}"] = routed.astype(np.int16) + prompt_tokens_by_chat[str(row["chat_id"])] = prompt_tokens + request_summaries.append( + { + "fixture_index": row["fixture_index"], + "row_id": row["row_id"], + "turn": row["turn"], + "input_length_trace": row["input_length"], + "prompt_tokens_vllm": len(prompt_tokens), + "chat_wrapper_delta": len(prompt_tokens) - int(row["input_length"]), + "generated_tokens": len(generated_tokens), + "requested_decode_tokens": decode_tokens, + "routed_shape": list(routed.shape), + "prompt_sha256": row["prompt_sha256"], + "trace_hash_blocks": len(row["hash_ids"]), + } + ) + + prefix_pairs = [] + by_chat = {str(row["chat_id"]): row for row in rows} + for child in rows: + parent = by_chat.get(str(child["parent_chat_id"])) + if parent is None: + continue + parent_tokens = prompt_tokens_by_chat[str(parent["chat_id"])] + child_tokens = prompt_tokens_by_chat[str(child["chat_id"])] + prefix_pairs.append( + { + "parent_row_id": parent["row_id"], + "child_row_id": child["row_id"], + "trace_hash_common_prefix_blocks": common_prefix( + parent["hash_ids"], child["hash_ids"] + ), + "vllm_token_common_prefix": common_prefix(parent_tokens, child_tokens), + "vllm_full_common_blocks_16": common_prefix( + parent_tokens, child_tokens + ) + // 16, + } + ) + + args.routes.parent.mkdir(parents=True, exist_ok=True) + np.savez_compressed(args.routes, **raw_routes) + payload = { + "schema_version": "qwen30_vllm020_trace_routing.v1", + "environment": { + "vllm_version": vllm.__version__, + "vllm_source_commit": source_head, + "torch_version": torch.__version__, + "torch_cuda": torch.version.cuda, + "gpu": torch.cuda.get_device_name(0), + "model": str(args.model), + "dtype": "bfloat16", + "tensor_parallel_size": 1, + "max_num_batched_tokens": 8192, + "max_num_seqs": 64, + "prefix_caching": True, + "chunked_prefill": True, + "attention_backend": "FLASH_ATTN", + }, + "capture_contract": { + "api": "LLM.chat", + "enable_return_routed_experts": True, + "route_shape": "[prompt_tokens + generated_tokens - 1, layers, topk]", + "decode_policy": ( + f"fixed_override_{args.decode_override}" + if args.decode_override is not None + else "exact_trace_output_length" + ), + "contains_prompt_text": False, + "fixture_sha256": sha256(args.fixture), + "routes_npz": str(args.routes), + }, + "requests": request_summaries, + "prefix_pairs": prefix_pairs, + "phases": { + "prefill": phase_summary(prefill_routes), + "decode": phase_summary(decode_routes), + }, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + print( + json.dumps( + { + "requests": len(rows), + "prefill_tokens": payload["phases"]["prefill"]["token_count"], + "decode_tokens": payload["phases"]["decode"]["token_count"], + "prefix_pairs": prefix_pairs, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-qwen30-vllm020-profile-v1/extract_routing_fixture.py b/runs/frontier-qwen30-vllm020-profile-v1/extract_routing_fixture.py new file mode 100644 index 0000000..2021ddc --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/extract_routing_fixture.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Extract a prompt-bearing routing fixture while emitting a prompt-free manifest.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--trace", type=Path, required=True) + parser.add_argument("--row-ids", type=int, nargs="+", required=True) + parser.add_argument("--parent-of", type=int, nargs="*", default=[]) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + return parser.parse_args() + + +def common_prefix(left: list[Any], right: list[Any]) -> int: + count = 0 + for lhs, rhs in zip(left, right): + if lhs != rhs: + break + count += 1 + return count + + +def main() -> None: + args = parse_args() + target_ids = set(args.row_ids) + parent_targets = set(args.parent_of) + if not parent_targets.issubset(target_ids): + raise SystemExit("--parent-of must be a subset of --row-ids") + + trace_digest = hashlib.sha256() + by_row: dict[int, dict[str, Any]] = {} + by_chat_id: dict[str, dict[str, Any]] = {} + with args.trace.open("rb") as handle: + row_id = 0 + while True: + offset = handle.tell() + line = handle.readline() + if not line: + break + trace_digest.update(line) + row = json.loads(line) + meta = { + "row_id": row_id, + "offset": offset, + "chat_id": str(row.get("chat_id")), + "parent_chat_id": str(row.get("parent_chat_id")), + "turn": int(row.get("turn", 1)), + "input_length": int(row["input_length"]), + "output_length": int(row["output_length"]), + "hash_ids": row.get("hash_ids") or [], + } + by_chat_id[meta["chat_id"]] = meta + if row_id in target_ids: + by_row[row_id] = meta + row_id += 1 + + missing = target_ids - set(by_row) + if missing: + raise SystemExit(f"missing target row IDs: {sorted(missing)}") + parent_rows: list[dict[str, Any]] = [] + for row_id in args.parent_of: + parent_id = by_row[row_id]["parent_chat_id"] + parent = by_chat_id.get(parent_id) + if parent is None: + raise SystemExit(f"row {row_id} parent chat {parent_id} is absent") + parent_rows.append(parent) + + # Put parents first so online prefix caching can materialize their shared + # blocks before descendants are admitted. + ordered_meta = parent_rows + [by_row[row_id] for row_id in args.row_ids] + if len({row["row_id"] for row in ordered_meta}) != len(ordered_meta): + raise SystemExit("fixture rows must be unique") + + output_rows: list[dict[str, Any]] = [] + with args.trace.open("rb") as handle: + for fixture_index, meta in enumerate(ordered_meta): + handle.seek(meta["offset"]) + source = json.loads(handle.readline()) + prompt = source.get("prompt") + if not isinstance(prompt, str) or not prompt: + raise SystemExit(f"row {meta['row_id']} has no prompt") + output_rows.append( + { + "fixture_index": fixture_index, + "row_id": str(meta["row_id"]), + "prompt": prompt, + "prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), + "input_length": meta["input_length"], + "output_length": meta["output_length"], + "turn": meta["turn"], + "chat_id": meta["chat_id"], + "parent_chat_id": meta["parent_chat_id"], + "hash_ids": meta["hash_ids"], + } + ) + + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("w") as handle: + for row in output_rows: + handle.write(json.dumps(row, sort_keys=True) + "\n") + + pair_coverage = [] + by_chat = {row["chat_id"]: row for row in output_rows} + for child in output_rows: + parent = by_chat.get(child["parent_chat_id"]) + if parent is None: + continue + pair_coverage.append( + { + "parent_row_id": parent["row_id"], + "child_row_id": child["row_id"], + "parent_turn": parent["turn"], + "child_turn": child["turn"], + "trace_hash_common_prefix_blocks": common_prefix( + parent["hash_ids"], child["hash_ids"] + ), + } + ) + fixture_digest = hashlib.sha256(args.output.read_bytes()).hexdigest() + manifest = { + "schema_version": "qwen30_trace_routing_fixture.v1", + "source_trace": str(args.trace.resolve()), + "source_trace_sha256": trace_digest.hexdigest(), + "fixture_path": str(args.output.resolve()), + "fixture_sha256": fixture_digest, + "contains_prompt_text": False, + "request_count": len(output_rows), + "rows": [ + { + key: row[key] + for key in ( + "fixture_index", + "row_id", + "prompt_sha256", + "input_length", + "output_length", + "turn", + "chat_id", + "parent_chat_id", + ) + } + | {"trace_hash_blocks": len(row["hash_ids"])} + for row in output_rows + ], + "prefix_pairs": pair_coverage, + } + args.manifest.parent.mkdir(parents=True, exist_ok=True) + args.manifest.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + print(json.dumps({"requests": len(output_rows), "prefix_pairs": pair_coverage})) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py b/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py new file mode 100644 index 0000000..0d420e0 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +"""Freeze vLLM 0.20 microprofiles into Frontier-compatible CSV inputs.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +import re +import shutil +from pathlib import Path +from typing import Any + + +STAT_NAMES = ("min", "max", "mean", "median", "std") +ATTENTION_OPS = ( + "attn_input_reshape", + "attn_kv_cache_save", + "attn_prefill", + "attn_decode", + "attn_output_reshape", +) +ATTENTION_METADATA = ( + "n_embd", + "n_q_head", + "n_kv_head", + "block_size", + "num_tensor_parallel_workers", + "max_model_len", + "batch_size", + "prefill_chunk_size", + "kv_cache_size", + "is_prefill", + "attention_backend", + "is_mixed_batch", + "mode", + "seq_lens", + "total_tokens", + "max_seq_len", + "min_seq_len", + "avg_seq_len", + "equal_seq_len", + "seq_len_variance", + "seq_len_std", + "seq_len_cv", + "is_chunked_prefill_sample", + "chunk_start_token", + "chunk_end_token", + "total_prefill_tokens", + "profiling_precision", + "model_arch", + "quant_signature", + "measurement_type", + "is_true_mixed_batch", + "prefill_seq_lens", + "prefill_kv_cache_sizes", + "decode_kv_cache_sizes", + "num_prefill_seqs", + "num_decode_seqs", + "decode_batch_size", + "total_batch_size", + "total_decode_tokens", + "decode_avg_kv_cache_size", + "batch_composition_ratio", + "batch_spec", + "projection_policy", +) +MOE_OPS = ( + "moe_gating_linear", + "moe_gating_routing_topk", + "moe_shuffling", + "moe_grouped_gemm", +) +MOE_METADATA = ( + "num_tokens", + "num_experts", + "num_experts_per_device", + "expert_parallel_size", + "routing_runtime_path", + "routing_assignment_policy", + "routing_weight_policy", + "routing_uses_router_logits", + "gating_runtime_context", + "gating_runtime_context_impl", + "router_topk", + "hidden_dim", + "expert_hidden_dim", + "use_gated", + "num_tensor_parallel_workers", + "total_routed_tokens", + "model_expansion_ratio", + "tokens_per_expert_avg", + "tokens_to_experts_ratio", + "expert_utilization", + "min_load_ratio", + "load_imbalance_cv", + "max_load_ratio", + "load_entropy", + "load_gini_coefficient", + "load_distribution", + "seed", + "moe_grouped_gemm_backend", + "measurement_type", + "profiling_precision", + "model_arch", + "quant_signature", + "router_median_nonadditivity_ratio", + "projection_policy", +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--linear", type=Path, required=True) + parser.add_argument("--attention", type=Path, nargs=3, required=True) + parser.add_argument("--moe", type=Path, required=True) + parser.add_argument("--router", type=Path, required=True) + parser.add_argument("--allreduce", type=Path, nargs=2, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def load_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text()) + + +def stat_columns(prefix: str, stats: dict[str, float]) -> dict[str, float]: + return {f"time_stats.{prefix}.{name}": float(stats[name]) for name in STAT_NAMES} + + +def zero_stat_columns(prefix: str) -> dict[str, float]: + return {f"time_stats.{prefix}.{name}": 0.0 for name in STAT_NAMES} + + +def attention_core_stats(raw: dict[str, Any]) -> dict[str, float]: + # vLLM's benchmark result exports aggregate mean but not the raw samples. + # Preserve that mean as Frontier's training target and record the proxy in + # the manifest rather than inventing an unobserved median. + return { + "min": 1000.0 * float(raw["min_time"]), + "max": 1000.0 * float(raw["max_time"]), + "mean": 1000.0 * float(raw["mean_time"]), + "median": 1000.0 * float(raw["mean_time"]), + "std": 1000.0 * float(raw["std_time"]), + } + + +def kv_update_stats(raw: dict[str, Any]) -> dict[str, float]: + stats = raw["kv_cache_update_time"] + return {name: float(stats[f"{name}_ms"]) for name in STAT_NAMES} + + +def parse_size(value: str, suffix: str) -> int: + return int(value) * (1024 if suffix == "k" else 1) + + +def parse_batch_spec(spec: str) -> list[tuple[int, int]]: + requests: list[tuple[int, int]] = [] + pattern = re.compile(r"^(?:(\d+))?q(\d+)(k?)(?:s(\d+)(k?))?$") + for segment in spec.split("_"): + match = pattern.match(segment) + if match is None: + raise ValueError(f"invalid vLLM batch spec: {spec}") + count = int(match.group(1) or 1) + query = parse_size(match.group(2), match.group(3)) + kv = ( + parse_size(match.group(4), match.group(5)) + if match.group(4) + else query + ) + requests.extend([(query, kv)] * count) + return requests + + +def write_csv(path: Path, fieldnames: list[str], rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames, extrasaction="raise") + writer.writeheader() + writer.writerows(rows) + + +def freeze_attention( + inputs: list[Path], output: Path +) -> tuple[int, int, list[str]]: + rows: list[dict[str, Any]] = [] + mixed_rows: list[dict[str, Any]] = [] + seen_tps: set[int] = set() + raw_by_tp: dict[int, list[dict[str, Any]]] = {} + for path in inputs: + payload = load_json(path) + if payload.get("schema_version") != "qwen30_vllm020_flashattn_raw.v1": + raise ValueError(f"unexpected attention schema in {path}") + if payload["environment"].get("vllm_version") != "0.20.0": + raise ValueError(f"unexpected vLLM version in {path}") + for raw in payload["rows"]: + if raw.get("error") is not None: + raise ValueError(f"failed attention row in {path}: {raw['error']}") + tp = int(raw["tensor_parallel_size"]) + seen_tps.add(tp) + raw_by_tp.setdefault(tp, []).append(raw) + + def pure_reference_ms( + tp: int, requests: list[tuple[int, int]], *, decode_phase: bool + ) -> float: + candidates: list[tuple[list[tuple[int, int]], float]] = [] + for candidate in raw_by_tp[tp]: + parsed = parse_batch_spec(candidate["config"]["batch_spec"]) + is_decode = all(query == 1 for query, _ in parsed) + if is_decode != decode_phase: + continue + if decode_phase and not is_decode: + continue + if not decode_phase and any(query == 1 for query, _ in parsed): + continue + candidates.append((parsed, 1000.0 * float(candidate["mean_time"]))) + for parsed, mean_ms in candidates: + if parsed == requests: + return mean_ms + if not decode_phase: + raise ValueError(f"no exact pure prefill reference for TP{tp}: {requests}") + if len({kv for _, kv in requests}) != 1: + raise ValueError(f"decode interpolation requires one KV length: {requests}") + target_batch = len(requests) + target_kv = requests[0][1] + points = sorted( + (parsed[0][1], mean_ms) + for parsed, mean_ms in candidates + if len(parsed) == target_batch + and len({kv for _, kv in parsed}) == 1 + ) + if not points: + raise ValueError(f"no pure decode reference for TP{tp}: {requests}") + if target_kv <= points[0][0]: + return points[0][1] + if target_kv >= points[-1][0]: + return points[-1][1] + for (left_kv, left_ms), (right_kv, right_ms) in zip(points, points[1:]): + if left_kv <= target_kv <= right_kv: + fraction = (target_kv - left_kv) / (right_kv - left_kv) + return left_ms + fraction * (right_ms - left_ms) + raise AssertionError("unreachable decode interpolation") + + for tp in sorted(raw_by_tp): + for raw in raw_by_tp[tp]: + spec = raw["config"]["batch_spec"] + requests = parse_batch_spec(spec) + prefill = [(q, kv) for q, kv in requests if q > 1] + decode = [(q, kv) for q, kv in requests if q == 1] + core = attention_core_stats(raw) + kv_stats = kv_update_stats(raw) + if prefill and decode: + prefill_reference_ms = pure_reference_ms( + tp, prefill, decode_phase=False + ) + decode_reference_ms = pure_reference_ms( + tp, decode, decode_phase=True + ) + reference_total_ms = prefill_reference_ms + decode_reference_ms + prefill_share = prefill_reference_ms / reference_total_ms + decode_share = decode_reference_ms / reference_total_ms + projected_prefill = { + name: value * prefill_share for name, value in core.items() + } + projected_decode = { + name: value * decode_share for name, value in core.items() + } + row = {} + for op in ATTENTION_OPS: + row.update(zero_stat_columns(op)) + row.update(stat_columns("attn_kv_cache_save", kv_stats)) + row.update(stat_columns("attn_prefill", projected_prefill)) + row.update(stat_columns("attn_decode", projected_decode)) + prefill_queries = [q for q, _ in prefill] + prefill_contexts = [kv - q for q, kv in prefill] + decode_kv_lengths = [kv for _, kv in decode] + total_batch = len(requests) + row.update( + { + "n_embd": 2048, + "n_q_head": 32, + "n_kv_head": 4, + "block_size": 16, + "num_tensor_parallel_workers": tp, + "max_model_len": 40960, + "batch_size": total_batch, + "prefill_chunk_size": 0, + "kv_cache_size": 0, + "is_prefill": True, + "attention_backend": "FLASH_ATTN", + "is_mixed_batch": False, + "mode": "true_mixed_fused_projected", + "seq_lens": "", + "total_tokens": sum(prefill_queries) + len(decode), + "max_seq_len": "", + "min_seq_len": "", + "avg_seq_len": "", + "equal_seq_len": "", + "seq_len_variance": "", + "seq_len_std": "", + "seq_len_cv": "", + "is_chunked_prefill_sample": False, + "chunk_start_token": "", + "chunk_end_token": "", + "total_prefill_tokens": sum(prefill_queries), + "profiling_precision": "BF16", + "model_arch": "generic", + "quant_signature": "none", + "measurement_type": "CUDA_EVENT", + "is_true_mixed_batch": True, + "prefill_seq_lens": json.dumps(prefill_queries), + "prefill_kv_cache_sizes": json.dumps(prefill_contexts), + "decode_kv_cache_sizes": json.dumps(decode_kv_lengths), + "num_prefill_seqs": len(prefill), + "num_decode_seqs": len(decode), + "decode_batch_size": len(decode), + "total_batch_size": total_batch, + "total_decode_tokens": len(decode), + "decode_avg_kv_cache_size": ( + sum(decode_kv_lengths) / len(decode_kv_lengths) + ), + "batch_composition_ratio": len(prefill) / total_batch, + "batch_spec": spec, + "projection_policy": ( + "fused_total_conserving_projection_by_same_tp_pure_" + "prefill_decode_reference_ratio" + ), + } + ) + rows.append(row) + mixed_rows.append( + { + "num_tensor_parallel_workers": tp, + "batch_spec": spec, + "num_prefill_seqs": len(prefill), + "num_decode_seqs": len(decode), + "total_prefill_tokens": sum(q for q, _ in prefill), + "total_decode_tokens": len(decode), + "decode_avg_kv_cache_size": sum(kv for _, kv in decode) + / len(decode), + "attention_core_mean_ms": core["mean"], + "attention_core_mean_as_median_ms": core["median"], + "kv_cache_update_median_ms": kv_stats["median"], + "pure_prefill_reference_mean_ms": prefill_reference_ms, + "pure_decode_reference_mean_ms": decode_reference_ms, + "projected_prefill_mean_ms": projected_prefill["mean"], + "projected_decode_mean_ms": projected_decode["mean"], + "projection_sum_error_ms": ( + projected_prefill["mean"] + + projected_decode["mean"] + - core["mean"] + ), + "representation": ( + "one_fused_FA3_call_projected_for_Frontier_with_" + "total_conservation" + ), + } + ) + continue + + is_decode = bool(decode) + queries = [q for q, _ in requests] + contexts = [kv if is_decode else kv - q for q, kv in requests] + avg_query = sum(queries) / len(queries) + variance = sum((query - avg_query) ** 2 for query in queries) / len(queries) + std = math.sqrt(variance) + avg_context = sum(contexts) / len(contexts) + row: dict[str, Any] = {} + for op in ATTENTION_OPS: + row.update(zero_stat_columns(op)) + row.update(stat_columns("attn_kv_cache_save", kv_stats)) + row.update( + stat_columns("attn_decode" if is_decode else "attn_prefill", core) + ) + row.update( + { + "n_embd": 2048, + "n_q_head": 32, + "n_kv_head": 4, + "block_size": 16, + "num_tensor_parallel_workers": tp, + "max_model_len": 40960, + "batch_size": len(requests), + "prefill_chunk_size": 0 if is_decode else sum(queries), + "kv_cache_size": avg_context, + "is_prefill": not is_decode, + "attention_backend": "FLASH_ATTN", + "is_mixed_batch": False, + "mode": "vllm020_batch_spec", + "seq_lens": json.dumps(queries), + "total_tokens": sum(queries), + "max_seq_len": max(queries), + "min_seq_len": min(queries), + "avg_seq_len": avg_query, + "equal_seq_len": len(set(queries)) == 1, + "seq_len_variance": variance, + "seq_len_std": std, + "seq_len_cv": std / avg_query if avg_query else 0.0, + "is_chunked_prefill_sample": (not is_decode and avg_context > 0), + "chunk_start_token": avg_context if not is_decode else 0, + "chunk_end_token": avg_context + sum(queries) if not is_decode else 0, + "total_prefill_tokens": 0 if is_decode else sum(queries), + "profiling_precision": "BF16", + "model_arch": "generic", + "quant_signature": "none", + "measurement_type": "CUDA_EVENT", + "is_true_mixed_batch": False, + "prefill_seq_lens": "", + "prefill_kv_cache_sizes": "", + "decode_kv_cache_sizes": "", + "num_prefill_seqs": "", + "num_decode_seqs": "", + "decode_batch_size": "", + "total_batch_size": "", + "total_decode_tokens": "", + "decode_avg_kv_cache_size": "", + "batch_composition_ratio": "", + "batch_spec": spec, + "projection_policy": ( + "measured_FA3_core_plus_measured_KV;reshape_assumed_zero;" + "mean_as_median" + ), + } + ) + rows.append(row) + + if seen_tps != {1, 2, 4}: + raise ValueError(f"attention TP coverage mismatch: {seen_tps}") + attention_fields = [ + f"time_stats.{op}.{stat}" for op in ATTENTION_OPS for stat in STAT_NAMES + ] + list(ATTENTION_METADATA) + write_csv(output / "attention.csv", attention_fields, rows) + mixed_fields = [ + "num_tensor_parallel_workers", + "batch_spec", + "num_prefill_seqs", + "num_decode_seqs", + "total_prefill_tokens", + "total_decode_tokens", + "decode_avg_kv_cache_size", + "attention_core_mean_ms", + "attention_core_mean_as_median_ms", + "kv_cache_update_median_ms", + "pure_prefill_reference_mean_ms", + "pure_decode_reference_mean_ms", + "projected_prefill_mean_ms", + "projected_decode_mean_ms", + "projection_sum_error_ms", + "representation", + ] + write_csv(output / "attention_true_mixed_fused.csv", mixed_fields, mixed_rows) + return len(rows), len(mixed_rows), sorted(seen_tps) + + +def load_features(counts: list[int]) -> dict[str, float]: + total = sum(counts) + count = len(counts) + mean = total / count + variance = sum((value - mean) ** 2 for value in counts) / count + probabilities = [value / total for value in counts if value > 0] + entropy = -sum(probability * math.log2(probability) for probability in probabilities) + sorted_counts = sorted(counts) + gini = ( + 2 * sum((index + 1) * value for index, value in enumerate(sorted_counts)) + / (count * total) + - (count + 1) / count + ) + return { + "total_routed_tokens": total, + "num_experts_per_device": count, + "hidden_dim": 2048, + "expert_hidden_dim": 768, + "router_topk": 8, + "model_expansion_ratio": 768 / 2048, + "tokens_per_expert_avg": mean, + "tokens_to_experts_ratio": mean, + "expert_utilization": sum(value > 0 for value in counts) / count, + "min_load_ratio": min(counts) / mean, + "load_imbalance_cv": math.sqrt(variance) / mean, + "max_load_ratio": max(counts) / mean, + "load_entropy": entropy, + "load_gini_coefficient": gini, + } + + +def freeze_moe(moe_path: Path, router_path: Path, output: Path) -> int: + moe = load_json(moe_path) + router = load_json(router_path) + if moe.get("schema_version") != "qwen30_vllm020_moe_raw.v1": + raise ValueError(f"unexpected MoE schema in {moe_path}") + if router.get("schema_version") != "qwen30_vllm020_router_raw.v1": + raise ValueError(f"unexpected router schema in {router_path}") + router_by_tokens = {int(row["num_tokens"]): row for row in router["rows"]} + rows: list[dict[str, Any]] = [] + seen_pairs: set[tuple[int, int, str]] = set() + for raw in moe["rows"]: + tp = int(raw["tensor_parallel_size"]) + num_tokens = int(raw["num_tokens"]) + routing_mode = str(raw["routing_mode"]) + key = (tp, num_tokens, routing_mode) + if key in seen_pairs: + raise ValueError(f"duplicate MoE row: {key}") + seen_pairs.add(key) + router_row = router_by_tokens[num_tokens] + counts = [int(value) for value in raw["routing_load"]["counts"]] + if sum(counts) != num_tokens * 8 or len(counts) != 128: + raise ValueError(f"invalid routing counts for {key}") + row: dict[str, Any] = {} + row.update(stat_columns("moe_gating_linear", router_row["gate_linear_time_ms"])) + row.update( + stat_columns( + "moe_gating_routing_topk", router_row["routing_topk_time_ms"] + ) + ) + row.update(zero_stat_columns("moe_shuffling")) + row.update(stat_columns("moe_grouped_gemm", raw["time_ms"])) + row.update(load_features(counts)) + row.update( + { + "num_tokens": num_tokens, + "num_experts": 128, + "expert_parallel_size": 1, + "routing_runtime_path": "standard_fused_topk", + "routing_assignment_policy": ( + "logit_topk" + if routing_mode == "uniform_random_logits" + else "fixed_hotset8" + ), + "routing_weight_policy": "softmax_renorm", + "routing_uses_router_logits": routing_mode == "uniform_random_logits", + "gating_runtime_context": "standalone_legacy", + "gating_runtime_context_impl": "vllm020_replicated_linear", + "use_gated": True, + "num_tensor_parallel_workers": tp, + "load_distribution": routing_mode, + "seed": 20260716, + "moe_grouped_gemm_backend": raw["backend"], + "measurement_type": "CUDA_EVENT", + "profiling_precision": "BF16", + "model_arch": "generic", + "quant_signature": "none", + "router_median_nonadditivity_ratio": router_row[ + "median_nonadditivity_ratio" + ], + "projection_policy": ( + "measured_gate+topk+modular_expert;shuffling_zero_because_" + "expert_measurement_includes_prepare_finalize" + ), + } + ) + rows.append(row) + + expected = 3 * 12 * 2 + if len(rows) != expected: + raise ValueError(f"expected {expected} MoE rows, got {len(rows)}") + moe_fields = [ + f"time_stats.{op}.{stat}" for op in MOE_OPS for stat in STAT_NAMES + ] + list(MOE_METADATA) + write_csv(output / "moe.csv", moe_fields, rows) + return len(rows) + + +def freeze_allreduce(inputs: list[Path], output: Path) -> int: + rows: list[dict[str, Any]] = [] + environments: list[dict[str, Any]] = [] + for path in inputs: + payload = load_json(path) + if payload.get("schema_version") != "qwen30_vllm020_allreduce_raw.v1": + raise ValueError(f"unexpected all-reduce schema in {path}") + rows.extend(payload["rows"]) + environments.append(payload["environment"]) + if {(row["tensor_parallel_size"], row["num_tokens"]) for row in rows} != { + (tp, tokens) + for tp in (2, 4) + for tokens in (1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192) + }: + raise ValueError("all-reduce TP/token coverage mismatch") + (output / "allreduce.json").write_text( + json.dumps( + { + "schema_version": "qwen30_vllm020_allreduce_frozen.v1", + "environment": environments, + "rows": sorted( + rows, + key=lambda row: ( + row["tensor_parallel_size"], + row["num_tokens"], + ), + ), + "frontier_consumption": ( + "diagnostic_only_in_base_profile_only_run; measured lookup " + "requires a separate CC-backend injection ablation" + ), + }, + indent=2, + sort_keys=True, + ) + + "\n" + ) + return len(rows) + + +def main() -> None: + args = parse_args() + all_inputs = [args.linear, *args.attention, args.moe, args.router, *args.allreduce] + for path in all_inputs: + if not path.is_file(): + raise SystemExit(f"missing input: {path}") + args.output.mkdir(parents=True, exist_ok=False) + + linear_output = args.output / "linear_op.csv" + shutil.copyfile(args.linear, linear_output) + with linear_output.open(newline="") as handle: + linear_rows = list(csv.DictReader(handle)) + if len(linear_rows) != 36: + raise ValueError(f"expected 36 linear rows, got {len(linear_rows)}") + + attention_rows, mixed_rows, attention_tps = freeze_attention( + list(args.attention), args.output + ) + moe_rows = freeze_moe(args.moe, args.router, args.output) + allreduce_rows = freeze_allreduce(list(args.allreduce), args.output) + + output_files = [ + linear_output, + args.output / "attention.csv", + args.output / "attention_true_mixed_fused.csv", + args.output / "moe.csv", + args.output / "allreduce.json", + ] + manifest = { + "schema_version": "frontier_qwen30_vllm020_frozen_profile.v2", + "profile_id": ( + "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-" + "fused-mixed-total-conserving" + ), + "environment_contract": { + "hardware": "NVIDIA H20", + "model": "Qwen3-30B-A3B", + "dtype": "bfloat16", + "vllm_version": "0.20.0", + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "tensor_parallel_sizes": [1, 2, 4], + }, + "row_counts": { + "linear": len(linear_rows), + "attention_frontier_compatible": attention_rows, + "attention_true_mixed_fused_diagnostic": mixed_rows, + "moe": moe_rows, + "allreduce": allreduce_rows, + }, + "attention_tp_coverage": attention_tps, + "projection_contract": { + "linear": "Frontier profiler using vLLM 0.20 CUDA operators", + "attention": ( + "Pure prefill/extend/decode FA3 core plus separately measured KV update; " + "input/output reshape assumed zero; exported mean is used as median target; " + "true mixed rows use a total-conserving compatibility projection" + ), + "attention_true_mixed": ( + "The directly measured fused total is preserved in diagnostics. Frontier's " + "two targets are projected by the same-TP pure prefill/decode reference " + "ratio, with projected prefill + decode exactly equal to the fused total; " + "the split is a schema compatibility attribution, not an observation" + ), + "moe": ( + "Replicated gate and fused top-k plus TP-local modular expert kernel; " + "expert measurement already includes prepare/finalize so shuffling is zero" + ), + "allreduce": ( + "Frozen exact runtime measurements; base profile-only comparison keeps the " + "historical Frontier CC backend fixed to isolate compute profile fidelity" + ), + }, + "inputs": {str(path.resolve()): sha256(path) for path in all_inputs}, + "outputs": {path.name: sha256(path) for path in output_files}, + } + manifest_path = args.output / "manifest.json" + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + print(json.dumps(manifest["row_counts"], sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-qwen30-vllm020-profile-v1/jobs_trace_routing.toml b/runs/frontier-qwen30-vllm020-profile-v1/jobs_trace_routing.toml new file mode 100644 index 0000000..5b7d00e --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/jobs_trace_routing.toml @@ -0,0 +1,19 @@ +version = 1 + +[[jobs]] +name = "qwen30-vllm020-trace-routing-20260716-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-qwen30-vllm020-profile-v1 && timeout --signal=TERM --kill-after=30s 3720 bash run_trace_routing.sh" +artifacts = ["artifacts/trace-routing-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-fleet/artifacts/trace-routing-v1" +FIXTURE = "/tmp/wjh/qwen30-routing-fixture-20260716.jsonl" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +VLLM_SOURCE = "/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build" +MODEL = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/prepare_profile_comparison.py b/runs/frontier-qwen30-vllm020-profile-v1/prepare_profile_comparison.py new file mode 100644 index 0000000..d154272 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/prepare_profile_comparison.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Prepare old/new profile-only Frontier manifests from the frozen P1 probes.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + + +PROFILE_KEYS = { + "linear_op_input_file": "linear_op.csv", + "atten_input_file": "attention.csv", + "moe_input_file": "moe.csv", +} + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--mode", choices=("old-profile-only", "new-profile-only"), required=True) + parser.add_argument("--profile-root", type=Path) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + source = json.loads(args.source.read_text()) + if source.get("status") != "PASS" or len(source.get("entries", [])) != 12: + raise SystemExit("source manifest must contain 12 passing P1 probes") + if args.mode == "new-profile-only" and args.profile_root is None: + raise SystemExit("--profile-root is required for new-profile-only") + + output = args.output.resolve() + config_root = output / "configs" + cache_root = output / "prediction-cache" + entries: list[dict[str, Any]] = [] + profile_hashes: dict[str, str] = {} + if args.profile_root is not None: + profile_root = args.profile_root.resolve() + for filename in PROFILE_KEYS.values(): + path = profile_root / filename + if not path.is_file(): + raise SystemExit(f"missing frozen profile: {path}") + profile_hashes[str(path)] = sha256(path) + + for entry in source["entries"]: + config_path = Path(entry["config"]) + config = json.loads(config_path.read_text()) + config["mode"] = args.mode + config["config_id"] = f"{config['cell_id']}__{args.mode}" + config["calibration"]["a_tp"] = 1.0 + knobs = config["frontier"]["knobs"] + knobs["cache_dir"] = str(cache_root) + knobs["no_cache"] = False + if args.mode == "new-profile-only": + for key, filename in PROFILE_KEYS.items(): + knobs[key] = str((args.profile_root.resolve() / filename)) + + target_config = config_root / f"{entry['fixture_id']}.json" + write_json(target_config, config) + updated_entry = dict(entry) + updated_entry["config"] = str(target_config) + updated_entry["calibration_scale"] = 1.0 + entries.append(updated_entry) + + prepared = { + "schema": "frontier-qwen30-profile-comparison-prepared.v1", + "status": "PASS", + "mode": args.mode, + "source": { + "manifest": str(args.source.resolve()), + "sha256": sha256(args.source), + }, + "profile_hashes": profile_hashes, + "isolation": { + "calibration_a_tp": 1.0, + "prediction_cache": str(cache_root), + "all_non_profile_knobs_inherited": True, + }, + "entries": entries, + } + write_json(output / "prepared-manifest.json", prepared) + print(output / "prepared-manifest.json") + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-qwen30-vllm020-profile-v1/run_trace_routing.sh b/runs/frontier-qwen30-vllm020-profile-v1/run_trace_routing.sh new file mode 100644 index 0000000..16838de --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/run_trace_routing.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +set -euo pipefail + +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" +FIXTURE="${FIXTURE:?FIXTURE must be set}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +MODEL="${MODEL:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/raw" +exec > >(tee -a "${OUTPUT_ROOT}/logs/full.log") 2>&1 + +IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?fleet GPU is required}" +if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then + echo "ERROR: expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2 + exit 1 +fi + +echo "ROUTING_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=${MODEL} runtime=vLLM-0.20.0+cu129 trace_fixture=${FIXTURE} fixture_sha256=e9e7f5b4e0d3a788dcd99d432f939d9e36bff2a64e412649b407b0609f0e39bb requests=8 input_tokens_trace=23673 output_tokens_trace=6502 TP=1 MBT=8192 MNS=64 prefix_cache=true chunked_prefill=true dtype=BF16 output=${OUTPUT_ROOT} expected_wall=15-40m hard_wall=3600s hard_gpu_cap=1.0_H20h" +date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" +nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader + +test "$(git -C "${VLLM_SOURCE}" rev-parse HEAD)" = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1" +test -s "${MODEL}/config.json" +echo "e9e7f5b4e0d3a788dcd99d432f939d9e36bff2a64e412649b407b0609f0e39bb ${FIXTURE}" | sha256sum -c - +git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit" +git -C "${VLLM_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/vllm-source.commit" +sha256sum capture_trace_routing.py run_trace_routing.sh \ + > "${OUTPUT_ROOT}/provenance/source.sha256" +sha256sum "${MODEL}/config.json" > "${OUTPUT_ROOT}/provenance/model-config.sha256" +sha256sum "${FIXTURE}" > "${OUTPUT_ROOT}/provenance/fixture.sha256" +uv pip freeze --python "${VENV_ROOT}/bin/python" \ + > "${OUTPUT_ROOT}/provenance/pip-freeze.txt" +nvidia-smi --query-gpu=index,uuid,name,driver_version,memory.total \ + --format=csv,noheader > "${OUTPUT_ROOT}/provenance/gpus.csv" + +timeout --signal=TERM --kill-after=30s 3300 \ + "${VENV_ROOT}/bin/python" capture_trace_routing.py \ + --vllm-source "${VLLM_SOURCE}" \ + --model "${MODEL}" \ + --fixture "${FIXTURE}" \ + --output "${OUTPUT_ROOT}/raw/routing.json" \ + --routes "${OUTPUT_ROOT}/raw/routes.npz" + +test -s "${OUTPUT_ROOT}/raw/routing.json" +test -s "${OUTPUT_ROOT}/raw/routes.npz" +sha256sum "${OUTPUT_ROOT}/raw/routing.json" "${OUTPUT_ROOT}/raw/routes.npz" \ + "${OUTPUT_ROOT}/provenance"/* > "${OUTPUT_ROOT}/artifacts.sha256" +date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ" +echo "TRACE_ROUTING_COMPLETE requests=8" From 008324e70cbb22d9c7869626344253509ec4060a Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 23:20:04 +0800 Subject: [PATCH 092/148] Summarize vLLM scheduler step profiles --- .../summarize_opprof.py | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/summarize_opprof.py diff --git a/runs/frontier-qwen30-vllm020-profile-v1/summarize_opprof.py b/runs/frontier-qwen30-vllm020-profile-v1/summarize_opprof.py new file mode 100644 index 0000000..4b67f57 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/summarize_opprof.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Summarize prompt-free vLLM scheduler/operator-profiler step records.""" + +from __future__ import annotations + +import argparse +import json +import math +import statistics +from collections import Counter, defaultdict +from pathlib import Path +from typing import Any + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--cell-root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def percentile(values: list[float], fraction: float) -> float: + ordered = sorted(values) + index = min(len(ordered) - 1, max(0, math.ceil(fraction * len(ordered)) - 1)) + return ordered[index] + + +def stats(values: list[float]) -> dict[str, float]: + return { + "min": min(values), + "mean": statistics.fmean(values), + "p50": percentile(values, 0.50), + "p95": percentile(values, 0.95), + "p99": percentile(values, 0.99), + "max": max(values), + } + + +def summarize_cell(path: Path) -> dict[str, Any]: + streams = sorted(path.glob("opprof/*.jsonl")) + if len(streams) != 1: + raise ValueError(f"expected one opprof stream in {path}, got {len(streams)}") + groups: dict[tuple[str, str], dict[str, list[float]]] = defaultdict( + lambda: defaultdict(list) + ) + phase_counts: Counter[str] = Counter() + graph_counts: Counter[str] = Counter() + prefix_queries = 0 + prefix_hits = 0 + total_records = 0 + dropped_records_max = 0 + with streams[0].open() as handle: + for line in handle: + record = json.loads(line) + if not record.get("model_executed", False): + continue + prefill_tokens = int(record.get("prefill_tokens", 0)) + decode_tokens = int(record.get("decode_tokens", 0)) + if prefill_tokens and decode_tokens: + phase = "true_mixed" + elif prefill_tokens: + phase = "pure_prefill" + elif decode_tokens: + phase = "pure_decode" + else: + phase = "empty" + graph = str((record.get("cudagraph") or {}).get("runtime_mode", "UNKNOWN")) + duration_ms = ( + int(record["complete_mono_ns"]) - int(record["submit_mono_ns"]) + ) / 1e6 + if duration_ms < 0: + raise ValueError(f"negative duration in {streams[0]}") + group = groups[(phase, graph)] + group["submit_to_complete_ms"].append(duration_ms) + group["scheduled_requests"].append(int(record["scheduled_requests"])) + group["prefill_tokens"].append(prefill_tokens) + group["decode_tokens"].append(decode_tokens) + group["total_tokens"].append(prefill_tokens + decode_tokens) + group["queue_waiting"].append(int(record["queues"]["waiting"])) + group["kv_usage"].append(float(record["kv"]["usage"])) + local_prefix = (record.get("prefix") or {}).get("local") or {} + prefix_queries += int(local_prefix.get("queries", 0)) + prefix_hits += int(local_prefix.get("hits", 0)) + dropped_records_max = max( + dropped_records_max, int(record.get("dropped_records_before", 0)) + ) + phase_counts[phase] += 1 + graph_counts[graph] += 1 + total_records += 1 + if total_records == 0: + raise ValueError(f"empty opprof stream: {streams[0]}") + summarized_groups = [] + for (phase, graph), metrics in sorted(groups.items()): + summarized_groups.append( + { + "phase": phase, + "cudagraph_runtime_mode": graph, + "steps": len(metrics["submit_to_complete_ms"]), + **{name: stats(values) for name, values in metrics.items()}, + } + ) + cell = path.name + tp_text, mns_text = cell.split("_") + return { + "cell": cell, + "tensor_parallel_size": int(tp_text.removeprefix("tp")), + "max_num_seqs": int(mns_text.removeprefix("mns")), + "stream": str(streams[0]), + "records": total_records, + "phase_counts": dict(phase_counts), + "cudagraph_runtime_mode_counts": dict(graph_counts), + "prefix": { + "queries": prefix_queries, + "hits": prefix_hits, + "hit_rate": prefix_hits / prefix_queries if prefix_queries else 0.0, + }, + "dropped_records_before_max": dropped_records_max, + "groups": summarized_groups, + } + + +def main() -> None: + args = parse_args() + cells = [summarize_cell(path) for path in sorted(args.cell_root.glob("tp*_mns*"))] + if not cells: + raise SystemExit(f"no cells found in {args.cell_root}") + payload = { + "schema_version": "qwen30_vllm020_opprof_summary.v1", + "measurement_semantics": ( + "complete_mono_ns - submit_mono_ns from the existing vLLM opprof record; " + "this includes the instrumented submit-to-completion interval and is not " + "claimed to be a single-kernel CUDA-event duration" + ), + "contains_prompt_text": False, + "cell_root": str(args.cell_root), + "cells": cells, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + print(json.dumps({"cells": len(cells), "records": sum(c["records"] for c in cells)})) + + +if __name__ == "__main__": + main() From 76107d3e87ab94cc8b074fcc578be17370087836 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Thu, 16 Jul 2026 23:59:10 +0800 Subject: [PATCH 093/148] Evaluate vLLM 0.20 profiles against Frontier --- docs/assets/simulator-fidelity/data.json | 60 + .../qwen235-prefill-config-ranking.png | Bin 0 -> 135942 bytes .../qwen235-t0-fixed-shape-ranking.png | Bin 0 -> 155272 bytes .../qwen30-mixed-config-ranking.png | Bin 0 -> 187630 bytes .../qwen30-vllm020-profile-ablation.png | Bin 0 -> 221239 bytes runs/fidelity-headroom/run_pilot_simulator.py | 18 +- .../analyze_op_trace_delta.py | 96 + .../analyze_p1_profile_ablation.py | 142 + .../analyze_routing_mismatch.py | 152 + .../analyze_s2_profile_ablation.py | 229 + .../qwen30-initial-op-trace-delta.json | 87 + .../artifacts/qwen30-p1-profile-ablation.csv | 37 + .../artifacts/qwen30-p1-profile-ablation.json | 511 ++ .../artifacts/qwen30-p1b-opprof-summary.json | 2171 +++++++ .../artifacts/qwen30-routing-mismatch.json | 1853 ++++++ .../artifacts/qwen30-s2-profile-ablation.json | 5222 +++++++++++++++++ .../artifacts/trace-support/support.json | 469 ++ .../experiment-card.md | 33 +- .../frozen/profile-v1/allreduce.json | 824 +++ .../frozen/profile-v1/attention.csv | 103 + .../profile-v1/attention_true_mixed_fused.csv | 31 + .../frozen/profile-v1/linear_op.csv | 37 + .../frozen/profile-v1/manifest.json | 53 + .../frozen/profile-v1/moe.csv | 73 + .../frozen/profile-v2/allreduce.json | 824 +++ .../frozen/profile-v2/attention.csv | 133 + .../profile-v2/attention_true_mixed_fused.csv | 31 + .../frozen/profile-v2/linear_op.csv | 37 + .../frozen/profile-v2/manifest.json | 53 + .../frozen/profile-v2/moe.csv | 73 + .../plot_profile_ablation.py | 142 + .../prepare_s2_profile_comparison.py | 133 + simulator-fidelity.md | 360 ++ 33 files changed, 13973 insertions(+), 14 deletions(-) create mode 100644 docs/assets/simulator-fidelity/data.json create mode 100644 docs/assets/simulator-fidelity/qwen235-prefill-config-ranking.png create mode 100644 docs/assets/simulator-fidelity/qwen235-t0-fixed-shape-ranking.png create mode 100644 docs/assets/simulator-fidelity/qwen30-mixed-config-ranking.png create mode 100644 docs/assets/simulator-fidelity/qwen30-vllm020-profile-ablation.png create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/analyze_op_trace_delta.py create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/analyze_p1_profile_ablation.py create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/analyze_routing_mismatch.py create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/analyze_s2_profile_ablation.py create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-initial-op-trace-delta.json create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-p1-profile-ablation.csv create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-p1-profile-ablation.json create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-p1b-opprof-summary.json create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-routing-mismatch.json create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-s2-profile-ablation.json create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/artifacts/trace-support/support.json create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/allreduce.json create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/attention.csv create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/attention_true_mixed_fused.csv create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/linear_op.csv create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/manifest.json create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/moe.csv create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/allreduce.json create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/attention.csv create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/attention_true_mixed_fused.csv create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/linear_op.csv create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/manifest.json create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/moe.csv create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/plot_profile_ablation.py create mode 100644 runs/frontier-qwen30-vllm020-profile-v1/prepare_s2_profile_comparison.py create mode 100644 simulator-fidelity.md diff --git a/docs/assets/simulator-fidelity/data.json b/docs/assets/simulator-fidelity/data.json new file mode 100644 index 0000000..6df95a9 --- /dev/null +++ b/docs/assets/simulator-fidelity/data.json @@ -0,0 +1,60 @@ +{ + "schema": "simulator-fidelity-figure-data-v1", + "objective": "maximum_tested_slo_feasible_offered_request_rate_per_gpu", + "qwen30_mixed": { + "sources": { + "real": "recovered-stores/aituner-interaction-runs-dash1-20260710/interaction-mixed-qwen30b-tp-mns-surface-high1-dash1-d8899c5-20260701T095858Z", + "comparison": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/metrics.json" + }, + "configs": [ + {"name": "tp1_mns8", "tp": 1, "mns": 8, "real": 2.1, "frontier_profile_only": 1.1, "frontier_calibrated": 1.7166666666666666}, + {"name": "tp1_mns16", "tp": 1, "mns": 16, "real": 2.35, "frontier_profile_only": 1.1, "frontier_calibrated": 2.3833333333333333}, + {"name": "tp1_mns32", "tp": 1, "mns": 32, "real": 2.283333333333333, "frontier_profile_only": 1.1, "frontier_calibrated": 2.3833333333333333}, + {"name": "tp1_mns64", "tp": 1, "mns": 64, "real": 2.283333333333333, "frontier_profile_only": 1.1, "frontier_calibrated": 2.3833333333333333}, + {"name": "tp2_mns8", "tp": 2, "mns": 8, "real": 2.275, "frontier_profile_only": 0.0, "frontier_calibrated": 1.7416666666666667}, + {"name": "tp2_mns16", "tp": 2, "mns": 16, "real": 2.275, "frontier_profile_only": 1.1916666666666667, "frontier_calibrated": 2.3}, + {"name": "tp2_mns32", "tp": 2, "mns": 32, "real": 3.283333333333333, "frontier_profile_only": 0.0, "frontier_calibrated": 3.75}, + {"name": "tp2_mns64", "tp": 2, "mns": 64, "real": 3.2583333333333333, "frontier_profile_only": 0.0, "frontier_calibrated": 3.75}, + {"name": "tp4_mns8", "tp": 4, "mns": 8, "real": 1.2833333333333334, "frontier_profile_only": 0.0, "frontier_calibrated": 1.3208333333333333}, + {"name": "tp4_mns16", "tp": 4, "mns": 16, "real": 2.441666666666667, "frontier_profile_only": 0.0, "frontier_calibrated": 2.5}, + {"name": "tp4_mns32", "tp": 4, "mns": 32, "real": 2.441666666666667, "frontier_profile_only": 1.3208333333333333, "frontier_calibrated": 2.5}, + {"name": "tp4_mns64", "tp": 4, "mns": 64, "real": 2.441666666666667, "frontier_profile_only": 1.3208333333333333, "frontier_calibrated": 2.5} + ], + "profile_only_metrics": { + "kendall_tau_b": 0.0, + "pairwise_exact_sign_accuracy": 0.3787878787878788, + "simulator_top_set": ["tp4_mns32", "tp4_mns64"], + "real_top_set": ["tp2_mns32"], + "top1_regret_worst": 0.25634517766497456 + }, + "calibrated_metrics": { + "kendall_tau_b": 0.9668009539030813, + "pairwise_exact_sign_accuracy": 0.9393939393939394, + "simulator_top_set": ["tp2_mns32", "tp2_mns64"], + "real_top_set": ["tp2_mns32"], + "top1_regret_best": 0.0, + "top1_regret_worst": 0.0076142131979695165 + } + }, + "qwen235_prefill": { + "source": "runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_refined_comparison.json", + "configs": [ + {"name": "tp4_mns64_mbt8192", "tp": 4, "mns": 64, "mbt": 8192, "expert_parallel": false, "real": 0.05, "frontier": 0.0375}, + {"name": "tp4_mns128_mbt8192", "tp": 4, "mns": 128, "mbt": 8192, "expert_parallel": false, "real": 0.05, "frontier": 0.0375}, + {"name": "tp4_mns64_mbt16384", "tp": 4, "mns": 64, "mbt": 16384, "expert_parallel": false, "real": 0.075, "frontier": 0.0625}, + {"name": "tp4_mns128_mbt16384", "tp": 4, "mns": 128, "mbt": 16384, "expert_parallel": false, "real": 0.075, "frontier": 0.0625}, + {"name": "tp8_mns64_mbt8192", "tp": 8, "mns": 64, "mbt": 8192, "expert_parallel": true, "real": 0.05625, "frontier": 0.05}, + {"name": "tp8_mns128_mbt8192", "tp": 8, "mns": 128, "mbt": 8192, "expert_parallel": true, "real": 0.05625, "frontier": 0.05}, + {"name": "tp8_mns64_mbt16384", "tp": 8, "mns": 64, "mbt": 16384, "expert_parallel": true, "real": 0.05625, "frontier": 0.05625}, + {"name": "tp8_mns128_mbt16384", "tp": 8, "mns": 128, "mbt": 16384, "expert_parallel": true, "real": 0.05625, "frontier": 0.05625} + ], + "metrics": { + "spearman_rank_correlation": 0.9486832980505138, + "pairwise_non_tied_accuracy": 1.0, + "comparable_non_tied_pairs": 20, + "simulator_top_set": ["tp4_mns64_mbt16384", "tp4_mns128_mbt16384"], + "real_top_set": ["tp4_mns64_mbt16384", "tp4_mns128_mbt16384"], + "top1_regret_worst": 0.0 + } + } +} diff --git a/docs/assets/simulator-fidelity/qwen235-prefill-config-ranking.png b/docs/assets/simulator-fidelity/qwen235-prefill-config-ranking.png new file mode 100644 index 0000000000000000000000000000000000000000..34b404588a53dc0758a0d5129e4f16ca04ea0d61 GIT binary patch literal 135942 zcmeFZcRZKx`v$C?G*Jo}8JQs?6iQS?vXY&Zo$O76tWbmo$=-XE5k(P-WbcGzXK$Y4 z?(=&+e|`UZ{(Zc@qw*g2eO>2up2v9{$9el*zjk@w9_l?LBqaM}u1G17kdRrBkZdp5 zy%YapDN^K){}Z*7zGd zp)~H=`PY2pczp8itwNuAv#CDyx>~)-p)t@8%&9UV*A(E(5_B*Sa4 ziX&Hy%OuV%`Hg?A7xFLi7#1$rSSHA!2=ZeDfZ2AB7VN&rY<=RBezqV^o+7nx72q+7OUU$8k#OHLlr$N0nZikTV5#@ejHURvRUhdFO}#l7ItLVO%5W3HwKK+~?sK(otvgzXE z<3D`*v^_jLJiDMk`o(FZ>t$}sTZ>NN-x-93kAD345wAP1uYV#WB;?lZ+dDNiHHAe* zV_oJ)*;G@X)mO<~yH*uQ?CW39K;c~&pVeRYAGoPy%%%a?o1&CT08J1cs6jtiQ%dSzr})K^h%-_qaT zzkC1wOC~04zkdCC^8C4PdO8>JvewokIyySZN(p=9ynXw( zqws&H%eynjp!(<3RPdKCU)J_}W4E!b#VsOK}){^czzBI zB~-J0e($kPeL0x!^GH+Dp`%BAT^A>7WO;4x-aQ{oC&p{{=em=tYntF~6%|T{sop?- z+=C{7c_L%)G-<+&?(dhl9K|GTS}_+&DjprFTelc=N}XbI10`e557mbC4-flXc)0UpO^r+jUsF?)^<=jQ zt%L_{1dnc2OPbmtdit!~sGJ-=>N9s9^OP-Z?J03o<8)rN)o?s_?%Y9Y>eMFjG}W|6 zug+Ncho7l7o2>I*)T|}d8xb5Q&>~ z(Oyte3aJ(P{rk5`OX^{l#mT_l_pX&6?j7Lgl-s$F+RL)5aLDnfRCM54yAJ~KkmAR? zf4(!?jkhZ<*kAmYB4anHH~g<)r#DHht*xJ*Hrk`5rS1aD7BuR-aESy@Kdvhbf=pwt+&W_`0w0^a$wc&efzxA(l}GqvVz!^6K_rRzQ>K-UzEvn zob3Mi$&W&@&_X?3tBCvo!xys>*Tp-+!oqLV^KLJVBt_qy8I)14>HM%NH5GbNA=Yzq zgE(NgGD~{_D+vdTDf^! zW(=m9;(t7u%rYpuR+pgzvr$h}okQj+;?W$GQ_lN=^U4tyrx8Suh?1yknMzmn#@Q2$yifI~I)*2f1s zu3Wp;nwoXdXMYS9=4SS-`$4gB6>OZGWHdq+=PMuWoBsXn)j68Q#<)m>zptd>Q0O1CUBY2bI#_wUyq(~6LDYZgjbSXg-YY$HJ$_|Tf6Y19_+ zs(U@f%WFG=$E10|V0wD`sGu3;&I5E@=Ifr*zjlz5ljqYdBA=SHrXM5rKV$WW)CDca zJpo_8%FznnCF$+$eJ*Goo-R}!bo^m(@PRws?_yfo7pM9z=iUC6?Kq{Au@Y07gF~>k zA1M?+fAWaxQR==bR9?yOFMK(^Kb|>DLj2Lav(S0Va--g@8?BIefgug}gx zi441sin|F%Mn--|_+gC=c9{0CTbw?9n%B6V_EUAWK}`rt?7Yr<7jGPnn%9b&Va?4d zOUujsI9qoe9ouB0<6VgZGC$TPT^IQHF)y~dIaQUjZKL+b4<#ZRh@Zn8Sh7&AEajyA z#g0?^jg5zC+s>4}e?Nbm`rq#r?|=LUfmSSzBn^SyuWok~v{T!!y`*GobnnooEoOI2%`*l6n3oZ>Dxv^!- zmiA(M&XvWf=q&5xvjz@VuI#=REgqOC|7uiVKNAxZCFhO3DJdzr#;Kdg0erVUkPta| z!1~*_Z~cRVUhVBSv3hCgJiE761sxAK!mTl=8FKrAy5pX-gsTy;I9Pw$vZ8;?d=8+h zd%1(lHOhT`rJ&Z*!p-d*Uh8F47j8<)KhihR^_FE(JG3hH?_mL_^d`F7d?o3n5ssc)4?&8rcquoa%L@B>D?m^9m4(Bb+EzjuP@#?&n6nubE;*2!ns9KjIk+|4daNx?V5Dvv!|-0Y&~!wn3n(RNfws= z!J5$L5}xAq(GolzZJF97`LgYR9C@9Vwa?h&rky%awryDcvYb501bBl(QH^C{?y^Gu z)eXJVSLPnDZP)(fo;%1NqB8Awe#Da&J52cFDr0~l=v$*v@bvM!=Vu)=2f31Zsf-f#Nzq7aBf?-wyD1IU~HTIx0iw;?#P{bs|yoHPaFDm#NP6!P**HQ)<%4ShQaJAG=l9e?!geQ21Q@v6U$H~(Iq#PH_wUC};QBs$O>C7@4e>ypn`I;amo{Q66h>*&oM(T_ zwf@(TZ2i||{NWdLnKSwH`Lj&x3lUpZB+oFWeo7L=K>yennXazx+(Z`@K!T*1S*nSI zuKWC9K#l_deo>#Utq2-^If$CMi$(U?lXSK8n|ym|g!UrUT*e#zE=?OW7Aw3y*J%Gd z1hwx`P3XxuU^3p*weDA}Zr^?sbo^|rYTAuI-9`LCbYdKxYBn~f@dtlwlYc8J-2OW~ z#ol=ZriqP?@?Y7MAGE~vzBMA>^h~wW$O!?`IcQr z#Kpx!x~3lPI&gxYUwP<&J5c`7vkqKUkB@CLHa1SM=eUvXi-3&-@;Q3WDcjH!8`#95 zSCXA|mY!_?{zpDOmtP0HaL@BSJv-cR8A+9ezMK|F7+*%9K9P;rFJE$qJbap$$B$yV z!`yuAfM;FV$S|t$-=R69sV7xd>3vHc&bs&=9~VACw*GHXqLaWXH?(9rPb+VU)b zcU4^k4-VRWau(S)GNG(&8u`f@?t&)Y&u6SGEG#@1wmxy`@li&N%Pm|{#lObKA4_a3 zZNoa&+sEBMotT)|o^MJa8*z$zIj#n=!9r0eCMMS6-}L?aLEGW_FuC^nG^6kc^f=S2 zQ{`7#dsuNAPn|l|cjs^WJL@*!(_MLX`|Yo$m6l|`{~X-=hV||DNIHc6$vSjxNT(QjSbWL?0-0R!oa|Q zOsZl5duk1Q3>d(`&AqR_zCKx_Aa#5yF;Pib*?Rhyba%NYJs%(6m#<&FUcI7OTVFRS zX!!c|6H@Usj)m0a%S^1SJFzr(?d-e&(?(k}(i#JqNf>rNxHNI4;Mp@ut@kd68=@sP z@2;5uv%9*w3ZFe2hpTWd zf1>_S3!1$F4!_Pu=}dl6M^-_h8VhuYmi93bF;!I+KYoy-y8iw9_dLE_5BXtHIKL-p zMC0~tR=~S^KY!kmx^#(n6?W4Exs#Tbc23m6(2$wvn($j@7M70)f(*^KkKVq0o4x=v z_0DE!&j}``>4gbR;awdy;8_6{9fc6Ro? zz`%W4+S>h-lR-8^we0T?X=p!SAYlAE&rR`O|h8=D_BPTn8Cy*gvPx#3>wG`$tej)qLSr`W!wY(Mi5e)G;{o`XO@#L8pS zdVA?`*odIHR?R^`B_dm%bD7&sBmnFaQsLLHUvDAuaOOUMa(MmvHFfl&+bFL7oyWSU zPPs=QJ}N#bsq*8;EyPEurXA_-?zTIy7bjf}9TYnR;kQd=7$xBvjk zTwNdjvQVBt~IK|1M8*F(e{u6XLNp$@!kF$kd-zzM-_-qgK#8SVk1} zINhRZf!ZtXZRi&SJa{_B&g;m^%O7sd(&Z5_YrX=E{Pb}HE+_d$#!Ft$4L3$1p&JWF zM@y%o&N}WPfE?u>?;pj8w2B8i$ZU?62PiJuW~ybqHd#k^T<6h%z)fq%rb`V&TRlvMe zj`z%&i19XnP>=#)M8n^)PA1tq(PVeL0W^abIx;exNL7u7FJG>hv}H!!MMdc_sO!0{ zRqml}(ox+H>B zWimKdmz9-O5;($3uksNo?wgz-Nm`!|H$+o`m&lm8*-I>_dR}6-{^vafL5GGs-U6$6 z0rZg4(b36{i8;?mu_?xSnMm}x%?6G?ztdaFeyj;uYu3HUh-}}!!&QMaNj95v(`jH7 zB=H4asPcP|+QZ$`xqL*elU>|pN2&Mx-Rzn`RO|IucuN-93`vdMNWZDr{@(R%)zs`z zT|~v>WBgv=y?p6{jml-8u`%NBmw1L>)S8r(|MF#mUa^8+FjCpPce;gR0BJd@(Q!ZHwI5YsQDhp1)*Kj_j6Vm{@s47aJobLc_{Tu76 zX4T9-Wv$D>GV{yR){RIJ-Z*%i8H*!Dt~y#w-{*^Fs)5^?9#ad-&S?Y@Z+vT|+yC+5 zF8WT&XH&<4g%oxEqwU>qfGpUdtgKu^rUKZaR?-JjEv59Ghd{@~L1Unzd}( z;cMy!=K5%TLk2dN27+B^G^rBhUYt0#nXDLRC;l67leyXgQCKU!$DqOiF>NRrjKJ3+ z7an>pU>Y*$$cCeZM#0Ynlrf{1m)D?7ICr-464ksxfGlatAh+hLBf^EN@j5ZjAT7*SzcC&}!nWp{Yq$zBUt( zZatW}s$FdNRIDs3E9)TGJR1vpiQ)3i4bA*A+{rl2-SAV|9G$_CS^mf;%e5~Y+J52S z#_}-zNA&A)Mu6mdI)+a~mw^)S0*ms+h2AEl3}=Ip$TSN}SkJ>he*9?GxieZe7nbcl z_eZhOOIbAw{T>-gNQhui*`n!a2t~v4?65bYvrgkk72pQAHp4(}<5xK@^JWP(8XY`2 zW0dOZ>cqdR^m-+XC!h{&*|zgb=QXki3|>AyS9_CL7X=Zu>I1|-*_vNuZuh!wWs+9Ho`Oyx8 z)Q~>Xt_v3~NFrYu1@l?|k}Bxc)zEm!3+5%P?RMDv$B!R-S(PncvZGJrTz-)q1O*J`^##lj{T85xB}p+(Ac?s#`m z-gvx$JKfdjbIwWt7$JxA06(&AhGaYVQevm`5Wk=r0ak;6BN~nD*Zx*3qT9)>h7KUX?=l}8eNo#UJVeDftUAs+ycEO zXh5Pl#ww}LrBdn z;z>$Y#1C~rXeS*&!*;aXY8r(&-n2jv`4iwS1^q8;%E-tNbS%08wvum1OpUolSJ5t+ z3KBq;pwdn%yiV?MSe_X&`Tl0hWOq?qV>^Oa5&}Vv!-Pg)XXwiI3m@8aJp*x5jHgd4 z#fgI?ByQMlvilf5UE?#Tf^GEl^y57xY7VdN-6KI0Eth5;22O?seEG9*BSXVmXm@f~ zI*RSnOw`Lg)^FXplPTl0Ns8~50=Q|`$x>2MB62B_AQy_hBoX)!XW`Z--$Nm*4kF-E zua|qcN1k=O=J1N3(fMzG{chI1b{ZlQNTHpKjEvL;<-p$H(^@)r`8VEnBRzjvO!y_a5`7GjH80n*N8pZ&*q@!PQK&RTH8IP!16L2yX; zKzLeOTKYNKl4ceh=Lbv(&`2%px{!Wp0^!$K=I#pi;PTkb)BOC6z>4Z)>kv_-8x{$q zj9{*zRxsOS7Q6%)j$T#2`?$O99gdZCSc$J!QP8h7_K-ES2E;*B`rsx8ek5GgsE!-Sne(GK`5jl^(bu1!FF-^l#JLuR z+Q>bfq7rB}2~OkzgGo8`^S$83)A~rYa#Qco=By^XA5)M8`CtPq<(EPo0cD ztXfk6e!(X$F7A^1TZ;~*+%;5Evw}lc+pk@{S_$MY-G!`7%FfO%TNfWvl%RhMv?~GN zG)}241RFQJ7Hdik26{+~a*6;W96I<82OTZgFMT~257i##(t39gD;{3T7=x`i%6seH4TH>^Ia@(5IW5nyAbAra z9GU~I5{G>NLl4T!CD4BHA%O&1Nj~9(>UV%%{1ceuX~-&9T8|5wJ<~BbWINu@S7TT_ z{%Sv9$bEpDfYvUw`1J;>&7ceZPWGH_v;ow6#-=DaJTSWI8Y`!F({WFz&jEVba4K#M zm65)aa?eTdiUgwv3~v;o#sALFe?B0wc8?$&W`BRXh*sl!p0PqBeQ#&J={EHH1;5(D zUho_Lg7Rkew!~)0zjdqw(B$BeBZ&?rfi!3Izdq;FIN2PZkf68px8H1ix6I2|(1VDa zA2nA1^$u!{?f?zXy+yy*M1N7zgHV{BD&I6amwEcpqen}lX<1TrZRzT9r~Q5+XNC+5 zVRu`E%Nlx+bO?}w7A>vi+QIDv-;MXR%Fq%NDe^&vDm@xCXqw5L69irR<_$weRlPX~ zb}^5&>B||R8ovexkPHt5E_l;7*+nPvfc^?Qcj*W}4sF7jh`Pg+lwOCJE+zMzVq)4R z<}&vo=Aw@P-Ri(mr~;fXw$IHrNDSbTG=f>GQMv=5lBJc_s2sj=D2MK&mIZmb)f zY;w2w5ii4G<*RVz%J~w93A@vNm0&i({u>8|l#%5^HS+J>n{JNaCL}sE>+$xwyY|z5 z_;7z4nbePw;ZdBWKR*%zF7-7iyhpnXTBI6`is0nMCW8?GbOLX~+3U;5RzwzM?X>%Lv>c(MgzgH_1JCRVqA>=kpJO&nzS z*XyxTqykpGwBX#r*Z3eA#w;xlN4HS(WND-0nAyb-9TvGUxn($K=P%%GBdeG7u0;{A)Q zPSJCGcYmIb`V@02Maul@)vL%m)0LYbDmgV4QM`yg`WjA~|6xw%x`L=E?H3Tz(=`fY zvj&c|+FyPFh&klKld7|Fgq}VaLaI#b^xT}viX-S=6vge}2x2W2qC~xMr1aaf^`48n zi$W%1)Od@uf`-HKb*YsRQi$zZUsctm)(sjb3C^)H_fFTclk$(RA zwUv{Tv&IDl_)lMXc~xFs$T@<4uka>QZRO|ZzX;7p&(bm@cM|XaTX{gjgtjTkZ_AAv zH}2Zodjo{l76KKwIcf3Xymzman|YOW8~P}B_Qr5uaS6A=}ysI1%u$o~)^vv#Exml6Z1m&2GwZCJEw zX?lRgs#8t6$KD<(Db@&^r@Z|BsqE-vdk#}%RFv(ySXM+AdM4G(*^X+q$;rv=ygcP1 z?R8o=myAZRK`0bz9j9;=MyJd1lRjn*COdb*07AZsv$Cz;>~OSaYC7U=8;Bb9VKII5 zq~8;+Qi04&PB*u*3dY@}o$Cb}k4QK0RMNGu-~s~@Ka-d~?(W$bE3N9G)7aSfr#&aG z>Wq}R3zwM9;1{hnre}%{fhRaQuinr!+Ve})bF?{St9|H0@rJPSrC(&~)!nt?i)hc` z_qb%F`HwF8Hte|rB&P3%GM0m-uHXe}5!%a;A@CseVRwzq%>!@=0fmY}WI4@WKXp9& z7SAnM;*g8qvg1-_6a2oXkOUcN>vIE>;)td_srMh~GY%h;p zcB4LaOb_Y9X@2xD!3_hW5_Sb_X~={f-@^zF;;dmsb3l}AU60doaVf6tpekRTEIH?V z;Le>p1d4!!lr_z@_|o6sUlIl$yYT!Bz|N=V<>HN8VLGy$*0TvOMcCZRcs zH9E|p@&SD*uSGkry2MR)*_i7D?=I~0m!Z(AFZ1+ME*{u_C~@edwX|p}Dk=`*{jde4 zbngV)p>T4btE(#xcmVAtdD+H_x!qK+wyES7oIz>h$9d|%SgeV9-jYHqFd-v}1P$hAte@9z5!}6->&pOBr zyxRBqrl#gqI-iu{3o;yfuky^(k`Mt9S9rmZ`2)$tAO{r{(j(lK(9W=|5~@&rq)4d2 zdvT97Vc0XcI_|8@jc7EGzkDwr92h7qCAIyvglDFWYy5aA)Id{_LQ^GlC8>62pRb3! z;!2l=COjzj4t(7b3g97BP!J~w8iKDRd_%w_CtaBSV9#=#ruCyfu{lO3T3xzy={cXl zPQpor;slRIKN{1c{I?$)mi+yDH|lt$TB-^uSmSCyNIq;C(*C(Nc|G8uLrH@D?Tp28Oo2(9lrn=oxfv{%F5`UKq=_@i;9aS2%2Ls}pZI zs>3Urn<;T@D&AQS@Ip$Q8*4iNW7${5B&5z*1uj{DUq}Yht0?DB+(P`?r&V=$-j+B{ z`3JS+n<{VLx$^`&J1NA7%h0d&_4Nsy)?I8*hHmTQ-~LKztA*xP0AmS@%Mo1aC)6F0 zwL-e|W@Xk4KoHWh=g6e@H_YDZRh%iP@AVgkZ zqFB&ru@fLn<65{r?g(slAsSN@X-c?tAj2=;?WI7xqp<3u>-y7@7V$?pPJ97b?H6!G z><~nXYM`hqZP6g(E{qBIbyq!{v1VSNHmKS z#hB<)>BOj&ldgS!!gvuZY@FM&{DrpRTHxjo3Tl$`fKdmJ9(_L6`7E=Z1*y+>4-4V) z_{0BeXo!X6p!+(4l6uh{%2G|)7Q%E<)F~e=uAN_2UhbK>0>9UFk)YsUg-i!{%cwFW zaE%F5rsz3iEx}xSff$rM6$%%RUm?`@Qj@b15)xrkIOu`0T}XS1x!OclGT3OQ(sheD z9UiO;*@gl;pVWGEnGu;Ao~1W~uBOe&PeF91IP?L*-Y|dR?CebHxw-!K$?Mmdt-@n% zS!(9Asyx6J{Q##S20OQJ|6?adQh`e6_34vTK}23Ad__v9QMMO`%wRE~XUa7!HqWRa3e; zKO;A{yy+@Fnhb`qkO{0ydVWT;hr}wu7GbH_wR=&7iM~r>Z4-Wi5cdhR;Wp}#;9c@F zuy0UEPj%jC5oZ+^zG*XVp;Rq^iW_2?(8`b1ZFER&y^naS^Ve2}x#S6LIrN|kT<)_% z+>z?Cd-Q1|{~v`mt?M9I{5%dMELf6w5htt6SYBfI$&-W7zpl@fkW*5|=l7tem9Eaq z&OQh;ic~e__HS+6+F#P#eN*RmJ9&Q1bzgNzmQ$N?V!A9k^zYYGZfM0V`F7IQthn@1+_EFgbP;b>U|Rx4ow7P1qaRoE@N=x6d~ zu6LlaAc1gLCXKnjEFAol2i~G_0b3Vpp&28&qp{GY>@A>Q1;A2TzB?4gpxJZ)9sdQU zg5WQ#1>Y=jh%;A!_G9NS2U4?$v{r-C@xrS$m!Rl|gtOp)|u%1YD1s~;; z%rIi`NnGW$?26!2yJqU1GBKM3_Q4zLq6bPh)}A20sWs74uw-om?gwwKa?oSTe|Ae~xrC;e#N`{wC{=0Zhfl+Fjx3JptL7sd}+W{tY&`jN9pNmyyqUk#=eQOf0k zfuhlz@&Mhr>qWHj;hr8Dv#H_qBryQmvDE>HMoP(x7j4(%b=!86tVePl08^N&jg^-U z{yf!Fa`L8zhS^#XVR}1!dxxXwdF03w`;uQ4$<;rW(T55tNL`MK5j!stC4W{->^Rja ztuNDOPM%EC%G1*+wqtLz=vCsk2>8q(I5r7bsansm(W0j77rtZ14!?Sv1X(BwQpSG? z6=zUx1Oh*GAbT4&mXMDGEZVQzI7F^J0}ZOj%E||<)DFVq_1E9ifBP?@K_gh@ut1SX_WFtIyyRH09T4O zdeI;^3Sr+u@={Gtz=DG_5aH%Fg6yeM@0*rM6aygR*u2%PACTJ>6J!rZ1#zl*WS!kn z13k$1{_$t(N-FLvCO{hv4 zg~f@mWt+8dKxX9R&QeP7A0)b!hariC75M9GG0l+~c)ZV}Y>Kbxjta-)Zedi(8@yRo zRuUC4Jypl% zkuIg8UqiF@fw@dI_kH460W4@liJTib0DSVNBQMEB^TI(xF>3L0nF%t8eAkn%9j{q< zxO3kFxTt^R0-+J~-#~kFN79HUY62fv=*Agza&WZE=dU3Zb3|WZt%Bq}Tpx82m|D6n zub~>;CDDX-wk`H~@)YB^+eg{};X)KX)wZM$NW_lg&c(&`k9L`A!KX+YKc7R>=i<72F++G`1IftdMi620~#8dnq!lFrOu&aqRb?)#$DA3*2*_|I_Ufy%HyE& z)|p@IF3!>x@8Km;Ew%{b!v^9k@quei7=p#J8pOo2aCXW)T-wDdGoo)25)uYOwZxMe z;ur{ks;?hw;tXs~^g@j@U{=sU?2H}a(JB3a6ffv6-(z5!Vr%6sc^~M>S0@gpoeD2f z3IYE-hOF5H*{})#8|h%GfWw@u&f|&ZE5d33sqai)jhib+i~F%?qTwU_nKjnLY=^ko z{~34up(zWp)zz%>FNqdFWF!3LOMm22 z_oYoRCeogd-)3iDcbOJ~&<}qbg*y*!_@QLX6StI=n0WyHkRg4-o1?CiD0i&!bV~L3 z=nro)D#C{hE6nv=9lTuiRJDRGs@HenOn_m?WZOK~rhQC~!*l}c=ao6WFDvBK)c%lp z4d8{!?Rr;StPf~SI2s76MuWuW&uV(lPsH&85s|-t zlBIxrCq+OI3K5V92@A!)`uiD0CUMB`!ZOo1gXV(${oP(He+|8Af%;_$d$`gCU^AN8 z84NzzkF~(m45@|aJO@4Wz{kX%2_6hv({L0&Ntm9nYA4l=AtEFzCk5T`tNpO*J_dZs z61smb1HA;CP3eZmQM7Aj&@7q@5S=8UX?Dyfcyyn|PNmx~0}REwo4C-<6wNFys<~?N zA+6K8V|GI3R2yU_S=LpO^T3=e6l7S9q#LJI3sKi&aNV&(O4Y&lHi8&GlbhXJTy0WV zyof^%nJ(5co_tTPx0l!XgWD5$v?nXca$)->yqmeBfJnq-z(GR8xZyl|1NYWHI?AEp zu`*{`L-XkqOldbxKV>R!HP^0z{O|U71$u+Zog{ekSo)F@SGU?Pny&nwn&MHvqp$x! z+oH45m@4Zgp`gYlc5e)=NQN{+oSh33!h`ImkJ#-@z@etuOZN z-m^!^IkcQIWgr~6_9Gkl z8+K0(|Xp`)=Hx5g2^=yy)TSH z8m6cpyy&!W;uxyA1Xh>@ON1E2A7a93ABdQfqIt7`bOB)5Fu!ARxR21n3e44N=$Vn- ziO&eA{}ktk;cpjjE?j0@W8F^>#;~D;gR0h z4)_rkDYC4LZjCySNlov$dSSL7cp6BBCe8q-MiXzxTGCFw5MClYsp^uSO>Eb8CwuN? zSHmx%BR$k2ee_|~;4LvavJ|s|Ho$3Pt>{M?G1LPr^#0Rw(J`$5cyTbjMEE(@WZm+z zT5-xfJ7C-66n$u|uYUrGu;sOM)|Vu&f`jRW-Bjdr?kCdK9Z2FLqUzg^0#+310v zJA*kDYDTKqqvB`oiRh3B_2Nzb1~x7mE>)7L@-zT#MDIC#^>w&>1@3g;Y^8WLx85tD ztp?-b-jgR!7If`{_J%9)>oOn|#8wxBBo%pGN_c0% zLR8e%k)do!0S_>r)M5KNm!nXVHdUHRqeS))dC31-=3R7yvkF3rTa}Y=>KPM+M+pIS@RiX~miR@Ihcl-ONk*PqxHgPg1pE5tEPOciMts(dA3eyIZ(_r+-ssQ>Zm>C^jYHT8h2r~|Wt($mvv{pO=# zCq4>%DqRP66w}M>sP&%jzZ=64^VJAQOmwhnB*D3|gk-8JE_UG*&Zb_)cF>1X`9YE~ zAAocT#Wm{jiwxe3v#Hy6?V|c~ta#tPeU0e2zeb&|H-qcQ{I># z8*INJA|gVt&9Lo@VHNK}?Mtvsl7F?6APdffj=YRC{4j3FlH>j=vBq|kIfRvk#dLm< zLS(}{kM;Ks)#_lIP({_2ra@uVmh@Xh(Oap0GI~_Pqx6TDCCSYked!>tiGLhCy-1!p zm7R0W0c%85L`1~+igS@p+N*5sCvE21Y{|p%_gq%a&OvtE3#yX=MD zASwY!>=$8BVhNtleUHivy8P?(Cjq#kAY<+VzzF*jf5>^>nO9HqOuRTjfP}SGZ&{C2 zOL67O7+sQVnXajxZw`F>CUX{++v{UBR4a+W6XmH5+LNJTFaC`OZ%XeNzVZ}B=E^P= zZMjn0;h-w_fN!xcVb%+OK<1=trdMkZILVDrJ%MZjt>!ZEEJ8KaP(x{UX2(d zaO9W~EK+}=TN1RFtS(%JA$faJU%uC@CCV#Oh%Bo)G+$bRDyk1DX!`Nvd%34)jiDs~ z0`UK};hIhx%6R*vY}kpj3kx3w1dzdJ?#FoX(Jde<IdGE zlAy4)wS^dcJ>L=*hBD^%SFc{xrJfWte-oMH&lY#>g#HDw{fwryblz)x+UcJ6Zjbq- z8hj}g^-xKKc{}z@iFT3A4&dT#e>Z+d9zW~AmSGH}1Xw@9UDe=MEYql!HLGx;mnyl0 zZsb+u{Ph;mjE;`A)-6KH4G@& zFLI^^@z~AlJY!EDbGvXhsJVIpo&wcet;u;P@QVbN;gI`3M?o0s_SJCR5(oJK!!tSb z6S>Z=t^why2r5D&(pZ2dMYxsKGPP>#w`8Nez~10LMReK`|8*IRY;0FX9$)#es$Px` zJ2hXfC2n#iAd;SY#^sSe@b=XW63k{o8j1Efv|bKa9S5Kt;#Y*((a%5v%*LNv1<^)4 zK7RH_E-|4nce+sgc7!4<4onC^HjUTFPhiVN1$Umemz>8tc%fc4wh^e^*a*IfEnBy3 zYY7?yvG*B~$KA0A$P^5qIWk`I`pJ`fK+!VwhD3z}A6kha?2`s&v2S4NS3G~71ORQi z79fYv9V#oW;d6kG_6QJWh*-b@2KUO)q9PH1Hm;5`;+iGI{l6urDcBfigs(Iz)_+s( zEaXhCW@MIcY2vZV+1THh1uC9<(_r_)Rkfvo?6Og`?5F3FaNGy(4BNtZ26Y=uRh0BxJyS6bTtK1 z84GIvYQ#H+JzX;2&jLOHdf~A*Y?r>_9bH{%a-+Ng0?*v!tUb=P&f`-!#&$no5XpNNd6|ck zNsx0h2{ao^Xb)zw(&nvwn}@Hyw6?Q*#ubsbvE}-*A-U=RSuHV(bk%9H+BE zBWwT|lQO;>WqIhsyr@Kq=%jk7Q%>vZ)|6Qz_R-ACE3C$1D)t@;|8mWnQU3SP#ap9C zfR|aS2}=uBYZ})N7Xv+k2Qacj%%Wp#(7#tM8vL~5^D|bjHKsCPkNzkDzqoq_ z{&bA%W2`-ue*4vMEUp+zE0 zffz4g)TnN1q5vd(;wCGpQ#)lpT+Zs+dfv#6kLJ3^+7(CeA7S=%;%;oq8?Y$b9}BrO z@K;!BQOKhG%|hX?CWD^@`%oVbGKg)AZx9ZI5H23I%*e>MD#-M&&jSYMy=F#6I{BaelW5iutuNr^ zu8rqP862TFdF>~Tfyl;fm?r>YJ(fJt)c*YW^GRGiBt~Y6!UXcO1fom3!QD86j6Z;b z1;z-GO-w2Tc8-Q7lBS!QnsW3hKt2!{)ubR?U<7aHJn#|SXbyb8fpp8Oe3l^|pF@Ij zWw`UO(F8|e7W@JS1*u)8HAN*Qq;ox$iGzcqZE;x{eMpn}-qEja8N{H_8W3HGJ>d$4 z#i9SpQznoL$cnI45uJ}{0O8q3sNNqOmnGy=f+8%oGmQ#N3w)4`Y)J|8T|O7G?&%JC z5@K2k+EL@o+EOVo=Y_Sq()zU6c`FHk=9{`F1ay(j9|r{=QZ1|mp!!VwL}b1(Y%=(6 zzwmOGd08XLCW7_bSabfT0=pObWe^M=7=mZv?wjk2v0b46hRO5K%|O_*QSM2|Dt(=h z%S#L~VUAd;E=%YAOHRg7<&)PUpG1m%udcp4v^R74Bd#i6ZgOC(Ko0{t&ad6DxaxVc z>=cm2QFctJHSWiaE{Q`Cv^z3r;hf`{g;|%u=@#@3PvMtT>?2eHII9Hrar;2J&_hEU z`_GMHZ~Lm*x(+}k+otXx{r~c7hiPN%j+bYwPN^G%;k^mPwYCh-XI}!|w-SBY9}8!C9LhuQzihzNjS;<} zB|=NEWxmfzxd&v0@Z~7Nb1E&(J^{xt1(>^MkHL{$i!~xFLrqD+8l)w}Q$ir{5CIk@ zGr1fC0g14@Y4F!!>GUxHk7rk?cDyY!!runJJPqM51MzQw23(dtFqIT-DkhTECXSgQ zzGjL!AOE3Uam})jJYln z5F`YRSWwoX?U~BV6->JkEfJ{CMp2xsYQSSfurE=c?J%2w2dKzyTqmGSBuunzfxOEJ z4C!Y#rBg%Z!Bm|RWqQJ}`?-j1qTy=k%aj!ZINFZ#>3jXL@FpI$f#!&#<1)su=MQO| zN-eDR@bEBOKff~G!4H!S86hR-H9mDvalZpLpUU0M(ZxmdQV=b>NK<(8<0NlwY&c;7 zO)%0YeT?Vm5cA^Vl_!#QvMxV9dLL|kf)RMqqi60!8m>Ml)`3kgAbiASry`$HojCMB zj5(4=^{Bmo;`|F_$9;59wKO7Rp$Hp&Fki7l5^I0AAs$X?=%8^2uRqF^PYYg4)M1=W zV*^E=80Q9Ko4bOvjNEwgJLD_a4KK{y&IGip@mNt1QqBA0;ldpcE-U1W6!u{rjOqnL0cKQ4h$S)ADG8o&GdV@@mX{}Me!jHJlDI4y1T;Ac{NE2;ar zkFsVodl?iO#srNF3_=G+?nR4&=uE^qmz2Y5O<3|^gShwh?HMqa83A}!jKh9Q;IR&j1jbI3hMNb^?KlxwwNBf*Tmc!>BV-8OhLUKd3yjsEEi@UTE6xK*Kco6& zY{W)yfJ=cn zj+gf!h>cIEnk}+k^#=grZ@#2TDZC%LGVu`mtkI5xT@6HBfepN{SNk2ob`05BJx{&!1T?6YOUs z9)(1>pF#dIR|`O$LmpSDa=;k_i5MclrCCvx1;DR)t1%a zL@^c2>&7S8aD%Hph{q(LkI{oxA1h}Y4~fWs!5nk#XaM^mScFKNc26ZVKfgjC4N0IZX8H_m2xoZs zg8YO!6=bYEpnVED4LSFY)q5(!naKySrEvxw+dm2XtmB=ai8T~=K8zHFbrqvE8soZQ zPoQnM`Ko~7#MjQhg2@T__ZGCnkTW}*Gqq_jF;*WcOe1Kvop1|GT)lboC}GtH$sY%g zJJ?{iph17E1Thm%Bu@ykp-df@M}4e*T!a7XFEkuHzT&3oGHU8~=o)K~qtQqW`KVKY z72<>CtENMdn~RHh#L*v?$p`uva^f)lxPFs(Is|jgO!*x|+!A8P=3DZs2`mHw*Z^o6 z@mVOokiQtA+#o?kl*GO!yD7(+tGRvdvnYbPk3O+9s^s~Cp{SZASFMMT>c*g85A zB&?J%Ka9o1pfe$hz`!2^$C?Ov7zH^w$Z|m#<9-3{5)yVgP3UX|JmY@}pUCUW9%ewv zxvw{#(ARN;&H->F_*&vYQTY}fu?cup)dQ56?+^`0(K$ zhb7bvNu&*&pskvkU9{C7F@h_VqLg4q7s~q2sM&sigz&wBR3T=Ga7(|hs1T1BKpG;N zJi;&!tjb}P_&po~4l(ygu|hBI=8GNb2Y%hW+cSnXlW>S`tS{;YR=dv(k`fP501FfY z+YFvSv;&R{A7G=(Z{N79hdFm|C4e6^?yIbiIW&1N9mFo8x#oL_>F&~BJqS2EafMz* zMaw?9$kQ0ip@MmeAUc2_k|SOnj=%D)ovbvPfp~r%0J9e!2M6Dc_28F7k7>?ierIXZ zKpz9)k7#X8qR@ougE>1YWO0=C`0*{J)7~lWmLpt1% z=*84l0S;^Llmm{b^R2IpW#z7NDaHH##3>OL5lPXyhO!@Pu(G`DSBThBJC>I?VQFb8 z-KDOfu?HB563pEWl$nmY?f0_p^oXk0X=nl(n0;nh9Ksm6=xtOt>8=-eJRX1v73^_4 zK&ll~(PA=r&^2u3@s{22#Xl%1Df#*5Pjc%!a227nXVCld)GZCPA=L{mFBw5WCP(

6z-YnlFpTyLNI+T2!-X2G)Spn{}oFndm{+R zx#IkQtp7d<=aBfM*3L33%T9CG@s$+N_ca^)r#cI+PHYHjzqMn`T3}RK-IrpZ$&UP( zm)`KHwpJF-TMC#{3|soL^&b9to*G22GO?L&DS@Z1b&UP0&LU=jlX8f`yM^|S)d3*U zly1)}8DyYlO)o|n+(i2P*wLX*Jn-brbvBh&Jon%*ruU?(TdRczfT*6Xdmua|B9&8% zAMwLtmoOj*-&B4emiRgz5JA+z+^l6ne;N^nU>@;a0iz8R#BQspso{ByT1M`CogNKC zP-nu%8@|F_Ovqv5WwS0t$0ayW%%{hU~_MkTA4v5+e`d$m4O(N!dT#G!{I!m2x zGt2VD%32cLL2?^?&VZ*8nQaeZ=K*GOYb~Fb1(}0I$vS=G|6%XU-+Ipbw{M1-vBlW4 zBtw#2*`pdnA`hpQOU$5tKp6By?if@vfyi|@;Y3S`-uzxr8y^j?_nO~5ZJjAv@ zJSxUkv|Y@2F}QnFjF>>-gssGdAbn^h6{CE&h7D_?;?G)4j_%OCNgp%G4DI276&I#V zY(Frief(1C6p!0jBld*ri^x+0W)MDj5zQS~S707waWx?yR4}!zZ$zk;zv3;Ak?Rn{ z#Tu5XTtNI5xjhjvR#d`2)p3nY@#DCGY1Vdo){o~#YAJsxiC;jm-(sSxG7(n~nj9fl zCwiDSslb6zNUsdONZBxH@@Uu^ZMy9IQsC;#X3Z1jVMoYj=Z9n?PdSDc;bPrl)c>G= z;>SdItTQ?5?tYX(HsGwl*@0?2#)hUZ$*mI^(-UeHrnAy6{jEa5`giH!_Emx5%xVfn zS)T7^h5#m6R@gaIQwIr&c}ypwiph#)6F(omuv*7;gRY<;&bjt2VZyD@;hcRVu%n(3 zA8kx9$_rXY-A?F7zwenA3uY^3?WZl)Uf&$F{uTO92GnG0GK;CImr{l_4!*Rzq@)CN zE;uOmReLq(1ifbUe1&sa%zSEGo~3Rc|Ni}E>bp}JhTp<@%qG;5u{}c{jD(mAvM0eT zAt*GBk}$tQrxi5efsCx|JJMssmeO{Lh4EXzrCS4`zo`4YTQ^EQ!M=|P+}YguRX|^w z+O4k-#W}tJOQ)*$@$uRG{Ny(Z;rFd>6Bw!ai-F3gx7*1b{VUzuF4`zEe}Qrt+n@^$ z&$lvwx}wWCVzWA6#c${-Bc4d~@2^IW`#yVB@801pLCLLGG*PH< za`M;pDpxkwk^((HfUzh7%P-$r_h4b_zk{skZKT9Tebt{GVtM`cS*1wrspTBs~A1=1z zbI+9?FiA_3{#dR6hd>qky%}!26@u<&H1|gD{syp-@zN$i zn)stV=2Q16cCG2MrA$v@$Y5`l9+xAG!f@fi&HE46z;6hG)E~e=CJWMrz+LW_0K!Q;JHeGSA`M zFhY}?QRAUV`__Z(o>@;s5)6C9xvbu9+<@=Fr~0w2X>$p?URNg{!Vc5epGRoi zma!6L=Db5+Mc;q1^!n3xRaK{`00WCDNq38WYxLN)t|w0XCgh-*_5a~*{3$m^FY{Xf z`nim%FlT2z1H*?pn(|}&FMq-4bU6NvXjrxt>jC2lH!c)L)ngQz<}FwNR4>IF*1KQJ z*Kl8Lj7o*N>n6rCKr$63$(zj3}X-W{EuF<2*g*V^UP_zSg(Ob%{_VmyE zCCN3)x^y&gDv5e?)~6*IeG@_Oa{V>DT#N-K6ayo&*5ebC(e}3xz>$Y51 z<>k+R4|2igY7=EYmR#$uzR*3Fzc=a`!}4DkuzN$JDN39)F=y~6Ly%FUW9XK@51zDD zO~A-fTe!M$1WbAP_N)H1YxyLcvnCqu#QG_BiWj_u$CSz`MX?@wHT`}jS1cJb1s;Y>)mAjsI}sr>;D zD`wF*HC5*xGwq58vG}=Ruq#jVErU8x8CkczKaTWDIWyeX_i6n`Ug)lGUu|9$t!E-Z zxa*ZVw0K<|H)ty+Oz{U>OhM8OCtN49UxzU8Ntqk_HDtJ?bCue!fJ{!(ilkm$ zYIGX;j4)seZqK{`jS+O~gx!Gy*LW`@oX_f|uhJ_r2GsOav|UkOVQw`XXp8PALUya% zhttERyAYwuPDmeq*>*PrD=Nyy5#wvqX47Rxg=}bRYkP-!Dq_nQon?A8=Fkx~RaxWD zYi=T?EDSM@KOLITwxY(;1Ct&}CHuWs$0v>819I>pwpjgEQrE&a{fEQ$5QHxAVU;>O zz8C@v-qKJ9e~I^upi!!3C^U(&RmOGyVk{}BP2?t|O8D*PjY`jDqb6dzx}A)Rk;+x z&_iIr!XuOlJ(IdDJ|&FS6A9tT3o#bCyE!U zLK_`xTO+$5^#P)9khvrTVWP#1m)-l++{((CFJ!{&7S-4`B~49D!DNG^JKV zunb}fOjrw(>)mq2*&(zfQw+jHb;sjKisPNNC(=EbqC|X5ew~oEolhwMR)qeWbhxv1 zD+PoJx6y9!_J<05rBvx_S3q!p!plum8^aM2iJC#22I^Hmh14{{uweShY@f-n{B{-B zvAI`iH-%d>*uWdOX;r=sW~T1ey)?PhkEk+)eKKQyOZWL4kB?X4gB+(Fl|o}7I_W2x z9uxhMjJkqZ3YaLN+G4FckjY=4cu@|7Vi4{+%rnV%o6%)Ci%#2398R z=*9uy370>mkoo&QO1em^2oGc-pGJ6cL24tQsyrQl=ut%r^l?uKW`UxWzx%^1w;v7z z2DKb>UbYhrwiqizB>RZFXPy$MNt{||=e3y@->|sAYgqboM$3t~NiaWR;NP9Ol6}vF z$OsEy-B4p^5`5MBEsJ%8=YhE1L_SHyy>dW2KmGRGiN*65{e_1o2q#7@J*ODQNmA*} zz$fGtvpWw^?E!l{q#YK7_705VC(NyPl2=f81OgIvxC7Gnz{$Koqk9Sh4Tl>`yL!y{ z5>0v;)Pfi|&tzyLsx1OHcRr%%wtMvG@l;t;fnr1GcCe~4qnGuE1q&K(Dbcs@8UP%j z77#w$RC~fI_td9@jUt=k+c8A|vG!Bb@o^FUi~Jtp*_dva-#bD)LZl7gE~->k=Hr!s z8Ao+?b_(4U6~u%ET87@jE`{4GFn%9zFWg;(VGLrI4<7?o{rT3ovd;oSufW9-atbGE zw5;Dp>>0o2TAcv&6<(dZ${e_-q67tkD6smx}V=!YB zPV}>89X)@32sH5`mgb@?fbgDpJ#O%kvB<@hSBT|1CTD>3nblICw#S;j z@@o-roDHga@rfvN|6^RK>pCi9vN*!vAi{c)!g?$g9)JqM9vOd1p$4Uf_xfbLd;#_& zB0y4Bi*ZwY!1eX@AK+&weipCg@4JhxjltZZkI;og?jUM0R=8=jnyM+jTmqTmlnZA< zT1P4{gqkP#`4;&h9p0D6D^Krra;wO4~_N8kSA1yVt7gfYRLJT?bx%yQx zpvpCsPH+N-N3GKtCOuUzWE98EyRJPG$od?q?&E03nkOQJGYtyy^J>+ig|{5cCL7kqv`k(~jIFu=iwtAI4--w)5GiADv0 zyJ6lW{teK+QMeY7_$RN*`?VVc&|bjcE8USm`Udkhy1fAF&BAv6SJ*Uv?pMEOeNA4xd=W-?{N)6l&~R6H+Px zylOJw3&SR9b-#b#g5&@DOnLc-blazhvCN`i@zKB2? z96WX5!i7TC&sq;(DksX3jJlZboZzj&N5=GQmFgi*>i_yHv-0Vf#O?x@{@;JK8Oi^5 zDbas-Ed6(7)PFZ%{dYgvf7h*%0sk8c{=W|e%jbO8HJ6wa4diQCIeyCa!V;LFU4fd~MA9;4mGzr}^` zL6dszKdZ%H4308S1=P0Kw-ZAZNb{3O>AL*|J%PRo(JmSavBUu;;1`exv2P7iJBY#_ z(9wLyhoCZ4Z^D9b4gJ2esU81@kUj}RCpse7FS@y9fRSs!H^n3aA61RdIjh=RRFun0 z>1YI3Ak=IO13yO5Butf%3OvMZkkWTCnkFq|wBP!j7k@zfwS4Q~Bl^)lo$WWnq&;yQ zZ7Xs07^343&)i$q9}Gi1FDfkT!TwT*6SS>Kbjj1d2sIXRDb?l6Po~APhwm|Noo;*v9{ne}kx41FEd07K#o%Y?ZQCUodP4a^(LP|vkn#7O z8JxkEA`I~URq7K=-kvb!7q&fH_I%1XaMZ^DNJ1O*O5VJP|8m)f7++ZJb?b&pi5L(T z5Mxx^N1$o}N~L!0psALJbDy$4x-Z}8oUPQTQDV^tZmesAFP`k}MsEPxJU74eeDRv= zy{?v3^f%YswR``5kEp~6V<&ApAhqfqXt)48C%Jl~trO11KnXq^6T88}PxAX;XFIp4 zJZFu|fFV73P0P@L%}UU!;DsuJ*C`JJV*tRR zKOZE~4Miyyb4mgg73^b>G)I~?UF;x?IA@HhnwaE&=B=_Z_-?oN`um?ff1U{z*}seP z8C#fI*ewj53LQrojsBz>TxJjZG==rI@R>Tj7p9t!_iHSBw%rQBT{V5C&_3;Lp(k&sdxjNyRZ{bD4;LW=42 zB!VFcA@Qu0$@zGr0lFqCB@U-yAi${K+^Tn8LEXUjdo z(5KHr?7M3_6T$wFRT}T>U`7Dm#=b2h6AxoeJ9H4>4B*c8)Cp+&05gcU_VDi+FcxrP7dNXqY%k&P3+Ebmapk{*AfV}gy zoq~$yYsD37_X~Kx|H`5O2>>i79vH08D0=g!S`z>gFl&9t5u*_vYx#^JuR~vwhvAQ% zGpQMTRk+fuNH>h!_BRa2H6k>l&Z8?{c)JpNEByxz9eSJ-Xaas>n0!_mzD;PMMeruh zVcjL@0@!^Uk)o_{-}{Fz<3sRYk-qi$-_%m{x3*-%$_Y%|qCVvy$RRK{V#2)+fL9Byw;lQtWvPxC^cP7^5zAr6P5+ac& zh+iU~rKJ2uI6DeT#XO-r!$Zg)D*OlG{>NLhY>X7%bKn2+IdY^|(G5Kiczjb%<&%iY zUH7W0Du`)uNSb?{#thLS!_=vl^?*eaCX$wUL4The(7(?~xw+w2J}`au&+wUB4t*8N zp)f8#&eT!G?FHB#c**RjD+uq-fqd4qAMaP;aRDYL-8n~Wy9-`FH~;zHMExXSQ(@t1 zVAk1~t$CM&e4?wUpx^I1foB`8ZN_>QEIWWAg$%gm!J<&Mqd5>;*9rgxzIvyHo9nx@%9I@%$<5Pf`X>J#PpgCOBz0Ud+l88uU9qq{TFFh3IflU%z(L1+aPEI z)~Wmr^n?lOKV;Q1AlGvao2QEv4il%Uh#@ueYBcblOrL@>FWH{>N?a>3Gy>*~=9{I6 zYLsDn#@BSKb3+;Q>InBQ>D8;@;-9g;7axSp!3zYuP3$A6#{iiGZ(wRF zsws>!uH!Ey{7cb&jxQ`dE-_L$&YN^@yxj|ibh1O&?a_9Q5CY=ah{6Mj=_X6dJ`8QM zU)P7v`=R*$T=7+L7zjL6n#Q|&!x0onBpZbLQxLgN(&;UD9+7N1yyn^bjCB4_NKs+& z%Rl}Pk`9!cLJ7C$*@1u+#3(US#;~4PFC{1ds2~oi$=0pE0n7_SdS*Eu;dI63-JVfq z8T~05S4j6M!fxcerp%1lvzMaJ4zn&*PMt9FeN*!n0;#YZ5{DL|QniVmr9e?pe)z?QSI#azv65=hV$@kk|}GF)A1|VUROKlf}`3+9N!P z)l$&Ff@@`6$PsGN^L3ScasiWbe~xh(ge?#fj2=Moe4rli5GY544v2x09z;jh9`!xn zaX4Tgd-yEosVRluVC)RPOf zWk+jKDO{&Uo1bQhNM5MN$>S^BqVAwS?u*<*+fcUI7`HEP%V^tftPVSpb)_G>yZdzi z=k={tWDWl=?99pGBMPndb%=eubNZqjH{pS!4R@cuaWn#ihp6Dq>-22xc2?Hah3+kN zQz~nNyOqa+gFD5VlYv50M*$X&T-UE=DGFph)-g`FFk2;WEJc7QOvL>(=eOz`k5WT( z)tB1g2~#wg?1|P6HXO2g^=jhO*IjXSz1Wu0BGSO|>izlt7ll!u}Mk2C%YNLyQ-Umi>-E(Tn&}L&p+Pj2ny&vN78R-xw?FxM%xv`Bjd zBga{L@ul%MozfP~fVQBYfBt#KfzO+u-1|XR&d=E6o>@Ns&8_-|c7N;J&B#;bh4e)H zHoB0cmoI0e%nwwEO&Hp}8#*2|P}wIYc=CgV#8}KQpk!=(Yk2c^OT$E+qWd(@Ca8PF z_BkX{__}|0dYVVu0bSs@iT#t}AEE!H==vEIt+sFhM#DtqD--jR^hUt}>KLjSJiD#z z@rb}6AzqE)9GK?*sfDljTBzA-jBJ^cTk75Uz3>+Qp)J=_we=}->u&$nlvh^Xim0Ws z6)VW(tU^DQ*VayD>|jOnd?7Zr;&M$lN;qU7tj*p_;!OAm9>_LwZx6wgE#`EZC#HPV zBh$*XJG;0HrTgkZU^nPMNVB?U|NeS~99!Z{kLmdc%3oZo%ud#ah0zTec-K)99VPCkp3W{TH2l6gb*oIzBSJ1CRQ9#zvGjxn z@|CwB=_WhJbHNoyjth-;r8s{yxAf4si&%iP9ikbUE^OT+UQpR>Yjm(EO^Ap8`Queg zLCYl6>-zWtA~C-l__3^ZBO7e8n$c1T^~o{Vyi-pQb&Ved%&}ynSg87~UtwWkZzdYf z9#Y}S=^dPZ@9y26%vZU8)Z1}?4$oQ`VU+K?%X-$u`uZ2&OBMp+eOl6V2%3qz(Pp&G5TbHsz|HRYLox1;J$jLdBV|zqZgXvJ(x0 zs!BT3J*nZshm<2xvCYll%=l=PzWw^?7V5F=eA_5WXPv?=Zyz07QKN^=T66uRDGZba zHpX8NCZeK*!r0o1K|ls!FOISHe!unp-m102xVZL! z5aFUkpHw@OX_G3~y}ONRht8gf8H184dfPWzxnaVNy-Ct8+~Lg0lfU1_M5cM`X8eYM zn6x8HNm7>Iyuf6-{tfdxQ*ihrN&)weGq33ToMj&)5}6UgZk>sFNgzEXF<`bxi@av%UFv*z$9@+UR#;0O;U+vJu^JLRC# zXuEU5g#&UehpvO}E@mr%*g0g-MO%=QO$GIMnPYGv;uTY446DwpQD>-Vkpyr02bRHWku@9vHo z1|w|;MGHGJd*6PayLvqZq#JsA=+xVp?H}0DF-JP{Hz10<0|qW^nTn_bIqCx~9?`~I zy)5|e)bgf@=Cy`3X%66IyrB;>Y-tDwY7+~meO$}5bECl$?d{xSr_mg&(v3%M zDK=MQT6Hf-l!f`}kaZbx)d6IZ%}uKI}(XSXHze|Jo+Sxo~Pxr0mlmC-Pjg}wEp zy1q5^N7ylLg!gT=++kaUDhMCgtcZ<=TYsT!<^qppapD&QvjqJXu!l)B;ptBw_RBYoNkR!iNPNMqDT6RfA2} zE&II(5wZtBwMA4B)jP%zQv*K{I>T@oU-f9vY*q|CW0Qn$G&lSKb<*Aunv3=#7WQ)# zBT!al@80<|)`9%xD2kCAKgVGQ_a4C8ecQ&hMOT35iMvWErHmhle%R3bOxnq034iH> z7;|K}4rsVi5c{^a)~m)Ezp%kdve_y2_egO4yPE6k(I?&7?4Xll7fQ%S+;S?bt+h!C zeqyX>?(W{57u^1HedX)bpIR4V@gNXUZb91ZL{gm*=Agm)^<@hi)^T3M&Z@%N3)44V zosg0*R3gBBITm-szjt@nBHdzsl-PrSnC>dvNttSQOMNmXYx*S+0yLztD#U z7B?p@^Qb(JVke2BfA7Fy|AV<|8P80s>vl5dEaEHV%ab~OdYG2hoAa=gQV+|Ezxebf z6U2O4MGyH@C*{|rt)O_BDj~r2K6r? z{pE6?*i6$xo7u1=OGxTvHLvysqCq&{n=az%`9osq2|xKas*1GPSDBCztw(LGGg0>u zGzuzskb+}q=Y%m1YE&s}uf4*DUy!X!$PJ|z7Jdg1yuqY9Xl5kKE!zP2&l(*0f%bPv9M)u~OCJDq+tQu_+5+KlS z0OkoPrHtY_k*g=x-lqZn+|r`Pd0FVwlVjY^lxPtyLZoAVoUDP5qpFLZJ?qO_??;$@ zsMdUwKG-5PdTCRq!#B4Tldhjw{cp*^y7)e8duS`3^cmxGZS1P0J05sUI@$AjudIU} zC5tW(uG;ir{L#^C3o83x{W#rZ>)cl|S9%;u44IrCwO~($=fN#oGnfLOFT6;+7TRTv z`iF}vvOCW+fFN1l+`M{C-(I!S_*7iyiW<7N)Yq>?C&dhjt@+b`7ZF{VG`&M;xhYwW z{_GlB!;y#c&biB*EfYIVd73&L)NoAHNvAs3t=nUP6OBxw}jUgEjLI>UA$s@mZJ<|51iOC z2An*!oSzlQQ8l2+9oj%rLPtCQ{M7ZjtbWyk(|TW16UaPg6_7E2^VN*2N`XL<`a5bE zu(Sl2Y-1JNyST$z^cQ}dxXH8=sKx%Jo!WV^FSlk#d{f@~efid#Dc9RR-LIQ+lgJqe zwTYW>0E)CI2tgF`E0CiOHAj9Oz(sko;M80{P@l-lAKm|0@@rta!Ik|i2LVjb$jJDT zxHS@jv5?S{%(VTjl&{mA>&X}f!8PPx-AWd)EJ3b3CovRtH~WaKwnlcPq^GYWU(Qx@TRS-8r*xRgZ*INvSQtO?L*D! z)Z6PfRDeM!VshW=z+74I1!pHL7xd%5Jl?j!njOY&shjnc!}DiPBgBaFAK0rwJVf{t z9XffYVLou#2#y|%4{o~n`z6o(eI&g$#xZm-trz>zL^VX8I3;bVLnlpC-uI0q=4l`J zo+HR*Z*5t*cZav(H!o$tc=?c-9pAobW7I$fTMc@p1q@%#MitOnLb7-oiZ<_#GB&^6 zX30~8j*;xI1e{3!R6utW%5%OAp41yJ3x3s<4E5sIueBM97C#JJV;$#IG~)cW)XfT2 zfF7}!Ugd>s2&6Gj(a+ngu${q@U>@w&<_Asc~{{$$=%JjOJp9x&TwUxlgPYxy1ddp`K#4+k20B>ba%Thd%$! z6z5cqL&8-j#R>a!tmV%|Vw-4H5cV+aQ@-)RRosPjPz$#F2r)K)JU_IT4&LRHr7MS$~MYg!c$8Wdf*mykQ9-xVn%cL6Vn%F z^YH3dON!OJx^TOGNul!gTCK25iQx*5tL?k$9J}tszUC&a;t|gF+-h~7fO7@pUD|iY z=FK+(Xc>=}r#O$?s?|bGBau$%&ZSJ4=~U0}oqZ^74*~>Jcd<0nB%W^;b*}>tRt;Po zZUW{PO&O+_H+q6~?U$@(Py_eW{QUgttvhR`ue+&nlWSBMRC=~UAj7L5(}tFF5zG`1 zeGM~VjZ>q1P&)*3q@3jo`B*^wB>M@gE^pQ^$@6tN*!aPu!KO~uEmd66!1TG6dCyEs zLuqGo>-mGHwC2p3mE%6!Qa34r%6Ime2_Z$*#N(Sl_xd?vS4~oV-e|q)soI2*bx1zW z)RWm~PPxL#otHdp7&xcGj-(G~^O&8Q0m>kGp{_L1@rYS?v()8(6Ont2pBF;)8rYOr zP^l(}w7_(fBy$(ZPc^#x(xRApNI~36()+kg6J^KFZ#}Hk+)~?Yw%3k&FaYFa z8}~BOWE4q2!DtK1Xmn%JogCEnl9TSeRe*;Ii+zSz1>1iz4=Ys;R6TR`+$4tQ!!`cHOcnjQDfw;;2BgiRr7jIIi-= znfd$jGU8<}5s82I{Yd+378Mj27`UB+MLvxO|0rw+6{SSX-@c(U>BJ>ocsRd~v&#-E zd7<3?EG#a#wt$bYZriqPK~{x(rfu2lxjDo0Raj7PgFhjD{U|-AS#I+Gjl95Oy3nN^ z%|>O4ZT3aRc?q`wKs896F>)$1e2uTVV=xz*(AtW*?%82z||Xo9DtL1mZjy3cfYaU`dCIWZ+W~9g-Wlz~j@*c1xB2(_cAE@|PHnsBStj-9xPWIkSD4?>hB}wx0R^S{7nY zWmiX?Bf;Q2>jr$N`=3Hd!c9~pA{LO$`|@tv>QiDh((!DxFBT0p)m-R(uJt*v7*1#1 zS?MC7CgX7?rko%=^`@P@y#?1}H~UO3dQ8w99slLY$f4QzWezZVuSFV5Di2mtoZq&a z!gMVe*y_T{iS?vlUh`yf(!S{jp(1VTN{T;UbX{fI=n8^0W0c<~435M^b>A8RL%iMV z+aXsi8BYx+X2d~80=T*C>+pz)*?-2YG5Kt%Z^6Mc6{&7&FpA%-iwXB9OH&d$=j0-)Y_4XBZ3B? zV_0=bA$j3I19uMeWWv$Va=dz)xO4OJt)Ek^OQu()h5|T3>eC{=l_OOlId7j7J+SYa(Y)n_}wNt{g z^iRYni%g$mZ=YVK-MFKuXm48Q5#5|hZGCBDdwY8zaJFuG;lSa;17Hyi&y1R)4u{aj z)0QyPOtW?FV_A8D0y-0QPtf5?YVZJHFdnq4ywbgpTC^KC=zLQy>R$WvC#+vn!ww;f z!Cd;DWH#Mz-GG`wd0nZd07*h81nNkz2VUIQq}96v1+9xSU*L2 zzyK5Z-fc9Id)d^I)XZSM-2VM7lG!jRvrQfyRW*=Sx3BctFRsWH^HFvZUn54jx>SIf zK-d=VQgO}DN=!a_{J3Gf?g;GZml|=~aH94s)?5;K6TdW`-oMWk96Opk7&xACllkGa z>ZnGh!sTd%oT`=;$KyB?oz-DiH&`p$K7|dM3 zUQf!>fj2tSvxK%uva8#f&aH$0365QRgdl97k$!o0U~WB=cBl5V`lnn<0LNCdZzrW<@$?!H5MiN&QO1H;qeFQ;HHUUia+;7NnwI{aEJ(WqO7DZyGBN+CT@zB5>4q2 zv)WGA_!Op_HcTD($H%iOj)%)hDyFefodLX&6~rsg^z}6_ghxHK-fhkcLD;?XTZ3i? zyi2TOR{|?Q%$yf2TxbOM>>6d{&WWjt$~T_rIV`_(oN`d@guG^h^$q=`m3Qc!YfFF3 ziq`75&!0aZc;4{ki~1V+xeRgh=rla8*~=Kb4jZoyu(1{iW~&1kI9T?#SCi=q!)4E= zoY(p|-3@|^&_TT(XJVxm)Ev>{oJ1m&l%Pv*}q zW}3#vmuj7|^O_HG7~foIDH4S+1E934Ar1n|X?Z5x`lys|M^P?e9ip(i`J(}ilp2>j zoBS9Eg-}iE>fNi?sRjwT=Er(>cbOzL$W^t@zo7LfLZ-GfQ6zf<0VL{%D zoU@6^EV+KXOBzsH7wAl)EcM?%{$WgeK+)~wR8HV9%i)&ZbpZLnAw0tSoY#Ssv7bHv z_x0P(1mer;55sYJi-`4%8ZZFkwQYNHVn6gAEVs$l7;Gz8_PLCV$$grH<6UQZ_v@F( zCxhrxsRDn}A?OJhWXKf#HS|Y!6fHjU4BttmJK^zgd7JI{IleE74BuJMAxOCW9W>rx z#P~tNzKE8f4L|QP_h;KynRYeg7gpk`VAdoZ*~`SL#}v!1f9-76%T{zwH2CC8)ih_y z{&$ai0sF1wf9R6hwJAsZ)ZLaL;mSd!Fvwh{^u~qxd$z2)y3jzRALx=CR&$l32+2tQ z-dqI6bH${JRhsHbH}XMOzO>m=7r}i!y?;GtXqsimrKtMm#;w?8=dK8y1^L)sV^(0eRNO$i*pW9}k7SsOE@J9ce-q~@ z<R!K+-BCtfc^mh2;i^CU?&7Hq3mF3L zYxr!Xo2)arYU{k%-GkSswtaauJvf*8zEDzq8DA&HFlM&!uSY!7SMBC%YIt!yME$x! zIUsNLH2UPtQK5l!A;nGOE2F%|C7RX7k6s>B^NavjP1?vL;MH2FD5`nY2nQZy^`~sQ zs~##C>GSI^+8Ma0 zsX3+Fym?w`8dQ4%ZQD%ODb_Ob%}XX$<#&Nd*(Ch&E%9lg-L^v++pThc`;jhg>^YN1Ja&Y5c)_Q~Sl|l<-rJ#C{3m8RgZfdqDBBNTR~+ zS7P+Sf_DQ7Pqn^yiQTno;NXshi`j;bOXyrWcq(BvgcmPBFSz&=qOk$!<5wDDq9b)G))#%eD^ z>-eS%CH6H_)nBM>K$ClmU*P{spP~AzmcI%&J3;uh>g+x{RvgJ!DpK>lYoS)`ifniJnt22{w6({_Z7Liwq=7SZJbUJrXOf`~9ZY5&tA`8cDR*BC% z=ihnT_`SXZmk&faHm6L%A{2Sx@yajC%F2c)7J_^ex1_nN^HY!ej!UVOjEoMkF{ypM z*8b`?>(KXc(JNi*^eqRLEv#m+{Zze7HxBio#!NL5<(5N>Pd~i+^zq z?@0B@(^C~K+ap3lL!aB5UXdO~s&mSAJ3h+pS)2&X$>sBQy|-g zfE}N!R(?g!>WPNv{58e@z&R@l?56%W)c<-eFL<&tNq1}xZCX(0xZ=X6R^hp=rX61{ zdB1d2c>ltCBSNs3Ae^Ya#|lmlur_@DX&qn@{^m_GBZ58MimET_H^OA)m(8^ud~o2y zcbEUQSc#|h+eV&@CF%x&s3~lDwr!^+r=}V_^GK1ibJU9>E#Hd_vKrALp)UxUbUXWyhK7oA=Z(xG-p zQt6RlM7&DTWsjG2?R;JXhrS07k`8tr#>s_sM4UZ3WnMPzI@}14nfcvItUJGJcZ3!wt z&Fy$I<-&vbOFMqk66_+!eF&f?O{pBRDZ0um;4E8QOq|T&fG*07+ri&9;H<6QrDS7^ zuMGB!FF)U(%WvcwZSY*SYPx>dHNbTV*D{dKK!>|Z?dD*FNM5WCm!h<|PulXB7JxYR zxcUKv+d1IcWqX$#Lw!(R@!V2hbFK70(FuV|7B3d~`~B*b8J72Nbh<>1m(TDOMQBnP zx?43KYLu~KhpqYZ%Q|GCb24tJ?s9WqJTAY8`-$!uvx&k9IFQRNtR1p$XF!P|8A^>-vG;qI9SvP#nNg_pq(hr8Izz8d}0 z47OHyYNN_-D9cKE^k^?T05-tLiloA|iXzH5mE%QJDcP$iZ`tTes8E3=O!YK+tx`Rt z1A3?pi*V=SC&(D|o|2v87XINnG3G?n*`okx{KbxsLWyISgZc~UMGlxHd6B7wl@tn)smm1NkuQT~`_L{HNU5zTN1csI0H401$?|P# zmfl?u`leD2v*r*uW_T*YmtZTMZEC+(%6Fo3RIOk+pi(n-m88c72aSJIQ0N`}ya5C4 z4<7`yWu zI8J$eQEuM1qeh8|Q}T;=n-P~cjq<2HcP{krL1MwM;0pLXwZ@29BLN)iI@F!vw}hVqPgQgo}drRSx^^ew%?(bLsnkkvqC%ns6BgA+vZ#0`y^$* zMC!-UeY0F77Kdl-L_E+H~XkeQ;MZCkXiAl z*@+e(_+^o(%>_0AOsW%XIGYPPZrMG-R2aT?_QvufNXQ z_GbEh1`{P$zaSFizK^HSE681TD8`aorwuoj9v@wK1ORAz^3hjL`CYzUSw(9GF z;Zs?bjBau+1cIZB^|_36A$|ElV6q_28`>;txv3peIMlP7qe5opRgR^AR0 zCcEOo6y-KMk=#rUD9P{psDGbTEbs3}uT-bLnKa(JM&BtIC$i(&$WN?nHjI>p7kCMi zDxp@o5K$2a)a0(w;)HA|hj=f2<}561mCAO9xDC#KF!68rZV_oksX*33iQ)~AY5k?~ z+%U)e;U}W3*!X0Vw7C|vJ#buJqCd_>$Fp6zfZ!q{T-*;bb*OK>Z@C7H*&zd$6!!;o zpYC#6i;>~>?$gIf>k}NdfMMRwFW^N*l13KfkL?UnD*r%lw$FS00yLw)sOzdbSm_LNfZg+zRqe2492%bdCoevH4UOInh?@Vbju0JGXi} z+@uM9(ssbmHzjptxJty_dx8ZQ=nm+fA1Tp-#4tzt?NoKzI0a8s5;pfcw_lX(Fat(; zUtVsmwUhBbn-RH_k@A8Fi_T)o=u4Khl7$iGPRWrBQElhOhex+eirBhl6g;Y_4BwYy^hv6%0O`G7O(Gy3m(dfx=gm1{}secfI$sl3U?c;tWq*O}lW7|n^m(Y* z)JGSSG9nu#45Ww>$B<@0tj_)td#ORZzqsKw{2P~*9b%Ikk-=nr9(4^NMz4fRIfeJi zdGNCU0s|{YBO$v?0#0tV*}b92Pv(@(X!95QzXD(YmIqqRoeC-@^PQ%&jhO z&B8W*JF$d+mdg*KRha&y5e;B&@OnM|haP&cA;14VM(7`o9+l(3)0t&5&(~KvdGrSI zy1SySg_BzGMkX1-@$@)os9P+r8ejYK?|H60Xsc+q@9h?pj)$p+W8;UjgwX30=K7!t z*;Ea)kh8#srhS^j)}*Nqu*(`ng!3ukGSk{LGsKp2-7><^(w zx&@&A6en^4DG6(OrZH(>*Rz*6VnNB#ukYP7zmw&T7Z5J%0q6@WVQrXQ7zTA%z}SN= zA;Ri_%*BaaRpIbL?(AOJjW%-KdA=1T&=TIIHda_7F0wMDiqjz)P)43MOJ2!T`MlPL z(s~>8-(>~!(-B)=gaBy<5TDboD&40z4~6i-ziy@YylBg*JI2O~^4KCaZ3<;+6tz*8N} zodvqJ1iurk?ZJ+Xzc6$#*E501_$Y?O8Gq7(ux;WuS#o-Yc9aC(AaND~a&Vz#@oqqE zp(mo8#(SGRd)6^EI_UM5KmYtQXs&_GtpVefNxEEZ=a!2=N7?1h>tIG->|!(s9v3dA zS9x`7kM|ue*kj6zML+%YQ$6Y|(0MM!EwY*T635duVY_D72op~9PR5?5tk`fiF)gGt zedek=cZEwa9|&1v5QfIL88D->b4Ord+Bv#}fXc*`<;89ncq4(O>n3J$%yT&nNp=EV zykD0v(y~6b@zCacL^a75FHBq~+r(4?v%KrhRLQ!jQxV3tv9?}7=$w%@$_yUN94Gti z8|uDbBl;RDtQA;MPt#zgG`ltUT_0|Ox&IwhxL|DgicEyGy{N)dCeIPpCA4< zw47wIfz2x#q!buye<5{JduqF9&z;NPEL|!5Kk&h3#!V9{m`KQfk&S>L&l5>7#|;3F z+0K3I-MjZ%z+tl~Zm5&bIN4_vSFnI9b8$7!7+*}=IeKIMtgjC=}(1`(- zmBYkXJf}Q|p1ZZZy@s~qG78GK*?+&YVck~?T%1mGRy5{s< zjZOPZ?Y@4#&aLvH!4H7qqe#dGqY#4qq8m2MTaU`!QdBRUOWN-ml15A@VJD)qNjhaYk zrzlgzCX6QXh-9A03%Bwgk^Ha0sXb(Tp>i(D7<_|pNUc^t&NGMcAIWLMJXGL(vqAAB zYDRyj;{_5IC8Bo^>nkMlX(csssUlQ63EtNU`iVW{T6Giv1Is06D|W7uttmB|{=~#NKIJK=Kmt~4Nsz-%iV|0?X;Y>= zL%Nos`bu|M%}(prr^2nQgqqCf;|tQ3`mQO{owz2C+wY#KjeA;BcR>vGF1WxPuS{BV z*w8&Y+AA5*8WehWgOQO2_{?(m0=LsqW(6fGmPL)U6TR9FcQG{4x26NBq^Uc7kWo;L zMX+Xgi?#TQO)5x>7ZuHvZ@*3AybNkHsP@j@_fcL^FOm!QMYkMGxtfrEwo;4e9a(dz zw6G;6%5vPnm@+3K&*pqZL#G%1N|u#c&7DK!);th&vr}8`y=k*9jdF~)>TlV8OR3$H zP*^3TNB%{b`45^udViUpawdfGqE}lGq|Bfc&Byw*Cxgt_Cmh&mt#C;L$Y%DCiEAH+ zv<25n=eg`Zs62U`OYE<+F0IY`@?o1ze8rt)sN9Q?l@yr+%~c|!m79%M#>p*Q_!@|# zqep0BRN3RJrTgz=f?Iiaqpm0u<6>-he;Oz=SAWv^aSFQ)d~EPw_A|<^wdudiy zQE?OYex%F5?dc@f(UqACz=Cda^a`ccn~F?DX_ds~x#TU+L#NZ;%hwJ7tc%pT#9R;QXKz9kyVa`<;Axm{?;T z;q9Azf|g1^FhCs>_RCE~^L>jV_xDh(o&c5y1IH<-Nnek}yh5Ow@1)F3ZGZ#menoL` z--9Vh4+_@{+rq{ADFF3I$zBT+&XpBT8?Sz*!Jnix!o|n(G0R*K$wy6oa&*MZEoW%v z%e12z^01(SI8wBOBAk#p_*Mk7g5SwloR{A06WyPrkjhp@5Wm;LX-vJjpl!rgbFJIj z6O**;YQJ!(pR~3Gg37cJ{K|}CUqwIi& z`l#0R&$xhPueZL3?4+5QJz#W zFVXJaF`_A9lk@QdwIh+YuB8roH~eiwnK|GxYH-82n&m5q-MQ2iwTcPjC&G-`mpRYv zJ1A^-#k1}EN3qss1k+EVCvAyPe-2Ir(x69{EsLTcD;MCHfZ0Bo54z(>JPj1}3HhJ# z*p!fYG&{HGeV&$blE$Rw!Ye-}}?-)`1MiP4ujF(IZ;y^^Pf5QPP8wrj}R2-moLTkmysrq3LJ~cQO z6kG|yEXxMONEaYlGiTnrq>=(sPY4{UfC2iQ{!o5loLWE5$Vw`WGuhG4&yG8Rw(gpc z%k4T$j%6Lzw!<9%234oo*U`KPV$A^?dz=Z=9I6=V~NVi~(52gTE3yp}GZa7sQU1|+$XRw3o1uYhGP?ijU! zzKIR;;n2+T)M9zZvPvjR~e; z9ci4TnvXh|gYfK=gIN04h8)JML+LvBq9H3YJiVM9WmgAX*?VSQ^@ueqT2stcl$|U! zm6=E}M3cQTsEH$?gB1D}lK@upO3$`mC;bT;PwZhLL&sRY0^F=}(^S@?1rE;WqKfTW zm+unZ$}Af%=Fz+k-Aei@lF&A~{cz)3-GAJH>D}|+_v>~tR1ht3-uCMM(_**JQgDiy!XMZ>cO?bkN$v{u#W%x9vq$Hgen85Lx)W)!O+;ZI1nFD z$uMK&LM1W*9U%?iMEY4#^awQ-4fNyfs-BR_w|SzXI~V62g@@4AoC;gIHbe|JAOI6t zLQvG8Gd1K@2p>J%#c~)96f*|9ZJjrITp+JyL`FcXi6 z$9RLfYa2IT5l@m&CGmj>GSe9m5cz_is7)WcS#u3#=Q&p3uzO~oP^lMU?SiEK7HCs7 z_pAQP7cH6zcO)~Y(lusVO5Ra+!}h`^RpObMAA z5uc5se}Mk2q_Nb6$8tOHyex$dQG^{aLPnRZ&hN@&u3qS^1Y01uAuiyM(W9p!5fZ8e za@`to4nJlxP4l!xwG*~JBP1rzlhuKcd;DH0jf}V>P08t=b z0LW@6VX$b_(o7hg$+|&;etzbVO@DkI^;vm_*Lu-55oSeSxdHvvqM~=^EevwpMh_*j zi8#o}RYmpB0a^z-+RI}6%=N3HM zjAhVG9KZtTBn_EZ=Dzq*nTbt(R3qX&m&DoS-cTPD-&%1+1iK~g{MY)OW}wtM7-ua0 zRIVUilvQ@`+owxKa~}PM4oi~&_V`gzL9%1UM-hoMh{KuuOj}fTUa-nSLl09*stJHy z6ugqqLJCHUH#+6slGR_JIsKu>{6SO1=OQoT`!t3RzEY?z+$XvbQvG-iq8kpIf4Y++ zXBMaVsp0xCM1I@of<$5z0V>$w&N3>}m%M}b)zy=D0z;-5i5RDYjOYKb_a0DHo!_@8 zX&PgTy<$BGqKKe?pnw_?4^jjv0s@Mn(gg$sDJE8==>b6jK?RhKbWw^Gq=(+YP^Bve zq{Eww7{&bVcw@Zr|BpNFyZ1Uq5<}T%?{9zKDs#@YR;Rte6>;=*h(l@ST~Uh(UK~5k z5<(S>DI;33r?*!bTZABZcHiv?Mftz2qBENZN^3f72uaHTPCcNbk3d4^u2?)=O$cs{ z)|5i@xe+R|lwg8mS_$$!J-NWWK?A2K1VZrfY;35VpYjn< z=)4E+U4Z+eGPd-=%%z`BPZ(1~AAo?P(8c@|9PAKMcMKD}4>TtXZ_@@7t&efuuSU$t zMD5e+q4xo9BszukNr!+L5)xt#F7)9$fw&qk_b4Zz z%j^9^Z!WKm{4OA{w?QC?*-@wapb_*gL{U(Vh)bG)1TdF`lBY7(<}0 zI=`I#x9Mjr;zoUmOxLOl9=hd^k5UyDO{tL3!VZuhy1=pO@;!&wVS(^r-eAh#^a38s zFbuTjh`IuS4|RfK`PFS+G=ILw`+$QDX^w0jMm$O_U~@8c#U#Ib=?w!K0?)H>S|AQ} z5gEMFu2B1P68qE-A7J-Ts z0+xw5}okzjpf9|H=hl3nl&VJdEn)Ac()s2mv_P)7K~0eC=N!b0IL}=c?1!pzIAn z7*05tsM)`7Zy@S_$}Ru5ufhMlAieDW+CKVkT*FnDk)%C@{siQVNA>-83#hReI~Ag# z$Bnb5a*WM;FjPYj`&pikh;TNuZIF#{CA}KVhY;_&#Mr>G1$-LHhyj}O8LL)3f6#%5 zv=?K)$P*xwZH^(A)AM9J`>^}=$(vkTiY#!~udHggJXV~zCNoD{TN{;;cYj&5DI8Hh zQH==h6kfi+;H3lT5@v#j8FfbKEb2y3y2fCVT;gbqrc&8zCalY!h&}4#I^abRbdN1A zx-yG%NXEe=T3_O{w)xc?jh-&2yaGEb&Tmmf462>W^)l%7|Z-cGYDMm&ce_ zLHC;bFJHYPl$)spuL}G?Ms)l3Wu7$@2Gu;}VnPiVhV1(GakH0mr&Ue5t9LT=+iD_yx5YIJ4;Pn(zK}HC`0R(~SN|&eI zRv%t5<;D{fX+(hbV=BS9Myvxj41i(KlCZZG%Zy^jn3LT>awC*nK}guLsc@JYj*lXJ z&s@HMqr4P=0h1yvNHKVQ^d8CitYcDk0bIQCLpcXWW((H8xeR;UQFWoK^ta0#MCJS_ z?-ob{#~qj3gMg(76+_C<-;o{;ai0>2Tc9nmL!Stew1yj_U${CFl~{@rpi|7oP(YOR zWRMxXBw|wFpC4^h zk9mt|jBCW*UN|5~{=X`PE865f6#0^XkRKUoUQ4->T^a>0k#fOw*0%4hVtOBfuArcY7*}Ocic$KfXy5vVv_97a7>6dr(NDi0=Zp4s z-e{uWfn=F-d$9ZHihU<{Ai${RX~|=M=OMq?gxUmOSWK#Kr5pvRXBJ)_R8FDjk4F8? zfJRgumiSvu@ypLY-{{?8jiLodGUBjZl)$r-FA=m!wJr3@utOJyCol764$ZML?`s8^%j2vG7}i+6<>qYc;~uy+?kdctT<@b>Q>3CcJ|eb8O-Y~FkT z+VHLX^DMPtG#0Vr-q$G>jL&_AHSGg_M6I_d%>s6K3mQ>M=*c7kH<;<=G-QHzmS|mx zXWOWYHZ~6i<0C9%K2`W5yCpgp6b;q3;Zu32!lg9)4p_+rOP1(cjB4*tLtptj=voj0 zV4D};Hn*_YlJC@Q)*BNu7XDAvOSa{L+MZs;pU|;WH=fE^d(qotJ?7A+_d#-}|6os3>6c)0vYXguNqP@S}QgNF&ngfYwoX^it&mR-js0a^Bql7tw z&elMWsILYFDnGhC;6bFNk8LE?(3X;`G?R+K;S_jxy3=|*=p1&Qb7_#E1JlWGh_m0CF2vh6p6#c6}vk3x~Cett(&^=<(_C(AC2+J{9mnC~hmdcF!lBUH@`J zQhWBG0sO({ejNC%8~aha8H%@xEspkbYIt@PWR^s{BLpHT1?g6ifQJGYpdSG3$JaJ( z=FD4}$H}A95P1TomIxl-?AC!X&B7#ug#M(hAmA!`?UVoqzZ#L&YrC}-e!~!q1H>uHp zne+1Z_oqV-thpU|FA3%_KJz{S8h-xrBhXOE5oZNtvjiA4as9~T3rW0Av1TEO_Yh!G z!wIVrz&)ow3w{c5JudN;nZr_w8A<>)fmDjq?cvsnu)D}khxO%q3uW6j?7;;=h#RTd z0^m~vwF=aI;iF5Ph=ILzP+ndTnZP=TEWHoe`3A&q@T`SUC>ki9fQXb8h>aA63P4i) z;(c>cc?Cok)TENaeZ&^ja|WRDRxb$E?_$@R$TK3)I+)@UFiTY1K}P}6Kme>8{W>IH z&O(2|{5zCTU?C3IXBUFerAlkS3lxR|oeh8?hv4oY_Qht?SBbSY*=?t)0In$^Kj?*0fC(9N zNikq$@}a=uif|^$I)i#R1>ljiCcsp66%`QI=Vn$yw5<%*3|)6Ok%SGD2uc4CuAKPT z>x!i^L^MiX8MGchlfrh4_q==0m`$@SS`qZ#VpyX%2~LKH1=`+N(@}T`+U-O_K(=83 zbB+kZ+j=m#yBWrdjxs9{K88xr1u8QUfLM>m#mgg)67XDxjzHEct<24puu9#g~H^WCY zTLrj<&EC34vdF z!D&$j0E5d5`uHh|Eu2>0ZJ*b4~vI&c8!y7b2IbvrKYJuijMD+v9?o@k+^ zjQ_F32o3`5fMPygJX0KjCGdV$ukE;lnZJwE>&UeTMNnHx8_%kv{5~=hO@=Bos5(f= zycl$%VHb(>}O{rggZGooyfU;)bh+2q61zO zGCVbFxWdp!*iU4))G$EjEp7Ltiz-TlfrHQ&Tk#vgX*I!4e2;T3;EIRT~P^qB2iF9J{>W4fWYai_^q%Z`e z&&JZl$Jq74IoONfbpwI(XYjys3ZVKKMz4m&pyIU7$~V>0D<$p2v7cJOHiC>O1Xo>* zFu_R}ip)pU_X(TQlYt&iyKsbs@+GMa*tBJMt(bex4zHbo-OL5x(x-(RK2Qy*TEkbLY(kSNg0%@IM1=k)?D@vLcsdh6X7}6pRj*vU_*8t&lgJ|Y zGpddyzlSKFkYt1#ivx1NTM;2ZlspS_rPoqjz=Oy|`6`5bW4cV~Y2{v?Pb;MPHw>F7 ze8G}&p3hAqteZk(bXGVeuk0BJA=4p-DKp>Q4KsB}&xw=1jIwI9;>d9n0NDR0DQ6Vd zA%GQj2Of_itfd0Fml&-;2w?Zr4y)ywz)=Y%0;JI0qOQp@Z^<(jVEu)xDTkSZaH|Y~ zE8pB}fh_?N(k?>-;5FzDPZ&Yz8%TCNZL5U91=J>9XDmSC0MIt#v%fwg+(O1Rd(Mn zRFOH3M@Y-;wy&My$t~RNQZAte$Kf-Z?Lq*@f0QZp)OGdXwEMAh5xrY*f?mBl9J}8f z@ATMY;4w6hTmBSWgFrJ*y<8!C6y`pGxnkL`ccH1uO#@e|q4f#jV4Qb2pSZ$S7nxGf zLD4{_49xHY?(V5WZ5u3Dt%zcCGOh_BwJHK{1yUoZ#f}=Iaxsw|l){r3m)=^8?xhA} z5*t9ug5*AR8y8H|*F(E?MB;<#pE!}9v-dDd?Qf#72&gX8o8f#+1cfCym$}{44$Dc9 zSZ`P;E^!42RZBhn>P{zud!(b!+Gz3%(XWG+*qFu=NvrWLKRyjvx0*T&VAm4Q1TP{e zTk07IPX2aTClUo!X)XmLqN4#kt!o{eX~BCXdsE^cG)XyV~c>*D$K$@#emzx`8)wk!35^M-wX_=>! zUk;~u=CEGDL%C&am+7zoA_}w7Sr6|lM-Rz2i%Vja0Ax=Y%_HyYxT3i@(Kq+c)1qr( zu9Oj~MdU>xC>yjac3I~dhlm&y zo+4buNbOXv#_&rptuwK&!D5{&nPb-|A5icxAR7%!ep$GYSl~8fnyA?=%bC{2f1S^I z$I$z1c&0>+JV*|Jd6gklx9soI^07=WHR%+Aa2} zF9h^7CxJ-;oJptv!mI@d3fmEQ5|xZ|c0%ar)}X!sozr z=6^J7nd@brjJ$|KPX@IsYtA30NE9p^V?F9{G*WG?!!4!mN+afIgC}q1egLt#6h6-e zwQ3aI)FfF*Qy(hAPHU?@lpi=m!q=LK<@Z9-vTxr$R`m=k6jidVT*T~+^wB^YDwj3u zKP{W3lisZ*z2+MZExGpr_jPg9_)sO)LOzW^RKc`?V5Vj@i2h7(Z?*!}&2C4ZfS;yJ zNl&Ul$m^s}otxfWoZSl}o^@RCO^6?`!;B=k^HK^^pcFox+;(m0$0Cfd+sld(!uu&h zl9rpTvA;Y)C?=$L$M3bZG57D^k4`@s7C8{Nq|9pG!Q%IYr3`5!^_b&|0Qloz^Qe~s z!6V=&bJ>{J6$JI{J{F?Do9`GTNg-hYQrEyy-It&6+HyKm*m>JC{o)qUcAaQ#KzJsQ zleRzaUqoOIU>J?uD09l0^L@;=v3gz`rMR2;84@A^tA9VI>QoB!0rXQaid1UBgJQn6 zYw7DFR=U_r^gvu+86RFSxX%MtARAp4Qjb<(=!(R%tivfbB_m=yJa27ZRijB~Z~8LR z@)E0FOL^o(ga%?$5Ht@Yq6Fmxq{0C?%|ywK3Iaj=b^Wu-$0tCvK1Fxzl0kf3s9UE% zvJBd;f=DYPyGTc_w&e;+&fP|a54;>x!*7$4YsyH^jy5eSqfPL7-lwyhkUB?@Y=c-zV)5;;C%R1OSZ93j8Nw@pF6fo~OY5 z1;n%UDjCpjF2Y z*NYcC#cn5klDyxJ5Ihs%dV6&NbZyCv09@z4SP&w?^J!70a z0u{A!p4%Ab+`8^nhSRF>66D?v?!yd=5^Gq|OV_eb&P|c+&EK-ReS>)STTNOCv^qt=poUoe;?1jKtaR9t6MpNZLSC>-q=a@vHd6_`3t2J%}u z0KjEH3Q^j=3VukPDr2y(VPB?$S9tTg;g|y{3GGLbC!8z!tG#^#bD|$nmb59ifqHrK z+rZJl2_y&Ax~#3o(q~x2RG`xtCUOce3rQY|B!x@nyrN?{k~9JobBZqh@dqJl%=~$N zz>Iu&I=Q@pUv~h3ItAelfk|d&il|6pi3&cCe&BF5xvg$jLB6!>3UPz7q236c`K})9qj^>VJc)&T^{);grVHq zhk-6hTJ4MN&X`C?_b)!L?iS^7;TvxLgUm9(c-EKF5R7U7Kynn29BQbP0HO0ZF53wl;>}>@t3%(%ZKk4M>yQ-h$H!Y+1X$_KDfr(jJ%v;j}yeAjy@s3j#4s zbVMvcQ4p`E^ZjX74q7Dfw+@Pt0*Xa_=x(=2yKMAdOaK#`C#gWX7@H$u@XB>B>oc7^ z@ZKKKRUo)0N^JrQg)-YS=jL~=RFuUIPHV~c7jekYV57>DSkgjvfgOr@8s7gpPfpj* zjd2St#wu0T4TQUfpLO$z#W&H{z%YrR- zFsYYFc#~*`ljOJOc>^CjLGzD=fO731!V+j!LxCPB5mq3a+q&Qg^jGxJ$q;_fja9(v z9R>nbGhzdw&&SXJ5RA=b_<=U-T{R}3FLt#ti@}-k34Cp&65XsE#c|QcOybpqz zLqNSw<9sF;s##Enqv))!7+~^4lpr@ZH=zhvFbBMA{n!yer@3o6ya2xJm23_kUD`@705KTjhu_MvPj*$YO*h6eY*Wl>Qcvk>_e`dS>arT7+j2(?pWFi~rZ9Q~ZL z;K$KCCpnq%!&JYT`oaiCl$>PlMEBG**D>pLdvsSwuV3;llXUwk^%ImXw^29J&n__{4eZ4+u~BO8 zUJbrs08- zrZOa|Y-067j_ExwcJBV(IdKTCLP&NB59GlBb=MKJ^k|tU%9#!j{0Xj%y6*tW+mZC! z_pCj=-7LtSUYw|SFQUiw=IjAwNqvQt`f|hyp%oXi5Srootu~hCRgaHeq|yq0WFoy2j{r z##1301MMh)8$Q0`CuJ7(+kx#qv=Ik+oV4joPmg-E?*NdeRLH&*j3A|4wlWpP^1z|z zAJc&BP9^*h>>?zRy>LdF3^68(JMN+FzL80PfSZI&c*(i-)C3Yi7mNBv1FXn;kb{HU zo5}5u@ma=9PVMjq+zH5C|BTc-kFL1(;+U6)eEKDNGW+sGOO-(cZV^+4{eP@_#vJiG z!+L;n?K&%BrAc`RbDL|o@Iz>Oby6%Kb|i1-MyY%xo7@rU%NdzF5x}cpjc}YOGu9iq+~Hr8iNX> zRp}#_Y+EYo8yk;HbZ@;QOEk`#!3OHl7K%HnkiDWHOKlU}HlrEeyTt*gq^U;&^{=9w zrjTv%3h5iS%&?>N@}T3B1ZU(dcPb5Z#O9+^Mj1JkA%gNY^gWF^k&FG)@Q%PA057?| zH+qP_!kZ;{5q=(hQ^o1wL1h%dm{6H1zWL@9O`Qw{m`#OrNJc`?3!EA?Qez1$jscQ- zSU*of?qFIrV1yB~bQJHK$R{G%N%sm1q)HnE06i&X;Vs2b=}*FLcz**#D@ZRqtgS&6 z=lY3j;-^ai%TQQrfQW)3M~HgogBpZS&<7CItVdK30voQ(p`hbfC1N%71n*GUIn}m; z48IAk7B}z+rdJE*sED)@co~edPXwUfrUGYo23A3CrCsk$pe%gIW0@%KuE=p5;pasg zWeUk)9qv2=vYoY~^1Lo-&5_ts6hI2{w2>Ys#(LPL<^pyD+1nmA+bP}`HAjPcf0i!( z?%ntRgv&@K4GIGttr^WQAXJ}7stEKSjih7;kH%sEa?Hg59rR6|XlyW)2egOdJ8A)d}r}YB&#oWs5uDtJ4j)3+Wf`q0UI~K@ZR+APiYFA*+Dxf}ya5 zC0P&XgZlm;B%{9AKwsE$I<9DZ`_s(j+YW&ARYR370!?Z&OWh42A1pzdOT%fPzgH|| zFI$7jEMCTfhooeNLSnSe62hcH_S>{V_M9C7d+Sr7r=eav@|Ofz zkX*cKEW#TNfm0@>6$v9yDs;$$P!gm84UGNeU>67or5#EmtFiR(UAI8W=pB+U@Obkc zMZl4;IAPe~RP#V$e=7NT;EOl_oBiZX**+bWfW0$a77Ogg^w`S`eXwO9JQSS;DZxMj z?O8uJ0gFk^oloIi5l97rHRmyVERj*yD_C)R=S!W4y2N;uQB7X3BgP70JcH_z(WZu9 zoY%(xh!cpqlH={KgMZj53+A3?iBJ<-(g1)L5avE5?FF>~(aFT?v!0O3mq=QY^}-G5 zuXrJvg|w{GKIS>lG!+nVM8;AqfO!YY8m{ z5ca!j#2%Fsx2a1iDaPTMN?><$MW!!7v0lu+S{IoPJ6w!HU;r1IS_#y|yX6Eh_aP9N zB`B95aWb_Q;W;LFCp>+D58@V3ZHKAX15rZ=1`TmCRI`Ucv9vb(zwUu;C&>?OG)$YI zl$VON9vnu>iW)L=ww#;ei#ZvJGRLj0Cd=49T&GI$oPZw`fa-$_6lfMf|13nQsBkJG zK$!|Yz+FQ;qk^+a=_O?#K-;M)*-U8-G$;q_ah1SVnydy~fv|FDiXNV=wU$N(Sc;S; zCsT^~25JC=5rv>Yiqjk%=->kWM9HNSS}oB$3>0Fk00E@A5i{DUs~xfH)G7}~0lEP% zV0MKM<%kkI1riaV?u^Vz{sF3p0z(jQRG|dWoaRChElpi@4n5vUY%Gy$+hwN%FrjKa z3I|}Msg{G)K8nkM=#&_9N{Eb-lVQ6*^mLu&zJIBm@SEKUq+;z{C9u{Twez=h$il&B1l zWI8AWKwFNLR8vIyw%`1;e^mosU5xWm0{D=W9RSA3gp$nw>v#$a7roA-1q3q%3_z-= z!iIQyNU#O`1Lh;oL()|Ycz|F_+Peat;~3&bD+Mx386gY3B9eVU^8GeHUjg)9A@KiT ziWC8MZT6p@8VqY!i7*cbJP4iD*4eDFg$IJBi3`mvv;lvs0N6mS%zEmOi{pTuTt@wg zsh1Ui0#$$Azj%=AmjV#!gMI{kLv`U?+!3qM^JB$M0hFYU6mabB!JT6D+hiPlWv4L* zr-qk96G3V>CfRlJsX22dOZZl;r*4rj=+O*)#CpwTh!w_#oWId$a;#E>A*GfkEUM?h z9^$gtgP+U1`i*_$;BYj6*`7ep$WeiK))JKdk%Qxg?Z4Ek931?KK&x=_6yWy>odfyE z*E;yVk_vYPrO`0_pLPRoBRx=0p2h-VVDRx zgn$PXj}4^5U2hmI1|s*_JAwc6kg8l0U1NJwR!*A0nyr6DU&R4wso zf{d>hsTnDbK3*5yM4}xCD)zJBE^&N#MZ-V0-Td+)tAf@X9Udpr6c;!zYY<1=m%C5Sw$4c)Jr z^=KX&MZ3G&kIeigC!M&1X;mjEx`PN#oT)@^0A-vI{$%j5k*`vy#|nfJMIQ;v`iHrM zg~L_3Ghn=QkpABE@=|jagv661RM+OLWiY8LB%D}NTbuX2zRkP8Q616e?R=R4Ohu~& z)(YvFJA-OUDFQv3M6KDdkgPxisv(G9)<4Ka>eG(3V@+rhARAS!%0?+03GYKg%Pc1 z);6QD9Wg^x9|%740KQ8KU36>LfwO=fmPM8K__+O~YYm?0n2^W-+p@h>uwY){+BIwD zBYufW`0|N=|K&p9yXNNR8#c^xPf!=S_IDwIIFt>DtoC$8qtO}wFH>o$Zs zLzqmyF^l5lAEwTta{c-Ri42H8Df@J_L6NMhrxX&kKOcB?ZvDk@qL zPH9JT8RA$M?EksaI2yW)?VnnUxB(hnU`XryJn5Pj#^=sW-~vFNGh%}xbm45ktiK_& zWCh|_QL2f}OVMx?hL!0WZy)-(28-gn|H-6FO#HR}cPuxGsz zWQL*=8JeXdhp2=F2XRfSweBb2w96|`Akw3eXcMCnu!$-YO``w{f-l*=SJXW-5W28C z4Ie=qf|b;_d&S}So)8c2T6Vic?q3on1@;F z<3J>q$xDJwe3J|Z6^1Uz0uLcEJq1>vGS%)GRS!e0^&Z{ong_6#sc*D~OI_tB43tVE z0+8rhu$dISk{k!;65%HQ?}jtqN%daty*&A)4Ro)Dy9D=nmD-=+&A-Rt=T&@g>0cfb zNhO1cNVfnO9!t0K3|g+>2m8R2(QObyMzeJANf;li3$b_+)|mQ3+;&gn(tmwzP8M*U zKgxc%7DbX&NFwWgXN6h)j`>4frbsO*u%JtE-sZ6&Pz!3G;Bbpm}ZW2Br#N>QLv%Ugh*VAuX zHDxWx^_SE^&RaY_CLc7bmjkgbbI&g2en})>+Z9W{yI|RS+3dG(51H43X;tI?ARB0r zA!Xc;^l&02o99D~sc6b0gfmPHsTOqvu9*Oiz)+HDFrP6ecoAe z)Egoan1H(lVJ*whv`aUFi~tVoO^AVk8P#(ZA@u5fvq*;kA*hAj&Wby>5-C8L12kUg z8o~%_VD~~umj^9O<;G{|#o}W*W$M&0(1{b3$EQX)=FYY|TK(suiK5-l8{nPZ-j`Ne zkug!TTO1aeRAm6`LCR0$hcr!vQamsP9{&D1*NtM4e}o=<fm04 zhVge!4Sx9RH)LqR$SjHcBt!-hACRBrhnwMisRd%KU4PBFh%Fb&3l>TI%Mh!9shEOy zABi8$I6sKZq{-kEeVWUIOCm=lMH|GI`KsL=L-;sLHU`vxQM#OBD>El>9g*yfsCB;QosemUP(w&edqE?4!vIs zt;NfcRJ>y<;TBa?W>Gg2gsRKz_j1ggYPb4ZU#4wLwPOphnpBPvzAjB_<$4txi9+hm z!N>XpT;Q1VrJ;Irm^$>-r415<{q!EOIJQnHBsw(p>{S97#QInn$F4V)Hv@rTWcc1& z90XEBfHj|sz&~O4`6yj@1ll6~HubonGz04+sDQ)e2bt~sV}Y6+j#pDRTYYQY&QUVA zN}Y0Z%F=!Np?ZV#f;)Xee*{1-$EEn0%1rp4#V(TG=Yv+Qux{>P(-1v zm#zlly(X|HsBB@!Aw{YR2gy;rFw>Rg0sS_Xn{@_ug*E@XsrUJvIsdgzRy)c#cgA4+ zZlYK>Y~Zv|QjD-rCGj)hu(y#vN-?SI@xZWS6LR;3Mb2}#-$kt`>3%ef96d|t?s0jz z?9F$p%>G=I?wb1RWE0N6eQ)`c8ylVXZgr{0XzP1V5oa=(i0a}NYn7mAjmo>A;Do4f zHIo|7Kx#X;_~F`f*4FB>)2B^S1R}uqunv>B(`onlSYEw!>9zG!#pU$2k9NGDJ{bl0 z5VQ8^vYpo(K<1ojEu=HC1ENP8nv1H{hmNzgx|Nt6qJju;aS_ml`Dr^FUSwvj&(pJL ze0x-&6A3qgFwXi|XEmCUs1N6nsNN4}fu0pdZ*H`qL6k8dL7fcWa?G7$_m{=LU#KYe zSq_dPfv`3Lwcv}j&^Gz82G!va(j;tJYvi{OB}|H&3>XzWBw8aXX0dTK3ivY4IY6KU zloXm60b{tDh3!Dr!?*Db>pmk%y#OERe}Khq0=WQZU^-&d7jb0V=MUruH1Xxn^K7>{ z3Eu_ogBzzY7&Nq!sRXQ;8d3m04F<4<4iheZa96NVt$1RBh?5Pq zVzW zjH4K(5_pue7C#qtH^s#*!u%kC)0{(>Pk~U zGuT*B#d;SiDdmAS!EDpYUw^%aQ#LWv$`ej4!hWcg8{nu|X$$~0HIOS!pw_lY0C*`caZ~3Pt&@ffgryvVoGCxm;fBFTkL+sp!Tn%QH2uV47%A+;E5zp8GDf$ zO+oL?b;j~L#@wOZL%YB1a{MJZ*%#MapLFqg*h#Dily5N%eRy3uz(DIp5~&Xmsp$vp zkkSdAe)s&i^UoZt72n!D)jUpS7?dT7_g zX88QCLk+&T`oI1>5Bru{oA~yBt)~O^2I2V7hSHYg!-o%}g$sj4B}!F^{7J7kw8eTa zN6q^7#$HdwHd&qyDkU=V%odKZs&8H5g>m5)+I#={qpk9PHxicQ|KzhX?v}EBcH-0U z{`7I}^QOmFcqsm~M{1_5;_?^PGE%SZzZKwO1|?ntx*KC-v;-69|(9xN?o zc44!kBIAv~X_@&m6q~J;uJjEDWY{rIlpA)sBx{$&N2Rt8r*sdq)z8P8B$V}a{R5ob z+w1-QAGngx==_49x6 z=>-=b$Vy)M^?a-f3GJNH#zJ7w>}!#i38YRfNfIe96x?eGs6Zv>o`sU?P<+N^=-L(^)A5%|RaG&@BKlWc_xC zqN*Va$tqF^FEK^RVo_TAc1lSyvUK)9gBv$9Z(D=&R)gYGv9*(`bRJuMUp1Z&f2_gj z+XYe#H0B*>TT~^}KD?Z6(Hi8{td=&U0wBU2jYEzD+qZmh0<@EPTganG=>PSy3voAX z=ps2(CKBfG_12Hn@ESb({pk_JNJ;hAH-zed(CiHR@sXq%I71{KpO~y2qrF8-1ax07 zRIzyf=lwg-_OqxBduu^J#=59~8dTwm;r`F9SV(-EK#|C@mYcnv0~B(2X@-ibYBMP2 zNPF1ncYLAE7&aOfbvJSn79Yg@`0jC3g_V?HaWq=W=PB$kb78mH5*; zgizDRMSIekmTyyuui2^fp;XgUMW;KNo!8Ep2A#x7WTzz-;6All49CX$)1m;Nl%=IA zT8vkq#JhBRbJ6d;h|EqRG4N4wf0~`|*|^eS)`n*+tJ}ICcwj!P(0_(xV(F{&TwU!@cCT4ajTSLh^4)# z4;m9)SbULu8(V4J>oTZ}3&|IKtP=ahuRa$3zKTgfH|dd>D5v*EmKAO>0^BDw3TH&) zkzp;4k2)HUu^O+fb$uK8tn2Btn#fS33(JI3LqmC?%&F>lS}bRuz!)E}c$fdZCXQ?L zqz_m?FX&HX3vKPdgCft4gsaDvZcE%$n%tRCMKitnqD(*RnDk`qX-3=?$K1q2qmYX| zsr~4`g58>A8PN=~+q1>*{3a%eS8QAN>r}qo^wq&%$X$)igialysVg8R7t4^x9s+PF z&!V6f2wr5YkR-a0R-E*|3w+aL_oJ^L`TXJBa(VNY)3}v&P`_w&_ZXB|1#Rs}L`y2X zfg5IXo~_2jgLWLfy;hT8a-&VZcQT}xo9!Wrn_=CP$;~d@&mUn{jSXv#&d#c766~BU z?MAs*SJ?0Xn^0g8jELS4b?f?n`TFrjOL3W7w{8u#DmZA24q+FE&D!ozspqBTsitks z#Wa7r`f4NWicW(Km^_*L2lm}R@b;$k3UyEK@s5Nlldx2)r<1w`6}yI3n%s-Td~pXr zn5>Ky=Qm}|+0wQ>XnE#IWc`NkX_C4umy^cwV+{mZIvNJgLu7rAy25g<^%FS3-R8oSoJcUwZVZ zswq}YGVvU?s-j1iT@Y(!Z%@qZ&wC%8mD238tXN5eqv_+aOcnfYEwNa1vNfhj|8SbNw@{?NXYMk+ zmQKyU{TZqiXZt&gj`)m^8n%9UB#lJd@y58CPY$n~#x1*b1bc#*Vb0bc>hf~}XSXyv z_}Pj#aLebj+&uNJs>-o1|6r9nZK{>Ll+i)me(3T z^0j&25+j$A;J7@MR*bvq&H*(su!dG2bXtf3%|~~^=5{>?cpxFHka_j>W7;uUUIEG z`4`KLU0ZCuc=;u1|y>!VKK8K{l7fUK==lQUw*g{k=O2iB3vNwkU`p*zshgd{e^Mk)g$kw zCw=@q+K`i7cJ@(=LVnu8tc7Z3*}w5O`*CaQ@C80uw^qmdc3AiFlwHRhtjl7Cv-I7Ju2rAJkibEy?J1_#9NVu6eR6#rNC?lZyL#?(^Mqw4Q8r z@E7K%tq^X#-ypmzrfGx45id@Q17)1-#LQUL>7299o9t#C&zFy|Q0|w%v%}8*imUyG zHwt>=D;&&pZ8(#wrj&II4$i8Gwv3halaTD+@v)LOKh?b0ul3J|6-!%EK9o)^8MW2* zvANZ1iO&r&X3P4=^30YTPNcWj$6hr4>F5=4k=5LNht4`Ud2Y&Gt##qlFw&QbybMh=hHFBLAgvl^WP`g7J_ zwKs7&HZE+fsGA z+CF@vBQaxEN!0AxSi7;lA;}1H?~Wf7(xPkc=|01cwVAH)xzg88AMD^YQkLS7`POTl zikte-$DSR7J%PT}7X5vx(nlI9rjEam_{U=!bqa6%{X33jAu8;>VaY273j;@*`uexn z{;H>btZ~EAkgE1~d11+2aia-gE}~->wA7}p?y5cdOvR61C)zyzo}Qd$ zPeDxbV86-_9?!a-bn>o#!)Pszl$zBR)4<#RFkj2J|74C%%Mf>eb%0plfFnO=t!2Df zl$>|8fbkE@UPwGK`4ZtBG5m`*QT?AO+i4T#V{zO^8sbXFpETTIWD2X?s<^LGbjBo2 z>uGm_-K>t7-*BEw_o1)X`rGB$awQoZ9~6!)HHodf?dBB`eAB-4M5V7(Nu7_vnOd`F zLav?34OUComBV4x!>3FCh%>D*ifHeC`l{`_+)u{ddOo39w#g$7bydX`(?uQ{A6Z*I zwJi7RV)6Mtp1G6RQ68$kNj&zXw@_$WWvrW5)Z4%i3wFz*+FYTF(kLf}YQ$$K zWSnfkVP{5oUnx~fig_^fPmRwjyJu|jZ{}j8XMz-DKowq)hQy0_Q&=gkv%mHTIr7E^)sjL4_2AAwUto-5*pO@wV3G#Sd)t#T0J3IA; zFyGQ9_Om#-tHQa3;bx=C^MAP`a8_}}`M+vPnvKfhcJZCju+R`PjjK!sKT^a}K~t2AG;v#KOdwQcyFWcfegz=epr z2phPB?7HfmZT0JIGCFJT>s6YR1TwxP3)E7P6t;DHZ zJ#MkR5pI$-=h(PN+#o#3lAc~4!-(j7lTP!OyY7}%?RQ!?vj-b2sqZhZQnBW0{thk9 zdq2u#NSylmmQ0XHaB##`MVgJp=f8-r+3T{cr94H6$7o={)Gx*%^~@eGWuxbTYx_Gr zR{C3<;@wa>Ue-T5ukB8!W~BIlb>WL=LH(=G8=VW-$*h>#`A^#R~{Z*oO zD~0k6-A^>SFt=>cYaYo_f4p@izvj)*(Ccxh4oh-YCA|I4ziPGg**a0|hwx5M!`JuE zpOtmLQg!@&tf}71;n>(CnHlOyF;7ZA-deCgw0por)GAF&P`6|IlK!qv7MU;tYyLoCU7lBJm06`8r9wi(-$rgX@p3=k<#+LE z$Dw(qiw;el6g#Y9{_;lGth04m`5vGgOw_)(GoaHqqO#F!PDNgVu0aHs4TK|E3kmHR zpFS9V2)Y>X{;}yN@f{iknN>Z<|J=x%vlg9p`BCQGp56Yw300A!$JGryQ%<)ZSsDK9 z%XZzVl`!V8fB8SL9I!aKHIs4d18d-}zW(9*fBmMC3Wb(FEw@LlG6jY^m8y(6ikE(I z?gzXOX@0%be<#xPzu091Llc&99LZLUmwP2ts{_BL%%`Q=Y{HXxr7uj0XTHI+nZYWb zUPSnO39-uOW0j3=>~0cn?MtaD9q3q=W*?qYb70w&@!tLV#rL!ka>u7jT$}U@69?zZ zYCb=C3+MRDpPi0B|J~>O@2_}V`(N#w`&2>}j-mfzYJmmsh7Hte1x@0f{K)w=_2=+V z%ly8I^icN?lm{lE3@GpSsVi<5$#3Tag}5JVlHrr>Umlp_LSYz>ZTKa>Nb?}<`pSOYPAVZy0XFHO!@Mz^5}-W{*P0jf!KKH@hads`=SrYvTO(bx+>cB zq@m~aJcU3Uv7l3yLXeB(BE9#6RPzKfW9qF)%2T=(jB?wD1J`Do3#_Hlb$i!a0F`V2 zS9&1K^#S)$A1{;-H3tW{X6Zk=>j}^yI=ycC*Bk#Zn`^p}0?-=wmOMb43gUgr1ji7# zHwbtD`FcwuK3rZ;PcK(!Pyr2RZsTj*xF@X$e^7MC0f;7bJ#9D77w>OGuLd9@D0=tU z&!4gtH>S}v2001Zz&1BRv6chjr9U9DmS=UNCN9a7q^(CRaTlhpsAFo$))_NT9ckik zVeQ|PX==~*Nd^{`)E@cF9Jk^T*bEp1Ez$fZAO&gx+uL0yNu5KW9Nq#T(Rqcq7$G#k z1_fcXAOHx;D;XKgckn}vHb4l%3_m*QLq@B4-(%1x$lSF~z$}PLBo}qzFMxqWb6;=#GP5uJ2562aVfI;q+OQYLgnvfrupkR z(t(h38ZK8;SHFWT{A371z>Dp&xpkvmX9?qBHD+yy$|39}GiB1I7CPqivk}UqwlQPS zh>;a8iFRibXjgY473)s=nx(Jnp9B&Y`|_=MEDjQ!!Nj(a zi5WwwLul0OZSO8vx*C&=>dgBaj5|<7uk#<$au-5}g;%n-HzNbbo0XtKThu?^?Y3>% z-ip>MsD`->9B>mBOUJ%9U}Y1edgXzBU+?yUORUt72`F0enOLj+F=LvhDJphU4HtEG zsog8J>4w}`=eKk;xc$zpK@SOCfYi6PcRg7PJIQl!5;*(Va=rSIXP||M{7I3I zK9mF7c^k`Iwo}cJre?Gq424|w-@tXd(D-2A)9=d{ELhMABvjk>;~PdM+MtVwJsw4U zPUP7fJb7pt#%Y*AN}8Z3#1VX+(8#?rW!MV#97fZ=#68a+@(iZ6Xl-05VmOB5aT^le zhp=Ko#RY9t^vLPwhn-{ds)%f;o{4c;H4dDbbqbZ0Swz5$rcJzziW^pH&}q53EY(p_ zxn^m(PcdMMFEY0EP0%@NikY^&+^2<>%i0F4|Xo zkj56zwH!$MRDR5jEFDwOto}8+pzGglkX$mHSzQ@9n3Qj4a;pqtC7l)xe|rfqXfyQ} zeRU<}KmP4XLX`KHJxTdgxd9cb3M^;N?ZVQWmc<(X37+?P!|I*{_#1=vPK-<-@TJyk z2JxTJsIo<0_(FVuu$Gju$bm(^r(aGhDTC|PY$=CZi5q$fwKWJWe8r;irnG9BJsPN& zCDl&nPd;Fm62g6H(s^tLEktBPu*A(FXyS9=pyJZa=&+&RlIG}8&X)Y}EkbZ&)otwr z@ES%ugNAyf>ug$>yqz+#qZga-uY3oDKNNI27L~1+;{l-Ok zqzC&#!^3kF9EOV8%3AK8)yBx3)XFE>o1SfqO720|tQ)`0%0%&7T6!PKn>}{lzp?xN zi+342Fj%ne%d6m7q-@p8qhr zksK=;5+ql~qU&ypNZO#B{PxZ`rNn4$)LZ1=Sp8cM-fk0~c%pmqw?rgZg?t7RyDvKL zV7Otx33#Eg4_74{DRh7ok{#_PDYLNL4{a(oC4YDcNw*-g5N$C@6^G)7OsL=3IJ_r5*1u7Q!hO@z_YY{J!D-5c zR93746Px?nY;g||y_+3I8dh#W=Sx`(52UXnQu0Q4Cr)7)juRLPaGX1c7%@R#;TcN= z8K_&fLgU)Iq@Be}MZxr0iVHx96_=m?x}i8q!;k;oSvNk`uMY9tFPC?tkmeSMyC8`4 zOY+g%^=_11r=j)x`&VO76g6Y%RvyWL7qewb^7ewEI0%N~kx|bGxRQsUDfA)+^-;l` z&E<6%AyiV}-qJ~LLs(8KWT8FADgN!+#z*qU!{?UDv&db7fjK)-?>NZzA3q$bI_dTd zq!9GbbK%xfhf1Po2AkpXqI=^Z;~T%;$#c&Kct?-~s!omm8cJ{^?EPZ(Cats49PsCpe(FL#@BarGn1DMqom!tKi)3ZxbVAzTcraro=MJOM*v-#+5M7f7U zJtwVzfikubO-W{*)h<4veat2=JMfprALv}#n*gatd0c! zYTdNNS@tWd&;9*(EOz;U1wk6;%CyeXk%VPRsW23WU1zc&(l`C2@%0`!nv`Omf0`q` z{AT0I@u>P-QN8o1RwB0fAkTbrIMdPtkT8Gw6QqWxgOb;sf#PbjvHbBtNvytteUG~> zY)M%7q*+^!x$t!7;Yf3s+;=^6qi;)&eRNAg-wY8O7YY-XY&~@V=TBa&H4ug&{>QtM z-igOvyf&tJu25MZDgc!9%(T4!s7COUTI+}$n$X@@*;D|e`DxtZ7B=;E$u zcp9Yo9x?w6p2iAXSgKYCu?8}hCHe^L@(?2A_)AJMdnyg1$dzig`{F4X`lyh?8FTeL zQ^*W5nY12a zjt^bf1kwiH@C%|acL!eOAaZO$D8d;`Y?#?6j)Xe{}dvffP9)UjPMOp_3Nng2q8?*%jwIZmt}8H`N-V z3-*htD;qsEO_)gPV{(P70uOCG0g+D9$Mj_u*p`n_R*WFQf}n}xNNa)&CT-Ju;W++d zad^PyKc;Vb3%~7)q`etM(YZ-2em+z@g{$G6PCyT(>L-KfY9C+Evp^kZ3x0f~^ko0d zrRe$m(<5}Lqr}W+Y#>9D8tW4{uqkz*z5{ZF^$VS-Eux_A>uZP4{dKt&K+P<%_xDDI z)5gzhJB;?3Cr(mtJXX(IfxFAf%35yScyln<)ZQK5Sp;=$=Juar{q;zIg%gfSX$nV1 z8o4Diwa<6ns@j znH|P2;VDT8W}NsBKZp1KSRQ(+=T}Z)G4r5<_XTc2&=RTIPC_{rU$gM*CV4JqJm>dL zg{l)i#^#8+n*oD`cTSE(N<;Ggjv7!3DT-qMGw7A#M@UHBP-$WeI@gz38GgMS$FiXC z-#anBMiidI9tYe6g?i`=aqLzLknH@US2;a#DAnrD!{V72$%*Kp3a|nzon5i%>qVZQ z2;n&yjSdzfsG814xM+dB8}(_VZmhFT5U?SZO@(TaZt>f-?x%O0_3dx3H#{F6K`mvM-0}Vm?*@`%YBsHYUsT)Wa`G44Z@35@StZSS}CUK%sXC{_dqA8CeJ&LF( zMNRVo6;yh)P$EqQK~Vt{V=|Fod1xvKnusD@2u%>w3@RWHQ94SZlt>j25CQqEbxguB z$;|tk>;1m#`)6`xG8Ue4&bjY>@4eRAYs(}Z7yE1&N!-&!c?nj;)XE6Vq?g&%`Se-0 zRYA~|40KD958a#x5GiNV`xC1jU<DE~gQVyA2b&S?1b1`LgC)DX7@dl8hQhB1MnZntS6!%gJfA`HKR?Dr01et@Q)= znS!cIlFu`{IvdSxv!rNKv#EYr{3=7d>WeqL%n-6E@HDmc(U8u=v1{c~q#>2P;Z^xo z9uR`9Ta_4CmW>uGc=rx@D}GnL?uylH^QGz*Y$T8 z;`g*U+ZF;NJBne~K`|3XUcdB&5e@2CjDAH_A`mQV1c1RI+GINYYmn4@Og98E$ggQ7RHlev^ zNUN{bqaP5-W2jMCar-{pH`5NK=yf1#$NUD;{R-jiY8&G<^2Ox}67W_AOMck< zqI;RKsr}&NPH&)r-2j*80c86akpOq(?#}xGtlU@7kO0-wo)*ix0vJ-H0F-zCR&=)v z|8D<^i(fU%JycW@ByMXL=HhW?J>ryV!Y zSyLM?N^@L7l1Cwo;!aXh#&nl=n7J{}T%dToYaj)P77NuDYE9LXi13Q7s6o zCE9CGjWu2;Fcd7Pr&)ENkF}%(XKGRF)RCKE7MlEWS|CLUI98Wu|Ma!LJM5woJ#IAV^ko(`i_U(tTwD)E zB@7NWo94o)>OB%CuTEd|E_Kx{kGWx)1mQ1OTQw3z%HNfR$w;yLZjoC7#K}0(+%(x{ z=LXe_k4lC2LE1QP{iVHMP1T55UEYdFv|798c2Tn^WvkpXu|c^P&KpgPXxKJg7^0P+ z9(O_lfOccV&U;_d9*j^%7ub`a(~18B(cy^W+0tm$$b&kWjZ-n_$IquFV;+YP#*iPL zU*_S3Gm>}{QE~LH*~)ToD*aoABRyJF`YrViMy>>A<*eZxl3I78y|s5eTG4gru!q8{ z(-y=%+4bJ)A)8y-DyL?Td>2C-`>`&5?q5GTa~w-=tYo~N~k##iz&IrEh~OUF^LuYN(ILv;JGc*Cv>mv@r0PRd`S-ADd%v@8Vx0o zsKDW-dsS|z0s;r;;y_kdgy8XgBW^@!FTuuHfeO+Cm49PgN^$k+yVsx@xH~seXBo}? z5umDOmwnydow7gv8bV-lTyk1gbXe>DB+sPrFIXz&wp=>i?;@9!`f$gLxDzD^&X*hw z1SP#2j&peY6~T-MLl0$4n5Qh-x_G0lAL z09ffcxCKRhPY>D-o8KqQHy@)B_0Xqky4&mWOOxx5J10nCHsAp}P_S`lU^S@;DK253d^U+?evGOXXMU zypz0i*k}@|9jUWEb9S(X>(<6&j*TR5;VOC@y=RT516LQY(AjiQb2i%vpi-s_{f zQ_#@P3}(N;wjG#oNxWfC-UX65ewzz9>qqT(mLL?G=I??RsFb}X67vh&H-l47F7AY7 z)VUnD>Te%RT9=ZXJkQkH(|DZrqwgj8D50{YpS1mp&Vuc4$Tf$cqCK-_U+G}rOgYOJ z2Hs3c?W%hOeUr{#T-sXq3BPb|>i(1Po2zqnug?M|Lpq!r?uf6@bzLXZ2*%!pPz-_e`-OkN|b{PHNiXVafc%V72(>LfXKVPU{^rE zntwyBR9aLGK*Qwo*UIXuQJ?S^<;6`e3cm?h(|f5`z8JVxxM#V!H?qhs212A+*MX*U zhqNNc*92^T3(G0_Q8KD2I}GoX%3y?ehT!D%;rvR2MmGhGz|&uU{Wg*fYaXqVd@!6K zoJ3*P<*3CN7r%{XPk0po?l+3nZUrncMQ6`WnH>&7GG*n9z0mwbqFZ(v$;9S67kVbG zQoiIq9pjJOYl<8DDa{9SG=%;^jm9Bk56 zGgm{?L+qcAm4GJ&Oz)GrE!5h}96Um>P*uo~Uit>K;86n_eGC77G{f62-}xZyv(szc z^-&L8NJ~G8PGr&&h4|qJ2@Pg_AAdJ9!BYg0B=yG>SZ@Ubl*vA;LKWEc6Lr1@@MAH> z0RV2@VS(d_XQC*z{SU~OBfNE4;mWfkL2?`KC}Ppv7kaGJub0J_}-z zmq&WuG^Z;VprPM&*yHU_9m&*At}NIEpiSO69TO~>fP?f|_^}OI$vt4n*@V; z@7qACoEG5J#tf6yIMNnw)*0CXhEwgeH|z#T%hJ9kz2oICpuqrOB9);5hH z_A9JBR^b39<~1Ep$i8*b$W2?{Z4{hhhI;&IliySMFQ&V%An}i)Vj-d46s&#ILohZr z0JRwod`p2XmG}^}DTQI1r-VOOT%{#PB%P2D0_aAz>41EL{63a^B(hg-d8s>|t)QoX z!T^OEIj^%3yA<(O9+{S+f~8W~9x|EyoPqW*Lbs$-FTl$MxjN9>f(llj&Iy^b$&2ZL zN2&NQY~+&U!lGi*MlVEqJJ}X)4gdjJ(Si3YM)>|E`fogKr_tRdWV@r^R$dJ57Dj zu@PjA*CSi>X~zv?Qj6_T%r;ptS)nPS_u89$PSNo^bV;+MH~ddUIneK#s@>@Vrsp| zq+YaEK~UfwaMCs?i&uOs?)+Uy2o8LKGBx2GDFDfmQm)C+jbkWK;@x`wTET4v(K2Yn zG8ovAJ77(m2w#%KKY;r9l7H04 zqUCk+G?o_zO?%7i&|W)DqXQZq1Hl37_feV}4+-)4g-&;c4CDb?L06^!(hY~3U^oxE zk>n|molI<*g^z>Uh#y035($LX9d|1v*I4(C6KtLD3i-cNN(W7xmOO8w$`Vx)H_0F~f78sdx z1hPzflq$zi$)ZYO8r#G^WD2o>6O@_wkK8)MaU`Eo(o6L?PKrULS%tZZ{yyk^Xp=<0 ze`r2SoSXkvz`oY_zz2tPp8T{{ISl6r9MiHT&Y}Qw`&-$c(;`v4VM$N5PBBw1e=Nc! zdFm%s&n(uV9gWTW^5P|^DjzgEpy5PS*%iT1F{;9*6D%eW=2hxd(J zJO^83ool*iK_}%qsU-P;S!0cyFH~A{S!q$Hw5Pw&R&|r3=%#3V^L6b@p}@&O6V;>o zFTe}Hnr4(IGG;&x`XLgu#CoiD)g7^-*M&v3%DY4YNM&6YW7~25+6+wy)Ka*~fvynX zn7#7oVX7Fs!OI0on7M*yID$=)SlcFE1)C_=bOj{^KlrMTs&s#bLv4B>Vj9>g=HT2?{Nxt5TM|jsJ!h9@hi^a6>v|QB^`${C*XJm(lg^m z(JFCyh0&xY%~f_fIobW2MxS|41N%j9VEHL77VXcMB>B|2t?Aa8;s=njVyA)VhoaUZ zM?g+fgso335XQPczFgD~@qs#glCb_>ef`~&|9nGq4{wL#ZMk>{TXLT|!$vp^fW=}| zB$iF+r@q#7bkji=r`;DjeAFQMGdi|dbm~*2X0{!Pv9pgC=&E{Ty@k|+Lr>8(#`f`Y zRsd`~7o($i+=a;KLkJ5}7rq+v7yvga!d7ypf`uq=irzNs6_=Sne(+>lrX!;z9!QBe zB#5^6kAmC6W@FaE^?@uDq@m4$=hP`yAWD#Dw+va-WoT%teSy&hknHJT@c24-wjSn0 zVBkIdB~VO({rB^a?WaN?b@{V05{&{e$IgtbmA#&d`LEQFAW{fX$^8H4BC^H<612=8 zJ{?mqNt}y64ZYMmoPz5C^C&M!D!)asLIMj(A&>W0XJ{>=2o7%fx1jB1J_QlPxo!pWD;J@KW2yBvU^ynE&cb?FAkuJ(*b}oLMUA@Lv5U`AijiL7 zTn)t)dqBH;Agz#>Qe#xq{@feBP}6@+T0wzQPYV)3ZMc3}yVgD@RVRY$nbruH=Z3UVIp&^tD_WC=AZoD+vD(U8PGWl;1tihAd>i_7{ zlgN(AqxWO?t+MEgtcX>>bXAj{9tyngcCr@132CLhW8DiiQ8C^mu|OLz#JFWPgvfkK zMsC*L9wq5|_^BRresy$Wo3Crw9{{xQyDuAL7ALDG$buM_zN*>PhizM82-aM>>r`)l z)s^y1s7%aF%xkP~uj*Z1xwsPGR&br?A46R~rkq2{?0p%OF zV-Go=#L2L>ly_3K&Cz0J32?U*TH`f$$a~8Z0UHn3)K|DZmxPSM2&$Z@uiCj_x zg^(eBRx;Z+(sus(6^74@O7j9H=?rrK{=#fY`vTACc4%KextW668nKD)i>!5d&}x! zP*V(V@LZ|3YxEyOmBHW2%jprZK%dn7;rdaRLM{QIj2~R6osp%|sqWt+ zC;VpapVINzXCI z2=f+=KDf!0z0mIb+)ecwoYlZ#`u4GFeX)(UNOtTrrW%&2CN z2;2u4`c@pUnIM5u0P)C6NixeG8Kb9%;-9j_{{$X2cM|0Qo(9I6{v6^grPwAB-qB|x zVtl7oqIU#uexGQb70Geu@(6z7}N_ebIr^DWpFnXuS)$FbA|5nM58`G#QAgarwpgjgDp zmUcVx964*Qt!cfSj9c=qhr3Xcm7R6$gTYd&H>%hDbiRZ7WoD*|^Gl2fFuVl(XH!EHo&(X3zrdB-bkShS(y6)6X zwUD3~G=HM?2LkR)9X1}GH&d>9G7`8hH503(WC`gE(vwDfhgC945ecoy>>!D`j-b_F z)`&4YI?`k)xk3}gB|(wV&qW|OeiXZnKv!=-K^IO@B$Q3qrYle)RxH{AXKdtARp5Mn zQ@laRgg@*nZV0uFz?T5~XHTYG!`TFtA+a0=bz9eBbis6CGU03~sh;1+n1Gu<_B%xp}I&9 z%6MnwUhO^pJDVAJqKy3gZ#zN(~h>oXmDR)>Q4C55Kw&rQgMZZf3FULI9q?qQwfD&3@=#j*8Mp ze|t3+HC)?K<~1hh^1fkTX!_(lLwwl3efmsL&u4AV`avnJ4U{goJK54Ex40Qq=3dU3x=qml#-}R+;$hbj5XC)ML(=}5ij2lv}7G+66TUQ19v(?QmjWVW^d726XY*~cMYfNxOm;eFDf;LDQ|w)K6QO74Iu31_1p8A;S>v<;7=tzLt{bpn|0ug* zA)Quo7F8v3VKIO)ph?$lL<&`m#zwYI;5cgq;_duK9WC{YjqTBL%-vd0`5azJ6)4Hy zpDS*N;KuQ3v%NDmF=to2M;g{1cZO7kofQ%*R-hM|pi~DL6^XEY25KE|sGB_i^0|GV zP4yzKATj5O)0ay`5Mi_?Ai_hh&t-ke#FcG*06CE>N%E2W>fN3)a#NNT5mOhqXYhdA z!9`)}uQVAthTn5hHh^4Yc!Sx1LIn;OdH(I+;R;>={sJXfF*VWNY=II+JZan&sp6NO z<4VSa1_?Q0C|bxsBG@vinBXefH$m@X-w>w7zL~M-;pOIaDHTmvkLW_qmPsCZ4b=z*e{8AOdNf+c*lt&(0v<=)WG(mURwu#*doyr%~A=@{L)Q;=Bjln{pH<5B&qH3`8zO+QjhZ5T_8JC4r>Mh%>US zj7muKg0GeVT#@7h_Oa4VgQ_viH7$RO-$2cDb;sC}WHOYswBM>B^Gwg38uR2+;sCWLS$q?vLXiU-?D-uYEj?b1& zxg$R&mo{OR=7uFw%H-G_S41X~2Cu9y5;E=`z+FNEr)@q}dTWElAl7TpyyYj((%vu-$-&H0IJJlUp1`BT=Qpeu zZC(ws2fGR%fOX74-d^FQB)S-luZv;O2)9#5AP;QaaZMK$1BUN3uM~IV{gCpZ__zNX zi9T##MF)p99Aq}Pe3F~DGQv*lHKH!glESktTI%9@~ zCeZ!MPg$w_a&v}tckh_t(6n=S{Gj%+{D>9G=1!&)js&g!;E~eizx{3E2k#mFO@FH1 zxv2QGzeE|`eEVb9u^+tCX7ItBw^k^><@$Zl*J%eDW8Z)K%hrMZp(9%kezkITP56y} z!c|$Lq9$sMbMp1~_%rnwW&(5M9JzAne&B z;P2Q+B@I*_+W_Q3dG(f5P(0!#iVa4hlLWpiqtk3rkkul?R{(?gr%}@nqmXj=CSZX@ z!_dOot>4w+IkQZ~5y4y*)ctv*TOtq}II~h^4hl}0gzO0P_A*ShEo=Vj4^eeM#Yl!JR8bAs_4vl(L0w05HBm;b7eD*4%i3eIijlAn{&Ef36p>UjsWGm(y zvF-4_F+5?l#qI(H*QGKD#{i-SQ-iLMU3H z=<>|!6UT9KHxV9D-^30j`e@T-Sg}JFN4W?1T}S9pO0aKFu!RLf&wbd~5LdQol;Tt=u!-pn-_FXfy$aodZ8;R&xj<1PjXjQ z55yzpNTyFU&8gXPF9m)dO0vc{O-W6y1Y(~x*>v&lak?R?f&$gQ1~Q?`7CoY!W0Yw3 zQVsRGU>cBZ3%mlPMUR~NI-&4tqRI;wMZ2PR4rwL-S$!2f0PcCAGb6?|%jM?d>XmYSd~lk1@2mIdd4+Se{A1;2 zokW$}Gbt(jw5vMegFBKjr+g3ZTk6wbHY&QK{!PQW{ADL|0`Uj(qds}EdR)0K#`!5r zb7!elCPYVH7t&w|=BcXe2A*WBc-`g*Pa+~MtTfje%w^M&+d^RWQEfC6uZ6M0)m`ci z-rnBq;AT4NY=sU3#6+9;UYzh8*j}+yt2s?$X!oDXb=kmirGJ+0b~Cm7<@a4#$KbTM7e`v;&ADjtmRI=^Y zW4gxT04G%QVukXj%8P;TVZq~H@*CKS!}-XpPq0^z`BX1{Lsx7b)~*Jk-);Z zjsG|I>6}7KUQafLCmp4|363~(bj$?w)#1Bu?0f`P($qq02nu53>K#5)QnItP2u=}L zWn2}SO=rSrB=$}nK`M(jC3qerow^cIqR5qm`>x-NnHjo>iFRvEBD4i~3*oI|JIr1a2;hG`?O zjP@R#7Kl9cV#W}dbJLn$G+HRS^`XV|;#7}w3+wizWL!+ks~dU2r2`6;s~JBDe@Mx3 zWcGgy$vG!{ohEN+N^Xk$ITbWi8xip%*(o}RpW*qa*E~={mmQmn zPP5~5a}u<<_$F#}07xk_v81sHzlP!k;jWfr=kD7HRtg3EB^ zfddDaoM|>5-PDDSoJK+^a5zb}$^x~dMZSfNo=sQ7d7|U+p(3~}Z3)!IR5Q4Id-FqQ z4N<0Za@Xjz(3@D`tN>{#)6ITHaHvIq=#Jt!JAffAVp~1pJ9{2Ioo)a=Scrv_PfZpi z`;Bn!H|3ILcMHr2R^zf$?jKxo1_T*Z&bM$}PYu%@xsOR3N2kSZWrG!xW9$1vXfp}J zwoFKwI}Hj0DBjc&Pn)K;jURal3=u}B;r`WCnAxgbe6%(|e558N1jRk;fJ6I@ao@cR_T2&D8KVR8&Sai-d3MFMf|LHS%J)hvrew z*HROn1JqZ1;A*hm9C{!tE)OONz!O*C`BPg=ay;#<=rfb5^(>M|;StS6H-aZk@ve9Q zo)vo0QCn=!9H539*n1<8E0y0Q^840$4q{y`p&FbT89qfNmvj)%Y~&zsH+ugScU?l3 zfPf;A1tuD;6J;ZFzLcHVFaUXpk{iWVN-jH)-ZklC0)Y8zcMgqlg!EfUFSKZ=thn7XDzUaT7ki#pn_ zSIX+KhPQ@@>euxb*P>3~)d%C^>EQ^J zNyt>4J0s#>4euj$)n@-Qn>|e`8zEa$cAJUK9ef}))Cr{5I3zZxJJ?5{p<^)eocFF( zK;40WCzFJ^BE>*&zCD;=7)*?}_XHvcNPXKpO{Q&^N0lU29`w z)BT$G(H)UZbfF4UDudO+g@S>P4o)|(y$c34rmY1{MJr`oZNL9|S=g3Hx4Z#HcZ1(o z-Z9KMD788OsAJlkp{)=+tj>JcByui*Wsp*3JA~&l`bV&GqkRLv7X)X1zmSfrWo$F3 z8*E${+tz{`y%Lvz(duR~&y}$m2iAIKU7aWWE1?gl5Lg}}+VpZGfn&f&E(q+Y z>Wibs-Jn?#8~;2VZXm&ahvuhG`}~H}3kEQGd5EAJ^pPLYriTYd0lc5qywfcT;wCFE z_YSiRq4%61!r=rWv}r^EaiHj*(|yP#z^MX5zCI%8g46EY>aR5I3t;p__bUNzV)B^= zCj^!m)U>f5g!&I!qtIH);o%RLHPM1p;Uvmwm-aBrdmdDw{b}b@_QjJSW>HON12)qE zh%k1%T8mn}Yv0VlH_6kafSFOKS`w9_8cE$H>Uq9&KqI>C zmD?}Pp$z1(hc==$f0_EDF%k5udh6h`4RrjmjZ9 z%@v6zYe9*0_UiDXs8-__Zt~q#8=qF~6fiLIL2@RmFMG)VLtV_c2(^3Mw?oOUA$QUf z4r!vwgqPvzodZa`ZNBjx3AKBT*R82}EqV-KbCcS;c7dx*s(q0wCAimJ&}pPUBM$Z1 zQW|feH8FmTF4DtVy&C1+vpb-Ob->hh=lPR7BP%8~AEF=5GH=(Cuia$(pH+J9aWDRH zQBGj?v@Px~@JRDox6;%3dNm7ykz{BiIq%shkan!B1mX zNk_m|w6%YWtFSsgeP#|S&EHqy=?Mn~@iO$xWG0blbiuA=HtYjSrxxunM@7@;l$sbF zz2dviN5fDyJI&A!$np2N{SzN%FCBUlxQ$-#pG4L7ufU0u-oEOHrPin=mrO{n0}dFR zW}8-&JWw7rHxeQPLi2g*9(7y48=I#t9=O-?ITkDwPl|4b-dK`WoksLcK)%5p0A_b> zu05*npNg-0;+eV)eXAPBMYTy%;{+FZ3>DnHZ2%QpLRuTRWFuj>oS4>qE3X=|LRSs~ z(}?)pi%^KO-ApnC9y#n%RjfhTG6yzh)}8dbYq@ckgEJ&}^1gwr|6kEJwTRQ+w5#hw$E(PoDIj@m)C@O%AwdZ>& z{IjP#vgk6RUt?u zBbqk?q`uC!jqUBV-WO+xgaKv?!#8A&rb!RQV;XmVp1jtvc$&wmtD>4quVl)96GU)- z|4oI3ee>U>U$_lH|1EELAO3gcP1b0j=lV32gay)BButx8-y(wuhvaA;(1660O*u@b zL87_C0JmXU-Qy2QLPy2L#XS~5cNf`Swxdv@zSWP>qnO|F+yiBoZkPSAUjgYY44G z^dvIS>g2Fd)9hjTWIb`W`hw_p8j3T>Bu$16738rTC( z$nUk$VZd~g=s<1>K_)@7xouhCg3Hph0olaVZt;0te88k5TkwWMsecVX4(3H-4O~ofnV~5bpFb{di--|x4HmUJDbm;OfI2e|72EA2^J`q1si zz5(|(c7##XBPAYKkEALtIhWi#Kw?0L`Y4hs@{j)d!E#IQ2t}C^sTly@g8Z9WNj7mK zq11<&$~+Y8>>4c5>R$P*Oagt0Ec<8S!i6gXG>g+qrzD2tr1Q|Ck@w#lqfq%&7b7sJ zb3xj8tsXh!4Zs4iP*O+!6Keu()Ii-Y*x1qprmer4l=d$4`!;DIzKl);*O1X^_+@1{_J5MdEmm)Y&FX;~#}l6Aqp!6=bPa>+2+0)WcR=@3sy)Pr{3?*@CslWX zk@K%X7jSf1+pl5v{9kf@iA8)EJfebn4c zcX&11*FaNqDx>501ec2Xqt2zK#sdj41oxv{(eZ0pI|FT#Fz3vuzMtfK6j$JC@QLT? zq@8Qd40cRQ)gJ3~^C!m$Y2ejyZ^XPkre&1oKc-uGvL0(wNE$c6ZsSEt32}AhOWczn zI_vpjx|NT6($mBHd-nbOt9{S^N|DCoG4&HV`~<-}>OI_}4(V7Tq(~d6PIO6li|8AhzvU0*;!% z9_oiG^!9giDjw%_jZ%NHeE+ZNE<%Jfv`{Je%lbRAUp06yTwg9H$fyJ$cTX=`^3Go! zFhWiY$od%U|73P(xI!<*2dQnaI9gZh26KS;Lz%5P2xrz0DxH@M8*3NsMjyXv)|T}{ zr2ze>=01Ay(v$arr~fR{dUMR5M#O)YVw?;~8(5^fS=S=l@6xOBgT~mn-O_SX`PD(u zmI{W@i6L>0TP`qFJP4OC`@Z=2-`)j<%3@jJ-<37sd+SuQ8tWC-;wxlSMpHZ4GuQNN zqCq;M?0v-|Q8S&H&%cmveY5(jDu{Pnpm>zZ03NF_joe@h;UQu{{Ms}C>{>ivkq^r37)d7O{H_>ESi@?I(G`eQe zKb);09KZ+%LpyMH^aYWzSSyAA(&+3}7k~pieV}FH5H0$0PzR$))8pD3 zJGh3VG7d$>nQ;Gqq41*B4sB1^zeWB+7*a+wBx(U}R{sM7zvDF4ch-Imm9|HFhG9)$ zv~xQ&4TETCDZ!!J4Z7eA3J>lX4@B9s=vaTIeuH&=nb1)(t# z9`Wjkj8x)6im{oxQ#cRQ<%i}xiZX5&QE_G-&#M?tFLM=XZRF6O^8ZNIkYqDe*LM%l z8>jlS@KDEtQN8f-epf`nWW+i_GS4i8f+gqq9nhZBLpk_K4o+9+dd>bv;ZUwzEEqV5 zk*S$Ort=f2W4pNs2e(Q)$Ilwq^g@<*gJ&?U`|;f)Bp6pmO~_!V458a@r6yRa3oJ zli1hupeV`2yNrXcNoA{Y-BBQ;PDKMqWUUe+(c)cp-J|P)?=&O~M*(Ljw$&iy7)Mk z8bljF6dSV#uVecvw0DEcGN~frydP+3mZ6PJPC?>#$cN4BDXT2i>l2zM9`tLr#HEY| zv=KW$5+tTea6})dw+Ubfb4#D{w0eD>-F3yH*J9pQ3zb(hZN>Uf<|(1n)`E6XtIf8N zCIuBiPht}sYXR7fW@2*+Usb8CiPCV~%vYR;!d+6!ObUC3(7zg2nuAz=*L`3dyp;;l zjI|BWd>LuF_-b<1BIwK}re&dJWHgqrc|*mI@cG;jJhU=wLA;@fiAh@y{E)4J&z{|i zMUdDPPUF$00O1>UzL3T_W|m>)pihZAva#ux3#0%Yu(fOlf*GM@IuOg2h6i zEoF3)e9FFWpvM~e66+{U;zRkAZwTYo)G_1AxO$7HF^(!|?~cI*fI1RNFu|fDDe0W6 zd}oIzwi~-M7&|Jq7kt1X0LqF$<^BctVX6?xs8t1b2r4<*@W0T{NQvp|N>U{U`^}^=s zg#{241c4dWqyafrF^Js|xTNVa!6#8qrwA-;&a29I@37b{zYmj;TVDQ!YC21^a48yPrnAGA7C$qG3v zzY4@Vo~eyJ*ul?o0mf(Ag>qN}b2S92BJ?4;gMGKhW1#xx7N~LxA_2t4C+Bit2JB^a zZJR-10C3gr{c3yO;%Ii*&vE#7Hf{k^f?~`Qq9zk=wp~s|VI>O~eCCYWpx|w;?cM(+ z&Hgw*sI&V!wnGRVd047v5IqI^JGdTcVPa3Lb{)hb%n78~3mxtR?kfx27kPMJ+?M8D zImUTYuNRoBr#Y-F!hV>oro;$lvOD#jTofm7-&=T|38LPB@<3Z|b54;iqO+dk*{M2v z9xL|vA!VB+o50Jy!jTLzpMfoyrl`b*s{NAvG#HE;u!dcBo)0%<*^^#dl+gY)!f|mw zuC>Mw^pw)wTnL^fQE=}x5C*jRN3iw^gSo?f!!g;zuuxS*+sj$*{YhBx)-0wlB4tnc zSH?qW+tF1un#SipvU1oE2CQKsXPCDRpe@Q~Lk*UEVr2r-g8h)?C|5YZv2;K7M?z)0 zKHS43v7DM=Pxjo6?7{ zG-=p6I5afz-4~fyUBN+X((341GUGz)`PHZ!#z;q_HzBrb=O***QLXwoRcpBb}o9eBg1_j@bNNtp{yy71F4$y&WLk)ChiM>;NbGuJ;Opr}T ztK8^+1$XwMo}X?~!u4gEW^=AD(+P|F#O%fM4zI?3Xj4$Q{POX+abw25aLMTb*r#X0 zFLtm-4RB~Dhs@JJ9>#VFJ?5{Oa0=Q=8FwMBbtYce17%02hdR5u8K!|b8g$tD6_)5uN!w}Tzu8c#RPlRdd-4-V{-^VL%0G3Eb zd`?U%qRd3-5!?Ns`^jd)H^??*613vn(5SEqJaC{vZRbryRDs7Bc*^c@>{sW(*H13E zUTAQ;A=?VxZBNkMfctV5iGepJxv>*7F)0^f85hh;j@7F396EmGLUQs~yAcLcAg-6> zBN@-Rk`f^pE@0@-{d;ZbJd3&7m@)r8g=-v1RF-Cad|Tu|ZIe4(N#e^F!Cv2@^Jv4@ z{;G-*KAx*+0-V>>hE@(=7?&nF)v$cm#B4DRNHg;GW6Ko61YxEA>WZ8YxH-jl*81Ym z6($O^ANb|5F;5rS$Hu2|OwNJyFF<6VHZ!gA0;ZDJhA3BIJR>m#YAr|I0doo@CG00f zn(P>#ms3GjIoBE}uvaGbb~{OJ0h(g_0RhW~YG&zq`ZsWPovF_2LYp+a z*%sv0+})F+BMa5XbI->5VfsHy$}i*saisH|rnRY^c+y7JSLwr$*GA8|YTuBHxwD6J zp|INB`Calvl~*Oq@hKRRn7j99mkk|ufqu1wb1NnNb;Cn-&{}RC>o{1oXm+iAsC6=u zGdT8?L|EUbG_WkoJ%VL*&))F0YoFa#ao_rirrELSpEcwc&cglTWCvQ6%c#3w1(LMl zLP-=SGE+wd12;~$=J1HAEz`;;^R;tE^|zjXIr)8Sn-|YX%fH%C^z40pz3(5R+TAvQ z9w6w<(A;|*H^UwmN`zBSJ5`eZ`82nq#2uv};r1I4Inmj*>tq$0Nnzr!JK*XCG)J?U zL!yb>p91BdMzdqp5Y~6bhd#H{a2kn)?oSQycH>cW<+aUdmWTOt#B zTTr#=>!e48_=a_$leqv9CnO7LOh5CPK;_IE>(Oyl@v-1vCN$K{ni+z+E8h8b30A#n zZ4R`b70sn;sWchLTUdyUC-%{*rYNK=YqL+^N585&=6V`?%jLl7CL9N8-T;^Fy=6<{ z47k}g+5m}s7o4k$%nqv7VrGMEY$E)#EC$3vAb!nZ^`m%+TI- z8tH^ohOepyrA1=$A+~X4;_0Kq+V?SfbuGW~usXgCowIS?4)vPx>o#7PfI~bPFA7QLbcM8UH~`a({P|X+a@MwwMAheci!TvXtV*N`Sm=5>~+& z43?vYgO*FslTJl+dFy8zRIVeMFrfT=*9)T@gH_OT!l4*!!OTPtP&XYA`M@#%{Rpkt zi22w+)@a;LKf)rK4@u~+KB2;J%X85wPvda)#-i6lKoMiw4B-K_ukj^FSQ#o-6KEvK>3xAVoa$e~t8 zW7Xt@8>KInN~U0v91Q*;tl>PY1c7Fib$~4w!Ddy z)64_ZL{x#L15qw?JJv(IXtp^wl_%I*>#MFf(E2g#=9pdnv1QD^*@K43CW#z8TaS1k z7>lzp44Jhb2dS|+QDfJanRfXNA^gGF{dOC?&DniOUu?TJ_4ZM^kCzbukmPfM0{2wx zoonc!wchN8+I+X~KusMzEuA&eJs%MJ>U{MrO3_iN+S+FU%Zc8p*@4p4j^0AJYoT=7 z!U&P}l{RrDhLuG%M=m+2(5(J;dES3t{fmz#@!1a{uSwKANya7akbMgAEqk z*?!9fePsk%>CD4HL`p+{OUE-2pN-2X5DAeG6?jtfJ4nC-Z(6wylAQ?zymref*yT{D5MX&fgK25 z_R_{(rBeA>=3SrEpm0*WlRp1Gk8Hs>^(ukP&euN%|Ho-=CRvA&mS^oeb6(PqLi-qp zF3@v{uBxo%IR6jM3o;1J4xqi^Yci}0_mLjFKd?$3(Vw~J;xR>M;%K)rkpW3U2c#4FP zgRj;4_6#8*FM>1M;W4TiZvy-*!Ws=B^es4>cvpPJikS{Ma7zbPv6U%SqL&~-6BfnL z`aX1tdLR!^?rC1>bX&IzSYK3jIAD)$1Gt<*$h?H{$moIHP16v;Y?^>AdkT2x&bM*b z6O=LTH8-_c7yXKJ*fwbw21%<^4bO2krryIOlL;as^;E8U;XNoG+4C<6F8TX{9qqOB zkjY!1pR(CCoF25rOFLmtk?1v(Uy+WRc6=O<1~^?18#kXb*07S!;(7_AvEomS*sLy_fcuoJ1o!>T=nHXQew0&ofG5 z202i0Ka3<*DoTPB){SnfNmr97JVHueaGq0nH|p+td8~#1M^CXTR8xXJa>W1O`Vf`RGNy+}w;!;OQU6o|&HIss?KX~x_?iho#xS&DDHa1J_Sok8Inp#wrw@tO75L1= z+;IeOwjaqs?BUaZJZ%=C5E{cQWDJr8RdlUqJ;XY_$mxTVAUi`54p>EnWr4G{Y4ibP zXg)UzaoS|9#nDoeQ2i=Os_zcC(mJ@IJu$vwg30Lx>IsH(CIIj|aRrNY?TWQ1eN{%& z35j6meq<_>>0%DAhf$*ebMH8x$c~41*|D_iM&XPWq@X7s4`686xx=TEF26i&0(1H~ z0QEq_$5!^TYdMVuuLp8xzPtXCUgTLzx6G(gAKY=pyE_ozao;Yu41SBCHIarASCEif zRH8s2hHISjGM^P-b1%<`GkWGrfXkFsr_(S;kv0c;I#83=T){Inxlo7h3UjSSGE<|O z@NEE;wI9jYBw9*cfvb$s#|VgTNl&4g-Om6O-=sJ6_Nay*4uH;GX;a>jJsqwtQC?Iq zvPx2#zU4vFE-6uo1a}>L17(SV0Pmph$z&iq?x)Osp3-pzEb)R_;$71?+-#=)c>&YT zO+t-#4c!eU%b;{!00NIR0EaHmgGU)Uu_VOq658mh1-Rr*rfr<@XmQllo<@nXdU7uU zQ%RexKnbf>c7v2vnl*)ctY*lU2u6l@NE*#&v9>uQkdsqXg;`c0*myqnSC#mNu*0vi z9Raum*2OY>{qpOBrWK_rUj$6ooF@f2m{kt~K@Y@1%$*J_*Xx8_CHV~I^w+1Hf~Hsi zQXCMM0NPjuLbWR|CdsWaE=D;+a)tnZFxALf$368!nDvS)aYu8tI z!=Q7a}v zXqR5&(g*mzo@H{B`qDUk_-qnsB1kRCXJ(5~>=NbZpFt}dHYu{vkS1Ngf%{1G^9W&! zBuA@o01iVpKNCTjb6_b%w$@UPmQA z(k>k6C>=R{I#2!Pa-X+HnFdV%tmlx%VcgU5rR5WGDr$LWIyLJFYwM%S3zkmgM*<@o zq>P28Ks@XapM3){FxfP-j&_{b=;~4-F2g#=HuP3PDGf1D_sg5Rzmf}ZbspP0Wd`Hs zJL>_t@PI5GfAi%fd>|C+>bkm@X%yg_rwIU z%~Q{e3N|l$0?ttFX#mM%TGXdrNrP!r0kM3^6fl+;KV+dE_{CI-TE$B1{6YiA@&eVO zJ|y3n8CdFDOfMv==A21NE1!Q~Hp@Sdo;llgKN#f}X#)6Vnp?)Ho&DMTKgpOe^h`s6 zaMVtE6@J+{8GE;)62ASD849uJAfd(!cvOS3ozC}ylNz8}6(J)islgOM4=lCv>x*K) z0fxYN$(+g{$>%yGqJ!kvGj%{n1E`djI?x@jE<^m&CQ?8OgORTx5c{LRrVv~fiZnx5*b#lAo1563OP<2cr?=r%%# zCjdp|w}hJ_q(DZud;ct_gL#&Q3t;I(Ej14!#TxuaKd+6A2Ezln!(kFZS_{tZ_D|tN z@{wRK7Hkfg5(&mqocP5CF6ZrW5{4jmHo}3l%17!kKp>6I=m9;pfHc^E%sx6!3zX#J zc;g(Z^2*nh$URUAN1sj^AxruTq@XD+ZhIXtv80SUh?~J_G%}_Sfh3;+i^s0>BIjt! zdiy)q3#0rE^i!S6VybmQFZy8@2e-%iCo0@NQ`WERhs9Zf8D2qjQ6n`9)-?qEre%-A zIRluIvjVcENF*_qincklKnXZhhEtHu6ZGTaXmqjyxH0Pk4hp0VqH8|lXgJo#5YFuQ zn{#7ex8M$IxdyBRSD76kdpatl@NC>(M>|A-gZqD1^J&`to#IGg#2VdFX^X90A z^`7-_?pbbA?rYMm3zAUY`=J>T6>&vtp@U5fT_}Z}WvhCd2C6$&g+?q}ISk{ixX1 zjf`KD+#*mD*{fd_^fU5a~x8zs1cBLtqHa)qEY6%$iu(yUHx z#!5E6Xxtq>pzL#B8v;j(l%hS9jEN89W=8fRwxwrWRGhg>b*RxnrssHWZ_kPBxM|** zc-TxARUB@Wj-PH~S2dt}VuNRo#{TNHlZQ(>@YKjQIUrxjOPRL4rCf!4m-OmDo|^BG z;r_^BX>z^thS=nXNf&z2%zp3F?h9J8tt?JIX5Xrmq&W?Q$TOL29=`Cn?@t=>s!vZ8 zUOxWLs8fx%^Zds5sHhc!J;U4074V$k+=Z(Z)mEqtTu|}Zf??bD^c>%fEymOIckBS= z^4>(nwH*LpfdwkZ#72fzq-z{ro9yta#(Os90oklvSEJKaPA-a(N0UR_m4*_|sNG4A z3jboYTE`B$twUNnr0pn{G{Yp9!)CkrRB%A z-3Tf+0aY(Jf34GK50d6Ujav4@<*7c}$jS2B-tekC^|HgLNNdl|;X%$Uqb=W~thElc zoTK{8_!3l(JtR6BIv=`Df6e#8icVT?a&?mr4jzd1_nT>WtNC7C=JdEJ?tgVMu^#L* zoZ0Q}d3PFGiCT`$-Si>4*`r)x5Hq&u@6GbK1D|-F)k*jy=u?eNk6U98d2cRyy?iX} zHZ(p)XD9qrl!nZElf9XcJ$;HvU6}UwzdaID zjuXmX*RPY4SID0=Ue^`mJWTp};YUf@(b zwOQ3W^L^{bb+CRQ+sh>QLYw z*Vq(o-sS$~g~OqMr0*CejWslw4&TZt>JBT(x+0xOufPAs_V1U$d|Y2vkKlFg(b%jQ z4rHeus|(<_@xbTXafkXb2G=ZG%LaA)L5%V`9=3C0NKU8ClTR8;w@4G)r*%kJ9UrJO z*@MMMU`5#~1EMopC=!mR2|~V1JtX+vynkHkYPPiT!Y3Y8VjXVPtg*H_sj?r&H51j{u#wqVmVZ@G zcJPfQjVONfWrJXJY#!JTQqbH)2ACnqN%vLd38UHIPkGq{zx3_lYoFZh%LxVYJvU$W zb{@qrH>=(ua`;Tj2LQd1a0<$+5NA`EP+~azd}OYAGMaE8{?{MZl{5RI{)fN2Cpo$Q zRp%|W(CD`8TAaBagju9WVnevvKry$ww{a385H(d=6h-iCZx}x=I%K-l@^LS;==oR* z?7=DDb6bQh6991?M!s9ax=)QPo4ho5Ok$x0tf%=l5F4IB+03~N0vQfxIHdT6AX>UO zLx)&ERN?ZmufMx~zyEirS}a*{>E%L^Yq5>4^{Zn|;uY`sfgDt=C;@=;YVV7_J>Ov?O^m1!3W}w^OcJOV87ItkrNJ%wEuid-<7zj@=@^z(WhV9r; znRtx1aI=CBz7iMdv_uV;UPlRgKYSqEk%(~k2N9KIcACRARxtiY8XH&n6jXNMhtph* zUOY5iCyS>Ic}<$9p)ln%YXIlm0T4d5f;l6)+Yezw%7YIPs)ONks( zW-Sb~J-zfRr3s_C_ZZP|zMh5FQ4oYtHbqA{#4mWzroIYKN#Z{T2HR7uLY zpX~j<`}}_AT-W*U{Be%!diQ(Lu-5v1KA-2gpZmU_=i_%)bt&|c`qDQvos{lYrk9~1 zV}zsVQVwfyr!*0n&IlPXX+-GloPZ|0hE>{un^6J{sewtQ|=!yCFCIq_M%2h(sP-cHd-S!4_QQC89YKkMDIN$p` z=Lhb(nooy)zz1BWn|JhD?518rm4}^a&^1I-D;!xq-(D=o;0-xqw+<+=yU?;q)u?nLG2%u1|xG|7(DB*|_wAn2yf~i&-&M zu4E|0+>T*j++Z{XJSmud(VKDJ|N~SvvR*=fuvO;5@q0hs#en*UuE+Tds6siPR ze|0d=K`Z6HRm7#ytev#@vX! z_#~4_U3@g>3(~VPl%;@?0;!%l4Yqq7R1R_i5uro_X#6e)A>x<}k(g3^~>AF-z<3^mM@hk2Z=vg0PSDdd}Lu(YbW{UB&Uq=pbUY=SWsXGys z%Ua^XBr)fcqCZt^vvEB-VS+l#xCG1gm6L9seZxBY?3kua(&Z_*@#&HS!MQCd2 zc$BRAyLSTrZ&`V4sN>Z?{^7~~|4Y8!|GbC))nEvzXmqHTAB60#?cM}=NF@CyaG{|Y z8}?nX8rQl|93A3sY4#v>p_0U;Ihcwrcx`rB?3gvm8UODgTRmNAtUM(7jW{+35NFg6 z9ZD{L^<+KKm!M5o;H$@ItO|ygM&Mgu>NRleD_lE%Z3l?cxP~EeY?OY)e-F3m5a&dm znNJFMX&M?|k^iTjAvg@iQ}^-RMSmlgYVZ4B$34g^OxEPoLfHgGivC0U`8u$1%7>4k!3%eI;&9 zv;72OKy2FR@@3GW98dArmp$EU zDq;mJNK78FZwvw(<1{57-x&LxChFi7m=UEqdtzKo1Dm^ zgHppBnk+e@$!SF1KEk*(dw0|EN1TN!)Z{fjNQ_yI(dYWG-qED%nv8AtEeEzX3Ghyg zi_L9~Xx_0Mlr5%2-xoON5s@ZbFhN1N?_`rajRlxa>g!6TF@du-A9A2O5QXOaBzHna zO6t2G2(O}PHi*8)NXK!bZdyZmS8x~deDWSGM7y0~AoMcJvf+wc@{kQ}g_}HfwBQ3+ z1&u&-{QD0j6!AL~>fMP}tjk>T8$AD%e?{9WS2gt4+Wz~bh|Ir-81_j4-5(-3ARjf; zmO!T@<~l!(ycPajt(KXaam~rW%QiGtV3E9A&-${3I#Pdc9A%sgjOoUS2qC<(JYiT{ zlqb5HwB7M*$@}3Zg2A6$qpYJS?i3LLtelt8#Z$i|Yw883IbwT9l8Q)wr*;gQf;3L~ zXd1z}TujUggQm!Jf!ZD)e94u8-v#rhz2K2S>~{jY|H~#yN+$rba!>@0@7cR| zy^s)d!IeFzWy+yKHk2>?AoTid;*Mp%bs2r$QL}<~BCi5oe4&{B#&qIDvy-bwU{DT< z@WCkk1iTihWvuXtlcyigzszTe_NLqQ6K=TNyVIoaJG`RjLw}`Lq1T-4i~Cl%&*^dg zyrXZ$2O*lkDI)~^1}H<5Sq>t}KQJ)RLcjiKMOj&yO{jjha~mtuAaXx{`qz)MX%O&b zK4AtE7niEzqSJ=*GT_}R*cJFMtX#f)`GJtlnwZypP4D|e`DyMOTVJ1jHtH)M==6EV zGf6kgT&S_pnckRHLhn3IBj%Iya+>LfCdkn- z>#`3q!>r*Ir#6kh=eE~p@%z+KJ@eT54PMH{bzxSx!-{`!?V2^zZpRA$Wt!Y$v1hb_Qsc)&)Y@3HLE+SRX9;Y-?PlWR zB*-~?s|xo@L05P*Q-Xd$sTS) zW<^Zety{OMB9s^UF57yTHGk2fXd|EDJs|$=-(>RKyDT3l`FY2Ov3<*m`LSc<_47C zq>3Kss8MF2N}p@fkn!%LcHBh!tp|(k5?^}F=@GaWxFLvl;-kIer|O!#;j@paiQf6hD%%d z9$XwHIy&<0+iL_Wpl%1{kuGPP1zr>^lLT5a62~eArfJKX#1O#chZxcmhbP3M z>m%vvtJ&;a_x{MtQ5eQ!f$|c!P2gd@J}ZZavux()KRZ_=wB@V!o8gdtMW~Nfytyi< zD-X*^o#kD3WKBYvVS0|ZapQ)-^?BO|5h0x>V>2J?{cnM$=YANb`8J4%i0p?r#$tfB z3d#$WFuROpLl~^YL$OY8iaNMl6kff?JZQGnz4bmzJV)>MY~xd|E5zjU!Sep z_su*uWzHeFHCZ%glZz!y~WbSE!w~=z?+aUAc+| zJkiZBHqH#g?PS>5*d}czso(uYujMLy#pl`!`mH2sM&l#nUAN0#@oiXGH(XTl#~*(< zo60(It~R=LBq@#(&10N5pnVS?w|3DM*LdS(o#H0yysDFv;?erGCCj!id+aMe_1WU# z@{Qmxt$zFMw?(Y07*fEWoSYm~kY7-+Sjwt;6MH$HY@|cAk$V@3n8nvt@lp25(F`J!OjfM$gxXl)d3z!=P%{iHTuix5>+?lYjf|=;DQQ z=G;dm6u9lWfLysT8wdK%W|vqjq6kw-Nl6Xr208uj@(OHEb5r{(FMiJ)6no+wR*K6v#_ub z>;b5a?vt(dUwKkqsP_fOi7)?6{(97BWzJb1>y^Cb*C;9V9pNDRo;?j@lA&C#&~~Dq zb3jz*(RNlPV%PBSaOTtQ8#m@SZ9y6d5HoP}3{6yhVu|2^>@>SM!St^Cq_rjRamAop zC(y3w2k6>Q2l$`OdVN-x4w1;`KjSGn6wd2w!5yu@4C|QFr%#vYzb?AqilBjEHo}Vf zsGGaH1qj71aQV!?jdiQ|5l?A#WD_a0gje*PHLo@Sn%q3+S=^%(y!bNh~X;KTIam{Ng-{p-VAzIwG1Y2>lln*}AI+kPf~%{=+j z2h#fMADQU(I}u@|jeENu$-NU$@XDYI&{uL$Gg~(0K7|M%C^Qgf4|3;L!WecgoIGZ^ zLg+;O&87Q&h>RJ}LM-@F9EL|Y476d_9DE&6%^m8fIS4^yU!()_u{s9Vm4vQdy*jlj zG~eVMy2GlTdY)j$*cE{v&_t9f(emU;dG+6S(il!z&;HMU{#=PqGCQny?yy(tG<1-=)l&{tQ>jNG;68<=<1#m-kixyg zOXVDQYbWX-LR7$d3QJkns-BRMx_Wh!>D|#G==obVZs`u_bD{zB?|arl7qxs?FsBOU_*U`X{K$W&k>(LlQpd3QAj2M0?0SjZR{ zy4xWO1rKXRB%-L9qARk1Z~?HF7J@I1&nEZ2qXC1Ho(=q$nk=`WGys#>-XU@hKn^%Q z*jYyxpq1X+8dqDZgXnlLLdjnZZf(r$Sc_GBYc_gVWC7hX#5purJmz zWqVg4_XlS-ik^&JwyhM=9hcA|xoxJ(AR}1hg7Kovt$)vD`8QAKHmA4uAA=Hp{k#3V z|7mQ&_&+{7Kvl(m`6Kh-pI0sX`eMVme~x$i^>6=A|3Z7CI;<)>t)qK-?2$kVO_E=m zBuAslVugSC@?~b;Sx@J-^8B3YckkY1ZWTyJS4*(Sf-BfsLN7Ktddk*4@4xvDTz9b* z&F#~-o08FVL%V#5#L++8x1vR-Rvd;Wv%kHsvFzRTX_zJjjz!4Y9GP?a70^UpssH{bItgeHf0Ey(;1Ar0@x(S0`$ z&j;IDaBxt47))xb9zZi=14dW<=-EXP4*;ti$14s-#t(^Ke~r#)IQ!QQIyGgxqhks# z6ncPKyaTi+{~#zdJsI1Ex1fC=ANg}4%+Q{e6_LJUn0OPbpXr#n@l(9AqM|CsOZeTr zD`ID7H}K`%E_4xVsjA*!#K5ebDgS(=iL7@n?HbIOxMP?6Sd#^_nGN(0+vsu=EU^-TNJ++4ypsoTXu_=0jd zt_n&7O|!Bh0~?0_f`iOOPBJWd4>5n`^Weq?TR`B$c|K1W79XCS&XmWcO&^rTx6W_D zaHz-O%ErUsW^2H}CEy|+tlh5}lPQ<}eySrnYHBP)pJB7qz)hPrX~0qQ6v$riN%ar}y?G6llN^0#FCtzI7?KfeV)pKtZgRvVQ zr5Ym8CR}@iAgl;XTE;aVALMh&x~GWy*GQigBXAt7PSwfAI@G6_UBO6&5Q}8jI~XX@ zfk%qn#@0Z{%a*DXz#?u;AN_$#3>ZBO+%5MO8FGll@o33EhlIx@dhCWugwgBj*%}BNC#JtK`4_JM?7>r=NJsUy+46C}Fa$qNzX2sDgEU%H7uVNoMBhJkiU-I(4tQm;l%?PA z^X5s~tWEHRoP8$`j*_@xPdv`9oUpL4lBN?l-Cc_|U=Rv!rc%WJqQM6wp_+-sW2XVW zRDCnc;YG7b3kX8C4;;bIrFtERHaQ4U z*IJ>d@PKt+Ggd>0eivHlJfQ4J9eKtBH_Pj;UQ|!wWiGF8tYoPrHNj0Dhx!^dpKPOo z6~K`uB8p~~vjzPqXc|kfGFZ;$uX1|$VhEK(P>^>cy!7?=t3&u8JOk`ybiue*UOIay z)z4!VG?Ez1d0HnguaD|S9q2|AwIi8J>T|ubD_S>V5CGj;ydE6lj~_oY(yVn|x*Bxx z`%BteUZp&f)4Y~3*M*NDw_{A zpjn_2^rdFJ&LMi)WRv{LaOS)Oo#a(7dZAxZ<*1RUFNY5%gfoVQ^mk~E2wC<51uGeb zXrB>)AoA_)TU9B`N=0ypYWn(jUq}N(#(?zGhZL%fQCoa-=BzL>GU7v9Iy&Fq0WdkE zB#SCYb#7C8V%KCy0R|Yd*cHmF_*3);eTNOz8IHo4ujGpQtM9r$ozBa9aAR9czNZJPKZetv7m|ihK989JTZiD!%_vWT4PVSknE%8 zh`Sz$Mxa?|@d>Hzrgw$9StH@F6anhIAAQP^nM0*c%C`pP*}-QImmL^`W9ZH{bT9R* z_`2ggaK!PSH=8&(1DX$Zo@Yda{<;tT)X3FevFX$ZNk21*E(G-@X5)#Ex)&R+WKePh z9KjLTu?x+Y;M|#81JsE_F@Dk1)MV}nB~mKUDDU2brJkmWBMTQSFfUlX&8OjqH%P>i zUc{Kvsnez<*O|MyWh9^IHVgxa#_5)r$Ioxkwaw>8Mq`IZt5jx1h^bMs$@xP_Co@e` z6_UUkEC!7-I|-K}io@5==@IB0wL)l4D;-6Wa*-{-AzFkABf0LvK+{KL-Ms(}+_WJm z_;n-dvI#rSPrtxeEh}5A$YeS!I{Eqa+%N<_*H@h8W>YWB)sSZEJoR|OzXgcf6dNT- zVawSmyecOeKmg9#uwg^+Xqs(fy)1fM)yrOCDSQJAJqkZ=?Zvo&GZ$h`#QnsnJpF|pF)?{hLsnxBGdMJ}rU=+qX!g^hV)=RR_|HmN_6Ncd5m z6uvD*NG}cqleVRE2$f6Q*s?l@)~)Ht>Z@TGmVPyc#p1`S2nh)lpq@986vJrf0z{qVJX zchwVNwZs*5dB-DzUl60ackZw3)}@R+eixR z(o)1nAPuyyDjo&S*P0g_D--l&E}Dj*NpRxCiPS6`hd9(XFaXj$o07Ldo{vGWj9oQk z)F`^3Zi_2CxvX&1wg;dAhcIH$DFNB^CjY!Sb8bX9$=iQ81y9u;r-&+w)5yrEcH_m2 za^t~DSm7K~P&jkuOiE>1oB*m)oCmG_`-w-4fU!pfkg5{;L`!o&t^f=O42%vBXHfYC z5v1Dm`qoGU#uiBF*m6WBY(i;E9u^FBxwLcV&PuF^vp4aU1S_SWA|x0-Jh~_waR_ml#bSL$ zv&0K8!;82js>|>?!1i4^C(@A`Q8)lPy+ikr4d?pn*NT`m1T_z=9c}>7LjF}ng@A_t z`%o0HYt1%$e}7D(6Fz7h!YP0Kr7Kr%;YD$*BT%r@zfGS#TMPBc!?3UcjIrB?S!9(! z_Eq>SNH)s&J*96;kyNn6+nB7&^MxJ)-^L;iPoFmJ5V9U_^iD*_T~KD9EfXcn?%n4o zYsEsZux!kFg71q!D*XZ=u&G&p_(zsR($T8MQhqGTKVXPf+qa+_W2Eh2X^i}gLy&j{Y$2Kd3nvB zJNE(l8%ZR(eC0|7T6MqTC)#mB>^;ofg`}3_-h{VZ0$){N&#T0vH(PW8L$~guws~&e ziKk3;AX5zhxDv8Qa582EzeAEZy6D2~4Mtr(dPm1Y=CiHqWa9FF<@y_2p)c-F|I`AJ zN59+_98*@+mL~XO_iNw0E9>wu^Rn^w?b{>cgqt$g8$;y13PIu6AMCu7d6_IL%qzVL z53Jc{>0{cWF6Q~=@QM~(#SnDF~7@1t-U3($Ql@vBSRyICsgC;FTO)LND#*?LCWm z0i0ekmSU!Ob=*>AiOUN_Ej!;oWt->0P9&v$VPJ4jnk+dhIe4y|*wAbDuESRtBu?$> z&U@-n4K$-cPO6!vlC4}5H{@h(9p8(_Y`$lxsuo?$J0izUgm$0_0^_pgI0WC z^2wiLGsQ6oRT9E{h3`8}Ve6>TM(9YvE4#pD6<~Uqb%8hTL*O?-ac|s()c3eVp&2!> z-E_buaa6;pQiEH%Gg1FGW4s3cr%06$%K~}as(j%uRd%(QnAk3B5TqqKdcyBjU0e(u_5uKu=f}xbHDcMLolwfxq4-!?NRymxe1czw z3_VQXdvg(5s2H#LAXg~yZZ=J0=uARTDZu`>PRm46Zw4BZF5{N+rBxhl$y7l>oOBFj zhD3;IB#IlmVpj}yT9VB@+}eg|BelN2X1NZn`8N%*3;9F*3he`f>Mop!VprtC-u*Ki z%5&i8Gq=VQlITnqv+eEeEquIvO$jhqNi8}GjxKJFJ;Q~Z{i4AL#Zoiir8FC5o_gW} z>+6fepWAxsCOft$U973Cn#`U%ckb*N(z8}S2_zB4+}b?fta>sV9Ka)8)!Tl4NBu-} z%WtS%c_}&a>~t!&(a}LU5WLyRFu91E!X}ZUi}8N4j7W z;2(j9ukPv=RPu8dEjm6lT>I2BJ#$;?nd&W7Nk&Z~k=dy_vT;NGD2a+|QE{++pMtxX zJAeLBMG5LUH9m=|xi}3TtDm*jsAu`IN0< z`%ueAX1&;m5^I;F4^XLebNTdgZ*Om3Jned$euzEh z!N|levcK3|`|jS~{{od;0yY>)r=LB0)}Y$9z9vPv9Ob$&Rf);yygv=m;X<suFqU<0?HmG76O3_6y5mQjq{f-wXt2e zUgNcG{6JunM~CR-$&>4@avQVL$MrnCnN)nUO~Hlp=iAU_>kw;)IwpI_8q7`=*z;@~ zIE<Ne^kqX_XXvww8s~;Ek7ac>xW;05T#tFF_!f z)zgH+5-u2kR_9t)I3$ob7@DPr0Dqbq0CQ4HJh5}j0T-d(qG|ob7+&fN_RTMP6#@Lo zmPB9{Cr@M)S5q$Y&3OnA0A~lYyeCYUz!`*xE2Z&bHU`{maUIGY-&%#&vV$a39wM;= zPX#iDa&H^vxV_6MyTT_NiQ!k72^)kuoX&wiJ!oi{)@1TH_a|UqYnlUjaWL%UHiIY> zbU0j0X$%mC_STrA0B7u*O(+t{S_lul?}S>6BZ&LsPvJ%#_K`#NU6 zuvgBmo~#GZoxEhpf(4OyUKXIIHoNzQLt?^EGDG=_T13-&8@RgEID-H&si1I2K+KhJ zA3rzLi`7m_M6UdTm}fh z@8NrZ3;_ng!EH3=)6@MTn6hq6Wq`d~tuV8jCRCf(iMJs)-?7-VayS&F3&JqV1C zR*$;YIBJm}%hHI`61|uVHaJM(qQ%f1M*ho};UgOln3rFch$x=%ETXce00G1TVc|P~ z19b)9JJtbHR(!*QivtnEDciAQM=X9Jc*BP@-l`?QNQ-!H@RXZH)}4?+K*&D-mDO?2 zNW0a)Gb32;MLkfjl=pXMfN>=x{R%81-@=6f7M@*RqOhWzm8$_f4??kWD{PS37La3( z!&J4|>)Rs?6*5je-Wb&H=+{i*_p@i)@rJ$t`>8^sjGX))^iRA|-unjEuOB0`g~0KV z4u`_y2{O>qZ9e+=GzS)%8ob5@OuQFuIEA-fNg2%WRvSXiK&Flst8!8HDQNilX+{FS$jx?S0!ro`Meb-mE{jtLcb6kDJ# zhly%J9k$j`S7S?s*se|0{|3N?l(Cm09=yS8F&*I`5p8&unn{LRiF|}p-S2u?qqnCZgYmLU zW~XAJr%(6!V_yJ%HsRA&z`Sm~;CNGENK!;~ej%jfQr9gYe27|+f;Jzut+Vw@&h!N$ zUwZ1&bwt1y-ldh!$QYjq{z67G!=&p|Y{%!>5N6A0L-ra1Ux|RBFz*r+e9;FuBRN z7`HMi*>F^XTU&fIE!M>LlCQ5+`lN{y1q!x!it3kuvx~_fg#?k_-_Ijt3CbLVmN*C! z(`U}ypT6Gu@D4rHbRAQ?rgV(_8D#aP^C_e8=}2ML#;QXTg0o9&-g(=DeRX2<4LPYO z@UQ{{k2{5i^}lYLaxi|xbwH|c_ObfM*OFa$F(GDr;UEx10Gh;|=zh;qn{gjT*!4^h zzlyi7XmSgS`u%Smd*?rN7=jD{ODqO6Dl3VuvjH1VWG<4xe$>}}gM+sjG~VDlCaE&a z?#%%6k%(`#0w-`s^4K*KQ&WT@QLMNMg+mPKjRAAwjRq!nb{?x4bYUp;M z%e?ePIGN-yAy(pKqF#C;75*qYVzIH>40(xh2TsTucvAUYYOQcnnnM&>S(kUczP0u` zy+hR7q!$zgYzWPC9D*C7nAb;k^M+$T-=E;fcAT-%E-}z|32Ar$y9iik0>Ll@fH>dS ze|8y~D#W+FX8H3rgnKt%NlMy~Uq8OI4Hy5GmNR%;%0i7m@>c+=u4L8&dyyjE>6TVng4h^ac6YwGMfc?CHoSoncqu9qCUxoCYyKRk19 z@oxs@w}lg#uCf)8E>{yg?}NJ|8rlVN4Q58X7pc9s?PTuoP#Y{eS)U3<&JQ$MfMW5$ z*n^bDYVbfMCR~c@z_ZdxZ7?tiq=6ug1mk6eW|sOampBY(h{wr0j)Ry5;}g}*=smN7 zw21q~iv}|cM(=!xZ5_hG2{@Qx4Pi-Z&@VkqAhIRp}d zVS&*-ECz5KWRkj=L1e@d&yS^5MdQWuv(b<5b?g7K5$MtYmO!?JZ~{jIKua(p@toV( z5BH0`5I-c+mHpOzWQk`456FT?ZQnQs2;@4Xw~$k>XRJdPqJMsr7hjv>o~oNuf#4lH z`5>#g-~P#C+^Ri$xkBg0F~gjIqT8ZYwp-TMmL~K+Q5Zza%k1W*`QZtbC$Sf-7fv(Z z2jfWz7dlsFyr{PRt9J$fp zB%S2v4d^xxZ{*!^RB++<@88?eIQPDWa|jOcb;GqQq74r0Kx+gx#I{*@vP zo52yo(qbj!gLCaVBy;@jb`}wJU3Q;rv5mDrurMMNPLz5bdAXqD6AjNQG0q9IVlBl) zg18rfNHurZgI&XDJXeNKiM7e+huczF`|XHyxBRm1ID?#Z#FE~0O=6a|to2M`%{U3( z+loWNf~E_o%U*Vh2p~o?|Iwx73nazY!p$@ z8Sq6U{bRe_8L548wm@H5SXD+F8sGQJ zs##4Gv>L|pevWJtT)UPn?}M?!=I<^KI1FWB>REonGD-8R<8=;h-gs{8X))PU?fb;y zElcIyRH+2I73Qo$QlO2e@Ld!bnh@!nYYicZmT+rpYXy8o(e;6WK>-XV7_iE`v4IW2 z$%IicJ+L^m+kb@2(wspQ{M>@W*WD4M%pV*!MgxH90rZe5Q1q9WIZ#DJq`6AM*x=9NkQ@C9kt?z= zRP1h;=ADp|^N`1@Lr;zcaec~ll5#`+{QPV~QD4H11 zczUd^Ypw8fj9o<`!&DNrcjZX^P$|7vgP@u6BMQ9bflTBU3Oc zWBkuhGVzrngR-c6z@>!2A;A`Xh|PN$ceb4*HJE>T&A61J`4Oo-4&GD#%kLN#Fd!_3 zM?1VSDXW{Eo)o82$hA}`o7uv{^ZQ}u?j0JH(Z-3d-M}f54J-(#6<`d^_6uWA1pOSa ztEfFbnQM7DbBg(~x7Jr!1K2!;%&iOR}axP!q?m%Z;KzOBR51Jv%9GVhUq1= z3iC+}AXZS~dTd9>)JJbYHBXu{ySP^tnxQCi3o~YqGyF%zbLP#P7Y3THUX&4_rX6Jq zFArlpiMIG_JNJIt6Wn{j_&5Nl2S#@|+W#x&z0Zh^lV%gNDJ!^NT($iVz`zQM?<*kg z(b{8M?*VU-O@rSAO;zFeON(1W8cmdGyc}&o&N{r-o#lU^dodIRiD`PmW4@5)PoK!5 z$5cCh{CF1c{RTU-b)tt~QE6)u2I%a-xQEA{#Sr{A!PS5^n3av)33+nH&|`yqYL8=I zx4(ay-8hJmi7}YuaA$a(B(yl5eU87=E7@Ml)S9U%p( z4t+@KctI^vJcs+t(C?=MuAMvtuuO*|(FW%4IYA5vnaBaXIso@6J$Vdv#$e=tK8bK( z8>CF{aRmfRJg%@>Vq5W~(afzQTdi81__CzJthF+4e2B2*+`#lC` z9<#-al9M$^Y}6!`8u(cC!A^R)ism@-J;44%A&3+l=y;V`E>^grWjDAxVpI^-;=g=3 z&WxEHJvhDGaTmZCpagZnTfin03!`jFEjE4}c<#-Jx(J=I*fG)gIw>}`?{i{MFdWAg zGygaQ-H9Bm0@r&np06C-bt3vGQRp*4*k1nQ55R$cs)(CWhOSz*s<>*a4aV8C=;%;= z0izj#csPGFqEwcW^8&)0!k-*6ZjJyZ@ z(X{&j$s`V%89e@EZXtAvH8~DT2vY0b$UQYAB%%rD@zKKH5$KI#Ax}{UNWX_26iV_r zUAdx;idj;0Z^q7^WY+=DvPNitpJu5pO&h}8lsK$l+vSBu z_-z-@zaPp%LTm%E@D@A-uDBD&tgBrZuXKiFG<0IXmH7o*t{y56&_WVE0+U5E@)AQR zqQGOU3gP;;i0(tIW-Oo*HRO@xhVVvvj<7V^D@u5q^&z1{KcNc}=pEvF4q{m}pf2Ja zMj{A)!M5A z=hN)OB7U%esM?=?h1sJG`@{0>{nN1QJw##j-~sk;H=4r1m|M)8WEYQW6!RxB^Rn zgg|#u<{N54MDDPHL6S}Y6}@m+t3f|dg?9riQWUZXO){ox3HPPnM{}2~^2sZhGPD1| zg9mgBXp@m^3tMaY)TtXy5qK^8zu`KsJIix!GEfVU8Czv26zyPmq)a96#&sY%QJV9p^pmw%6ckbLGmcUp)v@x8EqXOG}8G(%zBWGp4MjMPL)PeNRtJaqi3GJ)pO{A@zHJ{>6tI+l^RoEkazvmuE~5@yME??J~;p;}@}YpE+{^H{ifda^t7aBvC$u|sT|zUQXth!haQ z+FHHEtmWRDqW+d*r(hP#1ecLDOr1K_zTM6czT;-pT>SOJuw-)6Ajf>z$eW~_S`Z_y zC_!PR8OtnPhFKx+Q#`#uN%}&vlk?j9{P}aVAn51>Hd@(~@Rivu*5>f-bCf*BU@#V8 z#a-M&sMzeW`&3l30*#*7uS+IRpKtGGi;Z4v--(Fo3+P|JH4)wx-Q0!vuMM1KRp!Cy z?RD*siE9-@I(Kc|Q2i4(!&qbbK@c?JMpteoY6Fs`3NQjI-}Xlc3BAjZO5W4hDJ)Eiaaq$x8z1fLnLO+E^osj3QDy0%yWZ}$OWAw#Qel!yaYaHAoQ$poB<9@>wct^^9UD-~jSYEI9jdTcGLcNQQD2eS1k!>&g?Aw92uO~yqeHl{D-S0wlBVf< zK#c>rElN2JPCEn+Rcs)nk%z#2#}Vufr<(f$1mNwFQ5jBxM49()yHJd3!48QoVlp&B z%n(AS7IobNpGaZY_<^3I#$P*7z@=>sM7yP8RoY~-@U0yxv-h3thAx$5>3@wTXKC(~YQQiP; zjpZOS2GN)hBEDnJRT!$GSU5NM>v2_SkZbq*1geDbLL4WUnanUqY&)=lW8lRlApbC< zxE+2oP4Erqxi*8YMur+ZeRZ-_pr1t)coY~z^X95O)@f4a#f!)Vf`g?AUW=Zzzlg3H zP{rX)#|X@O-4fi?a6>)O%f}!{y8$!eSEJ-SoaG#Y&bn~2N_d5=*G&m0DiQ9Bc3A(_ z6%}6_Y|BIVZUaMvE3z`~c@OG3sc^cx?Mk_*sYBb8D#3x&!=*$dJb^aEU@RGo2wTJP za~6nrI3CWoyYnI9SpqFA2C0K|Y|5#@WGO`qNx2R4K^&fTt^E5crvP}j4}>ZhfOD_G zCr=;{g|h&fl@^y2CqM;CI&ekrxm_xCfZCX0aX{wRB6~}P>8)@G70?x5{m6uoLbu%$ zFUAUo#|4Wlr2J|ko%vIq7xOoc{T#9z{~HgFVk-_A;b0}D%eX+2ym{-=FP_tX1jL+r zoJa-oQ9HTs?%9n|1EerBm>34CijRYTLW;X>A7Cr(*|)EC(7sC;107URTcC+D znw|=#2D^T*1^Oi|jf^+MWPu;batI#}LeD~|H+=lhBV!czXotSZfOT+Vz`nB$eHi!N z27AeE1_4wU>fIiq35h_6T6beBW?L*+i-ip!7x zZuPvtX7xlK*=9hq#jubvVt}VB4 z%P*(ThnocR@kBcE7aD0Gd}%yBehBZU0|}fa4L`x#lN|gx*8NlC(UCv(1wa$NIA+1FK%d`Wx1TfBDxDX3!KU01)&#A3+N(khJ zDkLBf5`oAHhZj#3hsh}`1O>T1IkTuNg1w-|;KeV?#)(V$8)vRgpYrQ1^EB8{d_&{s z!EA9z`|l9lt1yN!d((QJ8)U^J6gd)tY=UeQPm*nUG>=;foHC?R8XFrULX>hpHdK@+ z`ai+s6D2TUoF%^`{+=U0jQ)BRw{Ucmz^Lm^B^OP2Ks%BUdV-J{Re|UsIPpFhJN+X@2w+SR9*q|pyD_o(bn+@l`8-M literal 0 HcmV?d00001 diff --git a/docs/assets/simulator-fidelity/qwen235-t0-fixed-shape-ranking.png b/docs/assets/simulator-fidelity/qwen235-t0-fixed-shape-ranking.png new file mode 100644 index 0000000000000000000000000000000000000000..317b418d38f65a7a7b428a14ad203078c6b6dc29 GIT binary patch literal 155272 zcmeFZhdbAQ{|BlReG4thDlJ8nl%0%}itN3!voo_LMG=)`Z`mtkWhEgA$;yZ#n`Cd! zsRDSNT{}x zkZj+wgA9Lj;Poy11&M&&RaH9$D2af(DcjD-?AHRQBN@&E2#1(DxeR;q)cqXx0T|Gl&`_41_BI&7b zdmgL){++>d5Bt^|kA}-V{+twkA24ae>^W@P82#e2X7}@pi;ay@>H9D5*t7S4{UaA` ztUtN+|K%H=^Dt6hzw*ES36i-Zd*pxp>3ijbDn&j2*MEHF%JMey?fd@se?GYX|NrIx zkE{3pjb%4D`p-ZAsOB2{^VWXc+h*`nWqtjwf`WoCqjt_Q!kcRsE^1%D{>UISLq=6K zw5?ph{osKENktZ7moH~%U3nq560MLdo$oOHi0dCJs^Cijq@<*oTBWZ39rW=k-Q%r8 zb(yiTM>;2|si_Sn`}a0WYA3r>jn9;xI(JUi!DAaaxkia2)_!hNDoQ{p=aAu9bG9$z zxBJT7e22Ts-B$4{j^R(dif>f@E)imrf|!|^w;$xt{@}5>?z6f$b)1uPZ&+B^9R~;R zr%!iF1hIIUwx!`C)OTHFzToTcU)>PFyO)kG%#HEHsZ&*zm0Ql9J$vWw-D{&?6XnWX zmj#{J@%P8MxTxde;vPMDa^>>nt^4-x|M~0Jw!3%lE-fv^iu=>EsO7$@o4F|=@!p?- zcMl!iO{?^{5eDTPGBjwS^JNKr$X;v zoRZ1Vq9#7Yzki%51s(s8sAOr1s;biV4-7ni_Uzeuw`WPo#ZSR!{C3iuepgjhg@^y| z_mf)xd-TE_yC2kDa4Su3A z@66p!zWeZp&OGD2G&G6)mdi7P&2eJnrJEb8DY9|*=_FaFniH-q4aa!A_SkffiL%w#QmR~?QTm~$*~)iH}87$4>mEpzcodUH0t8rnlEu; za<8uMO_)9T-yd3^ljE@E5&n$K&;6>~e>3#lmTF&`Dy*jUk9S+7NVjhJ(Uor&dhzbr zG{w~W4wGF!za~l=ojnuQQtC8&&bt4Kd3QlW>=sXwJL@aZwx2z`_xa&s z0ms7awX4C zbY7r1EJ&-Au3U{3;M%=o+dbRiMqTU)i`HAN@|7_*ubITs<18%K7Z@pa?#y+VW{KHB zK@oH*0FRKVC3v26;rL_XnPxbscAg-%!SABupCf6?nZBq^93WN|6N~Nw%6!wd_Y<9Y zQi=kfo}RyOVeh_u_iOwa2BrsU57aN>f00pse357QU7$5pp*EIjMCvE@_V$<9Edflz zmd&o2tb&4!Ro;|h5j+-q4jwdqE?BYp`=TG!<-fo2@8-v^O=6{K&l~Ir=d}vJ`EC1S zZEd|{=g#-oVeX1jSqVM8V<)A;A8403Ti^Qctd%=l`|n^ikYnI67a4EQw8lQXNDK=O z_Wo!8$y@L4Z8=F7dv&=3XW^W|$312Tt5`5+Z+GT0<7A%+x5ANqdv_`zStdrEOUJCc z!b7;%ZBfflu=Ta=uypmjRp~TY`SK7$hIW}iZL)NffDs47|2d}zwQz>7q$_9s`B6z) zedoCNlXuu7so*JZN`}|BD!0B!y!PL*m1J!EFTQ9V*Y7T{7->mZ~~?BYUnyxmMf zt<Dv9Zx{>CaJl1qG!lpN9{LZ4c$Oik4gE=i__#{re%bQpbe}I-;OWzGyf)@&$9~ z1jr?b)4jBE&%ZmRouN@kRpzqDqj8vyj;_dcId@{EJyT-<#U>|pijR*LUF#$}-Cns3 zYV7y3XIMV3zxCU{m6ViUU*Yin0|$&IkL_tT>&M=WbQfk%>>A0^EWVG{dRro0HtzCu zDXD7oB*iNC<-w4pk%XtW(7j}x9*^)B(2Uq)=6rD9MXcxZ?f-3}Sg>u*^J9quMcJZW zWVDx@xp3gg4<0;7(nQ|I$ajZCcztd^kNLM)T94-VtG9cKIPu+vxw$WvJHLI4z478C zvA5wo7QPZMPsiPEj^}?*93hPY^U5c57xo-JoE6&3PZnmWPS|*QQrm{cA+R`Zl6yDmIB+S;!V-Va7=VGnRef8?Z z18?vC5FPhQjL2kKkA`q=qeIlR3=HXfUM47xhG5?RON~+DH&#D8DrUnjdEr0&3>B+*rPgK2+ z9zU+^EpwT!VX$IJlKF34w*Lp18tlcBtGJDi{HlAwY4|CK(JP$fU3)u*!y6I5182VE z8Ozv>w^O1YaJ@mP+57cxSkg_hrM87KEO*x1-y_MtlT2Tv5MUmM_ip$gF9CyGyW zxG~DGsx@6@_i$6pHMAb;`aw)4szU-l@L9y0qA$@fQGA@_8E;9J$?9iFzyADKu9Hjz z59RlgyR|qa1_M)l6@8jlN&mRsO4`T7#FRJmKOgwYT-pT#<@k5+M5+(&_V)HZcJADl zE6l6Hi4rdZ%R(->t=O?^7VW`NmR&o1^ytw*udUfyB}t)+zBFuK5+y?z>s{U4I6XF8 zcO3mgtZPk~+iHPc*MFDO;L?APy3yT{FGttb7yEp5(f1!?T%;z=H(s2&_5H2Av#@qK zi+})qpX)4Tl1z-{9ktoFD2hlyr+s&)davHRNf9mRbe}l;wYB8f3sFqcTV`fv3pTHk zlL!9%tlB|IS-rlxC>JZTWodaiPA$)fMYHI&s0p`e>&MtX+S=N*yq1jKREJJ8oD>k~ ziskYj93S_?!|S0PI(w-l@(Bp&p@3PG(t`SLeQ*Cw{PD!HSK1W2ofo_8O$|q26ZyD{AT-BXE9wugbi4@>zfF|o3`ntAoDa-sxH&}pSl$5z>ZN>OaYAU-VUNCNCg z&7sXGDk=(e$gL(6EJ*(OD87Z9ic#eDZI;8r?i4aIG6YbWAMa2pqw1~jU?Q45AR)%z z&w&B2G^O--421Bkxa@sLx8XE;v#Deq!M1X8a=y;X^8(l*9 zJ};K_=kTx(pj9k@;K};q9Q7YRzQc&R1EBGtrpBnLe|&7r7nmynUr$lZ(aQ>`!jeT^ zayd|F)z{cqRaf^9wi&yAZmd1i@AAW)VShC0cA+dVkckN>=(0c&5D?(F`28-WrEodR z>C-#0j#u%?fSq;C%=`*0y8k|Xzp&d1^(mR?D|oEcfW*M|gM+;HUqYp7yM$5FYEuwrXFHIB;~EbhO~(2wtmx zv>9y9y_gs#e8dO%L1C9X&b+Uxvc~ihbYG*z3Vr!0tpYnD3h#W8po6?KTf+i0q(+T)mT)=PhMHL->=F+vZ0e8>qRfv<&SY#NQ46|Bua*DmX7IPER}DT)?vMNqa_H3sGUw_#{O)=D<_%ZbdYqj{Ut62f^rAPO7pMA( zpWh3PH({lgZcdVKWZ0Pt7ny(eRXoL%FkFlXeDNZ0Rf=dZ{`B1JGZ$~W8$2$!&%d_V z=K*AV*&*iUjx9LV)U5K`=EvGDEZsHM*D7+?(v9qod;0E;wd1^}?=8clL)UJ_3fQ7TiU1`c-%hhK9jF zL_N*}P5t*$C)+lz(y}s|7E~#hNf_zYt<pcFb=tmD8Y-^j`OHY4VXk$rV4o9YKN-Njk1}A9G^q-$%V!SQ| z7~+0I4H3H-7#QjVaYT+{8lJ?F&#O;XOrl{^rqwzicIc9`;@m{10@nMsfHWF1i$ccRRDnidrag4K2YT=@N+4J43lVq)S%)!VmkeIM^X#!eIKaDYQQ!zsepz~B*d24CJ@M=^!NSg?aHPD<@VG4qa; z+|Df7Twl~_eyuB3cZWl#KB((7CUq0l-J#EP%>@=3bpxB#rHO)0GgmM@ueD~VDJ-lt zt)Sofxv#H0#(}1PW1(%7ww~p1mUc$2Te?_xlBWsH`pW9U&si z$Ky@ee3dUPaq}j1OOn)!OIh{;QMbF^r1IH~JR2%SBR(6E9lTNNhuL%zjO3*6L5?iv zQb#AJ$BVr#G74%y(h8k+7*c7=Z%;qi1F9S>W%lEpr}NMJ_M@OvdvIJ|Pz;ZZuy|9@ z9mDya`|7bU5f*yh(AT`AsRY>1 zJq^{-Q0}=G5tC`vVHb=)tWH&MtgWvr)$3tjFhWnBl#XN}P+~+Is`hK=d#M-{%PpDO?S%xxJz)ej`J ze&L3ztLq0~UZquToAc+-e*p5limA`_{plSP9+akovqKFRMwY&pxfG7HroKoL1e2+L zwD-8;lvdei#!1c0$@|@fR(?1T%hRt@LuzL= zi;nbqY`Ot5B!_FGjebDsP?{z)yj}mbQqAEJ0_^-aRUz3IW#Suu)z7r|Bsvy4>DAAX z{0coKcV%W5pY5ZgV-;GRxFA<-V_5Cdt`>9e9to%8pIeYsJQ+tr^O#ceWiqa6j~j=T zSV$9S)_~$adO)7U_3N^3O9a~@OLkeFF>GV_vbi?PlvyDjbIkYHD#6-Ts?+sY@3oq!W%{Xz<> zx1E+|LH^Z;mCjj9!Tma(hDmMEBErbO-r%^^ykLu~t^Od{d=_f@?tJAAm2p2NM#d-#X8y z=`n2%4w!uo#;F!1t=L#ePqK-=cvm`y z@mc_5vtNR;4>VmrKopaeXmsL_&E+Ldvq}B;77hvn)&>qL=%t$F2J^IIeqTXv%r#|VeJCFp7Dn2qi4r|ipd!Uj8s zT)%9Nwj{>`@q`y@fXPx=UYqPK(`X@~D1I7D{+2`Q?FrM4EDOaIl$IV2L%6mC^8=@@ zV)d@=*eE4N%~y8ps$(0Qem9Hz_*u@KQlQxBiY7Q#AmB9fN?S0o@3pzPInZ;Ed343v zjGvyn>7u99Py6xudQOuThlU7*Xioa!yh9S`~x2ocSblsw+5uN*2r6tD2$1Gv+O{2z0*5cpYRuwNZ#PSFpPs}te z4+!q9Sy~K$D=Ix3HUO@KQd~OQc=5V~#8+zLZ?AQU7Ifsn7wQe@Shw3V=q!7Rbnh(R z%+#rHSIswlsZ{?gJlywguM;^wUMu4;nAu2IJ{!<{gKF;PEHDs(D0l4IRd273D%kbb zVbT|XM(XGO{rj7fWtcM5^28M!%_2#M2#5?Yc>o|GCg~6A@%`q;qA?c9Wh{>G$*%w=JsIkHyAWBnm!g!;Vdg@*u|bCcZ%0mXv8 z8z-GJZe-lHefv*9NAYTz8#gNPz+Zq@A)OuR7+jvI=ir1e($FD%MmbyC|EQSvkxQ4p z$2tN7UIi}UwCv$)Tgak5x~)A!%?l<~KPoD`U)8i?n7Q42mRO{+9u9`KgY`ZX_23WKo_~YD4kD%Si9-y~}Xd#j| zWgwIgYtJ$evnYH9?k_Jji-k%-O@k4zC+c>=%mof$niRXd9Vs!iznc~ti9xonH@PC zF@b^iK^yd^`>VqTR9A#Djy@^l*n8lBH#CSJ;EN*a>fv%uI^fX6c3?vK`^LnCm1N$c zB9skGy&MeZg#Oy!v>fKOjMVFbnAzr+z@_k{ zXW~4%62ZfBrx_0&x{A(d(1UYTU!xFygM*Lnn}OBv?DVu;q^`)0L0%C7KE53wH3>5~ znK^pp>csqj2K5*P{3u7iHmz}t%YAK0bJ)ZZpF?4d3d?Af^;Jeva({Dkb3Y6jy@iQR zXnnGr6(C3NB$VlpeY|Y}Y z&!p0CXos<$t51LqS>4_uA*;(+Z;ijiV2Y-H3lkjFnXm+a6&d)1~Ko$lD1}sCVc3QU9 z*3E)0-|Lxxe66SZuU404T2s+VW(FvwizNyy_WO2<})?RhuMY5 zuk*b4Y@ih(L>`JKq>Zz`^)n&|O_qBMztU2B&cT`j5duL%R)lU~-=?@?O{u zxt0D>_tVze=4LrYFtqd87mOQOYRYT|ef88XT)1%c@jhmQf%-Bzom48$q)8GHD9!SU z+1lSziZGaz)}k+~ZiHvF*L;<4l5m5xP@QkqF*j3Re_=bSqItwcg2Vku6ziABb;hF) zXocNN>k2NBH0S7Ft1g$e#1nql_$UL?aIRxQxTbwS$p`Qv#rjJioi8Ss64kT|%scf0 z2hNNufKwBEI$_ZV2QpEh05~^1YgNAXe&W<-gjt1c(!0o;oC-exxfKhQ936`cdeFU9 zIirEnAiPGZzPbHe_4O@J;sEvMfBpVlOS8WS zK_7Y5iox8LASY8V)}`|(#HM!d9ExWB11Bh+5T@$&=SG@+yR|G<^UXWIrcCMihHceZ zOF2|{d51LwIXM`Dn!_(+P)la(?6!psfBifgzYBjOCC}Bgifyi1mvxzYNWj%gE7BgD zoPnx;FEcaKVCm1FJ9V8t(vf_g&{OJ$UZ8`3O^lwYa9id1-BWy=uC}Iz?2JZ1kme$^ z?BggZo(gujP7tenyu7^hifW%SCEzfKmxVDnZYa+m>?^vhuTP3O4s{`==925u>&m?C zRG3);QD;{NVA=}{3%@aK+X?O4zdu77CU#)dGJ7_k?a-6{?g&2`HgC)zhM*y+cQ)o1 zTwGiTDH1?&>DLPxYU8#vNXYO%PP=G_%EJ#NNIi}Z4JaG$jc|kwnx?jLWy8UbO$ik zn^P4|LCQ)CMo?kbRlfrdrIQsX7UApMYK-L19iQqc_5u`6A2J5K0q2T`*Tp~My0(;K zs%fiU;*e2McNlEblBe_=P1vUJ)q&eE-5N!A}JF8H( zk(`{Yzx3xPf7c;3>+j#cE6s8(ILZ(F_;F9ieR+Fybadc&)%*A7^gcYA*aS@QM694b zi5D?|xv4%KhjrN8U-HdV+UQs)OjP{oE|7uYyws{NX*s!uv7a{jt-HIbuTRIyL$yTc zvp=gs(htak9n|empkbh~$iLbFHJf|b`q#(3#A)T$Y+q>0m61V3A(mivwyGWz$@nWB zjQejn`r-{AFX+DC28&ISJ2E|;4N8Env2il(Zvck0NiITY{t#{v4O(sHTv8kr6_vCg zfr00|_&X}S(?efR&t^?0MfNE58FfVk4T8fJxh!TiJJsKik*U2nRhnQ@0Ktw4c0=ky zmAt(C7i?nPx=Nmr6u?bg&H8jL6HSYd>(V(m&L=wf1q6sC^_@;E?b)1}o4bPct=N6B z&u#HgV~lW&ei&3{R8dm+aCXH;nxm`6KzPQ`pmRHXRajkVwN~QdWW)fpD-U*nrb~z+ zE$oMhZJC}H#SEq~*5wJ}U*#ugL6XDU7ccfhfT4om*!CLQOwB4N}w=>r;RM?|D$%YRmZb!#K{4zpZqb7HB*U_w>4sK&wTv$++x`0Rw%%m1g z?-07~MkO0C^8};iyqa|n$%(t?{bSr$ZdjLLjp0`@cQiy@RGAih3`4M|*p?GQ zu~@Qj<3k=ap?R6NOc6{%7U&XqXd10_a!VYJ0Y1H)l*(?!s(Ib<5Htekbkc{-uLIvA6X0IVI`-Wn3pk)yhJv@ zSxW8h&0_?cDH{Vz$X;@I!%?yJb>z+N!-6R*3*4!6YxG>Vk1%MP)`Z{-^4v_Ryaph4 ziaf4ZAm$1MaOVCet?i|nxOL> z%f(*I2jX${O0oayCcI-#0PU3S^=K9IA0Wn4e5$2~Qg{{iElou|nl+{LU4RcVtGs>8 z5|5T=hy5VQH!O$Rv{PTacrnMKo2N}sYPn$a;Ot?3n^&TJ-}ey80c?T;#D8apn^f5A z;q&HJl`nM`+a`;4YfnzLAusd3rA2{f1Ne>Q!5*CAy}7zb>L?mBsDYU1@dGEbxUbK5 zFj~pO!9GB6?G*V$Uvoj&ID5g!!-jG0Xl7yT(SZIpa*7SGkiM$K~ql}KC&p=1)$ zoROLW!|I0}PT-4p5Z_L##3o=BwHO$sfQ3U7v&SK(z~5XHky8rEkB^<9~KrCnwJ7B z*A1mA8vR(F*b6SnA5Z}R@(9hxtSiLHKIt9CVcS46iU9me5e~)a+Ub{|-w3gs6$1C2 zptJZ@ZVW+!diKx!JPlYzL-S^l-Dn`VK!ToNl=LY=t6N=vGZB$tH`bbZ+ zeQjx}et!Jg6~H;_*^mg)?#R`3v{n8tL%?Yz!R%LEHfk{_p!(k$VrqE}33Fs_8X6kJ zF}DWlW$a!CRnVLHRp)!*DI_i#9+^zHr|8bZBTVotWQpywS4Nttrf$e$;-)7eg?VF6 z0+4bT2OH8>+j<3r9VS%k1PjeOk1^B&&P+ScBMSDh20BWKx!@W5>rl|Dp$X&;paLF) z|Cx3rgOXE&ReTO`K@dw07GE_VD)rf!Z0@W&^=?)8uKY%p73kgssiGMO;?U9Ns6b^7 zrMLw3!^~C7>4on2Y+c*ic|5aFu~2ku~8bzZ=1xTQIJ9M z%h+l3dr6_*%J1L52|0#8ARm-N23aiCd~=*1Zc~rawdSvlH1A)l*&&L;Oh!R_geZl- zbfaCv>had?JAd&V$gtk*!*0Sc8Us%!YB7kdRRBG+I-dBV$=lu+qh> zKIic>IRM~{9Wk(=2#uf3vI6vR_>unS1=8rg4nn1*<7fd%KB3NmTM!M>2064#i$3Kx+_nAZ)9@v%(7RHUyyripz|Ge zzu?OKu9m!URvn3@)Pr+q6_ljHZU?Gl7pw+L%vd?$BSbbn4J}t6)JZwaF?1L|T%@f& z29U&5n&^uV7*t$m8+d(xww9-3PWA4tRYT|_jaJ3-S>&E3585)@;ees37lVWz*3eW} zcjMKoS08JXO0q`n1ajb)buPKB%*j33L^a;1$(!SUrlCIe4N}FG^S@u~Rw)jk#i*Rm z>dN21UgsJ%PC~7IK+A0!WKIu}R_{x!XzsX{rsfAIcr-1$ckS9wi1Q}ws;@;jH=w&c zc=6(e`C7uYr(@Op4ilmMJoXL_!EUN&&YY1V+*Esyp*B%Z)7tomhzPzrKkk_dKn@^C zdf@KOsqR7<&fSR)&KL(Hoq5T6u8EbrrmNc>H$-Oryd<>kHaFb%9zGnc9~SJB2+nZQ zVXF5`m=U^b08ivseubo)bkj@I`khm!9?Up>CtNo^c;@P#lJk4(&X^TA5poah6ezeN z`zm~4Hiiyx)}AsKmGjZT<+aG%!Z_<9xr6`+97A@xj&GvCNS?k$c-TgwMk3QlI1Vn$fN>Je)7oAuf;FfA#>}Iod?gdI|-$gQA=1p zOGX2{eA`UF3_+|ww;4c$>Pk5bQbD&B#hf0A<v_^f0DCRqyA`S5eP_YE}lr#MwjJ$@B#JYx-=72QP2sy=$ks(d*~PqSF4 z4PFt=Kxb#?W6Tn%c^AUFL|OSFBs!W|g6z`koSf$k2qB6x3cG9aOv(taE9N{w<|V}6 zY|J5RPZ!ztwNq*mdih+hb(UtKR@B zFlz)uEN#|q?N>n$ha~JxG+FUUtE16O!V;w2h{JNw$&4S#hPVA` zLz<4BK5a3w5tD>toLbf(HClO<8)-H9GEp|}9lDzv>$yK6 zW`rF#$F%AFO!jam%~u!4MyMH9%9j&)b3QdV#?1|_(Oob~v3xNya=Uu+=pt!y*N?}% zeXa_eI*IEUbFtpk`bImq9f3)D;;eS*ZCUN$!070#rPo}sYsX*H6JLuK{;&q3dAo6V z(1#v*x?_aXLyV5{^#u)Zl%3O{;(kwsR<1k5w$w8v@)qvYjTZGJxdP}$x$?W_lcd+y zhkNv(8HigD8OqcoH*kZ_B@|gp>}wQ;IFCP;ZtS5-T9fB+-X4}42&%i@Zhq!wz+UX! z*U_P3Q_$Tb+ubB_AtS|N9Glnt%8=&rBL-eemLaE=Ig`#J(+=+9N>eqcKN);^&-3!~ zdNm$FsQMMGerOzx^T+< zCY?(-LiO%1UI*e-eoQ*by~C*MjfwAIu*=b_2lC19dX`>f`O?wRG4fU_op20>;uhkG zwfzZ8o}mwDWWn=TzJ9JjT_9OgWWoYT2H~3&Tt(*qH+^KT4v?lKHGq^#?)0HUhnUBY zf@HEg>B`1(cMHOB3-O0_&LB0Y;>@;@Ygm6`ASK^J665;fagksMg)B+a{pNNY=ZB7V zLHTDHQYvwljMFdnnLl_&{RBfYFlC@MWc4J~Y?hZwnN-g&ZUV0<8D9x*Merv2qzUU>Zviabo$?S*nLK3yF z_S7uewi?+fGQ5U0$zARFjxaHv7*AXxlbm?E_KoTMc3Ytl4`7?lC5$Y^Cu|+b@+CZw zB`y?3TyX`Q@^wPgRh-jZLc+IS#098y`IyMjo865K@e6l~%gWpWI<4DO2R1607hDcd z?c|!2oY&sCZQ_CE@(3eq2eP&av(2M+@kmXDlZDfc+fUU;%nD^Bl5eNC(K*J$qdc8> z2(pGk*(o9TDAhAWlHq)BIx>Td77u%ZR}B$sEi@O{zU#nePOmqS$kAY5@5boVFPK3| ztn=`C-hwL$oi;>t^?W1TF6#QJ;bG&ioo`;wl5=p(*r95RRSe>3PhF$F2JVnpA|q_QK{V>Bx6zBsLFB;Nn0;3f5PHF@@g| zyuSJGMH5zyB=nonGf$C+g-K#qQUB`3i`bS?YGDw>-03X@XI0YVFdq35=R1zse5T|# zLfztnCMn0S1rS#R2%ny&YWw!>sg|`N!k`i2A#D8obifSN7?g%p2w3PM#S3xff&i2L z10NqE3`hoyrGS+1X*FgkJq!pLGbLVxXeH{{S^Iuc7%ibESz+Afh_0>N_ zf`P~z!sFpii&S4mNR2>?<1fdYL|n9D;NM|631O08@VKbsL=b#NPW zZPNzq(EfFFZeoBA3!WX%`L&*b;b?gN?#8heAi zH!&sEDs`0Oq10bZfo<yQ+(KWT$tS;9v-sq#1_`#}DZE zeC}&Sy)%HttifWL88%2NOd~0LJu46TAE7_E7Tc;oTS1Gfk}j>TR<*S3r=p^A?h{SjeRlDpk2-@p)WMS# zr*2C*700~P`Hzs~_Mck@VM;`lsskO5Hp;=(wN!4oySqC9GDog8K<}yLyTQOE&+%utI;GqC@ zR`lN-4AC*AU;2h@B09)GJx9eAEF}#SZOf<~p~*Q+e2ZIo{Tp*faW)v2%pNlKxj}qV zEw*7MIsx%esP0_rhkFQb{j5&8x!uOHe;lc9PfyPu6nSN5ryBPJlo%Tg!K(-{I(tze zg_TNst)Ao~I>TU^W^o&lxv)ptuLeqmEbcQQz*mJtI$<8Z^CTrg@I+)C|M(>9HC*n2 zl6$H|+u~lfy4EKAG}pE1T0#j$$d0%RW8PPO2yyRsNGKv?eEUyq9M;6^y)gz&_(F+# zXvkAx!XHnXyOp4RDW2=&wcmpjYV6amtwcobiF$2dg|kfVaoyU`7frjAtjGjVnt)nO z8&xk6;;KPYqv@!pyvf;$=3vmFl%Xn@AF%Bn1~_e$4LoErUR>Zsh#bF$4)3}ZRv>T58nquvd5Thc z@YPSc0q%`h0YuqOF(jjRzs`}{{6{)mnik;61 z`F)*MMu-+6=^NaXwr`>+Erw0Gk=<(0&4hX&zZ&&99rs?AFgj>Fm>xfUdJVZngQnfa z#>O{sCqgA#`zn&S@Ne&|Em>D@Q3jyBjMN4Vxi#GYG1^y!GMMR{b>362C!II^#6tCq z$0+94wdF(^-QPTV=$~>O?Qq5!)=MT}%^-JAi3j;-pvJ!nYH~GUL-B7dSym-FjZCDbCGfoXZf)lyH%_$TZaiA)q6}~7KR~IJuRf$N)u9%=f|#o zfvfa4m?4V8oCEYxH;=P`bmUI=&GowpGfhu($=$T+>WU|IwF*yLJUPT5{aio!eeR_~3)~9B}wjc-iNwDpI(U?{N8nn{sP#!oo>~ zMCZe@S>yO_KNz|UMB+5f+~&#p(KeV=n7wH~TLb+6dyw8xwizham4YOp=a#_L z^5W>vC;XOREU!j&X~h)Nm8IsZ&vN97Gu7-{y_B7!9B~R4pa9LtYA-zCbW;@RE;b&? z7vybi+R3~vVkC9LU*m7X~b3c~;k3*T8SNj`-`6~SFh*?MqL zK3h+yr%@)_u^Tkdrq(bH`@~IZocVP`HGvJ*o>J9n^rY>4Rb1B-!{FC8gA&AU zi&IWO1*t-;Lk>^_JfU{vd`jDVP5Ueg-4WVVgL5>4M7{Nv6eb1^CJ(210sP-?kx9pf zUBkf#kpGgWR6x$Xx^j`@!eY*mdD%jFx7%s@LLBnl?MaR!ivZ@n<5Kx<{`{72gedeI zxiaMz)$}Tcx!X-=e+`;Nu$Jd_IdDH<9^09?{uAdOLA2{xlIRRGgY|ivItUP%3*dOm zbG+J`Z;{pNey-O3BYlU#na^ztjchaWmn;G-qC&dKYF_E5tv?URrdxi_AD|_9XqUwB zj_yY5lN?p`-y9la5MPl)_YY!Ip*nQY;RCp)H7Fjmpt6=ZwkDY!{skCDUE>>buqTDsW>x!cDoo^hgEcHK%<4+c!&oq-U3(U%n^8 zEYU4pZ?(8gIc`C)p<@I8gJWQD(R2WDbd6N2x?YNYiqHSz}GAZ{9*pM;5(fIBmRM)aCy3x9MK#jnPT8689r< z71n#g`RFz(CL#06ed{q8V+j~LzFzzA&!3{&X}R7VDTG-Kt9eKDIi#-7OZEmkua+^w zk_F09b#OB^J&lSoZa@Ev`SK3Jezq?W#ziGWXRaZdlw_AF=aag=C)v;_qV$h!Tv^$r z^34qwTmhg%t#gl2X5%FM%>gjYD90;5Wzc8>InkQ-FF+>l9TL>usPQA*+>k0JWZ)6V zIi)oV3_S13nVy*rQOO=CejYJil8D_z?4mCEka`^~y6L)8F}J08v)g=<1cqjh8vf8hInI~|8k_s!`jDJLqm_$=9OCjFWx&$S8->h|q8 zihIpIFmEKir|0Ldf`@UvRFKv-yZEb3|C^@8ROvB}ww9r9?M}rOw{8j~bul^3yVhqD zEo)3wR^ZwTmSA2X|{*+j^DmFRIIfe4o-Su82MT z4h0?$O_w_kcYV5IbA31P*?So7&w*|eqH%AsqS8>Y%OT0NPENbhIf7_TA05q~?}PjL zTn+h?eG3Z{*pX-u?9*9R-{0o{jga1GTsTNgePX##y_6kvAkZz{ z3<%X=V9Ps*Nk^Bxa*oE}GI6!w%>I13h2OnpW^sXQF`$#g<=RL7{x2z8;>5_;fcw){ zyBRa`8(NKIWW%eQ6k7`ldqjFwnxWrKV|bbDbU}-5Ak_^jum+yD)$;h~=-ikVu2fZ@ zHwoixrMG_=33>h&f-FT&vnTP3`~W5Npp+nMNZSinBj~J_`h3F=IC{CVCT)gBM*aA) zFSYt|4J8=`9ru9N5d=)zrBZWuQuT)qx1EoA{w*hCf7^RcK(pa>;GHpXtZN8+a22Y( z40s3;EA*GDV;IJ1essO$e4H}+xw0lAs4q2#p;_(3CCAQr^`}Sk%>uh9%H>fs z<$IAat@bHyn@It4pQ!2qvU`sMhM0o~&VkbTn@}u?e1&@B+6S~XXYIQz^Ruu8V?vl%16{<4}%5nq$zcTb*N7#PURiDGTcOfH5 zmbg0O701D*UCIZn7fUN{BKreSo|$}++p1S^)v8$36a1e0bQ1&^w(=Xl0n~EGwXd04 zi3Yj6GXPB_+l-}VWXw>HMl9kV+{Bb~RUqC_kb{_uAI1F_dJ{bLnNS1SvO*&ATY_*w zG`48N-YIkz*lVe8bHiAcF_TmmH1|}=Z|sv%m_JOdT;K7wfJ>RWuO2*j@XJ}&E0B0| z25d6o@+}ce!mB7Ki5)?sq*>I(#ZiKVwcb6Sk2Xa-eeSpqB}3BEkK$sZKh&2)e|>XY z4lS(V7paN3U`(}v{GwMs1?@Q!z~!``K>-f|-AtoB+xAGz|Ag9p2hw00*OScMnW?VJ zGrr^aMW%tSK?_8lnudJZfUsPeoqidb4)R=Ms*zv4#F1kZam|EDBkw7UUAeY%{}xeE zt9&oQ+$;-{Z-P`z3W)P5=b&s7CGGQ9IF9L49SFbK1O`Lgpv3(hc2)Y8(*W)L6Z@dF zi9lsF6)?3R&41fRgkXs}!gWGVM9U^t05(nwO=P|NI=wJAsvza%rrlPMNB9S(6))^l z^G$H;FFxE`y`nvc)T}ea!!0bn&ko^e+z(ZCxU_hqH{Rj!>B;)iU(jW8$6@ck;_E(r ztHT|6@OF=nS+ffZ;($Ab9XKuY zz$(zaKy&mU>xox$pUS8E_3KqqQkpvx(rgJ|d1{ccK4&)~TV0un%oQ_nOKfS6_kkMA z3nT>Nr|Dp`$U}zgCsZu5;jrlF0GJTkoHx-ED5X305O06D7^9<%H=*POALV!y5Fn1U zFS_~J;@*Y^DW0rg&vV4^?O(ybN2Onzu4B}ildZmQ&k_E3S#O<6w_RRD& z@uCCb9W0&adFy$rcX zC50@ZC;t9d0T`5+3KRqmK;VBuyvu=YL=w4A17o*F3pZMNdSAj2Rmx&V@DI7T7we&7 zSjivb<7_1xKoDu`WM_g_D8YqP5rorQ1JAGCvVbEEAt=hYE3C zU^9fZZT~&zZc-BE4Atl9f+T<64**-{L?lnv|h(lwR~<9>gDh6-!E1{%v$I_G%W~2kOAkgmz0Au_;v`0sp)yB<_Y6PK0?(F zDVpCG-%iiXDVo~~InV7NG8$R4awXcSZ7~2SzLREA(#b17+_VjetGZoNYC&C%_Xe}X zvKjKj7T-|tDZM>~%lzq!8mgABIjppu;&!toqd0sQAMoO>76P)J)axhZjnf9Mr~0Q1 zE{gGA-hni2-K)IN({d}BVv~0RKwinAcasgLwtL6$S{+8}oKjjPAb<7wa!$-uf@lY+ z{O4_CVOro;&1p(()#otd-a}kfY+1ovZHBZ{eGYlUT9l+x%W36|(1}=I)D&LU#95mQ zcWB5lRt=gXTr~dou#JBpE)r-G41|k+q{7~Uu-);JvnNlIfuNqdE&C0wHYZ*KL0q+e zwYw%<0yj+l09qYkXTND136;l^ynOjGA!B*V^F_ZwV(k!$tQLf}ob0&v6yDAB7 za0{8w@T^HEIRH574wSBHiytMNC(gUSHDNA{W-6Ga-z47Ib?GA5V>Mj_BDW2i6@2g2 z0sjT;MvjzhK+mBSbbLL*znF_Xf)Y)R(I^UQ$B;Y{G|E`c*_xPMNCr{1x!Tl+TO7_G zsDMgCMya`p7IqF7<%Hhov18>TZ5E9 z{%x%r;J-WY&X-&9fnnb2-}#I_({_>vwS4_bTo0-a6Gh?1q|cOn-nZ(t9=8^U9rqnS zg;xbZW71MkheN~XvXId`gY{-@^d^GLfBnGmWT^;@Usr6qBDu(*VSCZ0)#LT(tE2SW z-lc>zK7T8pD8ZVggQSt3#}DW&Rp>>A9U-g=;?t!(WN@+*><(+cwX1vO^@w8d=g+vs zk}*GE%=9IVE?~ZM04AOD0#Q^LBlSvjqx^{jBTLf*;l^6<;a?vp~|o;8l|Gfmd8sy_(b7vIYhxh}$7jdC|Saw&b`K zBuDY$;L7vm%>UcTSNj0Titxrd=P&PYMY860OA;j#?)QKF`ZZw>Nc9b`Izq12AN?+! z>Nj$np1|};HmnUji@6ZLrRr^7bX;R{d*$fm@vl~W+I?m8R2dWh!#$t`@38#=(AMH1 zpb_tRLHpI6n_wI|020J+J4EPe#M?f~H&)CLQ#nYyz-Si`nijiGIU@$j2k>v2mNR(e zNH2x-(3fl-VTk8fIB!DPB3@2GjGPT3wZv0~e1!S8*IOtVc&V4Oc9?2)N!g_wI8S|tbU$6Dp0 zoIpr+7Kg_T;8K-yw-uywB7JlYs%j845ugOQn+a$mEb4DknhihhhRMr(k)$4BcnHzr zc#_u+lj@yJyib{g1Bl23XAB57TNaam9gD!--%r43$J+)|Q&W*^rvMyy0^2YSF>kzG ztgv6~E#93$%XcSMFB+v# zCBFC5yyx?od4K0G_};$r!`SN8>-Bg(9_Mk+b*}3=?Dv)ATEPW4705G6D~BX1y+QJm zeL^6KSEsh$Ps+Oj-VX411Nhv-T)x|<+0;u^+PlzmeJ`4Q#MXN>;A|9~h~M`O)<*0E zBzg2OzJMol@nyx9s)gY#2FY_)x`9PM>MCVZ`SlIgjJw) zlSk|q$pOFv60J#YE^*|$&wR8yMpqn|b3rZbNpF8s)IN^&e3qdy7(_F$ktlMV*7XM* zW^Eo2V}%!q31Bk<`$I(I?Hl!5EU~W3#|E0kxwB7ccE%sz(X)OFJ$3HQOR*~Dht@|g zabW2dd{so5M%Gg+JV_tIWl4jkf8^J9O~tGTjOQWX&%RR)a;G4OZh&z3zGBf?-tS5% z4b+^+j9*DwonW$ZK&b6E+H4O=G;GRVBSRWWHwDV*j4yXXxoQVz4z|~!xP0^e{kedr z_G8DFyxx?RtT=XD(8J*xL*`u$b5qp#joyzjWiR@~g>H@oR6hL%Poih>1;*OoUB^ib z1kt0S&eK}Ar8rZ|e_2#_u`{Dj`WvK(in;TZir_Tg zxF^KF!+?1T6dohY9aKWcy;OQ!4WsqSsMEh{X>}`(sO%0>x|3Wm$NtT-#@nPT$5xwk zWY9u!8H{zhK}gfxd}$dd(~__?`?$DrN2Oo`GmNNBNjT}tn9UVwtM(L>juxfc1DC|J zDH}i0+d{Rxl?R`5qhXF&L-V>bJ(LGl*PW#>efo4s1xOusDVDFrB}t?lwSvF3ZW9(2 zKJnWB+~F&Jc=Euc569QLmCh^L)NZ~`#v1WjLa2paCce;=FmoRo-BZf^=&au%{4b5R z-F9vMXgA)R95^8~1kU|Z(sXuHlq5oo^*!<*{b#*KEgp~_lsD{3K`bthLJ)iw>Fk6i zp&iFL#Qx{_yf8P+dPHc5o6w_Asg(RHc`z53MMXXv5{41MJ!rq}?-yo-wZviJK3Um1 zn#+m>BoTW!8$B>yqcphB?GR8r0pIKkY)&A4IZW zC`5>yegj)5471B$u>J8h(phdo%F5b7XcN)O z<%~;Q=)n$T;LAoB(QC6fzHeXgr%%6jO^lA7f<=-c-PNXgdV290^|uP^*4O%^0p0qB zN3;trNW;{DPNdF=Q{?RuD;qa%+={+q!<;nYMRf7yYy0ao=9^o$YSja0hGvg95d-#o zBLzxT&iERg$no2kR^y2s+Mo{+Fq>xQ`7aKdIPB!pr}NToKp$S1iHdC&p_;Bpy)T|xg?j2#)kkqn%LWi-_3>y9qRw^0Ttn8-y%cmvF!ydeVorN zS{3m%C?upUiBa3k%uLUFsn~Wv2A%oAa-VIFkuUNW`)JzEkILz{W=*>CGUxImR)&yn zSeJv{M!?e)fNsQXR5~~-9gQE%n@nb(DnYZI0Ee8#?n{C{NDP8e${f zKW#fFZ%OYRt)MdwyZp-cGpu*ZAKmpcZV_pP6l3w|oYKt~gTFY$nkv!1o7UfQ^)I5R#CcP}5f^yW343|%+s zEocz!SR=Hw`7xMTG!f3t)tTkjmsD3q@1Jv_Eon^$ynZ}CY2eefhb!sP2XVDYyD`%_ zl#6<{wYPtXMMM^Yw4S2rXg)3W=eNR;MD?{%aDX};-nb^e)ygpIHSd!eijO zEtCbGl|Nm*heUpzIBe-8($5+5l^WU`ZNe2``r!dO6ET#@3(&jSeSnYU6&JTw6G;Xp zByFjslLCY=FF>8YT+xaP9Dpso)Z2?6d8PcIcI)5OdNT3 zhcsQXuNM*Z#nZ*2O26g{2giKSToelW6}#PE8Q#kNlNEKV)x=j(Te41%_~>f)Ti;8s zQ}qgdKGw0k-^EGMVi;-N?2Pox%(1s3u9z+pVkP7XFP}K#PyZjNPmDinG#=00^H@1+ z%40|hmoCj>&)h>oHGj(sn(l_E@kY$Wzvl6N>jph{9D|~y6Z4FkEb!ldDDqdrZqpO^TvDRDZ!IBa)UA82SlV40?PuUY@df4oEdkLpY`zP9 z{oe)K;&>B#pDG5DBXkZccQi=IM*#Y$K_ES5oLE<)y%^eiGH?9OfacVGp?s2vqK17#KQxIqQaw( zdNH9@Bh4-6Y*W5el~^6x#afOc%|6%VS1x~OO_&(0aMNFWE#Zc7b1HQKTI}sdu^=|L zgO*}YBVrrzZDQrZ_6=1mv!sc$A0O@@@@bEvhWOggkvYfyB#1OhHI(n zu4&*k^=_6v*x4`)NW5ycq$ppuQBy$(E36~OwSft`x9!(c#bdzS;52|J z>6iB^Tn$w_3M*O{;IFa6wEG_jE0{E#JYxzJlHoh!(=(;jyLvP0dfYGCRj_DjMGJk>6p~48>LpgrM^muB7+PI| znQk}xQSVfug?~<$`U*78M+_+X{t$rKvVi zb&u-V6=!w>YSn9g^zh*wY?sv*rBPThn0YFz%j;(d|x?b~SsZc@c(QsC7Lsa8bZ{os6egUQ53jgz1m&YzQ<PF2K0;4-%o*XSWxjwfqfQaTmovp4>42m(DXyI+F!04l!$9RhG9h zWi!dC8;8Iv`^-iBu8hh2k*$xI5bK|qkkBHkrlRZUjfJa&G$H;V5+-}JPSLy;Z02lme>(~5C*=iMv};Pt%+YrFJ7{@eWfGr0Wtb~KtI+C3%ve%l2a>TzvC)NX5{CG?Eq%B!uJ(lgz!0jTu}1UX=0iD=AP61PlTd{0@>};TUpaT z!0_2knR~o!XFvAsNXuovJ3EvTuI`d91uShgg^WPHo$AY@;F=s=8kM*4{^~cNv*|#c zv-QPD;*gIX9+@Qmua$><41$K*NJ{Y}*ImnSPTNkKj<$Bf$JTm@J>bcRRv~T6;^Dh4bssM!yBtW+BlAP`)_uHnBiVQz%p{n#gr= zo;Ks4768ampA)S=dnJw#g}v06tsMRZ*DCCvAORqzfN@gu^(XEkA{q~1_U zUC!OwMkS;U&*#y{veK^F%MJxtQ}=v)6&@1mTIbC-<=zwplUP4o_zyvE#5qIj*foN_rA~kZ;*y@xExKH_5HQn*SIw$}P>|8p_mGGQ;q17d=^Da*Omv;X^n=~4|-=PiI%r)T<1 zb$Or^WChonZCFhwzUP$3Dd^w-vYPI?_V$@?E38>m;PR%xI+K7BudmHovw+p28JOG?2*f~uki{aU-|NNZu;REut(aT9FZn)IkoYKAiwXiU_VT};%xIxui z&h7D)quL6`8urv#LV($j?Zb<*qLe6s4ld2Yra+MJVf#p52Sf%LTKCu+5~3!m7^2T| z9+C*3eU%)+0p;E8?KY;vn2m*EFx=r+!;4k#uW$>>UgETgXL4})4AeA3VSb$$x1~Fb z86B1V;Y_(`FO*j{lu)O)cXwG&_;D(Z*iB`=oHE^^Vp}`Rg9|ewx`M{f+x#?ua!s(q ztJpU2n2tplN{Htz?f~nFP4LLi&fY-|@rLfPr&)%>K`bxbE0Xn=tzg#&7vz$kj(%f9 z-VnU@SJd_Y$!Pa2lZ+9{EFAVbLccvb`Ip(In=s5~EkDBzen*IB6-W1*d&<8{B9;Ww z*c4Q?(Gv)I&5;PGQGl*EXq%ubPZf(oES(yH#XWzK0x9G^m5IavOUI|@Wt73jqAowa zzHdn%b1IIOfDhtIMQXo-{MpQTYfL&%P71y^G|Z;bN>Wc$*25+RCVMNQ@qH{Vzfi(h zx@>*brrH@deAx^9z|Vjq=V-o&$8zcSmO1!rLVVUAg zAarR#H*x=)aOJTsssi~FH^rd<1Fmd=t^i=fEviGtGP&KbX%fGAq}~>eFSB4l3M$p2 zyXNm?rDr*y;e#d-YT6fXx^d&iX#~CDk9$W2l?_kxBsPnO=R0ZeW5_0;rx>kPOlkaK z5givHpNw*@QWrIO-U&1f6XDW!+3+Fy86oC^AcghgH!2#m9{oTDef)++1k6tcs{3^-tL!#q4Hl0S3t zKjb>PN}oiq!1>*&=g`&GhfJ{zKrG^UOB;h)-A* zK->d+2`=&^_2p^F$?aiRjrakR!|=wsI?^>CKV#Wk^2D!DO~j^1t{iTqDMStRtlWd| z^78&5`Kg7m$87ZfuL@Kv>di?v?t#jSmx*(LHxu-S{U!rXVaZOdTGQ znXjj?#qb#wcYEowQ1O_YV7n~Ip8eBd3P>{|JW;lZr8OD)qqw*oUS3_Hzs+NNqe|~i zGdXyp&b~G@HgN5rSgcE{Z9LvHxF>UY3%bPy?upY2xaV*C_wNsN_3AGjQ<`g%?-ONZ zh72arBnx71&T4|8p!MyAKktg?yO*n}s!GTc<7}zw;BWKlii(PF-oC9vTVJjj@|4_q zO7#`&OR|6ABHMZK;s9*I-_dix8ARSN8R8BO))|n1@G2dRCn!+XQSAyfn-9*-b67YhT50@{t0KT4UFa?OZoo&yUfR-AJK1_*6(#Sy+eyQlG4H8%bNA9 zkBSx75DW3D?&o(7U?m8aXRIB2RE+O}lp(7@`g}6;WJqL==1_*3%aXG^!D${U&iJXyn z6xsy5H`&7!zEiNk*jSyGm8HbFm;$(C8yb+5l(Zg~-B*#$W5%=;KWPVtGfW1!IR8R| zrPkGvtOGwD3-r|$6_j^r&M>tfpC-fh>^?Q+vGDsIHMKs4h`*)ihBj9Wo5P@RU zrca7OrJw#a>lDU7T(2A2Et*E&G`T}O!4+|shgkk%95aB19tQL(Y>Af(yVKa7D2bf^fn8|hXA!)rFg zjvdAAH1Kwa2C&KgW?3#^)h|&Ce;EI1N^Y{0cGyEeLdI;kK~3w^r_UWaW|Xcn&*`|T zDFu15=5U3h&fii~xiZ!xgTra1F>Ms=ySI@Eh#mZOD}B87wp*9##Q_u4ehmXW=sa;U zc5~muaGnP=D`%5xGQl?W3~Eqe>ya?o(@bJ#%mXQ)NbAHZcNjRy&`?o~=^;M$=8%&d z?u3l3simdE31?N(J|3?qIs5%(qCjK4g6scjdwW*1n4ETX=4}l=b>GyvWYLfTuC2QS z-ZUUkbwlmQryP}Ce6=}Tg&3f#KB+iswIr!JVH&WvTDLOCiET^ z8J@!~i6|$j5?g+pFD-UH|Lb$=8_``HzFTEkq^4Bp51+Sctv?k*C)$OFRvzbJ(l@Sy zt>54ye|=y+!6O^{ktgJ%lpMi)=6Xn4d!I3zHH>WV?cU;in)Mpelh(}`4D{j>)PsYmJ2wc=7<@9{_ysl8CY z7T@ZGP(?i08~5~I+U&0%#jk#EPuHTSfBn^2y~F4Q#D`;V*O3Y7Q+odO54FFx{lC6? zM?iqetJkkZDk;~kjPIh|66#vZ*!gRLC-*zgphC-zzy0;5|K-;gX^U?u{LeT1-@iHE z{QnOV{A&aL&=_xv8vM@lJ^(f zzB-QMF9%6;K$*^D8>NKT*9`ycyR{@~d6rT#62!(p+_>pn%D zGFuGP&6^w&%c8k9t?qK+k+Q6Ke*1j)YyLXt8dgpj+m@RP81SPCoR^98S)!jDTBcv} zs`s~oU9BdLn8R!f`tiHT)$;>v0Vc$tmbA&sfA3_I?%=ipqDa;$GDFF_bamU%MQqio z)mD1nf~^l&7I|C$iA>Z$Y^Ws4DM^*N56Jjvk<}rEWEf3-XlFYx>zA2xBs#9X8-S#P zD4eZc-)v8JUPiKzqd~OYCRTowhJ3QNlPWCjjpE=~LsPYrT`jd(uiKqVFp=-j%qF&zadc-Fz`im-!n zB9H?3$57-@m|nF5iAd4xG}n(cY}Q$|KAG`hLvMh3hU&;wEu}Gdz+0y9P*AL-V&7w~ z#855>IAh_s_=J6;7bHW0B&{YBI13`Nm>F7_%PoIVY5)D}Fu@$py?b8JldTqPjeGNI zj|tnjn-hbSd-UKl{PC!VEFP7#n<>@FmZF9Y#&Z8lbA=gV9V6^mH##>Dsg|F=QC# z^Ynv7^dw%{I_tT42CU^)8_zTA+o=9b4ZK-pvK^VZXSm6KNA1JQ{#5knzalSc-LBmZ zq1OZqX1@zJpQ8Y2{08*GEke_I}_+UOrrwTqF=+HX9D zFO2i(t)`ZAx(uRH~p=1qXzCeMlLtze@en()w1Ie0?IKINOF8+k{qsK zLp$QrT84zC^ zGORFjl?gZ?{yWmE&BxQ^gJ%U|O;ogh6r}~(A5I2o7F=lLw zr>MSmx4QSH=u9&B_%JA@>J~2=OH%~7v`jX_^AWyp6JF#ic!I$4tm~xto+1M0mU?#6 z&XWFiT|zWXnyEPdKEixYQk6+ za$~z+4LgyDQvVHR1er(g0OFG|){`*JeJG#t0)C7ODcoU=zhIpGfbT|Yww2*MjJp_g&Rpw zn8u-&+MTF*hk21a)6WPPC>F)=|1DV7$o!Yr&eP0^bzTw2V8@q*IT5zGEd-8{VY~>9 zc?=9*YC{RTn|6Fi?pkY1Tt?!AeK~A1dk>mI2YO&{fN(j6#+bfCV|w0n48_W!!&5Ar zj?#_gqfG{E6Oj`rvp|Eqv!k;_y2AiMmgliYG_JI)-C;Xr}VI$_8$XT#}d! z$!ZduGgx!=#5vf}QJ4`-HtNb4!#*0?*-rp1+GjMU*qTHPk@}ChjtU%5Zk$I4A9D9y z-Tci)<{W|sZ(}@}pfGRVD4-}4mtmse1Mp8bb_XsvBc^@#*EG1cx~Kj_=;+fu)>G9;gOC^@l>-=|cr})f9=z$~lk{l<4NI3BsRxM5qzg$iT8vHz zMxwB{yH*!!(%oPHfO+!(2GO~75h*!~&LD%!RVPVzf_RJ4apFRn3WoS5H&>=?A{`cl z3NR2;zkxD)fSH?jvYoHNxSX-Z3yeva^;3P!_mqy%=DMKEzh7oV&{t=k+PsYhe*Et1 zo9!(H#N7T^di+2-=jfuaBX?2p@y|O681FeVX>XgE=ME^pMU1;jXA9wy8J+a`y+CQGtsmiWXuwnnmlIdUu14BogHx%Hu%xa zXPW#Y_2&-}AL<{htKhD*Qv9%Gp68GVa!saG#VM`;*QZ?9EW3vn;d}ybcH>*f@za7I zJHsT4267sH#0gyF9^P2C=|tQ16B`FHK6U#O)4~zhW`yf}lLs>-fPGjy#OmY6iv#!3 zE=soU4D&5K3%kB4qPR|4%NLWqjIj3_@Q0gQ2WA2cvv;!2ysbYX9v$``>fI*D7(Cs; z&1w}));>8Q@Z!#02e-Od7!r}DD-db z-3vml3?8y)&mPsUUpHd?MoJ$lBRfC*){X}2~7E+)@5 zuK9=)KeMxs7pR)VK6=!Ot*^Sf>AXPV^mRDKezY#t!iX$Oh@&@5??HC(Ht-IYAV zC>*ND<&Lm!7LFP-reJ3c=~DSUGqB1N@pH`{rtH>DZPnMt?pAtk&dT|hi`unn zb^2H(&v!EWXn2nF-03F#vOUsZP2?7ytqdU90awlHxJDPv}CFVGKF4U`@z|QnR1t} z+kf%mMZ?_VN)Mu=W$ve#V}D3%hpG8z8K?>r)-b0%!Wtg6v~%{nh0D61MCvV>U_+n4 z^L+Y-iP;L{H%@iuUKUJQE}Y@S!EvGTxZ6L1EozYcKu(1iYItHY+0Snu}IdOB@ zPkVU@?Sx|s82~Kvi^0n;=OOhMyCFCRyL?CYHPOEkqgff?6@4=2=+QxY%`hCI%Y61<| zb_FqD><~J{ORQh4EdbnAOt{ikk*R3Fy~p z$Pi1IK$*cTdEU|(XF>?nwetRR`+zJ(~3Ofcxbn;XJI;v zEBqd{Vk!5yuhs20K4hI&JU>4pO)X)tK65uAX8jJ7yL#prx=@tHG2pz-2OaVh(mS;a zoy+)zqlLwr9hWH#DzFoif>*De+GLt_*0AX#ni*~Y^CnxcQ+pA(Y*~=CJ+<_;)z9k6 zOdsPVGM^e^$Ay8*`T~bfB{N$`hb;N3hn1z*BsAPJGPLQ2h5vfUaLlwTONB2K+qrP| zGo$~mUC-(@l@d$51jpI53~E6&UL09{C9{My%50p%P5Up#c*>1)Tp@l>py8P&SyX#% z;Vm8PwpY#K#ehRvao$SMl;4SV;QQ7**B!0C!p4=7H-5dW8jEV##0h6!mi*Zpn;8gT zJ5g?0x?fXxL;biq^~y<|def*Z2Y+j2Wh7QVbT2OM$_O@XD$)Yj!WVb#+rK}CKrvhA zwv|Qh`ZbCgHuZ(nrFeDuAoKt!?a1bw=;Hm6_ISk_N&zuq&MYAfG3Gi5-fnKxXUBL^ zn>K2r9WJB^#)st{ODJIjT68&pCBZwAlQ2)ce%8ovvMHg|s@o5!UG{S5tP`Jh0A+jF{1I+u}p`HP%{SoLG7M zl|rL{?R9-hj*qbJf@JYZ=tT_~_$(Bn3F~D@cNa761~c;uw`ao0+X^SMt7a=zx?~8> z?$Nr4TM+y+1P9F7Z)H>ge0izdZPS;y^?vg${8`kNEz#Tl&;|JZ@@$&Y3a;0N(eq2; zQPP8Z{PC;IgThI$%!A&H62Qa|t_>TV(GE+(DxofjLul6%F7#|fEEfA?1&vP5|GBTv z|LDj}tIRjwmu;lHn7w_Z%Ig%DrH?}EWv5R;p-UxrHM0b#%hfLH+x|U{%kLw)j~Z}b z@7WydeBF9KgO!)wYzKh=*anRcwUan3kh@;!cmcG`o}34jp|;JJX%cykB7<>;|rhp73DL$4+^k@y*RYjS-Sm3_ zx$F}b=}kLr)hLn-Lz`uyz$obVG>+_B*f~5q*~>D-RP_}Af`SljrJ*g(#xQ;x17tX^ z*a96fR$Vf`cdyz`d?UiH&7YssC=|Ib(XbU)9Pr?YF`HVjVr*;k7cz;kkwHr) zfbJA-{K3;{KF44Wa*1j#46i=c8oFw`UtC@&n!O{NKQ4UCffN|AK&7(0Jeq*HUX=O{ zCG|O7V)R>fuP7}f=FPTDC|O+3W4WMY5;xi5%#;Q^w!@phZoz-vz2!8%xC@-lyPRlL zZ@ocM?_j-mIv7i8X>`Luj^Q3zM^MrPtHAS#&fo-^v407|_yLYz_BlLy{Yv~39gKW^ ze1yy6_!fN3(+A|X-*CG)10H80taz+wZr9sy0C+|5eElM6I0HoQ^ zVgT=Fb>4tvxu@=cy76JqQn?X3I^5QLIoIN|(&x<;FFIySU>~`wSwv_K7^j5rKdWFH;!ZwZ>W=4s_|O^09|yTG7&2{OqG zw7ftiHE)AdYU@Bm%`&wGgphmOwXLuhV8>E~_24v8LBG6d=a>COl%COP_j0foSMc6k@<(O~ zL}&rC8z$#IoHu4I19^JucSOu11IlEG@fT7FNnn0fXZ$V^nIGI zGWWGm^YQWNT>jrzH+b{TUwSoA*!u%mUuRemk3}XjN7y5ca(qJWJ{cDHjL7+T7?eG) zb*N8;2NPSIjP{JDJ;2KaJ~qJo3xlzcb7ISwVK=)Z!l=)Yxjcr2?*Un~RX@QeE5GM^ z;eyP2*4`zZ+wyyj>ervXtDE}j119>J4!PZ8KVwJXb2sC?spK<8H!*2iu&-nH?!R&? zd#%)GED9N6s&*h4jC;%s5>AvdNlj&rb;MP;4elAs@hqFVWz#^LZwjr*7E-Gtb#qk| z1WS&%MgSeuxdJdCd{9VA>Eg!k6C+5trau&I4y}g+`(SMR3buMKT(LoucCDx@9=8@b z6ez)jA^wy))923B%NjpuZ$R(8vkmg+({hFAuxehqc#CSY|7Xk0KGp9iMtwR)vI!k! z9JSv!#a0u4Ct=jyrA3PBwvK2!@j_DGPWngf*)5KF-A8YjuBxd}Tc2M~e!^$-Icf%; zY4VbXs_ijhPv1am-P98hi4NtKo&{c+?lO*SKqhHLd?ub9_V10WJ;Tdwt~~A)7O<}- zyW&LFjrHeDZc1Xa&lcmA*im?$KU%17MA6e*OqAv_jPmlD?I5_WWM_}B4YC$a^U5US$qDOy;KO-7 z8Dt)X<)G%UJ9@`BlpDVU3N+ollzQ1{2`OYRYKCd2#DVJjSfa@R<@c~J`*n^aW;?FP z$)8GHz8;t8Xlosz5Zmio@9ZdAd*8vEUM~UYH1>S>X!dKr;h%WdeWMaF4e68g`s#}9 zCiXrR$9rbyXY`t~=Xbhe1v`JjT<>#VgzrQAU)eR-dTiv{Q^w)qP!nrU@bPFSsc`H@ z$)7AGjc(@78P1{9W*kxNR+(qseu=%ekVyOzn4x%_PQ&| z^nE5brM0E4*0bjX_(xM!Z@;oLyppw)0yWn1nQL5fipSui0|Qs-AFjIo(8b1&sc7zb zzB0`K*nXSI3A!aYH#Ps>pOD{|<-eJ=&4fa4_@T|VBtx^@eq!7y7s70{1`-=M740!4 z-jC`T*_0T_YM71H*!1ay$!%GxHxQ+tHC=@uop^E%Vok6q2jtL+wBCV?iD=E0a8j1pCGrGf44?j4{wKh=E<+cYj+zNBl;N)_$YTHz^JEV8hCGi&2Grns)rq zL$_{zVg76s_Z3Q-_C34;FHw<-w@2TjLRR4r9oZ*u9ZI%E!YYO}z^vckW(scUuD1QW zuP>IJb8OQc(g@l@k&Az;q~*8+>@rWKm{IWARtprwBRRqpDB+prU}g;^mnGCuSdHV1 z$=9R%3t;hmjPKV_0rb$rf47taVj5uTMBT7oz&t745Jv3YzrW>)lP7IYr~$D)b?r#R zeeq(!_R=T&_l7STke3$Hp9pg<>^AkG)N`37l&nm~9Iclnr0Z4_j}^9U)5hJegnsVK zU$;2;_E9Nlc5=mH=^Tstmk#61gp#UYu(MiVoHR@*q!Hzy_U<7^>)aui5EYOx(xmQZ zBeyGcw;0T3Z-}E&{Ewdu5gPHtKPsi9MA=0_e7Uj8uBSkCbojmJh~;;-DymgRjBDvR)F>u$x z^f|cpZ(LN!=Ma$7@yU^@Zjv>#MV_IykOm6!g2;))} zIf3!Aj0M+Dgz0A7BKY7PeALAL(pXzZ$I-}Wj?`=+gU0H{zheT4rw~33KG8M72Y!IZ zd_NEr9Dk>6j2NDt{-G7U+UFJS3k)*nIRwT3_hFkH*br`zpC|UG;uPA+jC?P+-q2v4 z+*y)wVY^AYA?oLku3QBZxqGrFy&U4YZ(sbEEhM3ts&BsC|1qRxro%!cBr{63g@M^~ zykFfIMtiKx13t9Q{8Pd;Y~1*Op&lZ(QGj+TQBV-wej0>YG$7g~&t1HD@x~LJmwc_s zF39Udf-TF#5IxTk)_VkIptg{aIwn!uU)?0z_|WM<>APHTmFG>BI&|ip7CVT{-r5`LTT+An^Zg+gluAq?Am^DF7G8Un#@zx}oj z9>VOmIr$ff)3K*4ZCY9@joCPB{;BZSBZs8w>GKu_d95PNQ9q*cgr1r(OvUe6h7~b4 zYU{_39}6dpw3y|bnZ59#;o!pqiu#A!*~Cn8DKD{8chz-P8QFN@uPT#B(He0Y@4<}z zj5k(cj$jx-SCVj(>KOxJI~KV@_`Y3Jk>OrmK(DQRqJ9AFFx&4@Se+vNQ?A}Z(iK}V z!6k*>l;Za9TXSRXaB(SoY*|*o>$`1RIONLFF73XSoW74SOl+yYV6EFrw&vDb&#>%( zFikv5B&S_(1U5c0c_^XfnL!1?M@|_6qzUAzB8rC@Rja$|K6dlPfyH_|7dVZk{30DV za62+1Elm(Q$Mr^$k>tHg=D!X+GR*ELp?0`W4Q;EulS76aN}n^aF`Inib(IZ5SQw>S zbehj7!{T>eit{KZioP7t1ydJ^w<%auPb2wNH!q?|MZ+ufWjb|4qI6n0$A_ zHj{QLYv=z2cUhPe>)GHbGHe@X@HNv_?oVX?Fh^wo+0aH~7N?=-Gc*m%)`@sk(S-!_lQt^;>ujyc$?+xY=?>&)ocv(@v8 zZ`+qu%y{|QWd6~el?5kGsM#iJRXWFiN=P_uTZEtXFxQq}mN@Mz*3z?mubFV=c2##K zpoP9%Q*`FziHnQYM^OxW!kDoRnwb|-xc-W1Wkh&biS-`RHvN-o5!RJmq|&B6R?iC{QX`L1xG_Ghu%sJq-o+c$SAF9m1TQ%OTc!kAgzeRb3T(B$2!(^Fk^0~9#Krs z^E%;UHPQqNt{_=7%xM;o{5%5%iK}3e$Zrh)Hc$EV_U#$^oDCW_H0v~pa##6>{bW#> zB*QUGk8Ibj-C^HEvB_Vw@8vU6zgZk4Rm&_4a?f8oqCnc0ott~KU!79U0N}nmiI>VE zo7*FG`e|yudN*A4T$9WcqfG|ItFW0qzQQ~Alv9GmS%=M=Z_5~WQLF(oR%QMCae;P< z9|*vR{)aVgp1Aoiw<2Q08kfq95{<_JsRJRt0Kq3N>ZP~wQ-6}gGQ9jv2Tf4BrRo@g z>_5c$Tvxu+Rm{?!ofB)OssL_mV zS8wsE1~u)=D6KuU-Z%%+BQr;wVOB($@tibZ12<#e{{6k&+p8Ju{H3KYcbWyTQa)m> za_2n z;_O9_W_t`W^yyR2cTN+3u5>JnmoGX3qEVDiA5A~l8%+|i+OuivRi{Et_P^jqh?fka zrq{-2vm+QEUq+-8%k|M%p3T|uZQhNnPd;k*JrZ^1gjd?hPLIw%j4RGU`Z}!Rtje|; zM+O{SLTdnj96RnsMWu``PLY`v2ad)JPOChIs^Y-8N^OmU{+{zIXVHm2UR9^{|2|y> z1{u4sD3*u4_@} zz5_<9YPbK{Cl8wUqDsu+!xm48PG5Yy^B4_{w0DziGBYzr?6Y*cDV5cl@sUk(JEd$g}~)Gfl6@}hq9#-iwwhZ=g1 z8<}=4{mtjM_fdTLqTGCD_2bc_=`f1tP_dMhmWJGj>V3+RTDU_7)3uH5yvwfpFl2eA zYwUO!{(9s`ap|S+PBD`KA{9hs1sVmpmMA}J;ZjWPP`rj{2>=5qk90h9%Z*SxE1{EqaUI)TvXK_`h1rbO*ooPwM<_(|uldR<}oCKMFG0 z$&6OJS<^ys)PXg2Beef;1c3-SHl1{`uUGf;08h80YbH&age)q&S!LJ*-6@{cGpM>$ z9^Qat=~peRZDZX%D(&Z;zjfq_zm|d`BfCJfyh2ki>fBe3Jp8zQUJVam$-dsWt%>is zxD0`&SZFLRsCnUAcz=1PZ!nn_i#V8Tva8?cBdr*9*%Ia19`sX}!IMK^5@O{LQ zle*t&jOcf1`5dsA4vw;G%YLK*erU?Yo*~*fzt^7q!@V>F4_{h zk+~|*X9!~DI?m$2>qbS?1aAgE~nLrd$)f2ZHEw< z$J^1!{;ofjnt>hPs&4Jzu>V*`Tm75f73JmeW#u)QN@X(Mc6L_H^_j)DveHAf`bSl( zM9h(4Xt-aOSsv-7s;bL}F?&D4{P}}-PTp;QxlA-paUBx5I>m(tD*}LN?r(yhq*u-m zNlML<3z3cdU^zw4d_fmik6SN#G5)G2Y#VvFy^2hAIrhWaX^3T|MoQ#Jn*x_#{xX~pM`+Q5`ex#On7Ba{k%{2qbmCy?!0em1f<^)H5L&J z78AvO%BjPL&&$=s$6Zv}ylK;ruRYHB?EcwDfAavXTgvfAo@tWw>Csk+OJ}M4NcoS2 z0_hY0hhH1D2(;g7?}93RvRbms+INPF&M#OMI~^5jPF|jQ%Bh>btsvqkFRJ{;beX<0 zU@PVQ=-jL!Z4x8O$`MZURELaHtMKTWyMB*S_o7!Iqzj5Oup%ir8&+1D4;iYs5u6E`Y=S9IGq0cV1+%;p2hS6QJwv&6mV!!MGdG-OyEZ!XcJS(Fy6`N+))#;S zf9~CJ6=8?l4TybLbL-ja7mbuQFAw?&!%HPI6&!i&w7~XAZ>NW~Ikutl)U)U7`}XVS z+1EO)I;*|LQs?WyHD=b(m!1;StZqh5!>tnIun_yT2M=6jz`tAHVxDrd%sd@ZW-=-LTl(?n`)ymgMaRO zH$g1rT#t0hjrY2M)-I%iV6;SW6%uWYZe9MqvlSI*Y^;8y?Lkpfqd&y45j6Qsr&n4;j;rJ@mg>Nn4j(0euA$>!+ zj4^hrl7TKAJ6@0|PxIk9+=hi<87DPtA6i8LBP~9W8pfM&}^8t8jkQo%2Y za115wcz%1@r3g$AI>b1A3eQ!i30K2)E}xb-ewvau-8i`}cO{-9Ssxa!iaU0O#8@AS zNecOw9SsHkVHuo0^@4(T@6M7erfXgw&ONFJQnxVN#uQ7&S#%0tuocSK*#p6os9Bmc zZ@!QZ3)&x7F?VUi6$E%)uo%_%MoM%UOoKes1F**jKDB&)qG39-u0_OCPNENa&mBGz zJY3`T-64>MRN%iLh}s@FQxW?sX6}t07;e{HTYp?7Mj6Y8>EsX+pkN%KUev1LRNizb z^`+?}%ZrkGAwlf=u~yrE27*8=xv}mHsa)K37#!NrKF$+OS6(jyg&S$vv}x1%#G!I& zi||NpB2P#C$E>yC-8Ajzjr;NSoRMQ*--VZSr@)xP=HiJ>&M{@?Xq=FDk>Wzw7CT`w z-&_1}8h~dwE0<_|jL=gKUi9qOp+o0UwJw6TN#szHy9W}&;*BldN3O{8RAi9pr9pEw za^SO(W&}u!n`>?W@mZFa#*q1!E@OeBtn|+C_a|nz545Uj+M)%CvwKv(Mt5j4M~vN9F*z&TiBD3b>8R0RPfcKcp9#_bk6Q**s}YHegmHZqJer0?P^O}FVzSzKdv9n5KynWe5d_iVtHEnDQ&5m}!ClSpg; z<~gb9po;}rDw6PXl*kuRv{r%t#XW4^q)G6oESd=FxC4>Q?SLTlmxdxe@6om&_SUUe zFBA``kVx7+3-~-C-J1SABz7Mbcy~T>(I3qYyP0AxGL`k++M2@DL`9RneYj+5+B|&Q?DSn ze+buQu&e5Il0SpuZ*q}6T!5Xc)s|D_ji&-q`wBHQqiWYIRFBb% zIT@>yex!`Oe;u0jX5X>9J;wG<`QXjJ&CZ>^=2T3YMkNwU^9`FeC6-m2^ZfbAVJ|DW zPCC3#u=-iOGc@&qI{;tj*RNhFS8TGiRl5STIr}Fl#!BSR!FE^l9oI@;bg5KTt%d^4 z@=K{_T9i}baT$@)yy2wl37svyFGxK7-u**P?lscV4!NG#AAJGFdWa@k(K{)9(_*t9 zSQM_F|ncnJu0h6aI310|acsEmz?i9r!!zP=F1d6jvb(1WVW zL)F(n)C$&GW3&Xa)y=>pDq(NmEmKpA_$e30Aee|a8jyHTN^Cc;sIY*$)a%11oK3is%xwL!LlDbVt54pi0I)BTPid$wni{-hwoJ~3#TL4_TYLwu z*N`Ox@4KV18y69~t@90dlwMJ5m7L3TBFdg?k0_7uJ_7N}*a>MJ1o51rqLZtF=)Ao4b>Wrp zH9ZN{>A({S`!>J0fu+q-~ zP{fY$o^iC#ZXMi4lou6iM`(qTQeJQxYj9cfSUPPyHM?=1OSqSZ!<|$v^6E;DCm{2u zS$QWlSYJjCHcBtxFFAADb@02FIpmZ}mAO|{_^n%(ICn5UVx3zIaniCuqece){Z1_; z1nGg%S?5<{Z1Umd?dPu_d-F1j&|Jh_hgdRgu2%VB?qiz`332uWPn+BCf+-C?Bzmtc zb$-%x$n68?N-y*uSsdD^S*LZ&ytBgNlam)>;`_AseogmFH8P@#y{h%J(*5t-<rR^<$(tqsy&)?c0jvz1fN0S;lR29Z6Tz5!U!&QV%bk*Si4wP^EwW zjHo3%uw*y`!?7ZJS3xm20e^zydy-q{d~i@UEq>+1WoY;2P90uVRlB;kGyhrzu&3Nl z5mEj{H1LuQT?(L&DnUfDCkIEzbj|&Dm*YedkC}0x;vZwwG8t_k*Q(9>^+D+uCG7IQ5S@N7k0edC-8BNk6rWA57$Gva?O>@KtKR&^g5V9KlQ_f$qGr)2g zD`$b|Lw}Ek7Db_mOJyG1b!%>asQ?ZiIB?*7X?i|rhxBArPud%iWBUc_aR?Ab^b-&` z{|BEi@OH_b0;s7>I!RD zMe|f0m8c5#aL@u$5z{V!NXCT3Wr$oI^DkWpR;*^6#BjWyiEXpEXdx+AZ^(Jve#(?7 zu}tcgA{`fq7c?%J`+GNkNC6Mq01K^Wd_o9(qbY5sQqVEFF~PQ}5rOB(R=La8g-(bv)Qy5_-UP-Me2QjrC)n4kF{AC+$zAk;lR(lx)8T z+_;2sBr=4rSkYpV}(8=MCJo&ihh1-VPR&Ydq|lbXXZ;4NG> z3=d{+`B1j_k!j|Q-O4m#|16j4+0cd$iA@z5_CTB?#0WGf^}WkMuiW(;6C?N@!SYAr zG>Q%4S(^3*APRt?3mDSKvGJuI6i7TDodApmZ=X?0i&(2vc7JlHr5>rKTKsj&RH^Bp zxBEUm4$e^p`Kt_k%@V6*2?KezdeeuZbxw%B%4fRBcl&`SOC24(JHE8m&KCNH1*!Uy zD3=JjonM?5Zjt;A+SCgq2xoRp!*@<6F0E`vtrXGLwfvcOK8sIqJ|QB@8#izx-}(i`h0 za74T@C;@v`2LC4+Faq#g0OIum^5BN3DAJ@*R1IY5+JJa+N@i8|)llitW4uIesq;wU zKSJn~kNs?Q+?Q~9+ft|r-HR=L4Yj&-QBd^2Qg#dK71$(E+vJ$DGDh$VFZbR zW5*g%J4)RI1>!PfJwp^yk`4tdDgs3gp_Ai_p_VQ+;!WU9d2Os-s79vpY!_0#+7#In za!(V*F0;(iEC>;=L2*9fa2?CLd`-iLIw+s454DjLd*+}qH_6JVH%*3iT7X#Yh>F1? zzGptU9?h+cX2seZfBF_G*_~YU4&T0g%Om~nP+9rqOYF|x z5re}%g>tH`p>!Dvwnk8vK*%Q_oa*-8WdW^!iKFb%CAaN^a67!ng`~&!!)AB^XD0cY z`F!5oY}ybxr|SMaZOK#ZWcz-`ScMzN`vT!$6{}lI)&go!*tS*e7gh#QtuJDkg7vEb z*t+krc<}cZQ1E|Jc*)AP z?sfmXnzm|gU`-=Hx{jw`;5=1N@Nki6c(CoB#b4-;7;3w9CG!PL2NYw*LAG z)ek1cfBpU+zizdj-sJB+-M{|v&V&Q)8vpf&zx}#(`~Q5-zx^%0Z8Sl*$3Oq(&p);J z?fm$E`+LRuGq(NP-;Zki*VFmuzpiW5r@=pVKPxTppOi{~m0lG9=f@*ym{o7Z4QtH!Si42n^3~S?JR`U7tJR~1Exw#i8 zQ6`crXsE_8LxYp6T9Zo1R6MTq&v#qEuUlvUlOPX~W~;koKsei8f@O-=!&5>)C=?s%c$0C< zWhR`loNuDN;N6+M#8hDdY+&i0^fAYTOaD~c+MlCo&#S7V(oAv$Z6dA}b^1pKSfR&A z`Oz+4-!_^*aXAfr$qA1P(7AW5hMFlwKpqm*Dfp9frx2A5JaS2MJqh3*E279!j<>1H zpD(ijs~qOk{>0DAXHK7fb<;n$)>h4TY|l@PFqQ8z&gHhbGr5j=o4WHwFz=i1LUtvK zY@>A@;*OWcr8fWk?mt(oZzx>}8!Txjj%Q0K4#ct3% zoPgH3qOzjm`i-@7Jef;q?tlc3qCO$T9>a`;t8??|$^G#M&*M?biQ0GHqq4S7J$L$a z2|Fjx{uxS+L(R7+{!egIP-yWxpAU>oUB$`X-sI&tdZ)fQXb%E$g@&H<-Gw1(9Fghxv&`!$$4ZQ82pn4PtMF+Ov8&AI=%C|*7$*xq?1nF-13 z3|(*l*3NXyw0{dxHC+w|Ha-Q94~Ex%#_^xR!*rJe=3V@+px1IJyQ=*_-3N!;*@M(K zJ5(3$Nm8NLA&gfpafG#RR<~M)9k-3iD}(`Nm!t6cF~lI*WKHd-JK)5bO2&UT=_bh? zNNeH0oUZ|wNNYx8zR)L+0<)yNe0JEDnOTQ<;;{8*Jpt;_;XK%G*j)RPpO>$htyC#b&#ml3nrp8v?4W>~Z*Q@(^DsSnjXpH{X{U5~Hxy08> zG1SMLObfco)@T>bw&!TO%jOZF02*ClYCqb7U;gd6zH()Q$+3meVy;EbNCl==P>E7G zpU%4bc>??;K`&$IMtIy^egN=*8pV1K);@6ye4YZp5$Y0OO{o1qTX#$TDE8?|2>IM2 zCjB$sXZPY7UKfR<$#bgiYe<6SzUK4mfNf#3bOxkb#qh>Fu2n%%iS+j`_LmohZ zdVwC!gEcS*9DyEeHvFg!al4y!c0afBue|5)uhzZUoQf82Ty#ko;aS$&^oe>AgM|M> z+?#;qoVS1fjhSH>!#!geYZztAt`I51Sgw?%lorZZqRmoBR5Hexk@ZrjC`+OpDGJFL zYZ9fXBr~B&5^W^u|2m^+?)yHT=XV^>^ZYyR@r zqP%sk;dWnLsA=Q+A0PIGwfg^uPVxVB8nSYs@Frjp@XGi1KlmgXl}FkFd+c2*S~OAd ztV-oXHn@`n2yk~~r@9Z=Q|1R$MbimodqleB>OlOaLxWVelXTvOO zR8RbHig(@}>1bY^P`|O-J0Ej-eF@TUlNm@aU7SjS%+V#83X+=fj zDq3_)iFg&X_3T)cXZA1;x1{YMC*sP}!b1C8&$|1ua`g81A?`sU@#r=Q|LP5&Jm_{5 znYU)4P%W+{K{9fWLvJ!U6-Dwk>@neEW{e!y^qw9FQon<@78Vbjy`E6_O{-QSwKLo^ zd{9n;?C@cWt^BGz&#Wk3epluaKl)zns0sOM`r#>;6hYL8*Viq4X5Dh^#hG>20X|RXg+shEZMpbWqxBgmP&` zYfuccViYK(WLCf=$b%L~k@AAH2|UQDAi7*kC?6mMeKV>v`Y>8P0mhU&MS94Dg(cPKmG*P_GA1nZ5V|FB7aA$5%}pa=f;g2TfFH%sd&Jn z5%aGS!^=6YLeYye7URxe2VT)9c)i8IrcVk^B_E5M!~*H6_2w; z89b~VxgLb_++Un?t6YEc@=629lIkAo1r46~Dh%Vw|MD7Oakk6E-`2c&^&&Wy*V%nG zzjyJ3RhJzN<;-N3`1|@t7z2d-A_Jl7=&h-#nN!^_r+eq&OGXbk^ZX~&g9oViZC=II z9p?``{+v@+Z*E(TTW}VRE4F7@+fGh@x$IMUI0~38t%UU!QDF%ZVYTwl&nel1Y%#K) zK@*}#MS2?tFQkC!z{JXngYf1!AomaQ&cTv_lt`on%P|blF$eF6U@a>z9=x_kydIeE zB>DpQ_*92U%!2|9r5$omG8SKOZMTrKBwu0*={fxAZFk;mhiZZ|9{Jmoqygt{6LTCx zm+jmxPzI2M?1z~LVA!rjH7rC)B$Us3T%-l1n{g@2 z8arNh`60|p2n9LIs+bizk?Lf-8d&)EyutO`ZWY+Tg?oCcNLb}G%pm78A=X5DNUped zB$lyKdz6lt-ZfkFq@qBeMNM%sSHPCpi7L_dWm3_KU`WtsT`4(q*_N!jz zxRiq<_To56jumwS^neHoWfCWX0txo*x3mY@1g;8JT-q5#)+$OQfwd_e57S>bbi!8k zkuqf+nc0XOU0Qikc@igZVvKggqdJ`(ClRTfwnx$O5|M`N5%T;A`Aos;>iaKX?V`xt za_oL`U!)CAfaGNUvG0h#hoz>f#wreN%NNP9+flHI5zZ zrO;WeQBkmK`2lmc;d+2q;}1^b)GK^Pj-c(6|!#9y?Xiz284 z&PZZr0n1*%W2!}SK?9FKs>rLzKwm)_I3@(6D?8t$%e)skSN*z;{Y9XDISvEdE6hig zTscM zJrq|meI7;32v$+4;ng_&Lww(!_hc2CMxCwj<6$`JH`_xqs;zd{u3e3tGt-2ALhM3C zV8`#k!FiiA0`u`irJULXO=Y7$mk_Z;mM0hu?=^#(M)sIg0qllg54ME~tJ*-Zl2~s_ zAFrHJVi`sWoXWny!+SpmM*%dZTzlGeEvChhFOAAsDVm{OR;z?3QU(Eq?k;E0*1ooV z&AKW|n4TwaA!8;u&L>JCrS4UO9#7QH;Q1!_L4`S^jqv-X5i2j0H-aj10#h2O9zr>9 zhxCvtXuGQG!>=u^vSyZ}b<@Cv9?nVPG5~eIf<-?qpm0{psUFk_9>Vy*K`L@`lXN`N zgsdk12oob0Nd)V|7)ltriNnnM!}$QD+(I`36Wt^&p8%`j2gD+=pW zVf7orKfMOGPU~DMtC;!koM>&Z#{xsU3ULp&=<@5s6?L2aLpd@p3D65Vt+zt>smu$x zsX0HNTqD7dK%gHZwx{@~k{mr-Lc^Cm32@4{kHCh(0D;r_!xcK#p_2Fl>BvwD*5CsX# zAZrOXZI8q_SV}GZFUU*x(BbU=I9{9W#ZH`z6V_U(a!{2CO$@H2S?-EDxtu#|xmlgw z0K(@G^%aazxcPKs9$Rae`e)n{6L_d0lRCLoKH3$-DOLU~$5N^(G^sKlXZd~aC#S?D z!OHcKk7l-{KYSI?C3=(KP@X=++LqXAdik0LYy-S!2u4^wJzgwI^!z0F5&AO!ZyLF~ zyNeu-9=Tg5&aSC>yNchE3{nzbVe7z@g9gpLWVZJu(FwHk77SAP^&u%inH8y%28hTI zh1O?Y13&UAOKDY(_8osMO0BqtnTZq_f?%}&-0n?K31fRST|s? z#fP|~Rk8i%Gb5TeC$m?<+;5U!Bm)CaN9hT>s|?~In0J#FJ_YD5yvJw3_^n3jRz7T{ zNX7C6Gmrk3E&HXxOBysX4AF3S$lI=MM)&Oth(AjGEDQ&y8+;A?xJ`9L;{G&vV89# z4qSir&-9HX<0jkp%cx^(?*tvX^_$o%9b_xGT?*Pu+kHOJ@d$@{A& z(O$7#O)QbTC+_fPctI%WlfIIPd|_lzW>4v%?w__mQU~+M72ffGSwGSDd@0;FesF3E z2uJ0H4HI@gj^5>M=P#lN;&!ToKNxS&>A-AG*nC%q(RPj3b;}XqwMC8#S*dBOMu$$f z83ZWUM<+M!hucIbk!b-KG9jssXoTJ?$X#$J5G#Nti4! z)ViqulTxHO#!lHtHHMrS&s5{x>YoL_qL?tqoyBsERh|3`Le;=Ot9BZ>R#;K}-DK&d zseKl_lPrS83ubyBC!t=3V`gs@=?xJ+_@y)bX&bk0oigkhiC4DV!PR;)aLO;kxgM4_IYL<8JRtHc&IPy{K{`>xhyA0FbM}l0) z>?uiJ$d_#IgW^TIA3SgX%%GVf75+GaBtSDK?tKBRP-aBV@_)43c*f$>!!mpJT>taW zy--J_=BS02ebXqyJbqb-ZKX)KCtaUZlsqg~bJV@SReF;or}t7cQlc=}&pY~OShKOc ze*0Bfrj|K7U%M|Q0HGqA9kZBSWsnqm-G7Ibd$~zix;o7{vXvD^ z)B)H%_!qvAu5-@bU25w$Y$&8iiMc!Akp4b3<3_foqGk0qcui?QusA>G-0i08LRvNL zEJGpM>6?u6QY}kRU`|6(d$xgd>$p?W{2X z<8$niB-zT!}plw8#dLsTTRZs9|<=&r_-5ZJ! zP+=rO9mLwps#dLD9Y)=y45DT-Gu*f9+5N3Tf8{s&{QVK#b*uT?b5(Wc_1z$8Vg8$UVHmdFj&abmp-$ zNfLaY?QWm!zv>1Vf%LHqEsvSc-Y0=rcJi>=xQVXcX4Bw&nAJR$)>Nt7%$M88JTJwLezglK-GUFs9fg_#dL^I$Y4IRjaup0+AT*?(tjNG%=GEn;}hl&2SBy zC`wXC;DFsYz#0RpX`;QHYqwXGgg2qKdR2EFX^#1Xe>MrKUiVG-byrM+K$q zbYM|hBkXG4RwIOpHJhdtBN;zOkeLSPJP1P!+C+}~6TQFQ4@@R4zP8r(inZWs1zYl-JqsvEh@C!Fe{p#Db=$%- z+Px3y2WVCN8UZjP%?_gXVj(>NWyHv8qiq$X#BzFv6h`D&>G~UDMS#PyP|Jz(&9k;6gBb**HRnB z^rk=OQQ=$1XB%6OJ-+Iz<2zfggzgzt`EW=|6bY=DK9)RBn!SjY*wq;K5X)Ya3}rQi}T2om|ew-AM0O-Q`cF zx8Hopp!(1XIyXdzrWA-&I4WZDZ9&Qb5d@`!i=2UA3@fkbnK~skQB(Ji34R3LU}L+9 zBv)TaR^<33a8-b}`}Tx;2F;TviNcApNln4As->>;G%IEjsx%)pN$C1momsi%`Yz?u zZ@YAf0_w@U?ZMj%qL7*sG{WBJT&HpRCAlm`%@t40fH}yv>?tl*Y-xUFR!CGgZP%{^ zP>{5ymVXq_ptsdkhes}YQng+nP2-o%dgo4P>2*C}@s?G&+g=Y5`yPYbv6K^89_Q<5 zuWd;m9pp*!0MkvyaFZkOaHbVOEzL>aX|$=d*nl!AgOHUOL+r$63GkEYmt-2UwHOY> zyu(4U5qV}>9LKz|U#j6B9WAK0mhF7(_AeZI{CB2>HC9t#Xqi&9q~toCsJ#lF4HBwO z8H7~x{)thC#@I~$JNMkV5U1%+u6g1pG*G>0gp*IA%)KYG-cB^-9n}?LAItgmv{UQ~bg=#{*X{Tvr_|c`YchqXr)Es?#hJ_!ydzdm#gso}{x}4KR*uF`W-SL+8$^LVmh5qrqYSE#)3z~lM#Z=IZ+U+!zGm8+{S0TbuYP}tE2k|DR zQ*>VZtY%9uEI03*+Ewr2jDd-ibo8!Ug%R2_(c2kX>X(mx@V(8zcDF$Y6fPP!822|M z=Ph}GrpaD=GF89gW39zugYs8h!3Ro%hH>0@h^-p@=uG@CBSWtfoKite`aSdSczl&( zbP!4hDQwjgcoZEi*ie7VR+`m`m>xWkJNEihmDTM*XF}|YZ>2|`D0^g1XZdb+g1>%rWmJdje^MA39PPWi#0%jReqNhNS}h&)QI~#-V@PT zE+8~g!3CXedXu}Ilzvz4sSW#CDIaii=aECR^XkIZ^L=&luf(G)U)9EgHB&fqM{nCo}4X%5_yVyf=<8uhA^&q z+Uh=g2eIVJF zpb%(j{Y0c{LirAoF85AQc(6VCG5TP=K4pu$NQ;rHiFg{M=^E`y2-xg!J!ZQeqDM(s zXcl`qR|0?d0~u+`iF0H*f^On?DhVsmQqp<&5+T!Ikwoq+*oJK&vWHOm5?!6#UPg1o z*mF>_E5!ywEFC!mj_@?3bP>XvDT>pP_@qfkeDRVrv7rv0ytj{{!{B+p;z=4Ji$?i| zf%J3r+n&8c@QxIZ$B-83NS+PUAKVS)wS=^kA|4K+kfLIj8uHWKp`KYn)kp)E9A;oU^LgmtUrVbe>?cvqA~WF0aTT9^C#g60+c* zE1+s5aMPY|;POn=%3$5YBF@p8Rog5qBWW{au?W-=t4*kdmidYNtA)}=K^iHVX=}F^>(xSN zb_m|Mon~3-aB)x_u;%R>!9=M#LWD=98O!46?6m=|kK3n*7*7DXNhM>IMpU5>!Gfof zcME@~ak)+lmstHlGdg;UOk%}!Hwu_aM84v}8uF9xvXkFT(;s)9AXf;`8oZ&h9|N;b z?tXwF`gfv*mx4jCemVsH2p7b)oDcfg+-TqV**Cejnl{(lNtQ)QnM$oDMmt{Bd-v?Q zo+YAd=z=hkA?bt{#rxS8mpa;1m*l5Jv7kzhR@JcMeFS$pDW9re^P_E1Vw* z*t63?qk+OjgD7j4e?LUHI-IGyv@ z`uvmuX-HK%MK?RF_vZOJ5wv3gjzG#%Ix_u&UagkH434?|jq-g0LGbZ4A@L})4N&#f zGN8wb1ree6d8nB0N*BZC_Cv-Cc_JV__e3wbLi%OH?RE!HEkFm??i&{Ga#q1IJ!_U4 z_|{eZcbX_-&hX}?hg;Oeo+}3cz>>-mD<%4n9|mXM=i|wWWg}$>*2qq~mX>Brt4JTr z{s94nDdU+;!S5JYbVp_bpfoyU=820i9<8`z9O>z)dK(S`0O$ zGYuH{m!A_?pP?%&3y)w?UxBn=BWQ!!Pi4o&V3LE%{*USV|MvI`W|acjjvf0z_a|T zG&qDUZP&c9JQ(TzCJDrJXr#2kfO#NqnZW_XQXI0mleQ7y{AuPv?9nhW>ij)Lm9VCc z_fAbCdj;oF79<=&cOnFwbf5`G!|il!dj~N32)KVzZmuy9;tq47`sJ3BKK0%U}f-7qo8&_Y6p2WeTaoy|TO?B6yCeHlb zf8TR~N04LLB8tz``5?)P>&1ndb`-tJAl#E}YipB_^s?J#yvkfz|5{C=krWB(r({}6 zR|tj%a?>js(?1C0Po>EDLc#=8d55JYtp*C0u+I`oZ;V{ArGAO4sBGKyp>|O@l_DMy zSTaSMS7Dtf;{B07)JY|{jHBOvfHG7%{R)^QjH*N(fZCHT!|Sh9#ZzZ|eG#CN=pM;o z&4E_YqON4(Czo_ta=cD+_Td-hyTqaYO+o+BwSM#~|GfoZ$p=-NDp1vb(mTF`X@(D! zW=pys?*I7F3-fKsuRH47IvV~5;6$c%*twEs-uAr;8^FP`J9`@*2-=ERoMN}Ndes}~Q|I7dVMu-3D zW2#HlKfX%7|KGZkq76hPu36aNmpz+&TPJ*Laeekcq30VcK7M%HgjWS0_SQ$g{XcK` z=wDH{qs$rT6+{BCJEO^09n@G;BgcQVe|VEChIMOu_$iY8e_xTPW*O z@Rxk|Kfl_#3uy2%Di6B;zkF(70Gx-rY&ti%i02Q!d%veB+O)N4AJG5zr?!VKn*a25 z^sBlj-tb@ld!Nof9vkdg@`?NCN2wRH2YuJzRYotWzq0e32g&!lHTdWMlVRdRO4;Bo z{$D)jh3BCSC|w574SJ0xVoudvI3px5H4LRpe%6^LEmcM4N})80g5e51%VNJq8m>tP zi8Snq5NW7hgA0H69}nKzyzBbH3KbR`(&;c&nnjT@XG}H4p%fllk~@MD(}QV;;mWnY zi*Wd#&Lj#ahiA9N`J22#kqTKOEX7RUcRBdOJ!~Segv;m%yVJ9$N0Vj^-duG$5BY(| zc6KGCub@s!Q7854)oY^2Sus|Yo?>#kCjczMHp0h9pQJ+ylQj3!PU)w5U&AYq6{)o4J}NGb zk`$PA%jY#k$E!&Jf*Q)kR`uOm&K}ZXvouK;q`$g8xi`FU4+y*<$_ ze*tmRECj_73eH%w_C9qTLPJqe=ejFS{Dl6R0wBiV`7K*+ZC;h1v@7LAr&g_Y9MWdo zqhS))Y6RCC1C1bP$?f|z!HOT7czkW?q8CXuV2M7iRt8Z}3DYq+U2o`6Q+gVG1DeDt z3=C*YXc+;fqnmw2^M-lVpL@ocF1eBQZGOMwKb2;93}4Wt`MVabq|D6B$+v1R(ewJh zHF4>LKdZj@7TPpe@rA)OuwQ6}DZ%9+b*gt89^LO=VMA-5M(UV>`1N^7kMq^EG$3Ip ztfBqB?Q1r0l5_C=D1lc1;nNckD@+6BlsY6Q&UpgB@pSb0L$q>P(}SNnGZX@QkK3({ zQKb#mYT}Up*Y6<<22f18#csF#PisN9-Lr_+W=E!W>(j8oH1X3-JqP@!=C3xSmsTH7u&}PUu}_Gmdgns3V0DeU7y)gUgn|aQso7OeIYP8tc-or8nG*F+IgcUD~Z2 zz4XRoog3y4d)&ypciz?O-XE_1rNb9A46!Q*4xfMT(fJtwu4yGyJ84C$k%)+bQ8^t? zM~305hA;4LpNkFWe`+a<))D_vx)Ri+U2+urm6-dZRf4@3@82Nc#I=1wi&bSanl~ zeS!$V9a8FqSRY@&5`+KkPPKXW8P-74ggFF+FPc~_=f?)M;w)={K)6PxLO;B6=Hpkd z`~GsCbo=%MU%LHXsVit(AAAjQlM>T+BzvE`?@Nxx4CsXY;Aqm>BTtgS zxx!qDPC`WMJlN@gf1UPzTlY2}{rSM`Nb7~)qh-4d_Sm<$OS^VQ5)%^*ia2#Ti!)b} zb8UF_++d7A2uE^93)*Nl?%DIIbFYtHMMY)h)DPWL)S95*HWqkEAFst6KKjXrLp_GR zJNiEQOZoP{)?WWVrl-PSt&wC9^?z zNlE{`i2T1*`lE*N&7djvyZ>1Fe+1Bdbf5o!xXC(d2;Vu-uD_*UX-@G?uEx`Wxusqg zt8##_S8r&nqFvua^8Qj2KfQS8&Yi#3CBA&iDKSNyn-QtLeR-}%MAU-xp1t5QJoZe6 zdF7<)2NjTk4)<|F9fuA{ltsug4sGp1{+56!9UOuOJ3H@S0JlK2 zB<{9g(UPgOIClU{aQod=#lH7*AtRo;w_7sRH*U&}r;HuTGblBi6nmm9|MBA<_>@W4 z2u;;@1hS{q-=OmyF;higa|C%SPOzb5z7P3PYAPDc;DIba?RSZdpi=S^BN7*a(7E%_{=_vVqABsk+HPy>= z65L1TxYKWP5HaK|=m6>75f+iga0ESL5D?>QAY_TS{!%cAp_<$;VUidOt}|kcv^@LA z{4i;_4C9-1&SRD|X+i!S!7>(=oPMhw22<%BkFUaPvNu+`yhsLCx1tyof`BaKmK>F%^jHl_{j%S`9Pn`duPtm5Clo9yW*MQZ zyq;xzuFFm)^86lL$-N0>efwg}cP-_p4yhSaix|1JGVK zgU>1cl49p39hmguB)=92?1!E*e|5g~%i?>gwZvUUHNX7EiXUD~+@!^gJ2 zzxCw8Tffh#VhVkPRM$CCunI#j9);*ZIAbLJu$X@aQ~ZjUxos5^qgnE{I5e3PUmRjZTm;Ww@z0Z0xvXSsjo#d?1adKt@?3>%?_Sz7VE& z_2}U&N6N``iizac8-Do(6~wgu{dMei1#D7{UjYw#cTqw0m5n4yzMIqV{1he$wx{h& zb1(&{M|AY8+Z%IoY4z1|xO?-&Gb(r)S3-lPjmV;xRli;iP1sjs19_2ZSMB9dp9Wpd z$r+Ut7!8z_e`Uc5T7qp-BFj*o0r=cxE?DyGp6coaqC3Fjb%9+Ks)p7bJKARqquW#l zoZyc&+io>>ebdi*xTEXH4zGb2_1WD0lTKd$#swr=fn;KKI&Y~Zs~;YBQQB}JCnf$+ zh=lR{rKJUNiA-S;jXwFY1=9sd$_(>_O9Z81%EQ50r)o5{wBd@0#v?4n;Z?(Ud{PgO z->Mox7fhDh^(3!P%~Z%_6hC}Fy`>ie*u{vo?a3TVLtuHIfF@-BYFUQS^Y)z)LdASwM5B!}9B_27AWPtorM>o_Gd-PN7?i zme5+x(v;yU)imV{%?WFaH|eIuAS=pe_s7-%;H{!r(C~!>RrdC4{sQ~u1xxQi=lWlg zIVt@rj%h~0Y0J6bQ9an{uRof)KR@3@BouD$?j|Uo$-|G(a3}Fagm^$gS(>I#<{ocd zOaqM;Jh=Eu;Sq>qRmSeZ)V@7MclWSFZ>)hpf*pgi{UCd zx5Gyr>C^i{pS6T)>5i6`PFVlwBC)h1bVEIt<8tx$vCFFTjHIPSTv>{*jYxQMTo2+| z4loG?ES{Er!&jz)ium zow3k8$(P)rCBu7s$+gY9Ls^`D+GR1bfpUZ2N^9t(0cV52>CEq`@_R>*4iwpe2&>tg zT8udbP3S%8g+z4Nb#abNJuv}}kIGbpDGYZKx3&-pMRULICuO9j2zt^VUJ>O1b)Ec> z?%P*$MY}1lO7$JBnJXi*nVj)A!6Ngk>ICWS2bV8o@7QENJKF`+@yx|R{b!!qhqaNk zW7}pivwhLAHK;C*x9bNSwYE;>{_a-m-0rLHq7Yn1%A>HVi+}ua1m&Q#SoQ1inS~QY zqKFFrgVMxTW5khM^X_wu-CSxL!F*P=~I7kpBy+f5}^6UIo^wBh|2oHvr{n)h9_ z^It2YKs}F8$%#(4tW(^4z)kU?67v=RXBScJh<5|w;)W&HO=z{UBwNCM+beH_F^V*hJPNo_*vh^H(Y`SR~HVMWezLPrnMh>80f=$CcnjhtNT~@cI{?XVnFD2Yr2;VrHKTYlq~a2_bwr_1QAB3 zaiDwat?}@GNkgVZaK6)Y-?%Q)8^^Oa%=8(}BEgym#-)=33i>H3tn%5PG^t}HJ=cAS zr+U4?)`KNAuG8}6S{=oWSA-BGI!DB(^1c0oEO4?YX-)wnmu1M%!a$o`?ie~l?F&D| zHkk#E%aOf!X|^n*3{%=-ksvEX;D%nn(eDE2F@ZY+9t5AYT@fokQAu1kA7=s_!Q6hS z76lfL0C#mFu|*D8{Y+cdyTAZ4+JK_IzoC%@()x=(k7j|3y)RU6QAhM|4<(7m^j!ATRS+}Ezfe5mOWG^5gX`9~{ zkz?>2gQB?nfau--GNVa%6b+1$vWq@QCNHslgvdxOlbs=-A9cu9=GKW859!NcTY)+VqnuTP zH;enV$Ha+7zJ?)135$!62ADqELzuWip&?+v#Jv*SFhqD>?{H+`SKWCYjB(T?3*BS2lKiCqcN$LyhipSN0Ha zig^tV85}uxbvzawT{fVj(_bYjUG#4Gt7P~+#RKfW5CPp6I8`jIaTem5ywR^ z`Rp<#)MDYbS#^{JkGE|Vt&Ni*yN;jg$3Q|kOLZ|W;c5?^epyT#k`Y`Y>1wz`=Ozyp z{?m=kGYB3kgp#(RZcP%MhC^JM?N#0ss>Uf zk3E9x4)nvCU#g{lmr;>@mW^6fc9RAohYSzuBtoC1!IRUqQ)=*L@yce29|n(#u8_Kn zUv9r;uVr|Smu0BK!X&d;)7Kq|1>~=TM_pdGkFr;Upcfql2WKi26JTOAi^$2OsTW&- zS(M7QEalgTvyfY2MSNU|(gcGUy>5_B*1|2ELVPyV76ycHV&(&1Jh2DPk z%a6}2&HIH_c?hAkf0h*K^x*c*&igJxCIC@;OLNS{B2gPp`eGTK!<y#j5%fJ z7(32h8y03&LWa`|dp539TNM<2NncX-RAkyip3b02#s3mZD}W?;$kPQyR_R%zwy8<6 z!~=_6*|L4xwiPvZPnRx<2rXIjrZ~UTXq$iiF!i%(gW9zDV^88Ims<~-mbTUSQ`_-F z&8(x3pAWfy`}nkE?MX@3KUrX4{Po3dOFq+THKFx%n{B?%0l8h?>bUA`sMJ~Dd8aTw zpfLJn)=P`R`^zg9S-Vx7y`$ge^ZtuJzpz!W(Bs}^ulNpQX6fFU5}@OHXfD3}Qx}vD zRd1rYa&{z+{0?VcT+-S?8~H#=%9WdcN~9!u%xN99Vs9tchugLP-nVY>%I6R4j$NEH z=!~AxsA7pTjn(^8U+&lb7b2nwmzwXlK9ft6}#B(%Z9|=dlwVb`a=F z#wo7pREEj&EhC?uvip2(mrT+sXPO=!4_H$~T#jPc?UeNBS3lJ5RMHM!Mh@J)xg?pOg zp-U^Ndh8vo?{6GNe29o~Cv(uy>Zy_#x-j$jBmIkFxBx#CbxU{g{WFg$s_-D+seS*N zS;pmWl|fT`h2L&h9szdbOU|`-Q%s*QRk;x27^B%1O10b?hn~mMT}cJfPox?_3WNMJ6+u0IsJ@J^_4Nk!f{x5HG*2H zcogYo=I_%lU}ndn6KRtR(!=g_*!-t!C(?oU*)utGhPk{}LV3jEM`ucJge`q~spI17 z5J`KdxIZyY&w6>!ImP0#j2D;@H{6jH0o#nHYobchD;l{d-Cm~!M#f0vewb#L7<-Q0 zh$2pxH-oGC7Mc-#3e|e5c&4)sSI_(X=cZa&7@%czaxc3TQxS)9+TJl2r?ZX^Gi}Vn zTF|T5*Ja%ACNU({8LtDes!DVEd;f;Wvf+Vl!3)y+6}!Hnxx)JKv8AoYM9vw@c z0@MEDy&;gwG$kD*u z@bn(b$b@m<&hkcLe(ut6R`Cs&b5%L7^-ue;4NqhoIMZs1&Y;}N8!o93+MzGIOOqk0 z^Qa6RiQqJVY$MA$V^PK0Ii5Co3C! z&#-fqhMNU4lpU86ayY;bAi#!lo3KB@iTIr{;zr|+4%^m#+GOX5SqodYu2l+X{y2aB z{7l=!dpnG%wc_^dQOqt&@|j^}e$Z?&aB$r7-@h!~p!a6KMb&RT19k(~Y7tVcHy;y^ z5wfjFQ`O6zoDDoGq65>f%q5Mfd^BK=fF`6042YAe}9ezs}!w!0GgDtY<%EBV!?PBud|JtZ=TsA9gd3x5azUE4@ zWnBRFYP+q;*Rm-ZqX|rRUKI~hV4{(bsu2{pJj1*<*jQoNzG;2z|E?>%RaExvr`R4T z$7sZ5Ia)h}9hS93OOi9jdVwR7=i+gAqxa!G$BmA06Jy$9?Ed!o2E8E4l(3vNgsvni zEXxh;KT!z}0P)KY_Ftq=W@H+-c>mNZ<_q?k6o%d{fAppGt1+OQCIg%uS%`6*Msgsm z^4*%nrkUxIv)ul~lfS-Ac)s4Py3ji6LioO=#vf_C6bUue#doq2$j;upzH=_})_pRa zH-*+U*FOJt=Fw{dzNXX|m0SJHB|2wCcKYy&BF`@g*sfzMlSI$NHR*vhJ9(NEl z6|}Kr^J2HTKBnKA6X;6vy3`Ek3T8g^NZ=4;63hSQg0Waq%s%P+H?ce31d zcwSHYluPO3!Nz;Yrk!=D=Iy=^`vxI$?oqF_e_;6Y3vgKtXGoWWa$#U=3&1z3Sm*RkkllMoAEIMH<53#H`S8x58{TVt`raM_d zeYg~d{uCUns~&&}*ftfVk%+jlypv}nl=wiw9Q>@NcX+m77WCN;*Q>lb#VO5j*dyj~ zCDX%DB5z0ns?c>SIK@XQ03HJc_O7WqS2GFRzr%266;Hp|drnEUPl;ZU9dfxei(aKY zcnE*?1+`$&dJKh?Xx?sMSIJd)NT~}qhHjxekV{@ej2}XUM6%bhk~F}G;HRtvate^w zqUK-q_QlpU^RDk6qx!n}eH&_EG2-hHf63 z)oV>vc5X>&V2#eOoMpzUG;y9`|4T+e*YY}VAD=@=Tnh+TkY2m!kJFpbY3UDed}%jB z#-7z^?R9a^*q-0#J*U82K*NGU$?fb5Cb{szrVJ}i1{YOq*zFvt$tIGDbnJC7rgpd* z;K>YoRGL#GY8HN(l;;=GCDQ!KM*G;erxt~1L}tJK({I*l6I0X2RAwPrxwubsOxC+u z-!CxrWq9p|;FR&Yc;-z>l1UYF%O-qkQ~wjALn<9~IEOxbX@0%&{rrUYf4A?qsUPhs z`Afr}4~&*S>%SI$_bz1C|HbH5`#1R+qE3GONYD9S?X7>E+Ha0J z{jaa_SDW|06&7{;_+YXxqx82!vmJIEADp{Zl3z1_EGe~R+bidlCJ&PVsvU-RSMiiq zd@D?_V$`m7+t5i#VtZ=skOLtvnm1R`9CygAh3SWAP-7~IOr-{;SBs{TQ*qnn3^Z>`Ch=~21$cy-0ujsrWYc=`|wI6d$Uq{@za**wT*1A}QIBixa&PH`$kWf_H_;2tre7rSi+!9=Mk=j^X(S6Hd>2S`h3hzz`a*&T z6qQL&=laDcZ+^4PTWeXP7;@-ITXagU=#bHp%p~kkb7iTN0>PdQD)pd@k&F?;3qO!V z?ZpX|Hi($LL1!N$i%#LEsvS7%Tw#**{pz1s?Pt3kJWu2E3?IoK;_|kPEFm+@r!cs| z?^!Ed@q}@@J#E4xVyvN3X3X~7yUd6dsU1wsdH74mMOT2u2BVtp}4Q*oQ~McV;7fxJ?=d39||%pL^%L zHy9o4alWB#QgvsR%dD8661UD=a{lwoA2)PXs%_ONnaGDsP(EyMuT^4fzZthy5N73d zS_9ScwyKUA0XeNuHs@rehMpU-^wFb7+K;@ALn{`a`tz4}zkjq30ok~`{-r3T!}qr` zYtKa+DCvex;DtJAAGeeG(Q46CPIUVsc?dWEbuB4Lmd&tb9|kqw-qGWj(E|?B9cqBM z1FpVe;$T$PeOdn`E`VzVCy?U;ORY|By8 zd*;<&liaMEWm>NfB$BW`?o#t=LbLT_@UJsdYFl9>6k3!_L|Xr%%ht-3xmrs}I2Rx< zFlT>=MG4~cK{IA8H>t>R$$g3m0)7?xd#^Wm?a^uJ@|goF(%%*^5M)*0p(=iJI28g^YPd&$h`@D_I* zE;!{JA2IXfSC_R!vv^bJru#R0%WY7(`I%36ujgbX7LRmkolNGEhTtK*CkFVMH#c6K zBX^QYZ-0y9b~7n$j^4|Y`}>S3&XU9cF@xtN;O>h`?m?(B*{}au?Hher#+w`iT!;y1n&ZBIE7Mxek#Z?ot7Gh&7+mptNM-w z&VBP=ZwFi2u1cIxKco&v+iGO`?*bs7`N*dcO}aAfU99nIPbzBmGW@!KX#aK!d05!S zMu!dkV}C!82#GzT(9({odWy-k2avrdEarwUH2HlB{kG<;lmrXsV{?`t+t?%Q@YG(f zvTc0gY&veS-h5wTf) zB(5tT(m*l=bG4lvYozXYEtzR(G#!|J0m3U!#S2l_TlCNjBKhtVK^r z<91=4sje7$Z^;F_&sQHBOoR$8dTr)gj~~gedR|gYo2^{eV#E=<)i@+1Z9WSs5*iDJ z|3LYuY-!6;WmD5vxQ81wP;QRSyB)6EBLn~hf-L~gUtMzV$ub+5vWR2z6DicTw%%!v zVR`}j0Z{>O>Y&v}0OS?-mR|0#lu>z=)x~S9W7b?BblF?yq(B34&*BS1D_Uwl+@?(V z0J(6Dqw}Vh_%vF4x+vBG2ssl|ijw5T9z*md=~_m z$J7AD41$yL*nCO~5V~j_l=BEuDQlaknia0RW3}3maK`^CIH)Yv`<32 z5-0<>0a*$VsFN=FAiH@;^^-!XWzpnAHYyiOlFMF{+p9=rS>IchW++BxF?k3au8~de zmV85k^trd?QH=*Qdvk@?9D)ma`ZGO;yGNVD;l~ZKbT~FH_U;_7Jsd+}qoluVG_@b- zz<}x5^_kS{B9r4M|DdOAJqL%|pI=%Znf9+&+XbEyuz|oV|73JXp6JnxK2*k}&goH8 zducAymkX`^@}7EiW1|H(_V}t1ol>0gB$cvzQTMO4e~vEFH*I|i zS2=w2eHymfg}3n|8-WE>QC+Dcr5+9Qky(21HR?4Q&1U+VHTXxf87#1)$Z$+9dNlZl zL_ZCx17QDYq5JDEP<~?e-svAB7q!2nG2!3pzW;Rz`a`tQlz~_os3~~J0>b3I;sihX z`ydj}?Ie?jZ#2x#J%fXS#;tvMfAwN|63TZA8Ygg6r+@ppP-cwPv=%I!T|r+y;qCf3oSF3UhZsvVk;@hu9;2> z!4?ddxq^1Bzsa@w*N|^+9+BWeSW_rD)gP1A{M9V)CBnr{ltNlrTSrygeM5{-p~`g1 zc0BnAz%s0KW`ifNuXitv^h62%JJbXW;NHb$<%zM1L?U)T`tAmAd9;5Y9_6bSd1Q&l z0SKdBm-%5^JvS z(T=9ttozy*;LV4^uv4K#NZNPUA`V4`vNL6w?;s>h;X{$)ee@OX#ooFrST_ms`t@#v zjR(yx`;A(?LMckH<;sp9{jdrJ`20g1R<8SM=tQODt@b(@6Xq^^cF%S5fS}1yzLon#>G(YoqPDxm;KEi z4e}@tvwj2yC}<%D$8wf^Gp}JlF&s;9+VPMR&j55nsU{?W)2lx>`Ycplq{ zAFlt_>I!DLN3ZKbzA9453};dz$qfielQp$aNy@N-+RCop5>_Sr?6oIzm1A3It07pZ zr9{H9p5X$Ls;L-ORf72j@NqT1kQXr9LR2NSKKEpXsp){4d-WyJ=udlB>ch9r+hHh7 zz+)zCqyS`=!YRX@?tYWUHkLn<7HV;9>N&^T~oL}9oN{h5dM zwB?6q@154StrStA)>4mo*Z+R6{Vcl#T&A>9JVj?OI;~#${PP%DFHnTZunK#iX*6?R zXPhj7IMX04?q~9$7?zhgn$e3$AtboK&90PXgI!A=&Rlx2jW!jclwK5z2m^aF&cIeu zJ-m(80lD1-k$p&R(#zz-AuBIZhRc+lcdjyd@!*9w_Yq|y_NO(_m3pp9KeDlrUL=~$ z%E!ab`I3|A0o*53;RGJsGNKQqpy4sI>p;{UMvSZfJo3$v833ob@Bq(A2D)%wE};DUYH!1+bUSSPjM{u?=8DYu!nz&j*|We~ zXrrc%zBRt1)-pSJ$<33AY;Yf@j?$A*`{fxk<80f{bvbA&Q+FkJlf??h3vzyBg&Svv zz>GtF{Qc+9SWu#og8n-*I^hsRQ@L+{USRY=HvCQijD7pl-oGOJRe*PT{yd?pBxp%m zOSrY%oG0ZC5a%nNh}}7oyL{)ss+r(((*p|{92TPw8bys;i}~0*!cvRj4?Kzcc9$i| z#vo~gWB{uxt}|X-$jHZG*41{(+U%7ZZ~Y%%srXFkWTlW}T3fJm8vNzA{SA~e5oHKe zNY{qHd}69^L*=~{*^cEqHR?Zze4{xu`HAy!l5OfvPT3BZg-HmVE8>Efxl$J4(pRa& zram-99Gmtvl_C;4CbZ%tyWHC+z0hT!yiV=ckqPA%t%t9!c&Zjse({7bP=FcZzn`rv zdzv{!F&H^@Y7favZ~h^`4@sy=6))o0fKf;Qvj=Cn zffCbV@8M(LzuUV%7M^($)9WZgOoft@;eMkM^h$1$CCl?O{Lzy(877V z^>a`-eX>ZI~$N-@V zQQPkZcX>D@u+m^MV>Ym{3myNa3sAP2f_*C$b?!gc<+S2)YZW4`cnlstE99<5XHi$1O zg+Fs-^teg@Aw_wkE`%5InRU%OkI-C0jx4d-XtIEMM<)xznEuZ5ga7u7v5YqQN(cDx z9t^&O6gesSnjP&bRWD_KBz)R@5g!mecM^pp*FK~#mpMDal;Agai*|6s{p@fk@za(% z1LbJJ#;r()xKep*tB1E^Y-SIh3E+pI-a-j42#bil#=7taTfJNf;yx>M887_dYu{BV zUM?4rE^1TmOLRFyb6RD3l!p%qc6ZhUuPE%$4b(8gL;5=$S4_s#P2?g1#kvR87o4>( zYdGd?>(4#zN!nyQWt0}}WH*Rb{`Qe^JvOMWT1W~mOBf+@{zi>R$?T4wZ`|>r2pn?X zR-C*ZFzR%Iaf4g$IsVXnuSR-{NNYt?B)R^4GOG52=KLV4G8z*Ob1=fS#-ZIUT|dCB zo4^kesb;#pxG36)vBAP1-CXEaw*Amn4_3i`JSsLu6!qzn9r+vHlD$8$DtYagl{r$_O35LrjFSh%qz3GkCRJ1 zo^kHAi3}dt)J0F1|6XL$Z8i%V;x{ePy#8wB4vwz2qihV{Bnha*8;`PsUFMUX$;x!T z+}TBQS;MMzYE*|u`fHP-A%XLOByIqvb{x@0Ua9`;$PB}e(7@Wi&M=WkDoKHQM}dsz zTx~c+*YnaocKQZx@RD+RA0OHq&#r-hsUs>{PoVekW5NWc7zy?KCi!GM=1tV#~TI7 z1>c;9+JpM^z>^uhT6Y+sUjM85M|c{C4SbJ}uja_l8F8(?`mFsCzW{Ez193RFK1ECa7xy?L#;07vic1vlJ?nBU6|vI zksf8%cW47dh!mv-GF!!Q!lYDyaUc!5I@VC(3=!-aHGZI)_Mbnu(%<_XFEn}wvIJj1 z)!2XEAhHArZ#@v^xIY6*J44Mag-~pkU6@ch1&~cLZ4F820u<}6Fp6@h91bmp``E(} zfq1KuGelIUoaJ}s)0E~4`YIGAb!@F-2dX)M**3d$Ht;weae>b1Z&BR%M5pMUepuSz z8Q)yV0XthM`L_1s-E*DTXle=}p+rg$bE&P@MYXFU84v?#E$Z zDKJm!nfFYmNgEl&q$MJg68QnBuca$k2WnSo&vN3fG zFVNo<$q7Wv{p6Jt3VT*$*~a~Sz-ut?*u0Cb4pBa`R43u(+2Z+_RTa>Huw}%PBehbb zQtbwpvCyvrbxwCdl!xXlOKY6G|GP%wQkzE!ALjh%aipiQCF@cs4EUsE|mb9jks-t{oEQ$}(Rb`WyuJq36NkJipPVc3B>zFDru2^hj zml9?VFVS2jKpQqm1JnT|o&pFF#P}-*#j!%J$_h;}?IwQPTfZ%CL*w(HO>5mTsvjZy zQH5+je%eX!6cya3wyihWb_`dTl9iG{{f4C2&jNEFpZMvE?k7{4AiU-&WPXsQpRI*< zV%?74r3@9{G(8=k;-t~^MDte3vw|j&G@0NnfJ-P#Q^lt4@?Q6r@^d6S0mKGF)tDz!BRrUJKk&YI|^03{cx@ZO{4>UDvo9a z2ASDMO<{;ZV$(w`lP4=XMXM@u>w-f+Y}z?Kx9m*WO9IpFk z1lRSwJ^eu3-QfdeMk zAa&zRfFAgMLQ`NSkUaN_SZ$_`C+0pBxdyHhn1I zc_`R{5UK}l;ZNk=m~GUNY<}Ek*t5z?O^t!j6}Nzk++ZD(gbX_pu!>GC)x&# zL$xSS(vwn}{S}ev;ruYnDgnRmDqIJ*%1qyQa&bI49=oJ8Dk_5cv9_)fW?z3B!Bn?t zdW$`!@(1><%Pf7jaV@npSY_$a*-nz8YPpe1$b%t1`UlKD+N1w zBISnqmqydKNTjirXk1`&ziW;e2^RkSRKr6qr#K}cdJriL*+$opp zuu0xY8OPQqg04gPW5EiR2e_HmSAFHO%Z;a`%P;@rX3@Vcx4t{9#!{>_*0Qy%a7JQB zn@MOuL$t@GQ45jsyi>|G*Io7Hvg|G=xo1?}<>j_7iRcd4Dr439XVr&3xudN!1;K?= zyq3wK2sOY3F=peh8m$_Vol(^P^wHW3(+k@j3z1r=I?GuiI!zBn$cWoFpIal-7j>S0 zR09K2J5WHy?zK(*#y;x7=h81;OpaB#Bsq!}uUGu0!DJ>~yKhnAm5Fl`Wx2_?2^8oM zN`%2c>J%ErD(m7f1+!pvDdAHnWMtUl#6Lh@P?dLmu}VNCqM^x&gC6Tw?n8&9 z6Oq_3PJx=%M8{&4pfk6U;ZL(UAOTP-PkVw;2;wVzy^&VJGrz#-E8gbT)8YDYlfS1f z5VY8}s%uK5@=3XBe@5lC?vN3yf(JvYWzQ!C{Q_M**k|=QXFV>R-Cug|eA$cRN<*Iq z+II)*_zs5o^sEw#d$>IL2kD^muN?w)K22B`T3~h{x=P>w!<4GZsO)^sZ+Gndg$wT3 zm90j;GC0Gze5t+klsoU+qaoheZEEOaz=R!ZxPXOWg>n zlpub0JZV~X*hGKZR6>f- zEX>%`VcA2ft3)NDAecFn|>-7O354OTV%6+^yX-n zSo)W@q>GL?7KN{HdfC0){PFX%<`1OWWj|1N0TkkYf7yh+*F+P)^`|dtK@}=aYcLk* z@RixzP%^o4rSOpdV##m k@_FZ$uh;WL}q)HJa!axQavu*TACuzk5$fkx5=nOSIZ zE1S9_ghFRN(ALswN73O~H@&U+^K^ruDabp3LmvF2KQX;BzQ?Iv-%>lX6_r4Ol-QfV zaW}UI?1GX}BvV9>A&?i^k|40DECHgH(&k4=YFw<|K$+S4Q9vJ2iHRb(y7&j28zGfJ zNvq`+Zn$8+BNl=;l?`rcZ%z5WqMXwlCKy*e4t?2LaWQk`eM9lkyGZsYu~&w zA>ka{CK>JYqjRHLvX|(_h%k*C`E_m`jaj;9j$5~0e~{X++53{Ix12@5;&C0Fu94oY zPA%Z8P1c5a-#=Hjz2!;ehaqxRcP0HJCfhHrksVB3%S$py3PKIL9ewTCV=porT1vJR zPi_HsSnxJj<>oW}H9h06VzTxBa2Tq%(=e8IOVr8xnD-yvfrSg*p;o(MI+z|YB?4Wg z;{^yS8E73{OB-LjpLON-p>p~g^j{j0Hl~C45X4FWT`LSwG}8@XjXhsocm)aC*ZlT+ zp01Dn8$+W26|lhAt!LjOAKO$i*G>b=6}=kbM|4ruaJHzYXdWfX@H{8pdk05Zlo}hz zcZvhnxhI?a{O?YGxL`BU@`W@4IPe5Vd`!pvCd%u0P6SS7E_+#+6rA zcHia>Wq+rCLg|ccFf?jgYdTk?ob$XCHRfVr{z{-zl_Qh(rLtzQMZ}huHh((JQ;_^) zSbywkg;>K;#UC1couRR(lu+WIsFEV;yuc2=s76altbPka5ff5-DQF2OV~HxX0aiqa{B$_eMFKWx&|THKp-~( z8TW>5M4a~83Fur76T=x`-)2l;5dL;Nalr#~9be~xkeEyNXJ($~umsgMOql(Y7c%B5 zza*h(T0Xr#*2jKIvT8$jZh4zsjmU(>PkDZ)Tr&d$W;$cUd5Q-!&&B5Q#)g%-dyES{N~qcTvfSVrssI^MMcoZErABVsx2>unBZ4E35sdW^IW?d5gL)49!YJ<8deo|fWOqdIMb5&*z1uff0X7#NJD+<#Blt9oul{o(}yM#Bx+pF4eJV@DN}uFernsg zrmTmD-EBF-ePMcSPcjJMz=AwUohnPm=gdBLQ%*P(c52A8yFgz-c4_NM>Ngu8A{cA#QV6=gOg=GL!(yw%4&}5j1Ejbz zsyy#AW!ByTOL`GOB{YxUqB26m~krJlv9v*8_Or2KR3Zt+JJlMK-W&jsc86s8LhSaVS zyxIH*ZEv<&zC;_*+t0vRg0+hioHrU8Z(D{t>k#6Dptp?>79<9@pQ|SNwUk{qXp$Fc z3rHF%JL>JSrY$jEMqF{Gyq2{-`J>xkt_=2z%1r*`$%YE?Z}{tvI@osoqchmrBX|sHE_2_Ercs#mv zd)X8mJ)q!8wsrt%9Xika#cypLbc#TZAEeGq$(i+M>`@VgpO;5mBxxO+Ln{%^EoH=J z5-?34363~x&}10p5y0(g0DjP;cd2y0{;qAH9+YKSg-%s(va>FUNks~l5M7ZxWzbja zm5)m$cEfXA{6U=X@*P9QUwHLRt>p@@kFHkG(24LfLP#D1{uf5nt*(r> zH*yct1mvNUCdGDya-iZZiO8uCVLT=aSA3VIrr zvYypt4izN>T^HgWPse@18+$W-womWQPyX}?U`6nUB+y|^ ztg@PGEa;s9pF7$54pe2#Qw%L;ra?KtE^)8`z+c#bF1eQ(Q={d;EeyvXGwH4nDTC8fX5PA5dymDLPH%URM)vcx0A7L?b zo{Nj-ju)(oaC68p^`EmMVqn2+#Cbo5~6tU5t->Wo6K@JtjKt zrWmeu&ZEz10=>D>Qt%7HRHVBNoNWmypB}$FHgHj_qP;{jVOV(8Iba;ZPNPo?6Z-uR zhd-NsKAk2?yAXVzh{_!->tx|M+gT%$Vk1(@ct#nO*h^yaOwT#chqUU%$R(l|RnKU$ zh>|{l(gOf#AkB}IAPWpfw66QCpNN}cUIXXi2S|j+>7a>L9AA|w>zwm-BSL`(O4AZY z0&VoJj9#{@mbUEDWOJefEsO&@!$r#df&*mIBMn}ls^*uFHBr5&)?R(P^22)Vb==y5 zwWZIaGE+KpjvnLj)-}}^xRr+7o)~a(ELSTwYq{4We97*%7c_s@J<~fj^K5txSE@b? zS&Zas;X1htX#EGCb9Eh!GGa$YF08XzAG5*8t-D)LIAO~5Ho%TG(m)4IFRrUjX-y4Q zEOa|OIe+>23OMrB#A}z0f48z;sZXp=tX6BnMCDg9Dgc5{(uE!;Y8d%s>Ht2ZL6{dgdfzh2T2@Ofoa1gx!1`8xsF5$355d|tDz#aqxx^ti6_yONks^}=r z5K;lw#$bNGXG%-uPtkVz2`?(tii9$=MSl}EuF&}IjHFsbM%gV}j$KXi-ttiWOAX>P zD{>@TP-c`YTvwlv={X+uZ9;8}D?(~A_Sq}(W^ziT*p3x1@$JdJ>iWZcT=M%1&Dw;YK z(&HR;mv%yG+}P)I1QJD@TD_jCbE)B7xyZD%@t$3@S5_B#&#XO0!H@co#MfYR2#+B> z5I1|KRcJ4LbrtjV{DXoB8&^uVi1K{eNsR8pHICEH;}tT(;{Z1ZgHoX zRTA&11~+ADwZ=oA# zI*1!LR#I2hwc6YB+Jn^atR8>)N`8O)aD%Nb=O3IyN3O^C{`Ny z-50mCv1habSHEG<@11c7h~#Cr zDqvcaW-|gK+cM_fY}#&Im|KNRg?GO+5Z#x{lDzOT21PQ_ZPUMZX3I zjm3S}>#u7QpT^WP^q%&&SsQNp zn$7k2s#qg@#Vu>3j~5zUY7V+J3Din!~N~Anxs6jJ8pWNld);iKSL9 zp3h&Azb%#O3(ABCWDSqY*B9>ThQp{^BqbyB1 zi-Q(1uT!%&30nNSrQ+l6KpjD~Z_lm??{>Wts6Bi_wYcu8yO)q7yQRF-DjfGOG{#vj z+BNe^j(&)MgoLShdi^%H+O+E?d#tGc7Q` zCe`koNK$Nh)wK8qwY+cg_s*oOABjWvCTXm2ljMb19xYr<@mQhMETk3bkXpDhw})dHhk1=3!T~Y4Hht^Z{VJkO01Qo6W14 z2QnAptB&iUOO_Xanq53%T&oJo*lV({9_?y$>owMv`j|?3lNmP$Pz#jP4(+PY&R8yM zb`g})!6nWoR}u_!Ef)>t{Txv0HS99=jq8YpbvHK_6%>eFDOnUiZzV&-WKsu4VtUXD zeTmK+K{phq`$gcb;L?W;f8_-Um|uk*d$2}(qII%UCjVdg7kWd6(+umY+9I~(x^EhC zJYZZlZgq8en&c7<{TmLW8RaqQ&sUG@wr7@C*#t+htb)@rRMI7tHw9V#fX%IrZ{k<@r!u1c53 z@TC?hu5J;#&@GL)T^UyTR|}_leJ6P)4kFnz`uDS{Hq6h}wE@!dV|1m&*Sw!f5Vr^Fw)8J(U z5M;G=1_2*Qspu3N%1*sH=QQ5EI%dzAEYNC2(LrRRmU{~OsEPDpx8BT$IbB|!E35co zeJ7Niv3aA5RBWO?M^f_9_-y+R-_O>H)B%AR1~irC-cp|M+0kyIPJ-vnhjM1E$17_^ zN`Idz{JMXmvSxX0t%xm~bHVEB#`FWF8rka)m|Q6pIrt!KeZ%GsS#ka-k2OOw4A_?= zFjA)MP+9pQLa6O5;{-3 zxi3Nz!dOe)RUg&THU!#`-_Y}ODdvDp<1@wdZf!v+YmnqA-IVA)Jv}-EGDOycA=p$& z!7Zg)sVLdDS)c-Wu|?JNp`2~-CNA@pxf%owa$^1GJ5~C`);oG{VAoHIxvjJX*WPCG z?WO3T#wgK0StV$sIRcgtlNg~i4tv;T7P<#Uvn}yu@-Hr7-QH_0R^;RZZqkvj2TWHT z&QjSsZs68Z ziRYH?9ftcngeppPJ?hqHl!cQ7h%P)%L02lpq@fu`njJ4 ze)co(N4w?^J=~}pt()T8yWx&@Hbmv%=NRe4sTF!flwR4UYx(D+d}Y5+6K_y)`#QqS z=-sHmp8F)Lg}upUHujYa0)fdVY881{bU;*;!c~OSmu9`XRropYN&ggOr$yKC@}Xcq zT?xf>rpdEPQ2CBzS+rGvU5#jHlyi7_pDG)lfMye<7{JM!F(-W=P zC~Pv&BbX%P33`q^hWZ*^yA)9Uj8sJKim?AP<8V?3>V|t~-q{}9swbH)nE&fZPw)_6 zE%CfNlugDpNc$q`0Biw*J<@oSM2^ka>aP5D}cv#T*OGjKHf3HMKwQa7`z>UwtrE(>-5DXyjUJUmngF7W`L-St}Y z%`KDg3ri9eMpxH6zY_=P35SE*3gu%JEL|>9EQizDC1|^(cLq{x zdi(tOe2M&}Z!9G!x7!kcWUk8sAsekB^BCWiBZZcIBCSq0b*t&c#$ z+8O>5t`4_GX5~KgCVV!TF#qhJmInJlAB;VLVX;9R^Qr4t^BLUQETVQYKQ{Nre=y5< z3`I75`_M3`P9W;s;l2~&+lbAQ%vAJHs*tYxwCddB&*$T&@qeYJI$D3p50kH?@!=H_ zr^E;hCKP}eCf3`~_$`Ajr6$e13>jqHNpWIf9SuX1e+Z?tHg6RCRCDu!`ES`|x3a7+ zUB3&r(9WM%sFKk@JQvph5!&|&1WK#Ncn;p7=oz*XS_;OAXEc$+AH~9x1eOa~)`bw> zlj+0UD@vz7!q*$l*y*r&GRI0p>aSgBPE>Jlf+WB}ZpMZ>d}UT6KKEOaqUc1CLVtb*ca}=&Ijrzrl2Rp?4QFPm=lN`h(mc!00UZ1M4!X>u!8C-k_NKKy+%3U zeXWF~8_xDMR5~m+<1#cw1Q}Mal#PzjI}sK0&MJG61cK8n!A^tZJz1mhNUGl1zY$j1 zbZ}^=SZsg;=}J??fMtIqw&b~V)zIqfZsRuS5DfP~L?)cp@u(M9@r%(`CA_Pza=bbPjgM0uN z;yvxC^`q-M@l$;D`_XG6_Fbko+(p17$(uDPpzrZT`u{a9GJkcb<9`_RT-O&|hzR7V zyv-KZilPPQ{JZCte#_U0eRdwkWCegB8)FKFpxk;UvN7*Se}r&2pTbyi)xdjKhtD~N z_3cR%G*O?CNa@|H-s=C_$mcFr?PYT54BqbQNF6QE1G8*e`Zc8ZEE&Awd^HE8nIU5gWqoeQ1u-r@pUlR2jpq@@< zVV#blITT7&frj?*n&O(l={5M9V@IHa_U^zcR^zneJ8V*60?ip$eT!++newgmrH}?{ zl@t0VQ6SY(yF^>pD4+_pQQ3Jiq8tiH@z?~^*O;i6=Re!_6}h_Yu$VQiVBQ3O`0ms9 zR6PMYEQ^h&E}xW)Wn2HbpF`$)kRi}CGd~vLiX~2rT2Be#f0c-yIw6@$!%1~A``V<1 zjwhn0GE_bk8-Su&M!}?}MgW67l+ix|Sxoym+|;m60zncdk2mc?d~sU!EuGru6T1BA$fGtHtcW+@Y&6d%Cx2sP7uN~J{5fLVS%iQbNz1n z{T~Z*e}R1Yy|+JPN8f`OCgIcj@9~N6%BlPhr8mxmmQ17SqX|jm?yBt zZvE2l$HOPbW1IP4s+}Q>_U3}bM3QP$7@%K3c=tESz2IWh`j$xxJj$$zqH|~%nL^6@ z;^yN~cLVI`o9|vaiElE#zx>sO>pS25AHMo8dE)Dvn5@^HRwi0TqL4S%uM@Dl&Z-X2f^zejh4H3j>ZB4F2Mey+E>-ESH5* z*4pwX$V3ZT=_*)jn~bxVP;`l;2_SC$H7id+_y}V(=ZT4(^G=ZfniAe0@3OkF9yn%5 zoTF42(ZH`{oQgJf(|gO;TBj>OomiK(B7LEFpE@BQP{9%(r=Z`@Fy}t8eCz+fTt;7# zL6XHAdl0*Wj!hxdu%sejky)Y5E4uicn`rY!sl1$VL7_10D4W@2&`3UJQsXKR(XrEz1_wwMLM`+q(T^b~yHmCj!Qa;{UGY1%?mm&c1|MMP&C8{WTQJ| z$)yyb;7Q(6aAjFasxdOs04ez!{1s_-(jN@jnF;2J^`{=p}38BZ@=8Rp7xUzL$nkevSn!+nbo~2qv64d}i0k zpRmP-T9?}I+3zkjPGTvfuip4Tq$Bhf$p&FPKK{#Fa8x5TNDbNNwlTbRbRRmdpA#)b zrB9t0#;l;z1KD31P-;AhwQiHd>mi2$S_sr)X{_=JB(XK=-5ZV?_g=5F_a09cAJKrm9A_uHc;XVKj@^1B(>C>lySX;bM@1@DVrZno+V ziCnmKu?KA4$k3!6T^~dhC+oc5N7&dvp-XoT@7jj6C!R-!>Zb-^qLiH3AMzTmBN4Er zPTT8_3@~!nPmYzcU1n4p@&*M8{nr&;fn%rQ!4ZwTi0G{)ZD(n|3e7 zPhv+lPNNIwlRGppI-BDiGMD=9z%vx_H=C~@d(?V z+J-%pye1iy5~)3OZ7{1`KYgLm%@s-^2{ovsg)X6bzwhvBdup zmrVjnT(HHxD={-d0^}g-P{6Uo7Cazq7r`tN<-kk*PV;Nz9`M%#&7yJ$igOk6x+U-n zO;Hbm5Q(F<=k!MU)F;F*DiU2fNWzPlkWZD(tdm)Fu(o~aE-u`Y=!iq;|t z{X5{=>`2nM_%`~CWOzwM2{5fnus(vw(3lLbyhiV|n!}2CE8Cbnk-1kdxlCk}ILU4bNfRW19Nl0V*4Et#YY`gPNQ?%2?ug0P;m;MzCR1WZ-z* zPCJZ;T!~Xtj7LA&+8v0{?9t{v0sQCUfYSir3};bj%p5_1D8?oX#b9Dqdx@CBX?K3K zYmv4c{4gDJQpxqKZUh_WL5_z)HvrZ+Zt}&i{{8kX>5H{7*nvsJ$6>TvT(~<&un3m0 z;psxM%_1S9GN%Dgef;8;1QB4T$gRIk`8C4AY<{i8sSvM ziDf4h!-OOw=72m~XrSrAiW5VNxML27G2N=F4B?BqR#fLd8pg(em!Py77bCyJ}@e3x-=pr&>%~nns^Z)t>4BTbcaer2D;J-(N3k`pz%M z-{8#YdAK<O_Ku%XGz0AHS*w+g~Bfl>ykvArQS26-d%Wjbdl>S(#b0XLXLs^8b~@E+G;^+CB}`%pL^Dq zl8F!qN6CLb*EF2#_k>~rW>14Cy0)Pq`*j~RY3Q1eL@X{YuE+&vqD#J9IF0l@Jpf)e zIw8YK9vpyYA8j;2jU?HZ#c(F4SC99l4F1iG$Vg5SC@CVjReF9_mNZf1FmfV=F~tF^ zt{R_)ab3momcRWzaI80U{D&>yyAoNXqriN2L#C&fUiEvWI^NfR^672ceRB4jZ~j!@ z&{HZkfYWw@zae9AOo1GLe)P3#8}gca_ug#RZEE9 zDgOAefM~F>gN*46g7H>>{Md$dw8tJm+y6 z%P4iyYa?^24pB-MWBJJDV=X^_I-c`RJm<#Ng+#k2BMlm*vx-L`j40^S9=(RGr8MH|L`9f$za*|{J@4hWH7 z)z;T{Vz0KF#utHiSopg`OJ9*w4fHpqG;0LzqF!-Rq**8X2RszWaBPREK}2vo8B+G% z?fHQ3P)E+9g?y1A^z>j@Nfiv9s==EgIl|bJVuh^T#WY8ciV9qHjpvK!GUWKG3GZKA zVJoZ5b0M@G=3jsVzT?onB&U=hA{>!6fw^%e6Wj)I7}972A`eI?Vm^7|o;@dNOTzw4 zb{8mS>`z^6f+Wa~g=yK=zY5J}X}H411Ws^Xpxhuq1@}cwZ7nm(rD9^W(eVQ~j$%77 zkc!oXuZ!rsW$~_lEc6&a7aa<%UcC&5eE9Z|$Rw3GX3+T>#tH`-)Ns|7E$M6kA<`AQCc^OY-ux_SdeGa z=tx33$S|;wYVy2MKnA2z=jxD(6dW&EY*fa9+%B>d9U1|JIFRdr$Ag_#UFD7ad7tC* zkN9><^*)kWfYEqxaRuo(x%L?OtJ5_A2Cd;gTRtGe*3m<(*C?dzjKh+Wl4^uzDTtd= zP@ur1A}5m2SWx^INI5Wx25}Vtm4OSQbM7gT zLNu2qE-8t;!?5p^5DrfABf8rZS1OSFbMhuS{ot1G5!WUuB}FzANVdivol&Sg(1E00 z$%$Dg3BtuU2`(TnU=NVe%27vNMFiIIqz4-o`9!@(`RueNel9tVT}DlMGIeby446Ya)U_ES2eO<2eTF+ZO&_!nlS{;$&XdOyTf@*W6urPU zp7BOLz6=58iB91BJP(Rq2LXm&Z;J*?*^4HOMuS08#;4|Ui|FO@%*;&6^c?XhbRtrf znaqqgNJ>hY0gS7}Di{=-85Aqq?gf=Mg%dJc0HsQm7MFaiBeoPNW^w6k`f!04_6f?8 zfPGbe_301D;Hm3_%xDcLQ-aZReuRhCP?~wb!}>b9RUkn%hI_mr+}v z{opC?qOv%0in$<)GY8yVPDW|^a5<;^J^OVsIC_QCy@fhf%~&U=PoEyy^6{HfIFyu% zG!sEPqxZ_yHVSl|Uwp>rW38ikb{Fuln1fQPE0y zPmU5gddNzr%XMkhG4M=&WkO04eJ&R5CqpSS<0*vrk$@U%Y5AOIKQ5sH9GRje9(%F6 zT6#wtU#xsW94jWM>1=IdQ>I5Jd22Hl8&gxqRzboDaIv?4g!ve2t7dm4vugmHf!C93 zN9PowX>4X0A)DRO!MrcM_vC%Ry}G@xF#x`U?C7wyw@x0)T)%nVyU$zmc*S_YT-(p5 z^kqZgv=sF%L53(K!+pe>kv8f}B07{Fgk52VBC}S03Tvo{AEHGy`LaPb0`u~m4dN7buYO3aNc?V(ObiKKQK0dt?rHH0Y z=t1j5%cwXuv(81=Fe38O<&FC+P)3+1Lu^{!(PSC1ZX)o`N0Fr-6L$xYrp1c;rQK@Y zl<}N7$*c!I&qLNZ4BEWXTPyj3?dcNvk((mAP*WeksNfJ&?@@ zrtcU;&%V|#cz{8KX3{7mj<>m{Qj(Gxng}@RYng?x*Jl5g?=i#x-RgsyeQz6|@sKBq zxByRhbZz$etiPiEjPfu|RoSR_@k(o5l^LG7tl1l7_?H{y|A2WzBvO%kX7k<;nR}si z>%oHu)TpYMAg92$&n;bSKE5ceK=o&Mg2L(1Bd!NJ5gZnES*S%qyPpZla>1Li+$iCBa4Z+kNfPflYJC0rTs3 z*Cy@ReUgS&QK0F)|()Aq07)0L0i9G2gSF$wBJ_hYadJ;0G%vK6@I)F{Mmy_%jPjw(9L>6` z08K^6t67k#spvqlO5{clE=9w(C8vH|bo~Q9J=594)cHHUBo{-n^2J(G82DAKohY(Y zPDL%8=Vu1+q@4bTL}9Xsbbc%hc+(*fd{CO!n-Ad=uPd^ui6+O`$T4B>1}9(eC4Fcr z{IRY*P}x_Z*%-*>a#t#Nff`hSXck6R&p4R7BLaOFEI;OUZj~tb&j)R zQ$1RW3}sN)J#qBG4-Wg^tUx~QK3XerYMw&WIU=*1^PkI#6UKy3Ly0hKCIslwqd`SM zcd&Agu4eAYaoiEFHy0TzG>dDBYRHbzPt+N+AB8yvHw#9*AQcTMl0P&}!=mw(cys&% z)_P&kU(k<#<2}jm1zAB^D*|JBX&Xt&NU_anhfd5Uh&y-QdbwmANhmyxx1lXy@hQh3 zB=nnRH+~J>_>&j0{KN3ED8EHZD9jdedaN6TG!0WC8!@g3xP)jmxGs>v8{b_dDJ|Uz z(t>@9B23DVd>cL>ww#ZW6j@Fqw;)>JAXC`l>^bdw){m=`5w>F>GJ0MmZEgYaSWsXL z3XrEY(`!XRL{8O1nDe-CX2f9zeaBppp(n&EUZnVX7qhz~3&EA66ppYCjR0aAs;ACg z9x75IcVj&9DqUv?pY90VO9ySF`8)7jzCyR*%j2@31bW00jS8LA*-Z6 z7kKX)w2^Cy7#cscKS8Lpm*dj<3*|;|MwN^}h;YOx7?II45lApJZZlwxKLLM4<-&}R zFmgg;lb_u|=Q{0*2|HJ@kFKyR!;bBxL|r3j?x9w%LzBS|jShu|gH9?}JqAlyGah?Q zsac6_Cxn`CklpIbmSJ;0F}t~I52_zBGBTpss2$*t`~d$vhxW0q2#1fy4MXrqDQw(# zp`2udJ)gXdm0Z+9%v1w3e676fZ=k&)+Y@_(nH95s>d$>>y_Fu^gv4|`^Dw1z&dh6N zW@uBbM!*<)3i8Y_I4ovMoH9dvg@8i31lpWXa*{`f@cVrMb7hix9VIQDJnBf()gs+I zv9r%>f55l1d-i%c>uq_tgFNq)7EObV6$m7n#a4KAZF#_TiV^hkiPr(eN>?|C91*D< zk5*s?w+*cw8=ls?L^@&F@&(Cf$?BO>UgE?&dZgZm`fm0mq_Jnu9)h6jzJRwYn)exd zzQxirn2qWyKVH1QCI9l=nIEeAVPqb9RlZC{rd{;&TR$>ADWnbz{3z!?TM(<{&>H^B zOXs}Io6H7}95feMPv)v%zDT#Zq$eaKgi7lU(Yb6LJC}xL-+L(tnx-4y&3|H!uyXg? zw0ZW3LjN|JJ^rCOfI#M$#9T2!y&r4q$#>@VJ2Qp!x4cBYW0##uYSR<^bjpz-NG1sb z!f(K`dF<@mO5uxa#}oBAh|rfSLVlO=ZwwcdxOGcIyY%~OLHqQ+7H zPk(vYMBy9M>iwVJ2ysMH<*%>A&#vwK>;Ln`{_9<)&wPW>ZKJyrzR14*jxj&~Wuu0P ziAi7ed+!kWLTT+{@&HBsbOnz;T8KZyO;NZf`%a+5R>MZ1wUgm}2dFDT=5breQQd}iU$@D(@UZ14Fy6qxIQO>yYQ4%(o+HHDjB4f!=&#Da?u!AcN11h2E9}mkqIP1 z2G?Bmz4H>%#bu-#9MYm&i0G5W-jl@|drktOC)aYCjD&Vt*0>k6a5RQ54bhhzjYYL{ z*c=qjbh4#5g_i5U+`1_@gUM_Vt$wO8(s6N8>JL&!2)+zs5&9WN4WL2u7>=ORHY9X#JQEzuRj-dz%QiUHnp#M!h zv}!z=&A=atS&M}|h*r1)l{X-E>u}uo`T3m?gga>BGUL(?0b7j(Sf}JeK|g{=eJ8_*SsH*@-u>D3e~jHJE*NKNAys0>A2ElINu$v6+k*}@F7^n7(-`4}zX ztpw16y(Q!V*y_#e_$92u^%xApDj_Ww#+6yeP0^+kYrOPCFk z`0bZpzwKF}TxD~B^rf>hrLnO?opiKi1`U+Dum1J=D!E6}kT^~|%@@UzO)qsK4So*M zmq8lK)Bf`14&r%&cJwq)5hXK>@D2qqbgQ}R#}}CvAi|Uzo7w2>Wz;I5r#RD)=Lt$! z=0A~H!{+r3U_J!67vM01UegE=Hqo5TaJ%f%P3-LK2z!+5Z$%(Z*lr0pTqm{_6rV8k zAVeAW-#qFY|4nugkd&-7ALDD~*=ewE63eVYYq&no6SqM$vz4kQRtYlV#|lNBcj4Gk zm$w6RKg>sVfL*}dp6rj=Om%trpg1lICa5X)BSi2D#Y%q`RY2C(N+a^>Z$GBLe#zB; z+1T3d=O8vljkuyw9GVS6(`eKPfYV_P3%m)Nm>z==bU24bFkCvdya0ZoyU2HzRRTI5 zxz=-PUtLAgL-xWI6LewOU}B&zKn!h=4I3u)>%Gst3$}v+FH}xae-*VYSt4YOO@X=s zAx#STxT3a%W$~y+u^ZfyI4oeDo8pF1xgTx$;_E|q9VeE(QK2V7_(oEIUZupl?ptWVi?v7OZ zYia&&6$PXB_xDq`3Xd`zKG&s$B!eJyORkw;(-$bZRv&73ykd7EhqceF)1kzOa#@dV zWfSY+DO408*@!>=<`<;U$YrrTJw4q6pV%UhRWu>VnRc8g9PPZf{EMa*%t^nRf}uPW z6990gg)U0{@bXzek0Qzh43%~X=rV0wyLN4<5jaeydrY;B_9W`Zhf6I9D9$}qs?j4g z@tgWH)V<0h_m%wjNf`}MWqeyEdTsv9*zBgNcK~O&?2GN+!0uCzM&;2RX+$1q=rKd+ zmMabVel1=Mhpre-;uFJ3og%0}L$WcRCbxO|X|A&kMa*nvjT)MIi-xWAqOUH3>`30e zn}V5LKh|q{;z;prK%iwt^R(}U@II@5leVb;ianHU^oPj97h#>qE{y#gNq@-TnZPt) z(3piB?p$9&Hk{?DjLHFp7)`T`D>b@7$QaoaGv1K>1QTo8Y%0sa}hOvQ(PwK6ePdm5Y7k)u2_6R z?BMNNu6Mwb;Th1%M=l*qIWEIyWjf)&>hBn|^`XENt}8@o4i`RyB&f2Xed+ro@+KO8 z@wun<(zWDmRta5nUJ?*hYm>h5ieRQnUo1gl51DcK8G^61O;Du zb@2#fh19tFFy7hu)mV4jwh4ZLykGQoNwB8Eqi|?g=JDrhK zi^Fyqe##5k9N-+C$dhc-O-xNqiO3<-AA#Z82|H+K9qZirY$3nxCGr75pVNp3oC~?; zK7{I_xA~_}PbWK-A|Q>1e4GcnF9Az5b&|$W(72KcmBYd{cOYe#bTPk_wHDG>{6k|K zqAJ}{7q=77gX1#)BuFz<4`%lW(6*cC4AhZmDp_Ml0u1{&Z?D0YYq^fAZ$*EDgoOko zoup*Nat*}+WvTOqxx!b_1nY$Up`6HxOt4D^VX`iypC)0!IQlA!T6aXJLX18#62!Nec9Qx;6!)^G>tT zRT;`m@w7X2;SS08z)`LXne&@u-r1*A3q?geiw5ef1Cau%gRO*0(0)bZqwqj_5fDmt zc(A#&6h{Ph3|joxv$|{WFmzti_n^NJ(s;0=?acD^19i8 zVsAb#YF3&rHw(Kn-`=@spl0Ch57HOv8~*n*jum!cQia4^!pRYGfs;pgIQ|_cXU(Gh ze}FBo@n(ajBmDX0{|RI0e?N==7uW57-azMBHj|GwWz2|D|E*G{5zGHiD`n)!NIGp> z)}z$=Z&wohrGx)Jy^?b)kWb|4Ws^dXSrS)zjXqt0OzBMu!Yg7jkvIRLg}|3MHhY(a z^nhH0L~8J2Hc7&_)br=QMQsuAmEK#lae}i8ka-vSNo>Fxf*^E!7V`ZizU(#@(N zq5OT`Qlr}yWK&d(F0FBDWmnmUr~TmuN8ie+!MF{f9X>|m*}Y*Boq2w#YAO{G({dYE z7+(z!5~wPx4L&FCePEBAOFY~2Vr8ROjq#d1&DP1@-we${Ros#Wy8e+dqO;0!cW0-A z{-s8J&M7OUu1~8aoEx`475{pN#FxDpO&-32o&Ea$4e4Lk460gKuvbNITT*l7TPr1_ z#oe5c#8B1l2+_vPPt#R$V>X|X8VTdu;`{3gld`2UlN(%Gr;t?HIbt*RySk$RUg60R z8}^*2K!|w^QfD7e&JsZR^mv0Vy!V{eC{B-^Uq9mkgV|Fe2Vbiu6cjywy>hM7>z584 z-qT&9!bYINt_$o<=Fh1v{-CUD{P}oV){1v`Zp*bI<+Vd~30F&3W~=VsW13pzA`zJ9 zhYMCBky2iwX79f33@`I=@_4Cn+s-*7qL>_EYHgzmc{}S~**4Fbr*K zu+~d3@Tl#e*H}u;qeoAk-&rHkiTv%+3}_m}a9C8Q`Q@;fxA7kADrqu?gy10%FrhXFb%A*~1fqp~aQ`;8E7M=UR~4du{^3+BD&=sv&%SE<=CXjm%o6$$a|3gOy{# z^R&KdSp8bS;qA>iyY?Mh zzF$giW-_8P`OX(sbuCGTLAE&^51>!q3z3)Tt>L{lc z*BIVJt9E9Sy*z3rklvevfhMqe#O$6bSNfD`CMo1o&<1_;|v+}2&m*v9g zLpNFr9>sh9Y0Te|JG3aKU|*M{q;!XTq1)!x=?F`MSFSI{*5nLc2y(2y+?04?UFfih zz{X{*oA-H}`PmjWD$dwl@IJh^a{RY8*CWr%b&3zVPMjM{@qQYo?c<;#E1=h;QX%gu znQoBj;MXU+ng6DtbL-c|o43gFYmJGP)?U90EtatJ6GPrE6L!I$D%&liZf>hrk z-}N&IC)O?3lWtgjl>HXf_L16+^5EPQG5ArClmWcDea{p~VfxtNKgk4zd}*lN00@k% zF-mY!{!~^Xv-UvXybKRg1p*Z>C{DkQx-dq>cp6m6xd|RNX@$rg0(F=ro)TBg^>qRP z7=%zkDr<<;MUE=0LIpMRv2H{?%)Q!uollwwXBBx-d@;ih*@naG+lT9T# zqNbr%FbDdy289&O!wpz#2?Y>C+Uvp~w&V4v2X$J#!OE>5F9Nd82e+b>W?_-ob9i_- zX`D2vwqBp*0t6rx<_Wa*S9un^E)ugqqk^$dhiZoDP?8E^TR-jcz)d; z#FApykL~>O7N_z5Sl1OC$jxf>w&FRM9WI%79C=8wxV%{QK;Ye^ znjOyT|G7Ztp6sr;+l!(@FqyX(uDGfRBLF3?RZQRMSGZL%g6AdrtKISdIP__gHLp+1 z!JIthdXqv0oN*SKz4=4kYRkzdJ5ceTw{{;iIcPF>MSiRUIssm!*enWfa~_Sq6!k)Y z*TlA_*{_^-NQ>;#TVaVGlAYaX(YV|b;*PH@=#mse^4nDm27}Yulgizi+qu)+oUP34C|XzbO0iU#vJyDSW(T8KZ>*u8}x_Fs@leQGTf z1)Wo#D=e~4=S_`FqMYZ4k;AqzZbLHW6wK^9y;MHq{L0JCGudKGqTJ`bVApuC-0J{ej#7Ym-M0LdSH9?n#;tI^O%iNW*Vk&BpaA zt;Zd1T$SR5MTX=oiuT7`D_k(Z&(`Ul`NKWB)?rF$OEiO;HWn=2ku=#<=$YHxv9rj@ za|{c&MKgU8#EIFLx+f;QKFZ4n?vwwSYyl0b&d#7-5|D*W)PvBy_@Lb?0 z9tkSfDn@dnGQ7l<+`){i*36LgkaP1s#VP&Za`$8xr;G9GG` z%V}}TN3$rf(Ku=%R(oS2&_#LPcQmOQxU@VV4HOO#_%~ z%~ex#8p@a5+MzC*)4(W2NA&TI8}CE7eaoPK@Yj*cZ*s{viJM4*RD1aziuC913Q`y; z2!U)T#>E0QT$>xMO(b3IwLR7E$S+PBP>*q!PrbLf?p#~y(wj1M$K+&5U<~`(Ay%hvv`*6^=|`~U)wP>aLn$vpI1VlB>d{+I~()5 zv%3#Umwm#?{<`bnvH-EY^E~prr6$|o*?|!QkL<{G5h)^J_1!)eFy#A?{Ncg=tf8gY z8ST{B520=&1+nN~tK*Ul(xe^t#6>;mlT0Fex&syzM_onYaVhH?oAS=z#@xNUXUen3 zxrul7Jrhq!zYsny#wnmDAY+8Rd1Q~9&hKy8qYVtxpLjZaVz^$SwP(u1PE+}&M&+j~ zRD|sspC^AacFkADJ_FD2^kW}_L7VR6i=jphxgH>H-TBz9Y*=hn9omr1UT!XLM}3=o zsMWW1RIJB_Y4|9ze}sNR19d}10EGP;?j4GY!zbe!`+{XnupM*d&Knl}GAj!z=}=2y z5tA1gD#UQf8KGE=j4v*Ia4p}s3HTLU4H%v&X z)V4RRZcBH}a855nJK{PxFT8kV#q#Qh>Gx7w{j0Ny}_g}<>b08z9IHkbi?4)}2EIo&G zuRx0wVQR2u70&8i+x|zDu|rdaD;3M`@*gmku@a3~WM>>~bqpy;z#-Wsz=OtAvaPYB zpRu9Mn$1*~m6;8FE&{>01Un;XH=~lA1PjO0V?q%C5&->ws;;y#E z7By!F@(wSoel&E$psKg(>w)q(o5!M@(4LH3mjSQ%lxOYVU!UCk-B8uy)}0oE=eDAe z!P=0unqo7D#Lx6ps%BkmOLff*T8R`?I&9cgeO(XNy60oBwZx+wVVcq`g>^(Q=KEmW z1Z;ocS%0{huQkrcN#YfsvCE;iN;6*=pFEHghwpLwX2v=TSr#tZQ~Gkkc*&EL#G^~< zzbEGr!zI#ST_H_U>Agb!^C%}g|@-|UQp)A%e5nuy8}Se53iE%7i*%$L9RFlqsP$Q_19yoAPl%| z^m(` zEjXsF`D6KGh`!Aam8C6!BD)*HCyz?+erwmq+2@`-?~G4+>vw)Lai)n28h%hVzpJ@q zC~s{$r1P5YTl3bPl0Ag>+1~XpNp5m;Z|xcTWmfM6f09(N7e)Gx!cxrLls7+jz8MKI z?VRxe3gOtszy-aseZRyeJbN4mq@PNsEFFERWWw@f3-8}svr@KYbtWv9Jih2$6fn8x zW<$`Zg|6-OM!D_Dzj$ig_cu#*u$*{tsMp$~r(C*y&4b!x^~E*44$j-|x9#ufE&A=i zlRPhV=i5f@uET4~ms=#dON*`)I@}(#YHAi;3w66Q2iOB@XSYW=Ey4Fn_aqBSn&t4%^AWEIEsNwKor>u< z@F;ov>BR$yn=dqnUF$lwrFPFGgTk()F-Ud{+I4ti8zB}qf<*MG3U;OJoyArwB=_BqF7E(`Fdd3bleRvz z@tzRp^?mp-{p^7Dw&G9^%-(7j^zVFV2rtO25b(Wso@$aab@lQHY#k*?d^Q9c`-^|J zj9!RkjPhf@w8W0zzMm&#UN*3%+*oFBw)Js+b{`b0lEOjm_{DNwks^oSS9i7~zH@4W z)74#ZMX{gumT2TUhFq;`9BDM>p6f5mo}N2q=R4YS=L)yz3zJITp1b~HS;IdZLo*Dr zz8HJJIn3=zkfX@$j(u{IMfrn!KXu)lSgAZ9>B>6Rs5$GBeEWWr4Oo3V11+EcjF@qt zH>TGN9-5-LWUSUsH^@lNjk)C92%-(vS6=GSQIK>NS2z83$max1%g!uXdEGRh8PDg* zg%$t&fO1j!O#O}HBA));_t1|}0nCdlHHIx<0Y}k_vIQLABHH2wh27mPHx&XEg%}UD(-{-Euotb#hNjz<_&u;Qg^R$|E0@ zX4N#t9e02;^WWX(G+gc~m=alXCnzM#=5t#+BHm#e{KF#)npy%B?ks#GQ9r%VBtpMW zC#A5!2(a8qV#c%QZAahZ^jk|UvO>K zM@?2-RB1Og5M%+R-kQ#BEPep@`A^Z3zCy?!=lUok&hJSpu(*Kr0^3mO{Iu-ZgMYy7 z`R`MFo~Nw|ed;-Lt+!HVlget$I3PYg0aw~3dmhdI3_b2T==3ve_x1b+i2UdDf2krq zMeCj?sK@rfC39uu<8++@h%{Z7L|8D+t{KDt-xi zjf$uND#k!Fu4|wd15KmxY&W2!D1!ClY2sX<(vng}M}~_({s_A-PzRe_!9>srTPi%{ zw7BimcAus%`+DrOlk%<2U2C<|ny!I2)p1&Pc}jj~%%z?sv$k}{MigtzZf!o(=xz~p zPV)KD;y0DbQtcilx_a6)cbWQ442;!6L9rkB#yv3U!ngbM7h<22UEGt8&aLuN!vBN^ z`+Zu?K82IBpK)Vf6}kD>54acqdVOhX>y@9PT)n<_n)S~iBX}AKmQoIQjPHlo@3*SIqkaZnAXuT01`92`o1U{1UHh_#`1;E z1K0-^K@ycM7AUshg~;N-*4NrT5Guov%`ew94ww7Mfdbmgwtq}Rfc`>Fb>3JxPWsV5 zEgq^bp6Ny0;X>Ah;FIl|IvIonnH#PCl87j_w)Y1ub$3oLB2Sj) ztCS6~j{qHI7MU(DxDY8bZ*GVK9vggWc*T)dEl;?+NMf8FO!;Xh`yD+}b>g#8JgCpjQB_txQdZU93IYFje z7#L2gWEu8}P+0(R3kNO0tRth9z_msHtsN~`=_-Q-x4OGeT;hVtN!`GPZu zfw*yp&)|zIXk>^3Y&?z@i5}c}R?t=lF$z_B-sy^JuRWGOjr7>?t7$#l^-P>2u*`HT zOi=_PG!68B5vvMgH*T_PS%1%=)gk)fB&#BWwiuI3%cLrX0S57vBy51-SJWQ?Iyad` ziG62GH{Pi;GM;QKS!aUpRQ@MhFs5&rN6`8IfkHPUHa3H5$5%wTx$D)?)`F-#W8;R| zhUIEEg2h!Pl64+2W|x%3n>pYOp$n=~4VG?< z%4^`l0bs_ld^?3U@HCb%KK6vt6^qt3V(%dmuHSk)bJNAwnMyxZMP8^G5hL*Ic@zYF zn9L#IC`j7efdX$p!1z7`*rB{R?P_yo@I&snMH&;<*5T35H~xd;DQ>d6 z|JnomUw!m{Q#Acwxd&yZ(q)&$sAOh{iUQ>zuD@8>1_y0pllI(k< z<1k33fmJQnuW#6J-~Ej2&-tIIQ?kRkVg#x|y3B_3V&WCDz~~hTYei&V&XiZ7<%UXwK?FwZ}Phr3qSOZ@`9|5_9@UXBWM~ zYx$E_G=A{UKC0-qYk-fMxUB*|DkN(;1|e*hnnb#)d5|knhq4Uf?++KxdhM;8ie?S+ zHe54YuaB6psAK5irYwi|i}#ddPp}MyRp|@?okJ^u@F#4 zKpQ%zo@f+ywAyMKfaWXV?4P_S6W<;&^WXcTjXwiO_?LeSQ^@O|dHOF@TiAE(2#iA$FTcyWXxF>Kl)W(~MyK4&w52N1KL@Ks zyl6F^V@;H>)(2u81GDP+LuMuLX#29sr24}l?F2G}DIszRT<0Qn#QbXMcWSzO5?=k1 zQ+3(4Leb65tF==FvNB_+ zuIL|c&pVay0ffrEgZ%+LnPXMK-LXnPYP_FL!tl_ZA>hCr))3Y~1IJK%V%2yol!~ZE z>9$buRAbb^pLHr;nq{1nawh{2hl@2f>%Qh5HHa$DvfPOgp`2bo>3mwjnV%^gpKGzF zZe&GL_395DfjksdW0Y>;JOrO`NNV7*^5MciJ%_PiB;T4p#iQXwV-MxB6rEC8BijQ4 zQPY@9@9GG#oS`V|T~~^SAfd5|d&`*pBeh4_=v@D3R@uBo-2>haG2EpU{3!f8GY{J> zGgkG{-EXkN>uZ}rco9QlSP$vz=#U-)*lVtowR7%!HVJQy#LwYzUXQH^*zaTR6)t(k~7f` z(QWSWEb)rSE17cQrYWa4Hf|rkPA6{u$qj$q?^)dWv(^)bs1gfpf>SNg%QW`*_(b$h zO}|j`dTg*6_fBDYeCu95$eD$iJT!DrrVWiQ49k>k{}hx$V4w$&#Uu}`&}}!htwR$f z@z{f%B_FE&>0asK_`n^JMkR`ZoTCaM^gJ)}_Ve?@l5d}yYd$}}J0Yjl9}<;_c()s$khfOV8sK1UrH;(lArd!u@DpU8pyNyARt7m86QuI zjLC^ViFnigF?{Gp=-S9XB(0)uXt0;Yt5_mc8C8e6R796m?3#$V`KQoleh^#Al-=JK z2Gh%c_Fr7{K(Q0;j>u2>*tKvzW|4mfl$}v{pLWo+iI9a!QWzWl^i~G`ttTiu`JzC6 z^|`lX|NkbHzkDOleR|rL)yB27Pkq{Kdwlg5)8}toT^Jd;>@BfFz}c1S=Z_b>t5>9! zq_%P0mu?HPmA`oVNm7$^&6-bQAOCnEIQq!6@2B0#=ItKJ^ZP)Gqna6K)FSA*Kx5dCD&7BXHB}+i73Y1bw*{Z^CZGh!} zAxz{0!~v#rfpQBlG);tv+!nc6228-gvJu#&-MQ4q@T@MRp$U3^+k+n(t%i9H!*dKm zqfDyKM+!%D(WZ0IGdAoZ?UPGQgUG8bZ*aByA?l{4= z58EJ{fII*rGS7DYv=Bo(;S5{h2NJczl}|RPCJaCYM?zUtkBRZl+bb6$9m*gFT#^mr zS4ytUqA7Q>Hf^??yROn8*3U^BeS}bDX>WfL{B+c!{4|ceO|ORxn>xtnLqSBO1Ak)S zf`YAZ%`E<6l#1-$P(uqn_6Z&ssvb9N+*p382R?N&(J47#Mo`?-I}Hx#H+7U^PdcSk z&Lq_*96xeZ`Nq1G^4{8HJFAqL^Uapee|Z5({U!;3j0lTd0N--Wy zL20%;*}_sd%USqAs@h>QIr;Ml9SgZh9=6;eTd}ooEtV4A!H_cMO8pVq54pgmjUI4r zam$-eSDntvqJ6qDY>&lmoJFB@N}HJamho{ep9BSH;U*X+({Td3QdiN#*)K~s_#xtV zabI&p1TP4KH#|V^z-BWBafZ(f?>msQrO9#qAPgBUy|AbONMwP$nd5p>JG_Y^8Vsx* zc=OEOppkF**^F6}ZXg#6MdY>hiuM6M0Pp4;IXfGhI&PouMj(p^l`3wm;*s9(>#7aO~$evzz8*{aU=Oepex6?^_iMV}M+#u{*%xVudDEh z#b+qC{u%hq2D>!6s6Rj_aIW_k+6%o|?Tzn0LLaE>SGn{4&DP**ntMFxjnf*(vvl=v z<%qA8rHb;={jh|G#r_e`3I$M!t!u;U>8o+Bzw2Iycj|74R42uBVpgeYWMXsb&DK`D z$PSe0>QND~VOL@WHAb_XhZ!G7OR#)ALXZ9LOSEw$>jCdXCm+?wE&Fb8&=ZHp7uESS zLvIWfW@kQsQZVAO%}?KAjFZ0TVTb4fx2D{NwDHYjD**`YifugxQ-4ES+@R>xSdx`N zVC&Xx+=Mq!4q0JAz_ki9@aSM|LxZ}c;b zk7UFKIlvYflN*SLHG38So6Z$Goz8GZ=s5wwwmOX+uI);r-*gck^uVoBg)Q`UhN>ih z&^0}r1pYH+`>dWmzTb&jeMXSm!Ec{&(n!;hsHiCJeuHJ14X(UCs-6w_Eq?4`5}diG zX=ZP4Yt==kgI8ihuEgrss2W=(I9tt6&a8zTT4tXH?Ls)p#6-b>C&HOdM*$x3Q1Uo2 z1KlML8}hwq+dL3k7J$IO<`0H*8(37cePcHn+pUFw*>`=c_i->4@&4QfIJ1MRE_}PN zS-pe{3jrk8;k(vyw9MVS^a-yx@0~i+D#h7KJvmd=w%|+Qu{PPdSqR71d1eCfw~3BT z(ZOfxgi}@%T`y@1j20Bs!J(TF%0SWYF_Sh4-&9Yw-u1&~(4)9bz;b&fHa$>W9+k3r zLl`vPl~63-!3Xwu*x|?pRy6f2?Zm#d*=U@dj36BV7eXy!?-RjhoITPE$IV=`8VBuw z>wOFelz2GjgmXD3mtx6WU`)6DxF&STe0bHaaYnsAv)3I~BNJg+Bq1 zHB854Di}8M<6F4q>h&q!^+q+rFaJsdI(o#m^dTOFLZ3aqKqb|j#`%3uwHW$gq0VnMZf*AvVuAP*F)8!f>rmHj1DW;8)k{TtUEMQ&?Qa)>a98jb{_cA7=dI1%rJ@Shf`0aK8x`g7Buof%%!u)VBPc@SmVn$k zsk`sTU3gRS2TwbVzH!$0IVOfj;0=B5DRRRmnkQzXo;4W1vRlK`x?@QXR6I!^VVlQSQEmPhxiXuh`g#Pbc}J|Ib@Gz?>2ur#YLtBoA$`V|I=2G2xDq0Z( znZ>Zvu_seVRCx8sEs!%dw4Tc5nMQFIpY#Y9vly0DO&*i07BcwZU$enUN7|4{m*Un5 zM1b4G#Y-2*p>`w(M5kmbAR*h<`;YE=!yUXcHESTYHc_(t0C(TXCccqjFsnM>Rk;+Y zL)ZPopb6k>&dJ!myuY#pjQgb(X5J`COWyzL;}e{4y9LNt(7_3y#UG^38t6`8|-%4F+_m6I}5E$8u_D%qNejGyZWk7o1w0q85BEw4(- z-RZEd2yM;`!wi+U9@!pP(^%y*{SnG7U;}2J+DMG3dM6|YWZ+Hd9s|AVk!f=q7rDaR zPEux|J43*QQ*@TF||; zo1utaZ#3MLOiK??0}%k6SHPP>!|!^gWq>x2}6Lf)!|atw^w!Q7Zy*K zRmbs}cf(=Y*7A66bd14vEmzg;#RDlY_6krPT(_y-UIekmp)~ydg-wh9xDzZ9OOJhx z!MG1yw_)oQvOI+geqjiS^%p2JmY|JzythK58VgxYAnSd$a{X3Bs}jCN+vt!HU-GrH z9d*a~N|Kt49rgXfCSx#OH4%WT$mFqRe|l`1TF=5#)E?}Z#LZ1PonO@yPj`+Umn!E} zSD{uMf$TML!PeDa6b5l~6>==+SRsZZZ>e*W76Ecm1Ih%qbInnD-I#=LMU0%M&`VJO z4V_K!Bu`kqM=r@3C9dv*krzH*{m$QXnhMu&EZ?fKhqHb#5b{{oAqgu#h?^lm>y+>u zB{(i;R4fv8d~(gBaTj4b6^v+Y9bTo0T%^0HcM0w%ktUBQyW(zc@$`tY)^`mbpNS{D z(A~?`;jmZ!^Kq$`yoECkXU8W=jUC_Sezsl7-7G0r1>t@M#yJcjo1pg7j+A7g>{$Y{ z&Cf+9o6bjgCGQH>FZ5o?e*?0fS?wvdL8{Gq7;9JD$Z~vPx3kpFQO3I>Z_F(xpDSWf zPb&%lXQ{7s@&h{C<)Mvt+d2>N0jE!-3H<2)Sy)3!I=XzG}_r-&yC#+kr#?UR3)`CAaMAhBcl$rk#(YYHHmF^q;_pVjj1k zV#!g_hNUFIv42&R46eI^rl>hGTx0wDdl?&MAMuEq71VvM3qJsk)3~F=8t6Kyf%_@J zb`1LC5xSF_yB2Y8#OT3ci!KJ#Tr8ZrqyIxB(HFt*hwOkSNF9M6e9L6tcJynB8|q>j z?u!(eVaP%f8c3nlalwX_lK~2DW$VR%SAG2^qGT!|D5A&~%LPIOsc9~o4Cr1K=Eam; z=V&a(D^#JH#1-673tObF6g1fAvi6h~y?>VOr&cg#r;K01MFdtn^#3a(x5*9yYc}XxD9nqEJ{n zCUs$bj}rftKhstX>)Aw99bvH<6##UmXYL zyM-&@8TI}VR{NfG!}~a(eN_gOUl(B5NeGQ$+?MSD;Dk^d96O?YXj}r>Vd^J(SD_&tkp7cEmEVMcq7toA%kR||Ey+!IVA*iaCfz!v@kl*ozkUk zGx&Y=!y76pE0v8Zs;m;k=D_2`xop6~d}3&VDOL)xz=c3e-pFYlB~xiYyEuNt9~sH^ zLnB0DSe$i}=FIUnldI;=Hw!x*_RsJ^Fc7@rYDG!=ZmyxV;iE`fEIP^2@;Q+=5;K94 zVx+0CZkvZsyn{yJGb_4L{;*CKENDg7i?vQDKaJomXxT^;XB6%Ma0pmNgpuaUF;J9g z&@tA;#AJI0M;9yyQI~Qg)VVisVy3rMispP>-S>UVlT(Zx=g@z<sKDtZV}jFu4W}3iuT0Xjdbr-&07M$gAI`%m$d1gC{C6qPYCg{U+k4{1gsK@C*{RFs@m`KRi` z8t;Q$J5Q?}*abcBV~`4Y-JjsE+d%8thamz%jK;-TeR>R0n)BW>2toU*Nbe82C-DCM zHrD4RE@tL5^%46PSR;0hN9z1@RYFjKn*S`e73`z2aoKiswgz|p;~`GkG`x(fHu!Zh$x>68 zaCpg90fK2G3xmiJV(f7RwCIR}*}P-pT2s%61syk>^y&iwrpXH(oD%MSuo=ymfoMBc zT4z*e;c;x>egbYFVAI9ch|F{8>{Qlp)CIEF9U<-LSwVO(+tKXS>ATu^rnvb?c1TdW zoZzUF3%rsS5O&684JXSnZb@ZLPo2*Y`&qj2F2Is8u<`3uRYDZ1p?@ktc+$3Pqe6vv z35A<`!GUIF;)?jJ9(k1D3ApoHi&5+<=8;=S+ZKTcVPF|C3_8zXC#c*uqnB;NQy zw*iET-@|6E`z5qln~Nt{S8|axpRs3an_+$8_)d1kN1R@&AgvDOCczX#9fDSYP6AyO7EkHodw|8 zL$+)PW`%tWnWUKTvVyc~*47YBz>_~p7%NGBwBg}Q=Mepx#A)AWUl8ggJar?ioOnln zDZ_-oHtdIrTN4fuj2eV4Ub<&`4H>RnO%)SoP-k$%@y=nB`DSC97N__(P*o2_ehmxk zWYq}FH;FDc5S0Jjc~hqYY4p>5l||>DS!^I0hFT=k(013F;$v(st1P}E1~RkaG55~n z#+U?{*akVo`>X*ZiDWo$Us-+6l;U2-cP@d?Jm?<9F9j8evBEsBOd4;I(c~* zJ-90!u9Ty>oJ}|H7v2ys-`5GlQ2g%Z=b0(chl7%B)-R(wzbaG@J8kp%d%k)V!B&r3 zxxkJrlOVJ)B9ne^oP#*`Fc5|)STvfmKT4h4e!`c8i;61msq5CLhh6zFRjw_M%W$Bn zZ-OS$$H77ytuiG0Bl`dQKIG&07-wn%D*Acaz}xF_+LhG7voNsWE%GJ5BQwRLxa1LPa-H~&tf4`?0x&xj zzVcc8#9(3EoKs#%L%KGE#!6cuHre*O*Hwg@D3!^Wq}W%I4dGX%p6K3bw{fc!WmxyKG=hq7oqE0p%Rb0hnE3klMkR#Rj*|?CFlP1&qwjPq|nDr&0)(0n)F#EH={2cdfvy=8UbTLwsl*GhDV;FGj*v!L~on+JWJl!5c8YOVsO*{v~ zI`&?CN>IeEQ*N6zB6t%bxLKqA{#!@g9^IYqtCC`j5;^QSG+;%zfRuv5s5f?1C6RgN zMYg0ottz*wA2Nn2=8?TmQk;N0foBT1+MeQ}$|_Cl{FNX9DGF4t{Q52YVX8X~*J3OW z{cd_QFvOd9?oP8cVrrpp633`w7d|Q6VoE2gs%Mh;OG~SzU| zqa}~tKpAsYd{#FX`rRUw>54cffeZaHj&4Szf9~}CX{V2Yw-xAZuGH|R?zGI3ud5TX9o z;BY~boQ%=8t)H57wTPj(kHLLyw{p_0FJyx%E~ez@2E~hnL z7DQR7&5P7nTniYt6tif}YT-8Ly@*LNVuWm)w`zLK&cFqx^X6ACu?iTWr>w3o1O}qJ zemp(Dg5m0*C?L@i86x$1`h^ITGP115(Nw#8Rsj>TYzmItivmf6(5FHn7$qJY%wzF` zj8A~u_-pIk81*k})Q>XIJKbZK#S)ZOPBJ71Gp~{otYTpXY=I9h1s9N)E|6U*s7w?E z^ugjaw|jk8?xWenVHZ^2)!r4a+a6sr#^~&*D}MyH`pTyRM~l9BE3zQd6$# z(T4%Pzas5;da+BBtb@+m)^b(vbZ$8#J`GL3;)pMnvTqx|a{V*xG%h5O5zpaH?wE4~ z-7`7P9oeTDjikPam4yk=t*m0_3I{|vuDB2>QvA%t+E|gLmbYp7g2jO=ih;4Fa@t~<^=*2ak(HU8GtEi7|@giK!KHo^Gt}N{_ zq=4KktSAu)FRlqPFzxZdbb+#Qe?>ZrB%;tP?(Mnt{Klhe?@JafPFS+tR9D$s;mBE% zkwa;EAV@^iZ{lnK>z*w;Jkc)s=q8XNN=JFOfPrH7On+!)7Ym*s_Sqsi7v1So*QqI< z%rQ&VRh}AnasZD!ZGAM4Agf&^nFL)KPe4@YL@`uu){9sIlZFX_IrR33zwUo&e~(zB z#OqQKJ15c`c0Te1nA@4r;WZEB7H3bpVcRya=(R%8`=G2;LCzF5gL8KQ|0Q{dBYD#T zvL?j2#5SmaRupgn7fpGKsxFcRgnqn`GIO>eedv7s6=8&x&n2ocPBZdwG4IH>bpd^O za&h^<9A0K4KDq(>L?{F5CRJt~{L*HmuKZ<^%%%-@}b?tkm_qD|y z$vIxi7@zDZo78Kz=ZEYOPDn7E^l+`w3<1G4W*QSr116jILUSr?c??(hF0LHFS2<-ZvD4L z<5$w^oYvg}+w8$BJMJf9o}_lA9V5>dC@r}RrH2=~!4YL0Q4d8)Q;zaSm_19ssu_MV z^}-LG>@_)iMpn^6Ye?dhhN#R~LXB5lZGr#D3PhyqwiB-y8JbS6+bNyqS{!OU( zb=}W=jLS+h{S{UH331?nFSUz&nu4w`x|C9X^meb<~L=$@C~ zT(ezrtO6Iby#37$5h&hIwd+XGeF;8uXhdZDP!fB<*s;&p{J8BbmL@9&BAWn{nWX)r z&aS!CZFax;fHckWhEwPJXg#Vcc6_`XPv;##3#9VleoffC`|BfTFYP+?^%*ZyYdIcU@TXN>fD!11n_c0bW zEQbhQ=g=65{+x8K%*80@l$HAd& zWwhM3J^%XXKcKnX`thR&Cu7=VABGqZxOq87{PM5BkyMdv(0sgmBc8)<@k_AoaKm%JI`T-TvrYlC~| z6rEG+utwUb_Lp9(t1?qoPsQAZ3eR$pFE>R2uZt3N7gkmvXA?{5*C5G!L-agT!3IOz~5d zj)*8h*OyZ6b-{MYbG1+O{Ms}+lugYX$&PKoONbiiMh2+E(9WW}$d9dCjIr2n^3E9Nb66 z7KKK9;(C9CY{CZ?<*lqSma^3mgetkr(R1xb0Jc5B0JPj% zb7hA>gm%w{;P!fFB-a9@F5sSTG-0fvkGJIKLW4FMWo(t6GNAWJj?=WgqtSth-L`E{ zxEpFaYeJFsFvp&IIl;l^Cl#sZX*u$QBRwLm>^xSbVK~*i?~O4xqvJD!w@S-@wVP_T zZ7NEZaT1{R`!uSFP$H?Fle!QpPrgwnd&tg}eES*Vf`KEbeySL{-5M@blBHWTUD^Us zg{4nMRbFATY;OrGUryFCCZx^=k_ev#@?J}YlIoy4q}}pd3Bc)$*ow}p7_1KA$#jgc zUjW+7teA&f{mt*3-un{C_a>&n#-U=dbwV0=OJCDx#J^BS*%O#~@ZRhy#XeR{*lLLZ zKX5RauOpa&!jnozrV~44S)nM2i&=`T79MqS4`*=Pos7|Mqf_w?ZyX$Y+r~B$MS_9X zm9C6NwmPL@Zw6b@tc2b9(QiQXXvE%-!+1gzs_~s1=gksOO+(JET0XdhBqxc}FDWC% zb}kDvXZHob*@k@G1+B$OfQjAs{yb`i%jKT{mr7K)f6mA-)2naDH8v57C!(Mj*_~Ue zf!o}OKvB#EAcWKj_Yr1h&?aA(+-MoxiL4oYuWB!ji`(FtJ$NNMQjhWW0@t5g)yP$0 z=&k&@a0b1$OrMUgr5bvdU0UDSl{-3VM!mf0p@5^={BG39mwKu~HqV={y6myZp(Mi&GVLcQgqIUgk>I@8f+&?wy1DoBr<{lM zp;-1;B%P7whCMM%S8ne7)iAfmIgX=osu8zM5F1Xo-gK&uUrFLQF4I@b3?qD5MJXALl)_K-0O znj}sUtmfJp{Czz6Vi2%{QrBUELcR@SYglBWDh)4j5xu|uQm3UbKD{|$D0pT>C({lRhWK-ww(nxl08RfPf*{NJx$aKa{KuSXN=FW#!+~~=(hCx%6Pq&cPE8gnLJwguGcHz+}Tdf-G>TEuKC8evHt)X||JwYR+N%0*ZG^U{xT!^f) z7ac<%Mm`~IM5UU1Uu?IJjvIBF94i7Qx5=SJd$?$m`Hq#M27Hj1&ZT8&YZ~wwOHj}* zvvexXJjW%IAwXxi2N_{74HSzB55m6N-_(f!NR$0|*FIbTv$nsrYz0TqHR8Q6Ma&Ys zWz2BoXvh@bPK%VuI9G@iuE3tt%aU$dgeoF(df2(LKoMl^Z2V2SlaAefRYIUtc zBk2#KasxuDJ0Uh=g4~&Pf(~$aEI2N%>N+NyudRY~3?N>GnKN<;fefauM(#$YQ7Vfk zPFy&+;pFV_jCBcyJLTjoY6O4(+%VuK@Y@qO0Ly^ie|4^XnhfCJFauSdY#BMqiDCrQ z2q4HyDD|OV7ppjGC0v~r*sLb^s8n>Lev_%Dyje6Zhzd7yb9l|Uwg;{>mzEhC8C{?# z39SO1tG1$DD1kw9Md>}K2{OV^+YOZ!eOUIO%a4NwR+;2!u=^rVnrKnOLq^PuK=TEh zjdJfNFM|oX%3$$549S%+rIStdFpoiHelaR)E+5vAa9z3NYVLNYj0t(Dgw+>*^bBlV z)WN}KEkFTZAy5}S1-3%W$19Y>#%o~%_%bwGV*j(%^(*z|E)}DbvIjra_Ct&;RPMc| zq?9}Jb!pja+UGNCKq7mk%)|fLx8RdG@BQl^$mRdz{EFB9VD>n~#ee;i|A#+h+d2%% zlYL}&y&?lOogrjsTc& zfa#y;5P1qMm}xL7qWrMMH7o>77J(-x>v}I!`n=>$Xpa^d50VuCy_`-fAu|Yzu6J)U zyHT%9#t?@{%%VIQgn~e?h`iqB@B{A`kzL~4Y#m4%<3xq8O2#TRR1H85F1g)9I87@% z#Q$DTVS;!~N#zmVuiy7Ub@@a)`#DxSIiPKH&-` z7dEBm1x2RK2vx37&Nd(g7GYi$r9yzS6BoJ}&px-pBqCzjLP!@B-@`hAVhf7;{AI1g zAh^_vp|TE(_q3l}C3Ku@XYW)cRGwsK-}>TrepSL{lZ5sKRYG%oP4+uG*#o(+=kEs7 zZfzs`8oOjApfUzJ;m@fN@9*NOq5m~BD(7@qZdQ&fRY6CY&GyOb_AF@L|z*j z?4O9_z?CeNKA;LM;Wl9K)ut(E7WV6zuuqgxJ4E0ZkD(np+boZ`V<|soWqCd0R)qwoq}bgY0)|_8pJ6#fViLI(Fb|vfqWia51Vn zHT=vl%zUXe@C9I#y4oNHB=^yAi=o#my^3ypL(602Y@y0>cqhDBP`C^C{$#!?YO$55 zll)UJ&N=iGimfY2#Td^Ua8+q?z#gP?Ul5K?=GP*=CI@h>qmB(~>rKKSWV=j5SaKF; zCe+S+Hv@mE>Ia}f!P9j~N ziWXeb^%Q8dJ@=TH7Q_MyLFtGePxqFqVN7R^CPN^?^PtV6%K@cb9}p-qKs(E%=rF?gOoDQzC*sOz zHpm#VH$`e!UFexqjR+FbGs*Yxj)0WdjGUAm0n$quIo4J&;x>nP*%ROp|M>JrFVEFV zoGTf3i+Wr|cC%rP;h^&IY6vo)4ZEs=@$DS85-ojbsX{XRG$_sJ+6_KmWa^{P0!%VW z&+j7hi#$C#f%(gJMG`_rFBT{0zSWalVeIJ4`bTsga6QWS+Tly{>T$aZpwXvyJYZs3 zOhHe7ZAI1DYsnRN)r(83lKA&td>($Rb3K!e6%`rl3QITNuUeujEZclPA#LJ6KZ#VI zpM-;2Cg+&~Rxp4nDomYw#um}qVaukug9t^s$NIZlQ&&`-cH$UcYY;{A#_g3Ff$<4B zll+|TBfk_v4*3fShXN5BXii}%5}DZ(&~pZN&D9eFCzi)A2(K|or8=X|hy69p*m9cF z`oTXUo%$?1jO+LK@tMRgxlLPR#L4!r?#kWJA6{(HU7cVBZNSvMdZmYD1Ee&UVchDrT zN~Ph2%((>@%h6x}EL&10EXva`+$R`6*~|RQfWt9{A~pQNkCkN9Ha`WY55G(a@lU`r zsg;wAM>9$*$J%v4@e8|~#ud1QTUoue9EoF&@@pgv1nSy3MazNTYd*f97ZN~RmX8ue zu8Mgq$~f#nzO$ek-Nz7p2}I~ImAO-!40|p}p9t4KGC%3D4k^Re>@L9>orAjA{hase z2yk`KBKmSdEs%|IIU@zAn>cOpVf8awyywu0=^P70y!jX?5-&g}_9)LF8;i00@VHsF zfZt^3726UNt&WK@u#mS{wb?86Uq}=)ojXT>Zo95?>`|IE0*qi_d z7Jh^zOC{)jD$Lee52*lC((}QCwQXVHfnk$=l-ssK1f~lphNVIjHM2$ILHJi@>tm>t zg%b;<9lH0-OQ>L>y5>3q11%J$amisASe?vY)Wymy8{?6Idc?Aj=C+`~FMqvx;!|G& z!o}Vyc;2#6CmV3S+%4sj+EY>vtXdueRPn{bb~$h0tk-+KZXuEqH>Zjk-lS5^H{N`g z??iLwY_|LLsP5*0)p8Zn==Zas_c5DWfk(a`UW3uenU(!k?N>Hi9iDN7ljY3R6Dhim zKG`gzeBBln8x-H@*@P!<(PQJq)+&uU=CUu1JBBX}mvj2u$u$;&Ts4P#8qMlO8g(J} zjuAFed&qvm|6wGM;K70B`xbA=ov8sbyC`OZT+?;%EXY>?%DsT@z%DgAa$!q@?m{+k z%)ILI^S8H<#@QqRXz3gtnWah$h*NxU$mdcn;n&DgjjOG=?H4VZvtbwr0&}!Tf27jV zwsoG99b(kuXM7khhPgxr}nD3&gT zoL{jeoMDG`8b}G56!3RhKVgfDjLCr^I)F}dA1D7P>aVeI-2*|8^;iRGBF;eesd|Nn zWJE-f{YjXtXyRPCVo(_ygQiadXHw=(j7?_btP#~Cy3YpK1yl;2&0-YsRlkmXNP;&t zv5I*#(q?zc0rjudUsm6m0>GGZ#XW<7!V+@?3IdMqnH5Y>ZFS;=V1;0NJ|bW!4t$wO zFZ)=`%sxRh#8g2CbO`ci7WK}&^<8o2*afZID3)L zQJvjtXtO`sH$_g)DbVAy6~EYuXS`8(05km=fM9h*gqm3hYzFYDmWbd{OmQ&rztaN>##$oNI0`s(~dim$8eR{nD-q;j&EikqyF|-hx)0tLVKp z#j$4WSkm^zo+H^!Nk~Dbrf_8n=Fc_{Rq(Y|1Og04M@yFjDC3x39GywMnFf+bQ$lLe zS8a3n(QVu2$cOmjq{nGxv*5!o$g0xP;N*+E8db>`cRT9yq^oh<`d2!@#4SzU?5^`ea0ndjEtwXvDqSW zDJl@490!4~QmNcSs1;ATu*I_@lN*5DHs)xk6 z&~G|kejaKwssB(7Nd*ws&k*}B1(hHO1YpIL>leu`#SC7GU2QYuL-0&U<1%jGQn=jO zT#p)2XkS#Ni+nm~WZR4>__Dm*wxU-OIuUzGJO(lio%p4OZhu!J<{rvaLeven$Hn-d zbIZP~09Bd_9!1Y4iyZBxwP-7bZORsYUw{4(KcVF@MiEhZiNSZE$u|sMM&gus>b6!e zwbr+d{1xrXqNp$&pa!!1kW^5XUAM;bfCtgKDWN%OR@*1?A=SVIEj-CxR5$i6Uf`$k z{?-(>@j!n~TJG*Z7}HEUuu3?V<<-YYhh!CU{^xsm9vaVT-LQ6DD8(H=gd1<5TJ_~J z^~JL4p(y-5V1a+BC0n+Cr_nLxUMS-u@eM&@&Lm;O6syG+iAAQMRxuOI3>2Ecm(!#3b%CAPSttv?==&u8;uE5vPnXoV>0k7quU z+Bn%j9bmdGu0BP+SpmD4X-n^CdiC3aW@0g|O*!%AbQTX7Q=NEuk)rdcs)*Q=JLvI1t4&~bEW1exVJh|TW^%iZFzCDFJDXRb0cHh>PJ=iQy!}ejFlwBh# zg}C*$Aob8fiRpKyfaIRia@$~se@;BlmwBW_J0EReQs z?BlyNKO`egd;&D!q;JoAXeoiVCGC=|HHwRgiP@EK90B&?B6pVVxE}JW`xzK1>4UM= zyyMS?$(>Pex-M7l=Fl=mu1uz)MkI{NFaq++J^(x}nmN%GUy+(!^R`2cBYs1-U^9qi zI?ZU`X>x~!Jd-7Vm2~?j)VD^A2h+7gMOX@@5_8A%kslv4>dI))YJNcxb2C)F}0*18qx*soAobIi{yAV^xT`*u|<$8OJb``KQ89 zlhi#?1FjOeVYu20kP|%OhDdbX;m%U||4{!8liRmy{@6E9>khlMPZ7j&kmEYdb0IWX zCzB>52T++ExyE|P31wq8^)((<;TNV1uS2zHg?YELVHSZ<8J%1bhf5J-xC@P3cM2IC zA7gAMega@W8^~sGnS0fn2}=r4jReoCrL|1S@_0z$8>G2dd`- z4xv^+>;{^M5k?Q{Eg5H=-m8{;WM^;x68H^A$0D3I#csZ|G~oMI6Lc(ZzJR7~8I`Nm zO~z=`z7l4KhG65yz=j2?0F3RTj8Y;yGp^IO<%PEhopriTL?DDGTJJ-y6u#}QM`}C@A)qHt zjH-E*rG3k8^Y;LotPrN){hr5&2PouP9oO$F-B;BjRLgIxMrye^(Ju7F*S~xxM)*Zh z?Ld;{h$<`!P{c`gjhcPr?6#JF@tQ0z%DnjnZHr(ka|9TEG6dp{7>BOG zk$-}ibODfu*oAE&qsbZSN{*F4|<_wlTw|sSE6oMkZF#c@9}YcqzrSeihJ{ zC2sG_?YoM(;s0vyOXI36_q90c|&Kf;`Y;-`z}14ys5{Dh{nIe8>6H~-`6(}d6F zJo{ABp2OD-W~474Dyl$FdBVINJzr!nW=k`6zw&g(voBVM^Ew=4^MvGtVY=tPu$$UY zG1p*SS%#%El97&C#1B|em}@@$#o|hK?$8xZ&3HcO4`gJ%A%l`xX3{8W-$XC9CjKu` z^4DE(0oVj4boODX?*AYE7AU*m%yB3q8`gj>Kc{17AU|cjGF3)q>t$pq#X*zCSSQ#0 zCo7&XgqTaHf~Fk;d?vyKyDEM6^P-{qzVq0HkpA3P&|fEgX#UOr=+NxBOZ)Rm{+Eq3 z|9m?Cn+MK+KAk_G&VL)a;m^hP=VJTcyV!QKa((i3BL^KyyJ3`lb(Kwcao?~M!vWx{ z{e72Td_$h)Vk`2fqe6jVm-{aG$zOcgn1?^De8z6At@-~XdAhd_4TFDtsn^dtIwu-X zRIi$lQ^y?d=!NlHxn&C*b`{pdF4wTzpJM!Z-1J7uQRrA)lkvClY{9JZjk9R1{+@#@V_DyHa zt^HH#6SQ9{U!_w$cx}wda3x<-=(-fKuEEv zk2^o0zES+Q`V%7~4OM#_7L5AGwxx#Gu2`_%O7QYiUy&QTF#K@StpCj-?L<|;)!j`c z4l1ebUpzAJOjH!cH|NNGeEMSZnUk?j6?hEVEm__n4H2R*>Q>C1U7J~9_}yp5q8B3X z`$eW!d||wOt8THW#x`rs^nk>O^_f?548G488QJ((rTAB)!{s_PV+`GoIIo#2v9hwB zD_PWV+o%4>q2SVi+=oR=%fi;ze(T-2cDuv5rtd_TB?EpDmwJZ1g5}qInzZ<4>gPu0 z#>JX$W+7r{-#WSdRm1ji`4wVu`f`K1y|VK<%d&ItB&=(CXmt9x_x4j)KZwuQ)%*AQoht{`tM-qCXkZT0f0hdWop8Y?QAGOZpOw#lRps*;HJYA&wcX2Zw|d+m+)$d49&cx6a$LXNTQtKx z^tg|gaj|LVD!&qh#v_Q`(LNUKrzgrXmdg$l2piOf5)I}@dih3+I(ki3+9XwP$#YsV zlw;u4GON}u_ML`=)aFEm-|c%UZ2d;_lx*4;S?{Q`?2J9JNAh~~$dB_hUKb9`IW5;= zY&>FpFVDWPJ*m0kx@C81-tvL>cC`2GxOvegV^OM|nO}E}=R~JVR>J;Nhs9MN|5~$m znM>iQ`cU7L(b{kKbauM0T=}`l^vt}(!zFT-StoSWZ>%f(;7i^5giJk&osXP3U)t7E zJ?Egr+q~tcP50Js*!$XS_kr`aTiky&Hy)|Iv8^CKN3QnhzA+?mRXPq3I z1l~P)^Sd|nPRB^SHg25#52vikE=-^-*Y5q7h(MiZ;eCB7plfO*jIP~#TAzF$sj@YO z2aAfpl&>yaez$XbOP2MZ-{{VQ$$zM~`&(ncZE;cS#Mh1ls%M?oDuC{YolQg|Prt(uoS>bb^w5z&l9gi7=$M&vdq* zR_s+^2#s@gISIyMYo8kBPjK>5I1*Hnav^Y1mMQC8CeG-k=NEa6M3|&uA`CV_*lHM3 zq=6hOiEJ6OB^Z_|TGx@!kmu zs7ZsqSy@oxk)YX}?9PE1Wn?t7o9SXj`2(Wjyi1NFc_-?urgfwt!mXv3+%CFtW1<_= z>;~9l86ig)War-W$nvmm&!3>06odpm0GP!2T-RRcrcjZO&-xo1Vh>2}DZ!A(th0|= z7=)4t6^~c7>Smd=<+Mx;qy|ViRve{F>EzIx9@v~-8uYkZn4TTY*CmpkavZtnMaPIC z%o$DDj3SQ=6K9)($QF$)*}XpQxn4sa6}>*_s1q}C*^M`qLdh5!Xlxz?$EFA~&ptqq z#Adb-5TFJj#rhNmX3JG3#OOS|hcZDXn7fhnfvKIHXeXgF1oM!G2Z}{8fW}9m2-i-L zdLSO9>Tp=hk-eJjRZxY&cr)(u^ngz4^4S7wCKO@FXGc{wig6L^k2w%sRr)c=SPs14 zknccV%KV|n<9IVW$eXDgSII?GwZ$4CA%_r2;_sxQahMKK^@J*sm4JfGfkAwKhOF~t zab7R)W9VBiv*%7SFega=FP;HuNi@jRJ z?ci&bPy!ayeJ$k=V)E=4V6&XPU+mWsd9UnJ*s&JHw85Ly2LVD)XF>A0PlcKC1vcQ&ka{j^rIXUggve*O2mm|-vE+;w(p!MOK^ z8Tv~w#4(bmJd}|i6*=vjaK)EAvA19afRXK8kc*?Ed$L|_%Wr{F+eD=|wd*+&le59x z^?($@?t-v2A2YrsEe-HS>hYWSKj`4s&ynijOZDub!*KAAaUCEDF`tJ+?jrdQ^we)a z;621huS~3&{&HJx%UnF#?Ubh?)Mh(o!x>@Wa5DW00Jb9?9T@05#Eq=L56^7N?d?iIq6fmL%ZmB?fy&%2 zg034<3?5;?Fa-#z*~i4W~Sk2eHKH^*C1yobr_?=b&z>4<&pt8iU1-r2BTZT z@97(4+ihWSYu4Z>;6;(x){4SDu&3-GYoD573E}~F4W11+;P^N;7zjm8{CG4Wtab>C z++pJv<1G`9t*O=}5Q5GjrOP1}YZO0*CTJ=7i-IDs*TN`nQ!_#}KMA-6>qO*!(=ebt zVZ!;7?A{@Yy&l)uOWxN;*DX*D2Vb~C>itbxH{3dw8a{SifE^&V#U{r;=g|i+g&k-5 zblpxqS2$HDB7GWLAS%v3Yo7BqM8C4g%yYeE4k#NL@w@N-Mm4?8A5IGyF6Usr z8`F+UY7$o#T)hbKAXRYT1Vt3KQ9PvesYPwCRQkvYGPE5kRC4%3?eA<7iy0_@J`7P=je^P`}K_0 zkGdWAxKxvW5g>#QAHx28##Pd_KN`fDsobC4u?KT=1tDA)e*)V%FCtCp^^Bi;Qp_`y zp$AbB5GJ2xTNY6op|L<{gvn@O@Xf4E8hL|+vTn09o!$Z@*;KtTT_}-Hjr%4v20H}i zmpeqhyDJ-k7Sp}sba2&qabAujQP#ntTM>pY){i?j2}P6<#aYwrkUcZ4e*NCjONj}x z8ClnoA|k9t2O?LLj5Or!ED{}p2t(0#J{K=w*DE5kQn3`DnOODnBl~BwAPIa( zJjm1?*g(fQtQV^8?P;VexrIhnu_sVOQ}sSt4g@o$XKVm@BL5=|FcvALk;>;}WKpH7 z5375V`&EsV=f%s1fBW_)56WXu&R_n@k+3kZ#?C55eN0k8tXIFJLHUYYf`Zt;tvFPE z>`{SpL#F%{r*-3VgUu%Dx(&{Y{oIo$%(8Z$tv{(*?O&Z}LYjCQ?&ch@?T4Xl1KUR78=s?$CY;C8N*|gbCf7{721*Oj_@>oYFt2 z8lAF;l!$F((1;_Q{(6LlBT|f`%+2jl957WT%Ywk{u!R<3_&0Qd~i3GTmX~ zo4I4}BRDqx_E;nm(Oru1hX7xE%2$V1o_Yoh*!SUi2wW6i1i&QnPzz8m)9%e%8|D-m zrFkt!5-bSUa5NQ3g%m+8(E5H9+JTXC$8Nte40+>l!EjAhekh^axDWNV*)@VIRzn^U zTCOL+Ib4E;RtnT%$srO6*Q{AniMxSh&%Qxn^NZEL)~8x#t6Gm#9#wU1&J_p&Shk;X&CZ$n{Svy!y2)@=oNg!$ps^Q)L{KbO^Jzw_me%Z5kdQvXceG z?OniiQG|p`hWha)dn49=Fv-MxMj%SZ*;h-BE^gU5cEP4(>OII1NuTvKcgT58D-KG* z^{|F6902k>%iKz13u2z|EW#&=n(*s2pj=NYsubEWgyZd`1qX3?DEJn!ox?)|+eUIo zt^7JZnMG5=rlg3Cqm`AFW8l#4rF4v-2pR8j;O`feMQSY`3XeDvup~j)Vqwx@b(pX6 zo~Pk#J#yqoD=K%ou<2bsbkO3w*dLO2{G=iQlWX0f> z{t)v?*pfo7;biewEw3C4gQ9}ksw2HAi5`n9@Dh_pE~3R}M%y`s^VXszleHJ=S@mw#E5}?WuXot*Wb!)E^}{e_@pe&K)v2b2fZoP#)cV zu>bp~r@{QG(LnC1!C+ZX)K!ohHQdfY@ip^mDv+2%EiKC0Gr!rM zI?6C=l4Lqhn6tO{1R}}QdxoG?VPP#7iU1dN85nc85V4+*mh$c8RBRG}-%OvL3qU41 z_tg!!Z`RA|t4g2Ple88`z6v#*fTuG5Sm`%xiqr?{=etdoh{p*mg>;liDpCbVuaRvd zLh{smR}Ri@hx(B4&InQGwC)UyKKW>dd(-sMOA9x2F*(S=1stGMdm!n)S~;%4s|@Zu zNv4ot9{3cSVcI?9ASQ3?AOQdeTTxh!G*^7c?iwPR9Sx;Bec<{5#V06U*Ad<5l-*;gC-e|QU5 zI}orjx963znM6Hqc{^*@&Fx988&wk12IsoYs6~mS29`a$0HG&N16@bDLx=z!lnR9U z4K7_cD5)zfD5BcV*)5k$mk)oqsmsH<<(m8pu1EpM|FDR%b2f=|Fl@s~+1;CXw${-# zc$0^QUw|Ie!V51ZEBuORM3|vagd>QOU~T0Yh|=Kp=YE+tuJrA>nCG@Ddt^v^6ooW6 z10z16^rD7ijnu8^&IqDUt-`R$RVa{{y-n-fwyP;yF46OH?;UTS@rS#Ri9HTkPoowR z<7ZygL!T?~As$zLb)PbPyGBXdxiS9+l$)r@0+#%3+X{C;lxJT>OUOO@D~K~~A^}b+ zIl~;%dC)vV-#Ol=Z$nR~)=NwmNMFtT_*#APF~=4}dL)Z3eHz$*g=$C(n{GFc#C`8< zcp$r{v$n+qsUaV?!^pTl5~;w4Ig)~43%#QQhF?Fv2(bE$@j)zJZS){36hj1`O@7>` zVEfMEZk>V&d-KuT(Qx;p!R zYUoM`<{n>`zFfRKA-_JLU8kjpYErGIKmQuk$xXxRajYo=L%yX|bya|Ll=o<6dsAZ`b$)+GKp~XF_L$vkR-|SbdUHW*qN;AW?l#Fu9)=n z^u}DoBto*n z<~T5#R12blFugCCS@0O-{h2mPQE2(h6Y_|-W4|zzPm$s|^0``2>nwc4&4?hD zclq&R-#gQW3uot_yJ-pXU>RF7=)35WN`X{C4gWwC^x2;zlw-_^rM~g9a_ZLsnLYy5 zX4i-9^KGwK4&hrn0H4|4tU0?Sl59bm3(*)2-oHx*I>i+Quv4UX+k?~X0x5$R`OJ=k zdaKSi)MOMOuk1%-T0v-p#|~98b?#Sm4Hx+Kq(wh=y0iaIrz0v5MkgfEoo1KzI8!s4 ziczrZSdE-ls{8^rUIhw+5CM}&R2*6Q(Ldu<${~vv4~kkD_6?cN)YQLY3ErzC4LcG8 zzBdO{!m#=uoBbFzdzMY6M>h@55DPydg^ZwNG-_jU#;|&rwb1+s>h*=_29+_acuH33 zjcmX+H<$RrOrjLkN+dePNGO5+n}=fk4O5fgCJIWlJCQ$YRJF3v^F+ zs7~AsX|u?Qm#d8sE5VRTPe-j68MT8Z!Fe=y9Gq8e)No-CZR@7497Ux(h@~o9jzT?Z zzB6>>vMa=>Y<~wmD&?-$MxB|CO=4?TcdMC?m)AGXaO_^NW3a0(P`f_U*&;^UA?c6e zS81@m;hsrTX7Xd29p3lW!qNIW-Y~w(?h%NcM)k>NX;a&E(B4}M=y+bX^&3v2kY~u9 z`N{?n>P5)TIUbQPW5YUQwykboDL?{ao&V^{V(LY&MjD%>Q|Xr&UVHvS$ z$HEHC^;m>_%6m}sT^%Y9ImRhVP>5L%7jI&BSZbPsp0_IYg39J|?>p8cGi9e_Xd6s- zDiD5;B85!Kxs|fvK;EFx4Meb5%-X2TOEF56Z9A!Si}RFkN5)n$Se@og_BHd-ci0$I z?;7o}l7~j@ABP5RG*>>>bMzRx(PiXBAvtZs3Ms+SUI}plPLzgA?Q}y}pYTGJ^K9Xd zP5Am4+?30BV<@tMWSD+4C(zDYigMQ~J5IJROj&8+!i6b4f~RLHzu9MyWIQQqNdGSM z^s`b*`oe*4V^>JOdGeq6Z$&(NCVcVa>Rrz|6?|dehd=-S^d|e1Q6a2HLCB~?9%51o zcXHuU67@gY15y?bGk2sa%xKGy1xG>J?~F}Jm^Mz^{_0eJjTc{ey%ds4*+m}W6tkKn zL}n@dvREmu6F3uB+aRP+mS9R+F)BkKzb#jSgKQo z$088mUz1axztBAWIs^xe(S8luP7STOFmxfuBu1j5q*MfY&QGj#$OX$@5}JVlT^j== z&)7T5BKxNgChEQ-vp%vqX;QU`Ec7OF8dXTs2!s&B$gE$Ala_MRAp#a8;j&+oyq%8| z+HRnuS}4KtBehYKI#5%g%);h4 zJw`bG5eXpwZC#1#T($>EeYaF5#dr6GX3(g@sCkGxHh*t<;I!G)=`Uv=HFUwIu!b3% z!$U?S717buw8B+N$5ZW)LaXiaO3cxf`DHM6oyTMr!}>$Ckrwb#W|^DJco@#Sx!?k&X)CoJ*BOXf)hzUdph5;X*jIQFv6zX*wN4Af)<1 zITRv9oFRsq2jHi(QK&)Q;Mz{oESdhk`1;51lX63G4FTDeA6#P|^l=L!Qy;{F6ouYH zJzH!kH!M{mIr+`&Yta2SLUh7Gqukw)We9OKHDoGl+9+Nm5Q3O`3pJH&&lZe3%`zNJ zM6ED*DHrDtW*C%b<@37O>a<;M95&e9V26`#v87?Ss?EW6e0HbjI)XA=NBi*^B-F7+ z7t7QIr`(Rdj8wmc5MZd8(wn;Z{D7$h*l57@O5L)vi|mp3QSXvpptTa zFit5eh~H42FB=uJ)`AImPVBsS3S!-$m%4oEig-?=2euGhes9Df8;Kv)Vg>@WI5i5y zf$Y?FmTv;ebxNrRB|a%hDimS6Q@OJ_x#EG42@2&Mq*Zl>B4y+6j2+SMjp8RMzGPOtr(6n!$?i%3%+wM17{;YRiXqbvp=wYfHiwrn$8hbG%BkF$y=VNyU9W9tHNje4{+5ciS2f0C{SKmIpW< z({i=i)`&j3Ln$8No?DZF8cxRGmaa}RO<}g`D^ZKC`)wSp_YhiK;?*_$8?0m+fPQm~ zT}lkpZTNARTQTcql83s=`OP8fL)>_?76lt+De#bd?u_$uk+Vd@&J(!0H38pwlzCH0 zAkSNVTxFCE8R^brTZI{mWDYk=3zSHvib+`%oqlp*woHoZL;T;$MlKSxgt zp{B*Xq##5=p0q?>W?y~E^8vsqfHGRD@-Kw6qN%FS+(R&oHcRe?mA99&ufYCARf>fc z@{1yjo-!{#M&>xPISCzpOnh@c!H>~U-USrWHSOaVN!pZ0Y5%bxcL!{7x5PdwasJB} zBIFzr%Ki+Eb6sKq9lca8t@V3BO$`&E*SWHkQfk98{&q{6J&^sJ>>f_6zCKpD2!NkSvZkOsPvg`URQZ{UzZ zkEkslp{tAKzU1LzbrDiEUMG5ZGMY9SQStz7e{HUX7OJTna)hkgKA@$rMva!0c#T|72TykNSegz+*YYLlAwr>%v0L1YE29M z!3&*sCZN)0a>*fzIjJF4<(7M*kc@@j&dP_9^X!9Yx-}8FT}7 z(AM0r(Y+xy*)sdWM^-tz?YCKm8;{6oIHVUH^N1S?J#yp>#*8*s8K!77*pE@Xx-=GH z@Nz#i=s=bDtK`)JdU|>F`4%6rE)bBORbTSj2!TD};Htp>25+T>rwr2c=A;!_s zxO9L6lzk=lkOOHg@b?2qph!A1f*{elJnzVHrBOshP0#ma_666z^R!baBct&-RQsmg z>-FpPK`PvoEFKAR?W`C;d&I@9n_?m%OHid5h&ZIJLwBeor~#7{cg)jw_B3V(e*R0U zZ`%T@eTf91ar;ks?d_jXxy_7dD>k>IgEd`gC30+-ZyTdmp6XCX6{20CXOU=LsJHFo zj}_m1L1s+vxno;y2lXlrwU>sc>APp3UHQ=o6b=gav}UGlz5F+h1%<d^opDJf{at}zq0XnD}Kr zN#oMWl>SK?V2FYu!!`4-fYNyO7_GpNJMOHtywbKSGi?!%H zPP@nYKBVl#0@LK{n9Q3<`Vk6sCXENgCI;1*Py}$xQlbFRy|8c@gCGDccoZR{ z^0RWAcCMX^l;)IWD|D*bU?yK zk`UsjnC%Hw(R-vl4>wZyd4!}A6x$)l8x@4Q{oxg@RmO%*w%ISpRPBB~joLSW6{tXv zs?_BSSrau*da&tfaLgPI8;EnF(i6*ds9%Z>e;U#aU=g;S&Ss|H9smcWe&lhs5OSyU z4dF6GL-F1sujq#oV+<@2XnRWA+R*S@XuKcA=6hdw#o%lH6ww)|gS63v*!&|-9x9I5 zH|KhVE!q-6F>NnwgJ}leM2}R(xm|t7ZpMiKDAY_HQZ~2qV}go9CLAgwAM$=u+86P3 zIabtysIXc@brp+D735Gw=z|_(p#i*=kf#Q=X{?c>i3yY?7!+xfhUQiohPUhla$3qv ziQ?ocW)}qXQQ#&7jA=qB>OvI5A-M|s?Se`Fi!$>Sp5NhdnAb5_vp0#s^~7~F?%Zw} zXobpKV`ZkhGG0@za1@IQqu?gbs!}@8@R#OiDSg>=-w$YoVgfE)-s{v# z%|pM)3p(T_L7e!jA*)6k!}GL9M|ILs3f?x_>Gd#P=mOr*PESu_EP1Qxnn?{-w`h*) z5D~(tk%#0V{&#beYHTMOxba}3!WfQtTQuz2u0s0aGG*4OpPF()US`FTo0BuRs@2h> zjqR#PX{0u>c=k+)OONYl;TJ!V({hO-UIEf}rPRg(hV^LOjAH_BJ=$MOMNr^3sD@~s zjD?4{VTAx#oUbAm9@>m0zWz9Gy&(K25)kGIkdLuTIhxL8=o!%tgOE0Cg(7$vPuL*l zvi)WboCG?M>YU2bcr?Z;Pd1=3@2t`T%+3t@C1qNjqZFB@t~m9zDMrTP0o`3476*ue zl~^2@tEf_S1R?(LV!!)qDSD#i7_?A&vw#`p9YxZ@0g{Az9bqbFA~gOf8@?8B+FMZe zLRarnVXvIbRv!~1b9^-kpX4t1b7K~PNfd||u`w@I;h^w2@qnV>h(Q!rc_uXhfF)i) z!iyHMI%{hpJyR}#I(DT@FLKhmt^XP)G5Xx=AoF+4)S{ixv>Y{Sj2Y9aYao|$@wsXv z{+FECUx+}3DEvKB>o_b4v_88{Enbn?ch@vE=NUd$Mkc<96^-+Sax}&=i#79QNL8t>?Mx*>_dXD{S4&x_Ql-HCq+WoVu`P&6f9T z)@*dyv;kk4qs}?tUlPuzb)41h&7Iv$9L?4!n>b&!v3Ityy!5B5nWK}Xy`3O0|1n;1 z?mw?MJ70B@;N!FXf8W7t?`XkyK%TCPPxA8=B&}QGB=^J{mh@cw(BLFc=C7g+>t{^cAv@@N_6E-NO*qd+RTOZjC<<`Z)~;P z{eHt4&3k9Y|9V=#eS5=S;&ohezsa8Lj|f#fpxm9FYo2=9HBx;pbUwr`TI}K|NTW(?mvFt_P@V)i1PpY%RHVbi(_% z#jQ!*2ciZFyP3+%->iNy_;t;it4>GNe(s}dP7MtWIex3YkrFOv`T6{Z%7b zi>E13J^rKZmc4uTN_b4Zd{pjLhW3gxagKPc`Dtb96b< zc#f#On+f3w;WKWojg)lx`u?{MA3h8YH6>5Z%xKOGH8s6OKj`IJJ*C?Bq#e~xYvobTvopjeox|hP zTwa*j@;v0Dia#?)Lwl}uuE&)5@=S}dgj0V&nb~k-V(j8rp~^n_0M)Jnm#NGU>Z|zp z;c)wsPvfPlX&jQS-@bn-JA3Y&Q$k8!US5IQc$z<_=b5z|m{cnt?RVHA|I|L#^Xe~&9U>KA!sWQ=;t1J$1% z(VSkIsvDl`@pC)GV)Q78T|8#5pn1ZHn;Qz|y1iUJzq#%`#xqGT94|@MHu5+vyY&60 z`rGf1oTtC3)jZ>un;C9t$uKE<^6c3&mExSy&U`1Bn~u;cm?)3|(poTv#Crsr7F4H_kPDujlH>g8N{*q&#fzGV;ZPFJ^x-K?xhWYxKR zrXQ*^d{>vu=n~sew&1jdiEdXvwMzKY#z4tFg-ABDykLGi=(jJip#O=-gqO zXSR5L-?3vEe-`}LBkZ=HI)NUrlb>vb4O?0Jt{(GNRef;d=1sM@Lj}y(dGoNzu`fh>78sk{+}xh>=xdQbV^lrW%(|@TUuBr@ zc-p>XKCa6fWq?(}Iic&KyR57%`3yTdJIee4m8g``l?B=hnSTFRgO!iI!OxFukB}HBWv_#^>@;URR8_g-{1PpwO^_}e$1$D ziBo3tnP-HYRZ?oawPnvv4vy5E9HHge4*Q{&^yn_v{@Tc71}&s8spHU}V`F33s*lNq z8~ZJ5TGU3UUAXY5HbOj>9xWl&u(PM&+O1HK z+DrXPFI>2=lZ`Ez{KSzXXDcf!+e>^3u{kc~rMV0@9~KvH6R1nk&E{U)$|h{}GF3nC z$X|b*z_0q$mTkdfzq|6|$NJu2Sv$AYoC|3N;yi{0LLI|&71<{1^y0YkK(-E^)i#T0 zMuF{rdqBI;eImv(EImCPd)@e2@qZ0pGzCsnjm!MEwXoS2y}*{$ z?@!jrjN$ZKa+n#UE7-Q@e46O3P@aksb9mR(m2>#dKaa~UPd_ej88)B)=jQhPXA`h9 zt>|%qVWReFpJEll`;}Exv`?M-7mwFeP@0EFd3C^ncnm|0mI zKZ@?b<^BD7jbowPI4MOLrrg47>Beq5=4enNz9?poCmK_Q9`Jo#ZaJV_tk>*(}riy-g6!6O#Jxq%!l&wBdEeFD=SB|(<1DOXS9T^ zo2vzmb!es;6(^BZ6cW$&F$BTb_z3(|$c~E&ur{*)` ztgo-{(5F~`b+eV_aX(TX@7=p6P-EJ(iVr;`BGN46^Y#7Rwi{p4=bH8HEmxN2&nheD zhL&7>d&6MkdwmBdCwjnM!JMmKPVPT&Ago`~sbA^graN`mITRj!=gB_B5AWZ9tgBPm z$|j<*I6F#?D=jO8*RUZ@Iouh&7ws_u2`w9I#$G%6#u*h94Gc!|w zN$$jn6G>Qdv{**Nj1N8&?Lz(2r%(7JobXRLl4u7u8sk`C5 z$yrz@(&h_rB))$A`sB%ze^Aq&@);YU)L+1_=o=p&cP_^L5zqo;vvbFeSZtP8adA~; zWpY^C$+@`}p1jVO+gsVH>gxEoWOP40pits?^r?rWqzo+TqBM^mzlK|i<25Y!*wP|u z+`+tcYZI5>(u+HHHtj!n5Y_qIHtFdHj1s5$Niz;vUjsZbf;xDOOHvveH2?-Hu`c*c zNk+v5tT_K~-n{W1^)tDD|2{q1ZzVC?qV9-@h!(2G=1rSaFJ63p^5n_JA`d4IPtV$^;Dlun;&=TE^iOENAk0rne>QMo*dPPDK%UP?e*vUYls_xz+)drl&*?;lao zJgKJ0XV1uiKK%FJCjkq|wn)-WPh;kgNYF^Uz`L-nd9zig3l@_@-0{?l7cb79IirfY z+1b^_=C{p6=)r^M*egTTp@L+w9zA-b5vR0w{rdHZM#Y|1 ztr;(9?T)udnZ{~v-m*pAz#sw(+SlLjIx3}8=xW>L+F|Q4Vr+29XO&NTdzBdEfM4<9DS78O4oCq!a7GSq}+zL8dxzp|jc(9^f+ z^!&n{r|48}#y+4qat_GSVV~T)fB$hvNC*nx&y9@w4Or{oQ8@3yp{ca+^0a!94R1&4=bWok`jRitC6ZFeCywT zzftt{ab!!9Nde=k)zyBKK%88Rtvh(J@QiWp3tSKcApy&2>5jTZ(VnwypQ;hx-&;+3#=GQW&O}F4<7B^LUI)BcEAz z7v-CA)Y|0*Wwp$q@3P)DVHq48EJr^vS+{XxM>R^nxM_Qir9A!$5&?q<{7ddOOH@=; z++zQ~zwHR1h44u>4xnM_&d$qY*U4jhTdrUiv0_~)wwz*D-yG6RdSc08Sm4Z$8f=v} zj#4KmAHZUwk!H{~DR@S}u5{Ve6u3eDf(&c8y4s6}1iEgj)y|@B?5N{Ei z){AWilT8bbrs(I@x_`dDk%^0otB`ikx-^A+>$lio)~Ws)r*B0L#Y;w9L1V_am7`iI z$FQ613p(|4t<3aP#=DC-#_01?13Yyhz@jE;vSoq0c@3`UiUVGGZPCZr=8SntEoH+SxpfLn!v)`xIK21_);iG*9O1EQBksMn0Z3h zZe(Urk5l3wD;$@JMG0q47ga1ralEAT?ATi|fVilIv1(fLkIFXwYfk~PS?r4@Zhv}y zVr^MXO-)w!g;!_ylBJ>W8T*b#4Bm-xWUo#}{rS8~_6I;akJ2`+w}*+4o&!@XHqyu&$~f<9@5wpnu|>QL(+r0Rc>= z-9;Yqj}Is-O=+bYb^Vy8s{~e8EBSZSZQs4ye0sd6=a6ARPEEnuUw;ia80G%%PC!7B zRvb=NoX=i?%XidA_cV(A07@?>dk%LgUSxOk#*F}UC)OPYl#Vs08%OspJ}Lsp$fgC| z7mdn}Xrsz3Pt(0SsMtGSdwT;1>G5nRj>1@&QQq;lH#f2^&W+dgHq;cOwe*&4lNBO8 z*4@LS&2aYm_3QH3DuMgYQN%js_4L9r(ZgfW?c!SA>GA<&$2#2u}{H^}&4=_iLg`o0#$!W0$nRRCZ0&Q(W)(eYJhn(!vYwXYq zO^asRt>ue7XGVBeQS#UTu5yYf!jCBwbMNU-3i4pnOgQrXRB5!;STh>sXXQvQS*V(n z)KIASF|XDg>U6i*^od|PNPdU$cT}^FZvRoXU+&($>+0toF6ov=pW&2puOHjvGTf40 zr=hMUL<%3-J=?x75zpUMWB}WMj`1bAF$t{~+I}bkkgm!Ph6FqFaP3rqKxzSyX z3s*0%7C18Q@n3qssgSk|*e|R+H$9!>Ri;7#`RmEy5EKfyfDamV||cN03>FE+(S z-V>+&Wh^K_a1uL<4lj=@Fu>VaZco&0x+4pm<50oj*5~P$wGiRf<)R%mr)_L(+S6v2~I`8}&6RyK2K0A?6TcmhnE2#;80PT|MP_R5$A-KTiBU zX=!O~AGiLe#wTRa$jXP)OYPjfyao!43Hq0_-F2(?qS4FvcsA?_t0>CTy$ZAGr07nz ze_jX?Y|(KUYOI`^)h+S%=uJ*9aZHd_DAz`HA{&d{uGDXJg|hf1kiu&?^xczb)23r3 zi(`^y#|(D7)O*nTIlXkH{rk*J&D4SmLwV#D$M1ds9I@*0=jW-mX6ELsg67p%e;Bwg z(@=d_gs!MY*{yNag1p7=HEZKvTADfI_#S&&+5SM@+$5S|x6hoRf4jrwfF}VXQJ9`C=*0>E3mfB$~x-D^L>=k;3uD0CgoT9PRr(gX+r zOIx{`5ILN`AW^kzG2ne`>xky>rltzbdD$)Ix(#%CAP%#*vr}_mI_f)1o$P8+*}y0% ztAL|bQ&chQW^TI3Pnq<%$(K-hoR-sXfIOezyoDLXocc|A^5eeqEnbVGdCB-BipS4l zR6N#I7~bELWBJ&!9dyv?DV218q2v+(`qPB5#RkhHIStf?_Gf4LE_*^E(ePWDw@9lp zwg-MG0wl47rl2CT)FywX$R<6*Zbe^4c=5!#6o9o?lV-B>mxZ}>=WMXz1h;SBu0%u6thZa7<%}-&nzJi(=_|*hNJvOHM4$fr`C?OwuAp%{bpWua zV5(L!*P@Qoe-hhF4~os>e|+Y;+&sU(-MEX3Yh1MrKc58hSyf$K?c!HM3cO>_k4YL| zu1%XjxFs|%I_RlwTz+Apbz7EM5}KQ37ZHE(C%NV|tOF9ut7tNc_+j~1tDs<(642Z2%-ZDtvASmbq6Bds&8o?=_(B6gJzkmMKzGD~RsMrJ6 z4KImWSF6ckVq!AXmK_gbJR>!Cur&Cl#~-_Q$Dynaf~3{XWL4e&aDUJ6+l^c~MIx__ ze2q-o99Nb+W=1=-2~vp~mO#mCr`jfi<%k*dPu0n`Kx*h$?e>`JxBUM3EpvLsk00*R zI3oF+M-9{etS}sIPF2Ugc?uaZnYwZp*sUqeP?8Wc<`4(-4g&wN+|yt0^GSH~7Dbt` zER1B=HA3l%gj_HDIX-d|XG%&|%doE2>Of}k_055LBj*2JM;fp{A)m9xe*BMHT z_eiLZ=wfb$G>!re>!?y`aOME^!%%BxtXxZn?!7XXCr9~3Tmw6e!cU(*oroSBCgGCA zus>x^2)%B$IqKEc>D_N^1C!%S@7r)J=jfoR5Fk5np+Eo1y zaIY)ZM!Q+7o|0m-!XN?{V`5_ify0~Jzuelx1YM0LV(dBiN-A&g2ud%3v<^Vxrf+Yo zmj~|USu}#|yzlS7-{q9>mHc>@g=Q~npQqwAHuS~^+o;LEA_@y zWS4U+qeN`mTMe5b08niPP!XLM1pCa)#~NOp8?ee>Tv%WQ(x*hb;KAe@rSCebm7~Nr zB=|nvAYY)xbccd${895er7JUFh5Ej6F=BE>!NDqNbr13M%26kHWV1Er!OV4{!mBCw ze}1d5&w_LOB5-O`_QklkI0@mkdHd2Knd?w>pH;jq67yRrV$jPbvzW7WMvaY(jBfk; zE2OnM_p#D;3BI=|mXeYR007gcTG(&++G)2DvROI!9CybG9xsRQg{nY4Kq1;hQL%3L z7RSz~4$d4Rtd1yO~rXvsS)?l`ch(^6AioE(`qByxCMACRvqUSOA zD#U=Q^m5g(=kn#t&+w}Po^2<>%YVX}Unod~_ZIy4rFaws+AmGfXj z68H+t51uK**H=$$llk!++rqCsp?$;PNgArvheGtvrP;!&>nc$Gk@}D}Hi( z2uz1HAq~29I{2H{MkIm@>Qf>G%rjPc7@6~Br(`3m-VQ)a#l%K zo3wV9TaAs4)_#Tg`H5g+L@(%atdm)^p|m4J7ttll7Lr3Y;hJ~({ zC~vOA6W_lRIeixk%d^pCd}u4ruvJj?Wn&y=CVy6K{GTSX!IgA*lZS~4 z25H36ZM8JL3Ubh}YFo^=H`mt^14vF#qyS4u6hW=jFmG>fgNBmm=;$L_DG7ktNsv>R zTO=L6zW+7Utl=rNnykouH9^T#0KEar+cU5_3Y=YC!I?D^R8K&@C+j{)KV&ClR`ukP z@Qx{`I_ZIWrcNSy;p3qKj`ZUOQW)y#BbDg@j4Zcb5PuCU*6C0welO9Mpx)Y4G!&gV zcaG-jL+NSJ(S;qNnRsD8z=b-thZ^L?5em`D(HWhZj3{h*H&LdrY1+mntMO%^bDJ>V znITQ-g>SLU(uem$8p6+2&I+aV_4OHy_pX;7YcFs~e)sMjHw};R0{Ku#46r|yOBa6} z%%3P^)wb1dSdEIA!CCJZy$OjO*Kf7(GLQS<`G|VJEUUbdPU5@d*mj8cPkMTJC4;gY zF)k@m@2CWbNCY$NsL>{S8Fl}~LBUkrY%;duit<6i7H&bzTsi(P(DcGK#w`|#jcchB|%4+0s;;9<+YfT zW^$!^2-;?Sz889d=gRzmZ5*^dqJvQw*z{?JJ3zy`$Lx>bs2^98BgX#mKQ8jUZsxv1 z|0A!{@xIvtw2SQZ6K7Tz9U70Y*wN#ZnTy?Gv9=P^MwI@V@MLJ`;@!}*@|Z+6ZQk6_ zVPBg1{rgpDo=2ey!XJkH5WQg=r;VQ7>dLamN<-19yW7LSBZ%|}SVZhz6f`Q+XUa>! zj=xk!C1UVc^g#Ua=>u7RKE3f3a8fvh$YHQuZfaUKy9>adQG^fj_ykR13=Q|BJ z11~{ow}RA=q!xSIG9Ts#DeNNjIHk}_9yfzw6PwpXrNSU2YU3V0qeh^B#!^2&y1@J> zPMJuL8#H@t)m^ZkAcEYweftRXZD1P-bRhJ^mu@{i+T;%8esR2&DH8#6?nMxXBn){q zPncK0eivaAO6f-dPU>)O0m!imGiSLCG1EM3?8+JsUw63CFpt&9trIf|f_zh^sRB(v zYoJa=X8!$H4Tp-<#J?!xsX#`An&T@X;; zzdwbGwF1l_&X}AnIpYLvLo&&A8#nq;QE?RZ6+TPCW$4M@fBy|{IRPucV_ecXX30cK|>Cv}*;)764?-=1n4eL_)h zdLg%wKBW$w3JO&^`p0pU-$p=L;LZf1lf!U9`ys|TSPI0SR0z_~s6G>4Pm_BP?=}YX z&uu}Su}*Qr67TtTH$k9ldhuMhwVoYLhR5cYLQ^QR-?Hmuva5Tc^Mj%q zN^-(_9fl!Cs16BJ5Qsk+mxq7`S=LOOSlr?Q;HWdsoAp2_`hZT9^?u8R=|PVY8N1Hc zndIt$TTY)k^$h_K&xOHw9@-K_J@GUiN$fD&@Ei$k=qhwSlZrvUg0>9@1 zaD<6FnNx>*aF0U|6*wIf7d<}YX4t~j7fx+~zf&hQ6|k}}%=hZmtHS~U7m!8Kf;9qL z^ol3_ICOR#t;=oZzJ>~*1x9MDY?5W4tg}&FtkI(5W*XCtjkoOPR`vESg6c<{68Jf( z#-&C?pyLt6;SArg|6iQz!-p%)Lm^FB4K}<26?*Ne?2c!Ma(slFTM5*O{{HR`5})zY zcmSV;kv(cWTe2e@;9#(1ekeJ8w`w|A#>K?nE#g5k-mGdP)Ry zrB$NoN+$^p=ekc^3O56Wx6>W_;?G4LnF@q*{-trBG0tx9$~q{~BhF;C(S|_CYTYMBAU3 ztt;#QDD+{mBM|s3b+P;7GoWDhyfmO#bii?T@$VL(BDl01+i={t)^e6z$oGw9F zA#D=HMgG?2T}m|Q7z5%}r%s>V3mhJ2hL#Y3lUJFetPsREKk@O$qy6XFZHDpjuCIfT z#OTG-5=C>d5G)3OG7aIL4?nrF%$h;A3X2@A5)9lxQ1D@ZG!oKq^_qZYc5l_V53YqpY81kbfx$urFsD z>Tx!ADP$m8-RBXH{duO>0Os{5UZ@{RbAcHd$7sC||1rg(6`}2JMmZZXj1{?3{|4EX z4E;JHRn85+D=H@qd~m)&Nkikw7o2Bj>YzJFG}hJ+J{^V@gG3%Qoq6E`*-&@_99jSkY{OQe2xZued%~3}hp-fYIE17@xvmzi zv74n={#LH7G)#y%#{cC+lx1rBD$#f2DcART&W>2-2SDt(gwKCtCyfOuN216^&=y3S zVynXw0s<0?GeBWf1hQFVr|fHCLsu6u44I__7Bep8@>(}0nCQ-WBCc|ELB|Qyop=(C zY%B1ylsni(Y@T(Qg>bc$wiJe@->NAe*QeTKa${YsXe)Y#bHKuE4<9}huiDw?veUHp ztEh+wQ;Tu9?s8NE)mEQoO@&3uil-^avx+Q8|DkNcK9-;sxVVc0ZSMt;&nenaJM8fn zAH&IsxAIZQI0+1?3hhj{{0nD${BrbpW7&Knj0A#`e|x&HX$ zVX_Fv3`~av)CD3YFj72De?n6sxiMlWdO}U)u_&fomY^%|Rxn@YfwXJp2{XM*!oUo= zNdbeRKaGm8TPK|Q z%V5KYtQ9Onn2g#RP_yx(sUWj?$l*iAs|z8P1*xIwzVnPdwB?sj+9(x2pw%Dw^UpU~ z#`AA&%KX%{@(0wkhOV(&dS`Y*#oc5va2J$vn(i`M7DL15^jlEp z#$qow`ZSJ=e$arZUFWl5T?95#644|E+#j8y;BNb$6=&iH&da*Q0RU45g1%;SGnLf? z-dThh6*yax)L`7AJYs+|{o(d{R1?k8r%PhY{U1nzqVMA$bY}<(sfLBkV@#tUAAj?M zcNxs>mrc)P6|p%aDk>Vkpq}q&M%(*-DvQtE)01FQk#y~|PY#)gKh88^K{TRj!WS8& zAnTc7R5L52m*QwItXtAZLTkCULzShMH+k=)x(IR8w(c3k3#J>+@#)syF^K4^phAf& zS)SFWI|3^dbRE2_>m!=&&?^s=uTOC+$FrRQTnmc>MYe=^NfHEI1<*&QqgR(I%-bPo z@X#RI7sk;((CO+>i!vDwhw}#ZDlYu!4OXsLn2p*AF30egAVI=fW?2CGF4)a+gMf3c zS6pY7oU65+M83$mGiNfE z3@?L9Q(A1$5?I9@Us`(l20e!78%s){(OMDHYinx43j{VW!w@ z?98+6Xcl@VBP1l$5o|(K@*u7w1ZC_yBZ7z$RNU9ItAP5pD;Z%;@njpg%OacsN+Gh$ z4T~cQ9f;d_*7wOWdIvYXMrywAF=Y_qzJRjeqFPlG)wUxJAaDqMD6bmv$UTL$iSia` zg_F7CBnb=4(HTTTe{9FB-JcHseCC#x?h@!~X=$=BNFw$Kw ztthbd2yOfCzh9iIU@TqSZ9s*1KlAOA93W@Y^|c$>CDKC=2h4m)AbcTljV2iVR(P{Q zJ^(V}-RBe)KZ*8uo9dN!$G(`NUHfG%_u2_3r-)XNXtPlKczf}@Wow2BWpMW2fB*e> zjESyox0Y!%b62NBabCA4ZE%gM0=ta2j-W+tBe5t@u@XQ~kLXiVAQCx^C_v;9O&tKQ z;2qcZ&CcV6iqV*?%}1Yp@?_^|TlPV1+fsmCe7{Xv)!7Dc>lsMeSAKl|GF1QKCeaL# zyzubwVCdLuUBB?bq!`gf?LE8($CSyBX}&nQo+hz@iRGfl=qz?$Uy}X21AFxruZo9% z?2{LhcCAXi@wyz@CXzZM+?J%`Ab=-bcy*8LCP0h+);%fZdLC_i#<0cNMKcWeL_6M+ z%nsT?7{vpY#we$pqn8YKfI~=A&1RAY2GYw#TRyFzpueGhg^mN3i)=}xv{!RZHK5vP znsK)EsX@OUuC~H7hjLgy+@)K-H6@227V$V<1bdz*eIvZji+kT6R)lw!Id7;#!RJcs z%6aw*aNDY@AlK4U?KN*5k5{j+SH&^-y@==GNfiziW`{59wYIISpzbsC3@2K0~O^cvwuDbj*DR6Vn` zgJ=gP*|Nfa<3A=N+nCaICk5DMP!yb+d=WBf%6GD$@AVb2>r7BU$hN&llu#PfJ`iQ6 zC$uoNHmluFP>b7_PA-F#G1mn>(KkP_Fxwiwzr$+xZVQVv59ohcHA;~Y5h>`QB!o?Z z0K}MXYeq{;k@lKRKtAYv?(k+9>21FX!(~Xqwi%utq_^ig#pAr0ixtOcOIq>3qo7RMq@~JrC_s|QaUQ(j@4G2C%&=(}Nr=M1vJrGeZm2UW zK=aR26gU97wf{tMk$A;|=^$)S%IL;nbciIhnI!$ZG`L17cpOPURIV+Gq}S!xb+tRs zN*pCbow!TkmeBumA*w#+`REHRC$9Gf0(^R71M`RZ22?6%9lqkmDQ4_#+t@4BcXCL3 zHVUQUNFjC}dagr1+dL4A=4I*XO2_^g$S?UXqi%fd?~lWUwbisFX&xo_6E_Guz|Av! z3o|p7O~;}i5=0Bil2Y`6FDd9;DYIJ#{Y2Mw^bQXvBYFF*`?(jQOnYyCan8{2W#2gU zt~;v$#Kf>Y>giHZIb6I(9F%D^w*G~~CZbie5><^le{{zXe}Dh3gDzbO$I1Lj|e&+9V$jqKV}A;>qa460$fmPkV;7tCrmasXO00kjy%0 z{Ub|HFe#3L^H-2DHG43JA`LNlM5E_kyGQxclpv>|AUa76Lc}P{(5}rVFt4@4Vydcn z;??u?*v)LFrY(-M?J3V|V^N!ib60T%aG$odMI3*7jHxDCcC|gT9+VM&Q>kl_<)@fi z60RdxJZ_S49MzrwrHt1-t0{=s4_^fBWYMJbIH)1BNFbbTxeYUuIH3Sr+D9l+Ua#O(Bras7Oaedge)u*+o8` zu7_+&BaBMhr_3W0HM` z7Gee7=sE>WMGH~MCU8~X(c-gDk60qO>xr9D&dYzN!xe-n6}hx_YXl5oV!VJD&Ny=) zIXN(~Ea~R%q1FpJ*=6TuE!?st{LWH?cW5nM|A*+hcBRwCSW5C-QyyDvna+5{C|**~%(J zvST}U?tH>y3r5>=V_k~*9Z~BJD=Lq6WzD`j9P`&}%F6bMb~%I<78XMEO(DB7tA@y6 zP~_G(9JBxgxPqhXYSA`juE%wSo8Km%y z>Su;kc6D!)nKi%8cZB57pK|Lq2&jPiHzHN$Kg=CqN@;q0f_*tN=~#0pDhkm=Y<}si zy>s{OC*+|upp=j^BQmwK$?1%ddF#)>H9akQ=IDU~<<89~0Oz-|iydq6`FPZ8puRIg zc69wfd*Q>1Pg17?i)}l*)$c}3AUyP1ils!%vB{z7VZ21wA<>E;&?l><9}}k~GI&bg zS5)Z765z`xfR(S4Sr={wO+QsOKxfyz`1lKzl^h&9Vp>EB41US9=x%ltCS*z_`-1$$CZW?1G zSJ!0pF**Ppi4UIWeoF?>gC~4pjdUaF5b6E3lqKJj$FRaH`9I?%hkn(g+E|8`0N?qW z=Tbj#|KK?XbybOujKRf)eUo>093*oP2*>vSaLfgBb#!ui93HM8q!1DJ>XjwdT*7lY z0bR{Bdkm>l_%lUWHYf~{OK zonJh+^gc72DSHu{osuu`{M%HiIc{=S!XAs#iKUE6C z$O5@!jSvvH07o5&%Z#Nww*^v$K|^rN8_kNQ&XJq76og7zK);pf(o#~>IPCiH3^fr+ zfT5X{nunyOcvnTt`|cgS2D!XWPHC0FaJ6YQ@L0$1S6F5bHf~V8+iYY-L1=Sbs3H38EO6`D>bH+MYUGd?5OD9rTFj7oQv&VJ3xQs5v$KEaHN!cS6I$l0a?6 zq`ii}l=%~-pC&~@YGw!#cPU%!X@Bhy`=2FDOw>g$>`n11_$c9255&zrLTv$4Q5g4y z5dTxgp6XNre@y`M4vkZgm*0KSKEiZn`q#>$@w_hAaL{$-QZcu_yIW?XQXX{ZIfgOR|0HG3Q9JhRI zu6&C5kjq9TYCF)6KBV&|L>6}8?vZHJk@@oXzI}2#4xVoW;La*GW*IK~!+bzWflQcS zXoyf%rY%2YMFBh(W}RD4Y>n>jZut4s z(6OYt97o1kTANrYl50OzhUMt#URT%?+js0>5D%}e+o5Tw)waCNv;uZi3}EiFI0%s{ zDB6O`d}G6@KT{MC|CyUiT~^=^XLtqiFcS_C>iez!FZ99uYo5KHWZukD4<2X^yu&Wv z#nCvAd-RgeALr|%K7RaY(DmH7wU-x)2stvO3c|IGA#-_v)MS4_N+fv}4jJ!OfzRV0 zjLpE6lcNXje68-L=0Z#ME`Hdl8^qoMMjsL{viYl2?-@x^wq#p=&|<`mhzVxKaoV!e zFWYcS2juWrG@EeFXoW!5S3rvSFR)wa|$Q>Fa}_9f}jWu!fkL`C?8CT->5GRTCVPban?1u_qs>3 z@o&)=z~o&Jq$nP7=Oe8ovXeteFeOeqSxXrMN}1JkBG%*pf{`4Vcz5Ya#n<7Rz1cRM zulXys2?kMhF~>^Ny+}p@={eyKi`z1o$};9xNT{y*-z(#ofvo@glN%%#%-g=r=>*2) zx%u`VQXS(g(e*u6UUWu5xX_CW_i^p31+01nBqE_{KGK>=o?DkUW&%W%CB1)SblLKY z5&?@g4Ogl8@<}*QczF!U##H-Zhi;!U=l{X zT)Ca3#i!MpaYMP@dP)vvd~1dgEX&oez|$O08rzuw3+`DQK0v~ID)=didsrOV8WaqG zj&x_}MzGxe@~AhdX`)gyr8yivdspb<{YB60cF}(<;TpRRpMhrs9?Z>gvH?9b|@jB?pSr)cEX)(^6GssII>MeBj^YP+|ayGpJ=R^_v?iARrLlCSk zfD$Ov!U7+o-@^I$>#w(yg`@bhm3{UnQRa@n{gceDKfl8QOl`IRf=|Xr{f7xrBDoBj z6kqZ5qb|NzsW<3|(rF_z!_(mfq6t&&Q|>Ao$`?r`c({(Vn(Y1kr)G;Gvp;EV`EX;C z@O#4I^zYug*RV}?=|OVVe5H-+-(CoTj`~I|MYZwg`GmX#o6wKbpXoaCoJ_e9ZfEI< zF5* zqX4mF5YCjN___%TbfKp;p<#s6>$-xGQwE`&>t&lOr!8ncfSevfgdJzC{(yi0b`^d| zJ!-!Lbepp_mLQ^8yfjIRGfTZOct4v+GpBx=+PE)GC-WqZd)y#&l=_3w-jCvCB4m8X zSXNv*az2biUqNJwL(CQs7^Dn{q}X*%qGE9uB6kYYxIcf}=}I^xJaki2Q=YY{WS^kU z6cp%->!jwTj5bt5qXSQhGsqidFw8>jgypi6g)AH1J07S8cQ8UGJTIje1SBKkQ~gkT zqFAOId#J3RZ)SJ!lp~XVk-NinoAk5`x{RTheV&F6fNyVWvtsr(m7q>zuyZB|DxBL zgedcJ9%HUb3t<_OL=CrG@4)f&RwnOt@9kZl0>)So+FzV51`yl-u_R;j6r!W=nazOo+?@xN@lLPw1f%9WItr5 zJun<&zIR<1hB}UrA(xt*#|KqlM;`9*!lhR4R_g8Vr}S1N;u)bh)S5@4K50R%n!;qsOtU`11lge?zDw@#T|+6kVYiN^6Iz9Y zJ93x0{em3_zeK=FC1GcA^YgR@q=(6dc1nhu zNNpq7!e(}vgszzS2~L~_M-GoG{r+x?>)Pe1GE6tk48}*d`$)W>K?~;b`=@ZiK$T@U z_PNe_YIqkrdU7&KUcYw8=BMDvoVemv=_J-PMoD_>X#lWq&T; z`Yn7#pX-@k5kDLW--o?E$)?pnNy=-@Vq;?1S^aj4M*GZ$60lF7hOfz6@oe4`G&h&2 z!D(0W3is?Wy0vakzThdA9#u>v8 zp>bT8hJ&B?y6hKQWKS0mzif(E;X*m%Yk8YyPH5{|!L4Pg5E-12NiB4NUl8tVY%=46 zkj-Afa_0at5-&Z|s7}f5Us10WOlw5co)*4tcDyGY5Vf;^E9=h_!=Ls1Do@1YVjs(o z5B@semEkWDyLo&4OKYV960RrZi@$u-oj(nAwGjm~25R@^bAn8AKy-SUmu?xke_2lwY-E5MNl+(gFJ0e4$XOT- zOCwP`XfX?Qek-x~@m40Cz<_|p8|$|qn6J(Ql|QQ+uwr1*=H+dZD=0)HI!+>bD56%# z#*?NT#HE3hXD;Gs%Zoj$S|s+oRxN9KxTQ|0hL|7EWsv(JXMgrb4^r=NN!5^!3s$1x zZ4U3pf*nb{nkLib?!=}d+7t0-kqfWR3WAE8jD3+mwsxfsiN1IwN7|L9W@no*Hlj&3 zr|l(ZsT9FP4=*qMTCvcmD3VtAIYW+}>C`EP&yGP>Vi=%K9l|XHfc<1k;JWWvMnZj{ zF8$SVjA>@9OP>fwRIXH&2(cF!QTRrtV0gl#NpwSc2p%h%I!|;eG-Xw1XXl_I z92-Ii$%_GWYUQwg|@_qD|U;7|r zh<@R-Kqkr8J0s{Zgd(Jk71{+m_P&Z{&feQEP&|%RILwd$dPiP5MqI3PefvH>obNiV zID2zD2{=I@o^59#4SfOjYa`xTK%aUQi#l#6zIGX~N*D!`tcrNK;;ea`^G`#3FB2J% z9flu4$42)VErw5j1yqbUB9|^*Qle3>d^!o$+*#LnOL+00vq&S6R{(?wn4JyU%c*|h z0@>A=R6Pr-*NFW%m7RP(yB*E`^>eJ9Y=?*+rkKeu#Rtm0=w71e$I1t0b&g?HJ{9E2 z3a518^UZBSek90#BftzEwvH?{KgQAbrUD$62!BPt<{gbCGYdd0P!XFy3?en~BH1Ys5*L7_CXbptWno z1&}rk2x-D-X;6xd$@rZxoJCPR00qV3EJUhEsuKT zACVDuNMfE~##pJN`1>3goMfn<-ttxA$QD!6GY#-zc<4R8i?*mNrvbogGK zd$z-)Hg%@#qWS*t1tjm%mpL)l7l#v_2>b7rL^qW+T5)wT$`V;~2y;e3K7yroG3(%) zK4t>?*m;;tydtx`RngTL_)i|1a3I#24Md3f>BGxPQBrM*GQ_w3QZP5t_BeK_yV#3^ zw-m1s@n4w6w5_;-1y(8$?d7^P4vj^rgB>H7miZ_&t)otMw2kZ5J)=xbOz;yR zgYoDl%zL@Ghn^9De)73KtM)!pNTJkmMADxi0k-M4-}pfzK7V)Rt4vUj4|@3M(WSOo z6p{e)mW)(dPfyP#W@ZuM(N5u2J1j^K=GX{g95>PBL3j%?NW)c{eOG(1+pp+$Cyl&R zd_4Jz{F9reHxJW{hvthJleWoi91@K}=|GK{P?PCY_(nGHDr#QL=wpwP;dbKXbF{|; zs@PIN0MGl^Zb54I7>Tc}TfbgBm%|RSzvFy=IW#+-5QpZ87fDtmM`ll(aTdZ@$Y z(N)Sm^L~elST5%%2i;(1FQ$i|B{TZ(O`?LiS%r#CWx3fLO_;Hx;ZOu%xK?~x9Lo`i z>dS^kEJ~)779x7Qrvl|%Fv(GsfT;Bs27~e4F_(46u4dOO!nX@27$Xj_7vkh0MOt=v zv~94@AqwGw2DuZE2YDF1k?jXZ2nb1=%_XUZgrj_>*Dxei^}>}*(BjI)bNDcg#fBHV zBwL2x@?fvN51mE4C<+%4fQ4j16QJ>@<+Z18>1DNscg5^;0J>`|xo6;Ub|Yr=ACb{v zVs_Hn*^C*lmX4+uJxX4#dl=Fr?0h1)5XmaLupTH#-{C!3G%B7N^0!~xYfxLs-PwRM+MV!34$*`RqjoeBC zdKS5~Sfx+_*U}1+jy!T>`CK@%pAt}e5NF@-Uy?kIH`3I&lF30F@OGYP7hdNkw+*)+ zE;`*cVJXKlN3I0!A4XgG@`e@Rz~jEAw|^!Z03T$?%q&TPh^CvcNFY><@gWZ1Y6j>N zDf!|}@eu078l&MiC6KYiZu|e<^)*VF17(X?9`Hn_sQJzx``D}4_UwsATx%Fs{CR2S z9I-QaPZ(a^0@V!3+izF%ak8yp^v|~nE&>=ie|fhauPf5S%d4E+cZ>1y@gY+5eryLb zF6ISDVkPnn0>%(miJUyIl<;FMF)GR(^v65J$$dg_dFJS3i`o zsAVVbco3Pw!6R>{5s7a9Q?aC^g!D%EZx{3IF%_qV_$G$WBHQEb8)010+Iq4Kpxddb zsfn8_W_xZ{Fgc^50{;#(-_`v3kvs&#c6K@Wj;av`K++n_13+SyFSzKN-=RZ4$Nue0 z8o@a=>Zox89ERb3+3V{Q*cc#%l=#{E zRWiaDRV7-JT?q#6@aHp@;1!pa(ETJES1 zH!GRUyE1$p-mB&3PX|ly|IYLVrBM~5KbK5Rw~UHSfD=kO^FCRb^Erp8G~CM&yr-b9 zbY%vzNy9nUDUxc&{Mn1ps|3FlV=j_;?u$7|27_Bv(P&4t8TThw+%dkChb9jA>2}Dz zc`g=V^-9;#FDc~w7^$_55vd0u?__LR*C`+;=zxb2oW*I)T9kIRga0=O`n-ATR!!vp z$F(uxo=9kf-F%AEm%zo;SQQahVGOF%;v0bPpMTr;Q}@}#ie^JbXotBXZ4bl4 zlbP?^0T9_Wn1$EzkSs-)H=*?NH?L!6hxomD-taRy7Ss;V5NNo(w{dd^r_Zz+K{2+5;iro~DaGXy0wVVdpH`ILibuTtKMuL*G5Uak+1^k1N6x{&#T@OW1yeIr zNJ2{c-FsSEv{8#rQ=eB1lhpO(fk>F_SAZ=rLSr>15w(RN1LB0m)5x1bFtq*ApL16P zVL0G(af3u3Jc*Y8kT=BmPfF~NGL9f`>B4K(oDaD#dXkx|8q>Ek?7uxaq(uG)=Ewo4 z!XYiAgM*1U4XrQO4x|r^jiuq7inCwTXMchzAs9uDo?cho+*(j(lTfGo)$44?+6HX)FnypeQCR(mMBL_>GBnQzp=$rYZ^WJJW1}A4N$(u zK-La}QbK1C5eTS753d(W8`qGTnw};zTaiW~klUI7etCi&W`q1bxO6gbSwydT5x(dY z3(dKy&w9spYR&$0Vbt_4b*zuS^&`#m)Y>b|9z*^p7~4*6Kf;{ znp1R%LM^FjhwmZ6hFR3toRL+_mUqE23klc6{Ceq|IXvp?Vz3O%+OLX)>H*_t z5P_C3Y}O>NyTL`CmBqW?Fx9m=m1}h;0DJ?yUz<3H4VC~QaVO5hAfkiOnCi`u8ne1G zR~k1+Rs-J#^ZT0A>T@GRmIFB?wRRoRI9Ah(y#th$1j&iHr2}e>ZFi9_+MvgfvEAz- z^I#N`Q{ZJ8UJ*yX&8Vre=c`p?pX2R`hpu(++=}_Ic1&fJ`pADe+t%YuJif038=oUX^Rad5^{j%c36S)?NcG%{JTlM3#54=`l1Q=j0vt zzdObqzuy>#anAc56?mR!fA?N{tvTnKi$;q#M$3)O@0m@&%YZWVS=fFe{&PZyYQ3NG z8$&9gIM)^@2*Ihh3eU{Sz$#ILe&k*014)zf!LnDW?l=;Ir-Ti34L^DK-mO{)4el^o z{sQ>!wxB3)COS@4J-hB`_;oL|Q6g)2-FjSV7fRbnt4fFbi)+jP=_w@N>B$wNbm$Kg zPc?*qfga9Bb;*(&a2B9E(Qj6G`4nE2T1jL|7g7>1F31DYlFs*(FV@~(x#`{+oX5g= zcTR6Q(Zob1N-JZ;lhF%rwym31_(y>lwqf3wc{0PG#u~u|nX%=#Rc^*}%zv`mtICBS zj$rM2o#xGTv|nMtbIi^C#hyp|xOdApKl+lryVbjy%~MZ_JK_D^sgSwJln}m?Hm&)2 z%B*a_tZ!UF59y3EIN1Qo-r>TWKfBBHFm#dparn zdZMUoL&^nj-BtfuPW>aqFe|G zS)6#PRlHq{cA&N>dWL}V0@5~pmddZO0W+4qCQ2^fa7}+eobQm>LBi>i8y9lJYQP{! zLk=x#w~6kab*@RhRe?(!uRrT)y!i*|Up4Pxqed%y4SxO@-GhI`M!rs@j7q`tc^MwN>)l= zU=+`w3S4A4u>sMCA}O4-F!8tS!PpJBEG{@U>Ib|LCkmA{(PM|9)m{HhWhViSY*E0Eu)RcaUUF&EIlYJ#C%KiE z0>&@r$oRXJY#UA=vbZ9UrM24@5=u(@TT%>Z^HEZqUaW20S^hAL8eTN;J$m<+@?RSL zzv`H`6wgY)k-NtytpEJP(VV{O7JG$Q;9=-u9Q{I&22ykBzxlHNM0xw0 z#=p!&C?t>|XS)Yl9uCKZr6Wd?)Wt(56N(o|P|l}0Pr7*dYA$zj+YGHsC2;aD0RF36 z&y8Su&`=nGk1QuQVucRXEec-C3JP9F-&wZ*q~%0|_0L-p(8>zN&0*L*qqd$$60#_; zK(NM>Gd{ayZ?^9Wjg6=9v^yHG!$ASYF`ooI&Z*V_ZO~lKuQwM9u&=nFOu_ogAjRbZ zdwE@LMyFq=ym)3%l)~D}<|~OY_vrLhzq6Dx6PM=TdHHr7HD`*SwCU>E>2+L4V&$qOgw)y1=5vz+Cq*zVr zj=N0sHXcsve_ifbqOMVL|+(p$O-lEdMHe#g?H)oft2Pr;y96h>s4%7KY z5J^=O*-I(7g4*u&64{ZEr?)mf+r_?;ZdUpkrPYwPgMG#@dKf^`F)Xu@4=X;%G3iA; zAHSP^M-xE@2}h=eE02GcQ$@52KS}J*M?M(k@4Zg9tQ3 z6%}W)|8==czqquu;|TZ@K%u3a6-t~VsE!d7bnMh=7~jU;8**Mnt;a@MbVXV$#S&#P zD|%$ek{>_%=%dd&cV5D_#D7w!=-!4EEDUer6g!vxf(+VUI3#*ti5_KchPTVXW@VQK zR|-1Wp!$qBppBefWLSt&TwQWf)W`L#I09S*rg2jD5L*lIea

er#nnL^Mk@k}IFH-> zzHYa%Z~F$XHSx{v(X*$ex+_K`f^C1)gkAWFwX{NCsqA}=iaP{Gn+KC-;Y=>Uqx`&K zba{sk;|Gs>c9rcbm@;7{-p82K00Y!l!Kv05V;uB*R3;UWRF)da+)?j91>cm6B^n?C z?(lh|lN0~ru^$yu6|CSW|8LExE_|oDI{)?hqVxlNze`rV-ac4eedkZlCD*03W&Vx_ zN}*sj_*{3`bL6P8vliPj59#!LhoIDQm;mt?Jtj`uq$;MjM_9PST%O7=@vbn;<*(9~UN6LLmg|9R4KzU$Z1 zoDm_<6hon5r1bUGus%gbI|L)&oj)OnT`11!)M;TzQZnCx?QKBSGI`pFejT=L*Wj*J zLu>3QrI;+9+z;ZKN_@M)O(xCGgj9K}qPHq|Mve?zIFG7D^|u$FY+G4aalQ|sN@NBk z4WEDYm3L}tDty64+U#Y|;S0AMfWq9BfS=37lk}~Ee6(#Q+uJE{_Y*jMJeRIA;_CV! z&`9wcH`_TFuY293se{UU;OT6;Nm&c89^dLu9nN^=`yXeK4qx&Hdkh=8g*dbKTNPX@6QcYI2AB!Q))su3g|W z=FIs^A9ZN{1KU^FdgWd$yu8mpse1Rbw6V?K`BRY#%y%WiQmp^kyWPK__7&+0oyRMR zR({ms#BHQ!&&jXzFK>BcG_jj{hweRuKZKRa2bBA_9sJtACpWxJuPRTz#1#@49z)?J z_ETRCoEq`Nj!qYnEO~+BsBt9rUXW%P{V8v<)P;2sSmMH^H+fyRoq7g$u$VxE%Fpw4Vo(wm-QAs3+W*Arc*3?|{vy)+Jh?Eq27jf~n@~+4JH&btsh5%Q z)7U(R@UBbet8l#F?%kc5KI*X6hnmiY)cqaQn7IG{`s=Slu}^`JmIKcZ_aJ!lmP1Kr zU^CA^yqqLr_i!rw=ywb|%%sMo>@Fmwwi-;{;?ka8}{*!<>>Ig(JB8}kN-`0`NwknZ(hzn zmZQV}rVssNJr@6ONYa0=$Nxr7{r75+v;DtlfLGE`(WM{)nP3cRG@>TzBTd+;qB4sw zN~qWD8%=TP(0tSLh+EH34FlB`6@$)S>iCaO+h!%6&_+3`Rwak--xr|9LLZquCG$cM zh@7^sJDm7kz*qp=Wt64M$OC~yLdBmB$AO(+4~#;NCBvgS76gLYS%K2p6h`?=54cqM zIK9z(u?*FtFehjcpM|}khg`VaD*Rmexezhw2NwiKkc?Jl8Buqfo7>pG!?sh(c=ri! zMPI0zh*G7U7M;lnn22z|L-7TaRu-UgE9z1VNH->}iS?V*e~UO>4xHA|CeHj~FHF;u zuD}5Fc#6V8YG>djv1%2N3#56OIM7F`$m_MekCoR04qSxXZYc-_qZ0;0^$3hYYbiDl z7X$<+JswIk+XJ^Xpn^hPiA47ZJQ$>cKrMnp^~W^jfiu8vfc=DXSlT~``Fr)>K66y> za&<_}!V}NM4g%Ec_@h>8jkDB)9V@;|x=UNVK+OH>a5~hN@l0>O>T!Z%ls?-96Ux8- z@=NpCmObH=W5jK?UtJ+W2}gwwJi-Mcs?3f|Hpb(q@R5mTs|r^(Fak+%NF)x93Wx8rwuG!EwE zQmHjMWexn2VM@{;yi09<+81HzYyf%hX0VWY7$6%wq+|@tq34s?sp z6@IHgEE~BZ`j615Vn@;bbw6--Z-(>D+P-X*3$;=`j8A#om6yLy%ZJ($q<7c$2TGNw z?xfO_M>(=}_ujor!E)k2j(2_RhSjSWsLAY@ukxpv-|Q}BtJ97LXWBX*fGnH|E3p_c zz{#feElt~w2zJ_Mf#HuP{2@w7qYoP*I@qz1(AeGqedzbq&ZB}ZIN=z#xD#N+)bb1G zzsDSIbKl-RVQEMgK{aYd+;8*4>#*GA%>qAbfHjbSx9$4b?A32+ptJ&ju)*96=t;D2 zl>jR@y;WF1>GVgjEvVeldP}$qfD?{~tBZ?0hw{sL8W8CES%(uzSasb7WQYV+lv#h` zb>UQ-OaH^42=?^@1BEim#DeRAlNjD%h?%iw>hgU{qIxoC2D+`}@{$~oO^&xN=ur25 z{Zt9s{tQc#aXx?tVuN|9VXL!q7~5Wq5@;t32S-NxF^_jC64gkQ9m6M1EU_r=_=SSR z?c#FmV(Je2`a0LPp?!FM=BTl!BB+5C=#=1X8&MZ&>44BCthpz6sCjt%vJs!-d%u8X zJ}d2jFP;hU*#7l<_73*j^pids)}H30-qTbybJXOQU&qnFD58N0?#b6{t`O%LQiJ2v1uLvXQu>)QP(>CM-1E+ZMroEmc<|t5r%B6UG~sh@^s7Ui ze@~#K?c2A{&s0`ad`>hAt~{6A=nIx|1{1oH`E7-%AI{L6woE#8SViB#Q!wo0p7jn4 ztcY*=YQW@UYBDD=rgareG?18)jAhV!dVSkl^GuRF^cL1Q@5ds@s z#XOjz^=AWNAfk(R!EMa*L~Aw{jviU^e893{JTp+|Gr(yzC0~a&OFy48`fN-#=Li`q z(+N%2(cf-h%2UIEdz`ub?bXmkAe(jO#`}>E(5E>L5 zijUy3w^n!MveP_jZ|q+Gb?45}`VQXg+?8yRrO1@|0B1xr0D^Yo+awh&F=uBF22XmC zG=twKXqLPQyoz|i1x_s{4lU77&}2ZEM-6a({vBeOMey}UprX)-;rx46w!OSDLKt-4 zP21QqA1ylZ_!%+cr}bejJ!rxsLWs6QY`A0ytUPaIauA%15d^!_wB_r@kiza^b7T=#q+b*JZ_ddoW8Ew zWQ+j+GzCbd7ij{92e@&(xL6(4Zws#Cr=T_ztSUS_nE*-Dd0gbqs6{*VHtBU6T(|Ga zMt#=F^i9!A#BYX;mGHC9oPQZfCltIi9Mdix3JO|{eM}(&?fBJKK867*Zgn9wPAJ|* z>SUA|i?f`N0c_{PiHVyM-sCn2-YOG1<#r{tkx6N|ZO9}vs6IV_(gsOsi1&CD+wO00 zYwd*yF3Z9_dw+=DqqAiKdln+92mNNyif%v(~^$!wXPnXw(vPG6feTNTZ=qE``vgXeCO_2v_ z z!?feVG>f4*g==T&BL$|@PmvDf6>tO^{)eVuJb^FpN$V+Fdav&pM7%BT_l3rqGLc5b zblG|I!?+zTGUX?)eF)6zx2hca;6=%Y4nGzwtC$#yiS9n!Yct)L`d4hxeEard zQ*zdw&=b~uDR{0U2cs#c%h8dk_!XRHg%*w!L4tlk-@biqyI_DhWXY*tW;EP`aGwqB zpMBBsk{ew%heFJC3H5#r!OTXHJKwxi&fq#^pjStA!COCzu8kw+A4Z#A?}f)ouxoUQ zo1G64<3V+G_1)LFGpIwBn8l{BImF>^lXHJn{TRhX)hA9I>pv$Z;k)oh^I508WbqTq z3J5X#h^m=Gbc6TK$UHA5k|e)s)kZWAWXG_GW5^9`HnqT51R!7iDX|WLswU@oe)Uk^ ze*2&ob@jXrZP5r^$1ZQ#OwW^yfn7l>{5_xkZT^*sr!jo-tbTMXvJm#`_1!>Ki&7W3 z>CrsW%twic1EQ_&e1Eil@H~jeXj;^drqFV3l$*$O5Wma^=pti%a8ECX+mF?M3QxY} z#Uko3`-0Yrwhx7I^9fgW-pyQvF;5Ir-IB9i*x%}rTTu{eW@73RHND07u1Ig@1meDD ztzL%`$MJDPW1}Wa>H5d67-JkSX;N0obTG4<8J>I9~l8JlP>qSXBQHpMC-fTj~w53XSjgRY`m&6u7@944K z8yli?mY`dTE^ec)RIr*RX_{Z_%f`kOr|f_y6-*N*i+ljqGL7ER1;mh1$Vz{Goj+cof96xFQv z7WSbEWUn*}v7wftBR2ma&ezfmPsH+h_JMh)OU0-+?+MNL6Fzw^;w|bN8m99qCq^cC zJv1R}bS9Ov52oQpM+$XDQuhlJDoP}xG1(n#Zjr}IG42X;7p(q;Uo*{2(u+i{T0W+N zqOJj}C+T^gVBwxKhUO({EAv*rjqYX{*^C~{v~Fu$Sl&&(l{`_U6)_r#oM}|m^o1nd zchDcJy-~yu92NXU=Y}wG&h>PnWq0+(b59DSg~amrjWdx1h~!;8lA1!Zq6uyuVhr*s zw4b#k-tgky)gLI)tk`l38&rm87i47#R}ViN(^_YDk3~vo{=jw2REc%9NuCw1BW+|} zD}~(f(yeIZ@GaBhV8kZ3y9GS)f`F%wo9?BFr7fv#AGYqlObQ=;VWp_^C1F6P*#E^Q zlpxZDXkT>x2Ojo8re|F>=SMeEEmg5g&u3wdCW%VxyXtda75bkUvR@5qHb2o}l`FET zg-2pHe#ktx-o#Dq4IxnM7sNOAYJ+`6O1p%Ke1C)J_U;X|%tFH>%~AbtN){-s7*6TU zg5 z$sYr_vHa>LIoBR~5*iYaD*_5ecoN9kZzD=HnPVeuhv;oL&Dym7{Kjk%pi93V-&8#3EQWRAmnP0i%GkUndq zWf`H4EI>lvE`jC*7ExE)2mf{XZWuq;fh^}Mhb9@B76ty$GowSsLmbM>){JsdxH_uc z@kSt$iiJI5`hMaYi_Y%Zzid*Bfza$y92CBJah5y$Sio=@@9^xyG1v04yW-);^|-QS zS}L)6Lv2a^)0T0aQh`{G?e|-g<#+h7+4}y)HsT%6?ZTV=;4EX-^2S}RDZ7w&@_s40 zZnfn<7tdP4wSt1|`G!$0Zs#YeJWU_7NLxFXZTD>HC>L)uKGg)M0uJiRl90X=`g9qo z#e3PnhxhoYm$dm2%q-MfQ4saPs`wsxLn?mE`?r;rt=-^$c+Gy>q=(Al*gwMPywB5` z1#hW+3aOe!h(((`V|1+;f+7UD5YpJ^`jmfj)KL7Q6CVMDP?#-&*e+nxGd3WKpFqe{ z1qj+c@Q5stj4vznO(FX<&OB2AkAhiX3wP<88_On(*%&UYxlI9|6+%E;uBDt>V{4)4HueQZp71Wm%-b zGFfyK0gj*gFcUyaS_a-V*7{Yu^~bI8>4M%YVu3iO$y^xH%8QtEqcx)w=R0{~D2x(& z*BoakoBFmv=%!G~w$BnKWoI}7VnA`#A@2CegfI@-(|7C3tSD481NW1Gc9QOJ^V+f- zsvXLTS8uL#k6l$dajNK}=xJDn22O^W$UdV?!$iS6p%KYW7|p73;Si*PbOTRQNs8KB z>Ppekv?HzhWz$+})bVE>l-@!)tp4TrI#jm zfzpwNaVbbA)oWy}BqIo?LO7_E=rRg9`msm$VPV2uI@YdeR7i>H5NUD{A5cH*oPb2d z(4VgE98p`*5K~}?l1jjgv83qaP4BoZ>^pU4#1c5yI`JGyX$U><)A6lLK&;>WxhFGO zC-~K+U%U2ro_j>ehTv%y4bOsKM7*nJ9;5!IHPyGOV-5#NF*?~P>0#x^=)^oxgMtFa zB&Tw8B&1y=$1i?xZqZg!iF2)~03&|S3fYrSHrLS!c$Z7xzF;d+#9A79kv5A`lCGEE zpMKeik4`&RYrCK9)RKn?k`T*OVlTgh8dv1Fm>ly?XM5I;KC38dd``61|ZPQT}s~swohW^!kWO z8;*|5ITF3C5-1NP6+`7);_Hd|`nd6H?RTU)f!H`IH#a+KmdN(R2bC-5aZyi<`K)0h z)YI*5BQ8s*bgaRdReA%YWmt|Ug{}S6m5L*4Y7gXQum5uS@%DgGAtf^$$?&5I@i;Yc zOcr8np&t-6>m2mJ1i!+*3~wY0NicNdMi(eWG+!LjqHIB=k)>ChOc4NsCFB^!I|em3 zH_PNBT5#URH$82e*7!nwVpv#f%Ug%s?}8T3P#H3}LuGR3pKs6d{g?I^gT9~TFmu)q z)|xXr{xEZowuAPZO0{3gw>q3``bPW5uAW(MzHiI&8l~j_&3m^3lY8ndHIDx3Ax|17 zw!Ci+NlU4%trfHCn>iz#UhJ2r|Mw|$8Sc&zrAmP8KP z%u?|zQ7HAsdvJ4I`IOjqWP}MTDprr`g+q*mUMd?-p}MrVSbwhr0MG16lO}oC3k0}5 z(lmF?fr+lqd-q#7lle7aGEsT9#sil@<7jUGRK{12^CN|@Zk7-7ZUw2}|& zsZT#`5a-+IfHSoNE7&Rf&^C`M-9#L*iB9Fur8OfJ6|aq_-To1=)~;Y)*xi1mTBAAO z;i(mQZ8ZPczB;~o9{FlC{E3C@gb5RRbL-hgLz9@Q?r$Sw^3C@L>CE5U7ym#!N#-wh z=dMtwetP53)rdM!rSNB`a7xID_`n|7+wt@6=ldKO3ofq}mxIX10l|ZMv(oR-J39f^F)OpFi%nmlstJZ^J%HKf=M%^62vk%0oTZ_fu}D zAdc$vKN@d3+Wo_O%~ENRG`Yk_2_6;zo@}_XBIH2B+o^5u*L&D=F&;f(Oi`~^(3if* z{1o8q$m?)Oo|Xz*wO34X4^B%l)#$Si0&#)H3zBQYdm|<|J|1=b+O=a} z#y{9*c`A9NUBm}-jE6dHjaA9BISJWShL=>l%oum283nWU)Tt6SPk+*?RTVQ6pYhxF zaXG|Xe6JsP73el%zy68oA3efKBqCu7u50N* z#{d3X%|51y;Q*ld8@Ie$F|LoU`{WcW_p8Mv1BSLP51M;u!|r@v3gFW;hNi6$BMm?M zNPjQS*SCs`i<8j4YUu9*GYQE^7|EpFS47g?`wNjC&TP5qJ2w0kD%dfTUZ^%BLf)5? zlcU$-xM|b45e9KLmbOt3cFx&NU-pviA8>xQsv#SQ<``Y)I`2htW8ZIrqnU5njK7Jm zLF@PUJ9VCG%I;9Ge10T|qD1A`l#w}N?FV>z>`32zvs<~0v<6(FxKyxAV$4ebDCFws zY`R~)%2L(53#2ny&vLuBB8QJUZ@KHc`q737aJ{kmx0EDR8fKJqUjM?C)$LQj^ahZK z1DdFGb-i8*%Uf*cC;)l>J81)t;lrQEDw!WnfgHLgJJ(^D7Q?;H7mrvJFj)*%fP0Ue zC}R^njBwiBAWBx$kJ2TO@JY@upza7EGtA6omt}@IRQ#+>V&MD#b;}08i7srVbA0 zQ0_}!Kp$VAa4Ga>+(cj7*w{FxR~P={xNcqaTkky~mcwSBdm5t?FZ?ne61LHAzUS(ufyWlJ8OzoqayLt=_eHCQpYM&hcy<1yl z|CTp5XxNUZVNPjhi|l>Vwq7`W+I0KYJs>xdE!9t1|G)f}!xK zG;_GYJVpZB#M{#r__(glVV(Y;XGI9?hoeaf9VljUOjdD~g{HlJTvn=Bi0JqD>Q0vP zh{#7)C%hfcO|zJm2-O5@1kGv1X9A6n3anptojoh1K(T8A=gsZygU`aFm6z-6Pw!dl zFy6tTP9rsNZf%8CMl>W5a^^oCmQI(J?8m*!fN9K=Nu9;ML*Q0Qa+|+~=o+nDsg|&t zBps6_ogVJ}3>}VmDbfM!imwNZl0Wx{$;M=H8K=oCSZh9__jgPS5ug`_qcj3B>nh&e zLTNY?Vg%$rRCs=@rUeKmrjl%2U)tv0))H8GyjPn6RK`lVOOXf^k{W`n4wLMZ-e+d6 z?^sISPW$4p1qFXolpTCLYO@9p`V}#-nfIr|vcwfbBl z9%>$?Kd3O`edTCoLwE(vRD$a6G19oKKAA)ccYz_yHUp*WKU?`Lb z+Kl3w3LQ^xMfU4FN;++gl#mwf@U~uo1rS$1>3L6ccMEUe7X&={qNYCn}t&exX zHy z`(|B5>PYC!ye2X~1G@@(BEnl1f~g8$t2ANZ(8eDE$+!eL$gt`8tkU&EstF*QR@ibx z&oBpyp;7VGo`hF)5gQRPI9Oy#IY0`i655t^-5|G!G{P<+HeoW7?8>f-%8o5uxG;2& zic+<$t^WHDd6OES9RfDhKV>Lfr$a>)(3v!8z>KM@g_F#(M?CKvrSil??ZJ2&j?&Sz z(E%2-elT`mSR5Z|V&O|`L5Ce;jYJ`DLi?GCZ{?j6#GV<&3h2Zt>_u?-YT zMjC{zh&Bn{x_I8#l|g4%+-eTJR#@M*pQhOsKIW;0^5BQ=SD6yE+d4TrGO~q#rjhD~ zS(@MRt$vS%Z_i&p?>M@M8&(>=N9uq_9_~1q&~J+KQuucDDP1nn58}ao2)BIoV5Sx* zVrYy-b{MTHX8Y{5IDc(7E7tm40pXJz)1$3oCfieE%j?H8TQ-A6NmG0H*TFEM#=^b9 zJg5bjN|OzK#i7hr$yoLCX1giREk^aK$8bP-{GtS#d~MQTk(g&!-SQnfPj1R(lV91F zB`smg=Ch5RStg$fAgwF<= zBUK#!Y_E{S@nGnoZw_o&*ZOXw#}g8eL?6|fM~_US-}6bd29y@KTO}maR6cGZ)z)Jw z`P>Z=+!-QI39h3U&*MQ@X|5+1JrA-pFxKbAJet7CAVax|q~OzUD@ZWaHoD}bmi=(-V3CXB_R+y&J}xY zv1gLr4*h1VOS@$m4A&YlVuTzBwvM=gD9(*=J5-kHzKXz+%XP7g<2SQ9dK`CoE;mJ`VtP?@N1b{y8HlEQ0#{)fPlU0P$pgOrWd zfHXZUog3f{dHy8FzqIOgknvGt#exBf|N8pt#NwQi*FjT{9W2}Y@Oqq+tCr2=q57)~ zGaoh`Esvgd0121coH=u1gFZ-4GdTn`wc+-eWxQLz{)HTyC1kX5Ae zdB3Q5h+$Clv2OzJZY_l}4d))(U0{n^ie$pfX;oJ^eX3mPjP87sQj5mc!Hhf^obmN( zrF1k6s#902)_Px&S<-65x9Ikp=e$xfG)wQ659|X)0~NKxG%JlNRwwwE&2%(In%=o` zo<7&1!ZVu#f{w=Z|-feciwIoK5Oz5 zX_vLnF;I9$@DP+nj13v~Hj@)M@BC!7ZaRd$yohsg7M0Up7|vUWbHt*$wOQAaqg+{3N}ERZmC$RVTeRnl&iF^ zRzJgU+&LQ1#q}kj28crX0%b}d*iT7~N~BjjFYk5uo4~_6?Y1V5cr+f2m35#)*d4Xq^Hg#J?{iN%^;xPuK(UWQXBQMoE zaE9C2P&M&L))C*$K}Gr}BUf(n>~kRggMIR^R}3l^H3iCkB?FMAG_i^Op_yu=>wAae zEYDd2rz7=&(kz4UzUeyOZ<>9tRC%NFCUR8~LO^?n?{wG5fpkEL9XJZ4c1hp@U((r2 ziu;ZPFq28u8yQXT+0^>W%E1kqw^W;2wTe~p9!*u-A75>1^7X#pv>9i#rVLQebnuZG zm0TxjOnTFES8&4CBzZ)__l-!>>M zW!6$Eqe;F5k57iZmiS7a<*uBc`NH%(xO_mRmrr2Pc!H}$Q zTwLbuxwl9XK7+>ky&r_jjokW|@{(h`P^V@_24%8(OWTUSH{HvJ-+wot@{f`lb4U!$ zQUo(Y^jnoSb+OINbu6eAR$Tf)Nx@_f)2vuUkt`@fi87)m8ixx>6vL=^eQ)aezBkpf z>GpH3u4C8pI_jlHHeCixUcr4{2t*_8$}%_T&f~{&W6-1~t{lY^ovI5K@YV`wwu*z8 zmkwD3H!;N(1kd&%^%K@#x7of@2lIj{S4zJO=wWGT*$Vh0I#8wqXM68ryb-O;z@XCj z9nt9JoxbyM{iCY|A1p`p^1Qo??Fa3d?X=r}+M@{LC9O|l<3S@`??#(tDqSHtD)%cTuJ)fQ>?*(dzcp6)|U)-~rjtlF*1B+jL2 zLTSMNJ}0azq`RPz)afBp65`fA_(L%sMUGS)GhpD&?$&N+bHuR&ZFMj%VUguYGqmBP$1KajuQj%h5K=+ny*NA`drs-cn zyR8zN3-Ii&uE**`Sx1gmk2peQOGLd%?YlK}nvNi*TYi@A|OHC~ll z0=;uvhS;W?I4U-5cDeIm=OrwHJu6OlEoC91kBtvdKmjvJI6{S^$&e;wJ31Fz7x;`^ zHPxf~NbdR-F5FEze0bHApMNiy@d(#B;%1;*p0;B>@k3a^v~j(zB}XeMiHSHI z>m^x2;*oFeUF7@ISfwtVc5o*I!2)QBKuNk4X)*l)L|y<>!3JZUuthN3&P`E01uC3O zZKD2YC6$w(GpNsRIYTm{hi<84;YH#|SGpLTh{nT0>$OPe*wU)XhnVRWwyCb&J4~^X zb3Ad=i{mop7>?Bj{+j(@vt5(fbGM0)ck!FsU;w98RTV~gB$uXw*SB@|E?H{tN%TH$ zU0Wp6q$Fn79P^y`$kO1vQ%baxo2QB8`h_=ZUV4fw&f|&H!D%y}f>`;fFB`m(oHc~_ zHynsh;pnvR>fASqL=e4h7p0IG{hct9dfk@lPP!VweJVy1A`RFZLKSB5WcKMLe3FGa zlM5ck78rXXF0+chjNf197bkN`agT@}4elYInxov6=3r9l`R3w!gLklET2A+YZNA-z z^WIAb7A7hcL}Ai@{Csa)gR)ZPOf(NJhbKYL4?@(|=AVp$+jg3C5mlZ7u`f9M+r z`$jJZ<@UMm*klIy{aTZ5MqRs6p#DO_4d}N3ERX!P_^qh`!7n(>Wys zNS7+&ui^(OVi>n$I61q8x;Gl|L06i1^3Tz(jjgUD3Fkk{J<|{D*+ZGq{2(B=qGvSk z{Pg54ug@a85}_i#k5SxBzWI|dSogG8mK8AN5_0zl`Mi;d}%=# z-8Zg$)bPZ^@0HS&wG1XM3RWC2PwR=~6?9=+>7~FHc*G`iRPt0iCqwDq^idBfj_%#B zUj!TVn5WgF*o2f<1AVtDL{B?MOdm>kM`Wfrqz%1E*Q~7mZ3eIEH$LKa%Ol;&eiuc{ zLE30UC-&?5Yui~qRC6YK2qa$_8cbfZnuq{g4>`VLRZg)oN; zw>x8GUdZvyvgn<#@tl?I&L^z*nddGx_5;gsGU3>J*VKA{TX^m8nm0VL6=mrX8^Ds4 zwz!9tG^4P%V6@luBiYa!`1U4B)jjV}TBYEp=J0MR?DT|5ll)SqCA%9= z)C?HWGIC(IzvFA$tf%^$rcVZbh+W}^#@ny9AJ0NL8PlqCg>KQ3uAoi;`0LV1hT`u2 zti}E5t-y28tFf=aEQLid@k{b4)3qg41INKsd0wA=vuQu}J2K6G+neis9d4{1bu7Py zYX~0a`picYn2E(c4yGP3G+I;JGzE#rPL+-=k2&%Aj^e7?)PWyL0cA0?;DXF~Y56~{-{ z^Dq*}S6?9G;s^!Ivg#cSAES_oE1tFHBPWJpU~y7v)<9GowByR^##^|WnzXV%4Cgxu z_*UGY34JYE%8~CvU@k*@}FecINjFSSPH3ec15@C5=^yxF@Mt^ zldL49mQ1=J;}pft?t(>&@UxMkRKFR0lIZP~eS@4SKxrf}1h*J%*hJvU%*)G*$Oe(_ znx))7RgBY>9XjglPJIv%Nw5)LY6Y(cD~*gK1?sMJbzZcC*APl+On@U;PW{N9-MgE; z`S}TvMqd>rPU7ZNlgfTdX#t<9O{tHDNc8x+cN;DL3N6-%_4HqFmb96oYo_I9;-wDc zmf8pRi8?i<`kvK*<8nDvw#a<##uLqX>LnEy)~DADsG|JWlp%?Ke%(*I-sRQ#41*tU zV%;HyZToCJ^pm;AE(Y^p68D@mI4T=DDYKamUkMcDdTmqfL zpm|Ddd9opwwO?8zr;qC_524r~fKvhT#V-vv*i&Th!$@K;y0@i}bOe!ZDuA7^p<*cE7O2~Rk? zYx>^Vr^TI3pmnhbl9UZq!vd5e7Oo_a1`96AWNvI>*JuuIW?#|_=PI-|Z8&#E z4)Jhx!LLZ++!ew6Z&DiA{IUA@+s?1>`I2{ET)O$;a8wqTFJD$2k$GtFE{UB;GLEVB zWikQ3BNz$E9Ny6#l-WYGR<-k3;<1^Xr2CN9>;wNMv*EbVB`FbTiv&rPM~&R!Igdfi zr^E@ppd1^BhaZ*&&HWZhQc=WZj;yFkc7akq=Jm^;mVO;kL zURS>Cv_tH@X+KeU*C7G{1n7Y2x?+xby};fGf|iH^8?(9|Y6eSrR2N-a63{=2O<{QF zH&Suu+ts|n+p!WDsQ^7c?3I;zF>O!L``PL}GvI^=pu@6iEqwNE};8A;@$Og3EOW~p25DlrR9J`r3%Od&(C3-~EddEib zpAWG~joJ8~2bO*es6;Vw)7rJ8Ea=q8NDq#$a6&k%P_5FGv?!8F$Dkhort@_`z`oqb z`?y>y$2b|h4*%Pl`(z+Q}iIdZDjjJ(u{8$Lf}9MbRTm6Ym{37xE%JvDB@SWMCXZ-l%UIO9#{L$su zyLq^(G);YKP2iii%m`IROp$}*3;V5HJ3%YDb%SuW1EtMY2KYiz|N$ZzQlVckV zm6`(MD>UQNO)8=%e^i!aK_tx#F_{rBJR6&>Rq(-^17c^>O;fh7d)T{W_Rt9>s7 zXYf?1zT#hq@!hA7Wb*aqXF2^vOdxLR&mTWNytCCx{i?`jQp5$Z1uDTPqe+Vmwf~s6o)8^3-qeaC0YDaJ=CCVB z8V7UFth)9NchcMqdaaVnbLy=ND&G77{VFbuJR!9akTKq3vC1a#y-7|>^6?`R*QbV$ z%A3y)kf7*mk-s2|MBxn7t#arZF46`NbHxeXKsUpon6;wXE^ignL~l~7A&|c@iHS0Q zH$Z*Ch{(9C2XUBo_fLi)v!YFEo#C zrTdObhS0bTcjgNM0D9L*_2=uWP^M5JgS)Q042dno3GQ?sY~)}z zd$=%EEp9^X*!oe7uaY6}xV+n!4_=`Ipn(ymMw@(PhMG{y&+Z6VCu`z}I*E^6YO?v$91qb`tYH{DwCVBbO-7&9(`5hkh_`X9@);UCO z&dpDbdgY@xSV(bL*p3g>_!*EO6$0C!Y6uLnO~EQvBnY%f%mo!zs8(!p2pta}ArMc4 zU<)}TwzTzp1f>aU=%lr0iaT>~?mVrSOpui+TYSpZ!@j^T5^Sm4UAX*gb=MB<{%GyH z`_^ZfZAx8!3*kA52^$_`XAvPk2cnW}ng(RgvN@Fd`CJ6jCN6qss>zLyu78P#!wohC z+judc?oyPZ?FqE@;knDQ2mnjC2c=q-fh(VX{<&BRQoyOu@-056RB#4i*Utx0lS=2W zbn8;oKWC8z>jd39MQ$|=&q|sJmqCUJjwH%lvGmkYrBUyl&|Y+%h`xfJw{S5u<1tlH z6*)ROiWYQ_f?Akx)_@i^43ze?2%_tJJGo1!)Er<_7q`3#CP~j*?Qyx6puikwD+&xD z(8LgwCZ{TrJFN4)04^y^f(gnLwmL#6aZwNEIC``)!bDGcIz{Hs24txEQZghqr8mFY zd{nQ;K&=FmP^!4s4GsAi`uSM>am(X(+DQo-UW`>FQ*lTTE|`B`h6K=Bd99Qt%#Ir9 z^73aP3V}Anf|QzoruHT?ekmo##~nYvfhGc&K+IJ!oH)29cDwR2u6@wUPh=yr6VHuc zA{pZBCv0NjD>;=qE>R-9k#b-k-I=sbUUX}{C`7w_JoHAeOO|^qyj{wlWcldI&Zh{9 z2Y#nTZzvChO(?`S*t(~G(P0DrC|a4na%6l}7tnL!3%E&Th(FzW>0<;6at>ZMWu!b2 z_qNvMQVwwm%r9kQO_E=R&_bv&X-5@7Oa9ELI8O|t(G$)P{SFn+36k$h_K)PRRK%wQ zh5SqYf_etskmEc~wPd4WQvQzcu>rr`dzZK9BBZ(LKPi!}`XDf#Dj}d7RqwRN5n?3I zAT}vW=f}(Rwe0CRGKxKLOl>&iv8UHZNL@@(bnJyIFs3vFpXHb+ zv}hf$HE~|EoyzBocrvY$46w+eh%UwM4RuhfF!s=+ymLS%S)J)5eLhyh>q$uG1 z%hV1tjXu(kAKM7(tXEf<_qqF3^EG%%iw0^#{qrS62`%|VVCtyem8NWdIh`a|HUx=J zd+Rc~8`82y^$Q{NrW_@k4wfEsaFE-)jQq^K6jwujyTAuf3nLj;KoKzX&aP>~_)={7 zewaOl+79(X5H(gMX_Q0#ZU16y0^xQxq zHQd!E$af5SqfeI3-DIX`jO<5|wORGN8oZDPl zy;MgaR@mbF%YQie4mUa^RQw?Gm^*ZM*-x-cD4+;*lrpzPI;o9OrE&2vY~x4Lx}}5| z2`ZR1$2LmvW(wb!L5}S|-u{24I}Z{=P`=D2(xEOlH^J@bV@>i(ItRQ$LM-2P{I8ea z99%QzPv^uA+v=a*zkh;74krcZ-5*!ge}4`7a`X(Q9q@HSLlCVRO_82}i7cg7-|*YN z{{4N3We&~#jafB(`OHQjxe@yj;lU?4)!de};jeu{9~3S8i{)m(g+LVSRB3up^y(As z{`0%0PaNHA-)z&_pZ(p$`0sz4?pSQT;opDw&;JklwDYU~{OQmCwQc_HUznQn&#&ph zGk^T^^ZeJ}57LFG{;xl`|IdR%4}J5WqT{cBy>a;8AM(Gx-Ar2%o1_2pHAZRI-mf~` zI>2>Ug^a^9NURx9E%omA?f?1C4lB~Tyc#uuOB`S>6hPXy>!bhsx$ke8_5ZK8-4(l< z|MTV3_5XRR`@gsOUk~a3hrQ|v0;#g&3Z%oqJ10B*uHJLg2f!EO*}Fa-EjIm~N>ywtbIyvv-$HD1|LI47sHy8~?w)2B}?UMo{nm zf2jBD)$486^_U}y0b_0Tp4>PzgStXmtlAx9w``dbTV%Wx(d=@#dqpjh;(WR`)h{0y z4@$2?dFpvLg$^c}NU?p)={v`2#v3ahop{;n&Ua?N{qg3RvhmP;icMF%N#TYK8Z^jt zd&VFD*rVP3=31+r>@oLi()N5XeI+f}zN5s#o!58#A_{o{7oquxdIv53dAO&C{epWf zGl?Wm$i0J_M^=u4AxRiQ*QD4=76O&QB9h& z5Efn2`pw-9*D0Co!yL=Fn%Y`GLg@OaXn&C#866bc{8F&A$ihZSkI^Gr-F|PEV_CWRHWjBk6fr< zgEh&z<~{pY-t4lAGUT2N^1MME+1TOkerubM@lEzB+7|cCruhX}!gzM?+VvF4x;Qi4 z^inxcQ&TfP^WhhG_dQn#7`Q#2D^l& z#zyZes}FL_gNhXIb?&Cj5ItwHoo1gd$jsGc$I2{+apjw52<8H}NIOLom6Z)0c)N|S z0!+^@wa1Xa(_~<$762SE48L&k;xLMNvayS9y!@vbH+FYM2_pR;RChP2qj>XD{*eFM zI(PFJOE3ve!Xa8oi&`iTj6|S+5;pZKG2>x??m+)YppeXuI?{2&hNT>Wr7133XgzPs zBdLwpUtL!>5@?~fy2Y`Z6fHIhb^4|0xsRI0uYbB%MoEjLnxF0Lt(2bEVuucq>Mewn zZV12sCVfDX^gLR6Zu*DNapiD&43tLB)f!X&tW?S z#;(Jn#TL7vKDKyPLJ+Mc)Z7-14OOu$HZ29WN!xV#y#di(@JNHMFfIFstGcz9LF)qN zeAT=Ef5q$$9gJ09S@v^%|28_`S<-d@fphoo-~aRpZv{h$0Ga4aT4w_VNH`2no5~(o z3P9$u`)5Q=`P3vbBtX3TgaZ_=^N)CKBN)BI;~b3dC9Ru*w&yMY=wYRK3Vza8Xa25C z#F`#zYT^wj%qy+AGJArCfGZI>te*x`^R!T5MB;GNy!Z5{xz_=P?-HQ!lghA27F`t2 z%HmlpqNZvQ)Pfedy{l9~f;&(Cn_I-o=Q7CqB#IMVeaar*GiuWlJV-&X9M^7kvff65 zDqZt3nrRqcS7JaqM9Kwl;3dCJd@Dq68+S`uH!#JgW{TIpg$E8f`e8*%qoG>6TM~}b z@<9P7h*{Uebi)rpa*Dj*4h^Q#aq9|zq*e8X3PwV3x5?jp!+Z#CJZqjJlMVKWQbGCy zYX|dlAn%Nv{$QI>L%Af5Kr+NcTA-QWrUT^Cbsuw#H_9yF;t#zmK%tg#{9-GXc?w-9 z)e~n)YJF+KhRso&xW+Lky>2wiDTt0{-7KAl{9^Ii5JK$JCtMzRv`@<7`!$VN)B>t0 zGjRPj5-bGLqi`&;6Rt|AA{pSqhrYu$lOqyzZt=uYW_WO2ST!CrWs#|pgQfA9z?ug` z9!ad#uQ5f+-u{9VCZ9e5AZJq^*g(l3v7H(upBrkvL#)jUwifQM@G_E^a-gyZ_JmgxBG)^|)Zq3-kF0qg4 z7Ml%LwjdD8iX#!P0xbWT!Vk)xIcruWy1?K%(9DDt?Gi{>9V+7|j~@qt@>Rtb8hZgg zFz79a62=CSkl`*UFjJ2oKVC0RE)BSmjX>8Ueor74lI!rW9#d$E1{@nd>6WDp*txN0 zC@namiK&cv0Vg67E$I!Sbg&_ZP%z!U9)Sui$2y)e*hR>_c}CXR9BNs35(Du(ky*o( z$l8l7!9g#sf3{yyX@Y*2&E;hs z&;JKOF@Y+^qNNeZtazv4)BmBhX%#KtMNf;^iIt;)E?dOF5>`5K`rfA*;AVs83F)nEE^E-kkcSCLrCS=yfTq14A4|2 zlL-DtsCi5qWiAG?ETkuex}T6DdtbmzSaXBq zaawywCeQNM68A;aDIgYN8o^ZY+B}O8>(^bo!dY{C1D1lS+BY`Qc_72a1&x!7j(cBw zk83ZL5fdh&(&L!mji4D6VEg`HQlJdmtYDsstJwEynq01}xn3HRdwh1oJaq z5{q@%G|T9~CF7u1Y3oFEHsYeyc<@I@u|B~AfhnR#6eVgCO7++YJrJg`_@W{JCKhy+ zgWb)w$dPR&qqhZBg~%1>S0w%glntT)6FgAh3{2xrq3Bnf^psb#WsS!4*(XEcW%yM1 zMzAPT=d%y7yrI#{AtrsRaqqne<$gc69j26$u~42SBRg>$-?;zVe_gn9FA~tzPS4o< z6euyUcm)c;`eMC{8FT)rJFYG$slN_XAm2zKs9tD`Gx2V$rHswSY=W`+>1wy9=^@&o6}@KqL>`2!V!PVX z9(#`5O$PWWR8QsHLQkqIy@nz%LiskfuRAHGjJZ?PB7{vOrz^g&j=vIEymBj7Ac1U5 z?6pc>Hk?~VY?BDM;!YB~OXuOaIeZ5bL=HY~NYE3fS22!M(h>_R(ml15)}LNIOcPE6 z?nF*F#8rO8!uQ4HN|hu(g$Q;m)KlV)e>e3-KP4^M^&HJC&!3kQs94P5#1uZC4^y%Aw$>mnG;X_zNL%ImoDMGZwBeUa3LgkIrRy9g!e=*uCeutbhO!_BnmgJG(<%vQM>qJ zdPR;-%?>U8ELH&?S0vYwO#sYgxq|!5GBDtM$2rg=Qer8w%ei zyk@`t{ner#@diahKG#zcKaqQBm1z3>ymI!RFy~*VcUzVk5P^)R0*~*iSm1)Uphf3E z9lP&W@Oy=yL`CV5@zPVg!F4|_ppa`|L!qS63HmMDY_ry|RVkOHD?%$5DIjx68-t<@ zVi@mR(EXM?saPL_F&JWNMQfequxckB%bg)HbGIXD(t`6fU=g^m>R3MwjgYodEN_FUXImYh(D=E@z$d5K$Yw-Ghiu(baokFz6yw4L+J7Bv0dsk#nR?d^I67{;eev_S_zl@^hylJ~%%jcNUkXtkw^9%ko}*!8-*KNL$HvAAKUWhSU|DAWU#vriLHYp; zl{aqOhLDHRiT|>5=g~^Hv5GSw3dK&)L;ghlJXLFql77g{_ll|9 zhrvmKLBGu#B_E^gg16;0>R(EjI1m8b71mz7ez<@gWvy2%L>|FB*ql~ZFsu#RCr-DJ zOr*ppg$q9K*)`9Lk5W5+?>1>32S}8i1}n=+IP;z*eVS7*Jz6VM-Bx8 zIqlAv==GaSO22C9EwffxN2M z5U9rnMEyof{!{lcsjz*)gj+Gb>B7&lzG%0pH6hWHOjY>xdO{6n1Rus(v zl#T^Zq$vnUH&H~oQbiGzDjWo9(mC(FPKsM+-tYUp*YEwV@1OC?%q0T*?EUQLS>?X( zwN`sSav!6l?7nl=t(NwR-jE)tjxg&{@TuY-f>7~`1+lWOy!9V&cfUC0*GfkxQsFG#-tRk#y;M(9g5axhk- z9{7TW(=Xr>g0NB2Q!@-vICx@h5W>)@@(baSMK0jRPna;!rMv@E8N}D+hQZgruOb=fU;WT!#_{n|8m6q9Wwe%_n}0V2;3w)v(Z;&OCEHH z2wK$&NF{*7gWXM{S_I&o5I&@hp1KyGT$Z*D<{#y*G(eU{a-u`Nw8Ddtd*=3vfa7!# zJEy{YB$(tG!!@qYn(7<5T2T7ogzwLva#}2v@sXNF4dQsE$9B%}>79TB@UVH18stVYeI%@~`y(%c~B$j!jv_HGT3a+1$rY;Kg+I7$TBr0eHJeOt

)+XNU7<6Zu=lHM0`;uyGpu*TzBvKUQ&tI^>tp9+zSXqFL zpvuw6)ZzB?Fh5q%xfNOoB4t238Wj#Tf+oC;GDt{82wvzwp(+(EYJ(T1!Fbe!4bmFT z`HuA;{*Xz`P2i!2#aqBCF=;h(nj))5vN0EuGF|Kr>Qj$Shb}Ruz#%uh${QOVKH%xI zN^<3AX#63O3}pFA)qf-fGztJVqZ%PV>6{%9e`?CxjxRdhJWhBh`ZKfGkW>MoI}6bc ziA0Lv_V6zp3Uq?a9-jB}+iO3AO#{2N`f9Pjgn5NrMzfLpfYmi{+(%05$FfnHn1%TE zCU)-{w1nBXIU8CCGKjxE`y$q#(7R&NjgEG8c7{YLHVgL55>&hw!U{d$Fp+XO#L)#u zx7zgV(dL0m^^Na38cRZ=2Il9LReG8ZNby5Tw5?>{9gBn%297=&LIai%mQ{y*yPqCckJQkH+)m6?JQJlc)$0AD`g=#t#Bi#`ty`^J- zWS^CvJO3cG{iIkr_xSuvV-bZ&gZ}ukc8Yrdb)%^d^o|=n@d6!wJfk+3LXnPAws=$5 z`_JgkJu4Ai)FRN_2%LrbsRFdB#7yddt9!>=u7Moc3Bk|Sa|K!GCy2O~#ip#4GEnDJ z4M=68kB0>uZ@wT+joDFeLBpBh<8pwl3$dY0l&D6Tda>jM(FAhS1PeebM8#*+5r`8- z#h81pi@q0wH40Wded^SzSgMZ^OY?5XS8~D9%lv8}E)jauJAF7s(tuOdC?+ztWn4~u z2oa=IrD_wE^Z2XDy>V#YH>-Pe#_QQtu%y|fLe!w3!uAPw=liL8ta%-&^H zMwGwn>Y^rm)R&%WSV8@uEyoge1(_T+6g7ndLlHIdCKrEUl0e^SjY0XTdiJS_=U||o zrU~C`V2DE4?YpQyOM*jTS3n_Ty9p5}@`X#c?{bO(rt{MVN_r78fbVm`GPrh%8C4Y9p$V)F{Vd0vT4QBH+VDs~nt$RWJ6_GS5%HfRi_e^+NG$U<2zf)ZwY) z3g2tKFj24M4($;KysL%4u~ZHvv-sUBEK}r;It6ZKFBR=8`orF|wnoFIYT=dE08N01 z=tyR}TVKoBx`#pR`RNIG!Bo()N}uOiSJR7*77qucP9wZk;m`&E>KdKH(K~>isYN!M zorE?yqZ>VP*n~oMbxC-SmQ@z^04zxhR-Kl;8t#k`9+6{hZ7@A`@zkABtHK`GO2x$R z#m%*gFFe*BqPm63_VY54G0`~G!HP*nSv6*_AiqIQt;1;nfQoeMKtdKOIKkgN?-K8R zbL(dFrAeNYAHokPAZ$MP*HnzUE%t8QYr|841yDfXUe?wJT7l}aI8pFWH#-lgmwosp z;9#Sxo1pt;QV~HP#lAbO?(DK9vHO5Ikhm4;<7F0!`_KZluv=L6xlS$Dt!$ z<(@ro34C%6YLz*MhR9b!ADn+L8<6s%`TCi~<1SSUkVfEmq6en20}{H-0!d*v&8)5j zJRK^w*0ALqTF&5*TxqgzGBdkuQ(oc@wiZITmHEw4h$mAAKq{zk5SLOW8a$_2i26;I z%f}38ppJ_`MBraEAzMW7Ob z6s@Dn=g$x9Qf%t+#g5knu~?mn%uwRpl=3Ezh^6+kVo>o3CA%g>oR{Z~Jvg+hdv3>t zPeYz45_t!k%>)K087edh5G4@cE2{%DZ`3x5D1sxP;+yp;iLA1=FI_#R#7sd#X zNQ>+L^V0wv^pvRQ@76;ER~bJHh(ntMNUWFvnj!QjiBNaG_knK-5N=>ioG7YC#*%dgHC^l`@9&82Jim*}ToI)fuO(i9(en1 zGdjf-^C@J^Vg4?EA{!V9y)Xn=gH&w=>7&^oV(J^{J)?XsymvgMTv6p-Pr2flDph{9 zqK(Q^s!CLD>joCr@x=)lpusf%o5covrwc^V8SH-C;YAmA;gG6P-9{@Wyg@=mqn@7@XwF-^ zN(ktP+N=;pb5efVvJUd9{Q@h1E;kBVp#8D8wkp*bNK)(gBqb?a$&*#(I^Np8sf+LRjFEoG4&w zozSFJzTKI5Ei)j2hF^FBA9BOEQSENRwN#y((`E~=$n>q^oF7C_5OHZ2SfR{WFM3fJ+Y|It`f<$JxAj94y2DzCwiwgsnO_I8Gf?wrm5y zP>Gz4`e>m`u{Ah6q%wg>Tas{4IzVxUpnw~mdj;D8jZyKFEp2U=yYHf$gilq>oT-Bc z!0`8l)X6De*btT4UrMeKMpdCA7*+z{Kn<0nPl*Zu3;@{VL4e^n5=Qr=4*aAr7DXfIQlfqzs zzoh4Kl+fX%MiX;KYLozIYGB%`wS^VDE)H;%yuFC5K@>>ZV!)21XeN%B4heLlVr=wm zr1|tAEnZ%zy&Q#ANyTss!oZPF?Ss6DA}}bX{$NQY6@H|iSM5#rMG-lo@tqyfbfD7c z2-38cSSn#ektT`R@r0cSlc~ZNdFU;e@06*&M8^ZJdIA6#;Jz-v9wJ20;I)GSH}J9G z2yB(p1m%zac&;fG^*pdgnE$5+MRX_bt`;m1F}qN35yuJxBB@ZnI)4>+p@=4{b%{v$ z0741szUf&Gl|;vi(uMP<^|J54xjVTZt-*_>2~E%f$!{G&K?E5eDwLtJ6GXZRR3EG| zgL%X(6HvZ~$kCsa!FU2nehA`IE1sB`P?Ib)K`;m8Vxy3{ep6SMjeS}#6Ix&TyNIIf(G z18r#>(0L&4u?L0seB-H4^lnoVhx5Kt2N+2zdiVj8)C4iT9~c+nZZq9-Xtj{`R-@U- zoOdZWQJru_B<7*HdH6gtXJs^r!}Sc*IL90vE3v7nF;tLh2W%ckn9um5f&?U41^nCE8U%5v=rgg_-F=;0x6Sd{B@qdJ;5q;1rv1~8XrAupg*S}w(_$?ga z`kYk{s;f7tnMBTCwk!c?bg=E;Q8`zI)*?js{UL+~=^BI5_&B0qkhD;Vomn$Ov-|#g zXp>6b&VC;N;)u{Of}Ox=E)53&sKbQfyT6vU;(%R!9hd-Adsb&?j-HO*^YyeF5kF<8YaK^W40E^doBzFX>Kep zzY9S1FO>w-CPOha4eccA2%V2b*+lU|#*H5SrVyoQ0Q!+Aoj#c@0(zdx@!`j;L6*{W z>C92RGfKZO@>2_q5(WWhurkVNsvDl1Kh%scUQAX2#6aCb$^U>uEvl=lGZQ2iMF;~L zED{bCqxUPoKq%3KiQ>UY(m+x~{l>nOOt`rVClQui3cGk8{0skj zDxFjTGEyA`IISvN?;egUiIX&Kx-F4a5;TIoAm6DKsC|tbw+3KUDzI+6EmiX5_~iL$ zJBSLbSgZ@hEA(2Sl6#2mTTPG7k|Kwy$#G}oxTuf=AQaUAp({ol0Cwzz-61wZuN~?p#)^cC6+LQ13V&bE+`hRGl^p`_~ej8SN(deaRm*&BxY2w4F0|k5( z9(?w$6{6hu9NJ$ZiV1*NfwY9*?TP|Q6 zPX3+vEacODDD-4gpAJcx5c3m19~UJ_5{^1zQ7U4Cp2{ucgCK_O0SZvzAUvnJl=|`J zAeL#g(|{*xFf12;!r$TQJ|*JtdSDY$SRxQfr2;}b&L&2c2z?B1KJ>*gSxb37>z<2EJ7|0#)jfJwFr6HSC6NGhw?1L8M;2`*j8eB_oyaXmq_XjZU=yheiBD6 ze)_r}cyEQ#bwVY0rHeziu_5$9lAdiYkNa0mPA(FblN-&b-{0K6{iaZsQ^zKM%vZPw zrh6ruga81UTb-70ak65{Auv5#!;e_{5eHNXeJu8&r^25nszECzc{_IPx(&e#Og;rp zoA?R1h#CeRDLj+IcA(IV#fH+8QiYy7zyIdP|Nkz3&l#RlqeuS*Oj}z6r#AsC61=v0 zZ7+puxGro=8R+Vz!H4mqRF&uCr20YT;{|{J+xz(ED0C@5K_EE- z*DJ(}Z$TESI=h6`mk6`AoOr<7>q8MQLEH+pM5&)Nu+B-MDU=oaU%?IAgruA}wRI^atwhN;- zj5sfBPy~j7x*UGAP&cg?k?k|sCLuO(btqqwsxv9*z#bD~Lv*kY9>jms$MV%b?lk)h zYLI+*BnJpmq=RMl3Y#QEQ5uoht3#jPhEECD4>opd;4=%=0qUTrdX&PdHJJ8dnOqQW zke&#dOq5Nn3nwSW*4FsKizC%MG>U!*r;l9xgi@nB5dVHoQ{}I)y(stuI7V|WeVqZ! zZgP67Q8@%Y@ht){vq2g;K)M+Y=|W(SWfZo3c0vSvR0~$zm@+KR#hd^Fd5A5g!7k{q z0(~EPohysL#2{;R8I9O!*Mra?O&j7LV_uRjtM{u;t(UpE=M~*6KWCfFahX37NLLa z@`b}mGz?}_uSw&9CnhCbxQzWaw7n+LG!fg#MWN1I8EF04W_sVWa)sjGl&`Ey z0NA;kGl!M&U%`bK$Hea}?@X$D8cK@F0u01Jb{j71xQlXGEE!Geq|qVq6F|lX5fm77 zjUhpGL&*+>3xH&-fW=^&v^WmM{ES4JfML=$)+PU0AOnpMK;A^eM%3Gp-RXj)6=mbb zKPBTmbRuUXD0L8lL>wSk%j8e0IkB7+A3q;d`|b6Mz?VTom!_Tu>QjL>D zCkk(ii~K*P4~n!WAS$-L8cs=uh%p_2&Ay*nu_Dy?kV1B(f#^YH6l03)i{|bD43Y<= zt9PLt*$py`{lq&!R7!RSf2CPNv1NGgH2s?@dqnijKeZ-4y#JchltB6`p_{vQ?drq{ zq1h>frt_%*a2@E6AGnG-rcuzxmC`p^b}6i>Me8r?sw5$Yb9ls@sBU0=z_M5v4;YE~NHXF(j0rqeow22=< zbw&{&LC9DLK*o`vJ|I{m+S}T&iuV9-VKk(~&gLp+8a(g9$#!*BRn?z*kQP!|U0qkZ zeMG|$w5qotK?$v+9?fHd-sUbGdb-CU zUtQ|rm~BEN-e^c5q0z8O(~5{6`fi2sIZpwh$qhpTnQrJsj^&;9_!v4KLUuCuygP^z zqF|`{Dd*5<>rkEnd1s7D9_x{?7DU!nka^A*HD zDun+Sv5)$i=wg=3hLmJjz+&JHzn&k2axT5j1)VxtUmx@urRm@*kMiuBzxtP+1-h4J z+vUR$t3?8Ka7Kq>s$K>vSVJ{|Xn9G!;ec+c^pey(Eh8KH2s$2U&3J<>Cek@HPUg?! z>ADL?5@8XJmWAjQVhqtpBHCT5e~>w@s4y{mC$5t)nG60Vc5$)A#v-(wp%V;B!9Q?@ zU|!BzR33}GP>S5U4NU~^yili?riaAN6o`s$cSlF?K?MvVn>#Qz@Sz+9p857U=*{`~ zx4VpH8}vdBE&>Ph8e}tg-B|RkBaEC%hG8>eVYrAO!maz;9(H$i?EqmKi5deHb_F2# z(Ca@BBco%*PXjG`{;Z9c6CZLorpw`jfYc;sXN1nr3KvC#p`l+!9e?(L*PJ+R0} zKX~F-pn(l8J>X5H4w-<8_gFpoY>V?`k1blZ#SGJAwJO_ve0-b)#i7lMDTmANP+YD7 zox_)n=@2}E<1$F#p+buq+W`)d2qaTi?Hf8@u?$wg38hKeyF#6?ou}o}I6$jc_}7p? zNK*=IazF+IMy07=gtO$WT21mXXec>XOTgmMxP-cepWf61??sZ2Zu*3VhIXVR{QTk` zFu`S&=IZL@PJ*+JW6j8}@#Wuun+AWKDtHUwA~lVNaiq4XRLzFChh~&Q)!vHPYi>8T z;@Fgql9q+!StNJ`bc;>@PSPj|t(sWFUZ}}Lg@`U131LG7ut?p7+m8+V z_)q*`@th{D*We3c%6rZ2)Wv(vU!nWuj?9ZwA)JVVqAdh&IJOIMg?^q_4y0j`QHrE- zgjMPU=oym?BVvVJ6VrociPXCZ0nsnVetij){%?efOe;bKZP@EE*bud|{ZLHn0#OE( zZ>8@A9}H}y>JSh9hbDQHm|gd#;)r>FRE$wQCe;M)CMw1z@Y5G0Fj*PoCi6GAwj-m4 zASs-dQ5`as*>{YQ<|eWE3wxqPh*ZLzu{AvxE?hW&wZ?;iSAb_Q#p)Mo+dXPujQBou zRqv{u!1>WYt(R2`NpKCwHz=8}?cjoZGXev7gW^>S+E1O3z6Aby@XmdCz?EcYAVCRf zIq+H3J?Tqer)tPXHW4n6xyP@H-G!;fC~c@hb(h&1ZspC3dO2uXhB9X`Fq>t%fWkA_eYQ%RPAP9B8GBj=(i*3_Q?g}&6y0UiW}i>A{! zFR42vpP5dIVm$)oX+jbKI0G6+MS-HRu`yXy_+rW+s4X8V_~1Mu8z8Mj4iLVME6&Cv z5*sGOhR>m47!^mwUgx!7EJFpDebvTL%n*bG&ivr#Xq@<0W@o&qWAY{pTMBhXK)Mn2 zh|mCTu5KoAB-D%PdN-k!4T0~*+0|6|mzGD_s|m`+;B;T=wue$s(ne4>E@^cYzkt81KXzHcmBZaf@TzmK*p^{-x` zW9cVv1CV?9I>0rb|M^bn(=I;z@BeRFbp5vML=~5XZfsPL3DP!JUuUx~bov$fP)-L+$$kd&)HNM>;Y5%wV8s9G5 zQGZ3OQh8%(bV;9^N^z%KoJny`L#ElTHBUCJeP?xP$(iYY7g^KxmGYW(XAFFvOjpTmlXPD8Ly81?%Vzrhbz55&X;(S_^%ef| zeBN|sv$TzS{x5zR7_C2yxh>D%%EELh17oU$6n_qj5%nE()J)#yfxK30^*kv{Tf_p% zpg7vWMwiL9hxx-bR6G39mX3eZi!;w@+G&U_kRGMEAd5C1v&kJPwKKaJeyj-nLzI_& z96%!A^Ht0d^)o$?)9+GRk$e008YE6nV16{k7WlK^RQ3`<3a~B)Aw={UMwh!SQ?!$} zjl)If7CM?ED1)d^CJ&z?@xyOFi$T_89^q<6#3K6X-}Qp)VY_deTzT!|u zc`NGS%F=TO6Ig|av>aa^O|L%6+#j76-1N@yZt;dM@}Br@?hayQAWi4WFfpOQojKx< zPyn?$a;&}`H{Oqhsc~M)cz>N|-AvA?bI^E##l~vs6tEfJyd}PP$BJB_eQXYNC$;t; zo3aZCbozd^?+p5Q4*9d-f)blqjNhwTIP>~!B}{Z+6{fwkHTdkBot`TIMKzX3^2Xce zO+SfeyP4R`_-zgKD(<#%gkKJwu{SA~XLBm|Zcgx<%dqnrCfb*+-as93_(r?-{{7~N z#e=1e>MuvDi17|}?;`xR=IVJ4XuRBh=ixV^TQwQiUwoZ2zX?%k$hK?m&(?}8@U1)c z@@W4}rHR-Dnv9RQlW}YH;8*#Fdgpqg4eNTqhRX0>D$C}7yp+#ZQH(7D*#UaWs!`=I z9#2U)+({FVU(y_*FF*dq)fJ+8D<*-jiC0&A{8xcX=Q2OR!$P+)i*6EU#O5JD7Gw!|*`7l>FC*~#cAF=L5Ty!Jk+=Kz)Z zUN%ljLDyO2HTLcba`+eo5_N}s*Cv-ry^I3-m(h7UP6F~m{YH1RSCi$&r&K^Qvk`C? z;(A{}o&-%yu?`Fxr-*W8v52sZjipM$|9W4PQ zLhUD1G2U$zM6az?wK$DbEe%1vDP*OF>{C1iac%yPfTyy+u-SMstS zOL%aoBFSX3?@E8OAWsJ9l}GG#n*b^S!l^Acc7GR03;;eg_V(=}$|%rHOZuXG-yTSC zFZX!%xYNJS@*aMuey{Q5LgOI3kPU1?tUU(fOw^!bNCuEw?_6s{b_cb3VPVH!ST0g< zyZ0Nz7Chi-!5AVZjbx$we}P=VyG(9k-1#JxiwYkO;M}F-?Qd$xYdRD8?` zkR!bpy~D82r4cGNL``%9WeAb*KsUHPpnB%0JWb5tr6zfh1!ghX`-R5G zSD+%RE^6Yhl@Sn-5}qHee;f^g*&OWBBu7K&gg7kxil|P&%rx|PO-h}?yO!k9Dk|H3 zznR0TuC87LUDv9fp=URXP2At5#s$1la>A6sb`1Mb=VW8fG+yTORm|OUM^;2{g_k3& zT~@Ur^^Fa^_t-iq=dy6b@$cCcFpy!W^{`$x_Rh{bt8G->d$e-ZJI3a_ld4VW*Z^=Mgo_>G7Oozn)N*Ni1yC?nA$$}qy4nkdpMne2>zs)r7EIurAcVS7mr}zVR zzZcJb|C?JoWBqKpm5t9p&(vSLx$WW28yMu{oYQStn*WbJH4nk%boHF|68bZTPx2?y~-8952+I-@biY z%4*hPDes0>EghZKv6(_`AFWQ6IP#Tsg_%O)kCJ-M_7w8wH%T9{$?$03+*k-Bk^5ro zggO1Uch{ z)o#v(CEQe7ga(3t{Sgv-1mKbzt)D5j9LpWTg$p%B@|eL*%yK}9VEb4j&ufyKHz^9M zY~Ux~_e4%Lb_qFl1*uW)$S>DDc-z)Up=zxw(k zdGhRJ5E$bFH$18u z9Vg%w6I{l37kD_W`)kecM1kAJS5a@EQfg_D$F}aeTd;gnEgY|Y`}5AuPI+{tVspT& zpbqn}{|AOweIBNL+3f^E6ycGpPhMe@S9eD`mdSFB4g<(4*V%8R!>yC5Mir!?BM_r8 zHT`DD%n~z}Z?e*c&7Xeqx5Wi}gVwJA9HGj==8;Be&+{k+hRc%@7Fzr0;^Iwf3YNRu za9YlZCO%R3>O<3h_$8Z!o$oKoVc9G5ym|WN_eDU08ajGD-8%JX#p2D~_YdQ3?avS0 zy+;RSLO`iZ55NBfG|^$P71dEFQoiM8_r;yMHGT? z@+Jq9Tv4K7-DYT?KVNp>>2_(?a^S*knD!8M(X) zbnng(W<#OwKVy-^k(|ce9~aC=-522;GQ0?uB{q3(Utm(a%lNmum;N$$ugnjrLwUpb z;%%_Y#Rv%{5D^(3t20fF7`#E#u|e-7I;sL_?MsQA5QFo0BizsfZcP1}#uGr7lIY)J z2~&w9Pt1GTu#Rg*hw~5d4Vhz05-HM#$b#((7ak))~D5 z--K8v7M>N32m+(pj`E)}$gMfxGj1ZO0KQow_0q;4&npQb@e?fNWfFoG}3B_K0 zIChjJ4%Vnf$Y(6zBItO0?E=O41a(LTVf?U~!d$RFNY?%Q|!B-*-T zyTC088*^d*z3(%Xj{#lNdO_r=b8za+!kc&bEe&STF3~SQ^x+h^iE)UoB5EXa1q_S3 z_AbJ#gE`g+6UEsTh;Q@@86vnX5BKA9p$6x&3Sqwo&(@Cqcma5jXH)9O7ccN!J|KQ} zBVs!^JJq1|Z{{WqrsqwpWhWzB_%}c4{r1}FRqV$X1$k#9=J4PbOn*Np9iV*HM*n5& z1O%>LqZs|?xqJ8Uy&6VKx2{U+KOs2&9hgHCQ#Hmd&`QIE7h~&(A_&G@2U*&0Qf?P` znIvu}=fW9Btfq=h&S5oOUY=DFzCq!9NiwEwYShE zZw&2=@e@+98VtDVL0epMfNu|CsSJ!uaD2SToW<_OR}_xb!Bthj(>e7tC{V>72hh1R znuRD6pFyq0*nFe6?bM7#9GtDxfPIXYWN}bgANioT$D6M$A=l3xh!!`+3uAL2iBLog zIj-N1FWDTz15wZ*OTiFS zg!dqXFnq@NU<;oj-7`EI!sCcfZ@u5_CpuKa_`c1qHOi^)(RDctHA;Ef#f%$xFR~v& zD#ra%4$*=$UhK^q@^0s+X}XL{cuWtBpB+g7qAU6`Iiis(DRS!&==I$YoLH@{FsLn( z&QKP-)}lQRtg)?KiRb!_{QCzi_T2*|h`sv#Bga{AP9~IP7{ao;B30Fbru+6^9v+`- z69ePYXnMt{FTBS?7{y5zv>b-s6N9pCL5Im4;$5Jga%HWNi_upz{&5KeL3@d0c?qS~ zV$d3B|H8QY=jU-=c>7-3q^Oa3tw(?z42&Exub)SUiV5K>T4D>x=+z@IA8A^c7liAX zzkcw~MX2;@aBUBFUM(<4yEX zqM4Y{7P8d-(YKh~xeuoSM?7%6bAMjZwbRQ(phl>1He@VZr$)}2N8VNNeti+WJI7j338m`=fQx=F4%Ve6*huJ?~q=yE>%iZ2y zqhsp4j3J2fk*oS{daocxMfxLLT92-6@myxYTy@QD{}~goJP0nrAbW_)%#|F*B zjAs}$(EGX<+Pb~e0^Bmy9aS4YmgcPVlQ|UV4#-a*Z$1K%Ma5}GEO~Y7YU>qVNVx@V z^zV+Q0dY~q%_q!=WKmZfP6>$PdtWdX+hTo@{u!JZS5CF>J5X(LuA}bg{-GOM3y;Vu zM8$}57=s+}IdvT?1K45}8{0K6p>GmL(H|0lRqSu%-d?S4Bdraa`Y{AZeaCo|nY-{P z^y+ZWWa#C`x>DF2zy|GC^2SX$QLv14D16)>Dh<2KVk1&7TF4N{`?w|GV{uZ%=g*(7 z2Z8$sP~`MRX;?f(X4|e^!guX1^rQB{A79|EQo`&R&v~l=&)JCe?k7{1e`te_j!y7K zOHFKn9PECI)kkW*Cu*zP5M{{nh6|K_kBDTOnMdNH&Sm$9{F4i#>F{u-^by35Q6t%30ud{0Q?hkF`yD1nojqXpjrIxAqBjLhn!eN&vzB}>Q7}hWY$(BkOL($-K zD(1TO*B6nT0@C?cQ}2Q1dVEt)VW$aiv=-H9QQ8KKr+qKNch5M}o5`^4m2C~MK>#J& z`)Dr3fu^V`i>_y`ulhd`g@P~*{(9a0?}*NYokwSKhjN9X7Fb#}fENx=R-A4@!^tvi z4CVQZ5A!KKfLxBEKbVCp)D-EW(&X{np`0F57#*w{C)a`mCGrTIuq%q@k+qFPQf>&G9O#}T$kp-jb|rOC)hPrF{`R$a z33@MT;-&f>XIyUM7E_T&-r7*ykhe1fpyrC|T1lt3Ps4uvZJH%s9`@s3P6?@I3Nr5Y zuT6%xO=1TSc-!j7jCvGwKWrP{eH2T%XrqM&dPx6*IM^I*H|a}6wO0^Dp_IRc><%_| zD?W(qM>*_wC`yv6Ol=r9*ZEJmwYWLJISvTG&7er6o9hQg)Q*l~rG$y&a=;Y;X|B3r z=)k6?cQ_>Ji7Oe-U*mMA8r%+w>-jH8iPx{x50#`jBIuDqbuUe=pTrmVx?*N7SB6<4 z;T)TsIK}1+7H@6VyEFrOTdl}^_}l56nJeDv+^Bd~_bM4kW|K|Zv0mcoDrD)oVoUjl|83q$T9 zdxLq7%L7wqd)Mfhx6*z_yM+`4PO!-*uag-Gh7XL#Q5&0_Gy(@VpyxZx+a=8DlVJwKVIGZ5dzJMa){6w$l1D^B;ItX{k0L5d`XqOB4IG@-i}G|gdj5Lo$H^!- z!ateeh6@Q82tW2h1c_%kgtL-pdQR4X{FPPPa>tSlBfl-o)kwB6IGS4h)rFUM3WSzf z^hEK6ne7qom&F)ij)1jp4U#yd8_Q%7r8>e|qmv7w`Rw5)U+bj?#1smL{kY8*79!cy zFcSNz)=(V2<{N*;hW2pN32=W1T-^*+tQoMMm3R$ssi|I2%ZYlAHw#Jvw?I05uYO;x z1sbxu!W3H>_aIM6P432LUm*(GVGjMJcieigKxuLVVj2|=VKFG+iVt)K#@7(Sh1|sq zMR=}nPA+snH?lU^-14>Q^&Chmj`ocrXfo^xLX=o*h>~c)99I5AH{SHN8!5uSZ`8foHDN)mt% zh?1cq%mH>DiANNuSOTu3GJG08fqMNvT>s&J3-v}r&?2OlUiNkG%zTKL9f@^B(LS5w zzgJEK?YaAe_5010#`HemfC1S#lIo6ODV7lc^uydbV_ot1L^p+Dr)GdvQhXM{c((E(#*f^XdlD9%Dwj&|CBmn~mq4({ri5^n;fSeW?g4d#F88(vplE1TX6 zPM{EyrEpXbn?iQRrhpHas?K8|0M4*1=E)QWoK_nDq0VuEZ$2SQv=#j+>MevGF9>fp zS!|3D2jZ8)_+$`8H#V9pygXaSF2;dHfI>;yO>pr#K==TZ4>#IokaHCsEMaE}%a5jOB$Ww4#OQ6ZQ1ONg`(c=!&3k4Um7ZDXOP@}Xk zgs~L&pC49|e zq}arm8wSM^aiOaC7<||8a+u)dH{L^8jY4c{m4uHVy}_E|NkM7A<{@0Plc-@cp2BA| z2C+Il0hxp!h?V&MFZUu$BTAH0;Z%HAByiO|@BF z$tvNkj6HZu9gZKh!XDF4ZbQfK0B{LM-)9p_%kBdV7Pv@b(WLZi8hh{`?0k((8zG`#w=&C|BY=R<0Og9>4K>bE=c;{#5J<^=f_2?W z4hx-6sMF5;$Ajlw@4$k!10+ ztC=AHiNe7utb{klH$?1z>&rmHFGn=uYDZ{LP9KgM3o|4rdTBO#2+;kBBZpy>?JtpZ zeDKB8Dkiq!<8~AGPv^p=-hTHg)u2m2-^B}-I+nP#17A^k0`4LlyPO0VN9wS#_gh<| z1%O44`UTi2cPh9@@*0dwO=EO;cyN=wQ&xI29AXMwC!oJJem(%YD8ac4tymnef!io+ z#eV!o=Cu#SE~cssFl39TZjJDx{o2|s1j9LkNil`KW2E9rZa|OMP!=KHr9rr(wL~~- zsVfAc0stRl20XF^(nkHVTa4XbqJkl+XfNiH$+0#3NE*kn?Ii1uUH zP?={KLgXz9x_bMUP$7Ebc$;)Tan=+GBJ_}e5}X=!kXJ(jw1596n5FMfwx36NwYRsu5(vF0G74otRtPYNRjZv7-ZE!9BK3>%`uiNZnqH{TOn z@CJy$go7>PJwWJz&C%?)c^&K|d0JT}xOq@gyJp)sWixOM5)?ku_9#nW7>Va4l*J$> zXhoD(-VRX{WsM^c=!DR&u|#?(dzk;OJPx7mB!T<)&_Dpcwa&leF+dotS4_gOvN;4V z@EMQcj z>&(9q@X6HOSIf9tfrDDBnc3#k??nC|ztpK|Z37KkgwQaOBo@T@Ba$N|Q|B>ZBp^oA z0duBQ>l#$=V`{BBL*+yssUxGr9^OlDFk}dT0T@@S!`>fKBKaUMbw8q54jR8T=1h@n zXY4r#yJ6sqrW6f9Rm3@T_dtKD6vYp?3Kcw{cJ0cBPqITjymjOgg>2#Q08-T;$QWrD z%Duu*xF-7}Yvs&I?FZN@oJB}(Q2-+k_ZdyZ+g)ONe+4>Rn87&M0w5!S{ z%f_b8-TLJWRCCKi99Ggv`m<2H(}>U8K1(5hBsshP&F7t2>9+^L3qpk4U)9fgBRX_| zWauVxXM{Ck_S+cyeDOJ)l5&hYf@lfwzbxskG>wdM%6cg%A{k@!0kIh9)hN{cH*3Aj zrG5h?l7GOOgz*!j)el^u<~)TuloXO{=d&#M*BIQzZl%$EWGg=dtOHvc-b2P`Nz^WA z!%fh5m!d~hBnwO^n*$-l4Ktg&2UI!yP`xX;xabotA0TV>GwO<~b9o>No{*z)>UzEb z#At#E0`_u{Q7qpa#zEK!>O0YhX z&ZM|AjMb|;?5qp64bGa06 zD~IY35y^vJ7t&ku@p?X|qFE=0kpe_(tgh+|0XAAw%h)7W->hM;P#n4XfT=}{*5qs( zM;(UBU9pcB5~x~TH}|7`8+Q6kmbN8F>;J{z=+rp-puYQK1_}rUniP^)Qjl^43H~iE z^HHm`uv9$Ym|-MoBRsBFE(YatBi$}}q2-?4j;5;2^R-=LU-!(o+qWH{plJ0q89zjE zCO@fG7_8EwB>z&m_hdg2G8vyBTNeW-ruCW(|F)3C0wMfpnglvs+5KgNQ&ZhB82DCH zM3pE__FiNa0wf7W0MY^rA?ET4dl8GH!9%ii1bIESU>%4IM@p6fOovVE>=DHLzl-w(p;n@RI2nno z8PpVX6jiY7NkFkYU^mL4Aljdm4$&qd9IlXc^&@43MReFhXbWha*10`d&}$M{MQSHq zS8ueQ4MsXo$M9(}twLxP1G&P+=bvJc?9f4xX zIN3%#XFyYJkr!lxrN8W69t`kL7+pS};I8_CZnXmcC`1_+DcPW95GN1}4L zLxc~Ccmi2lO1%lv=QX@LHe~OQ>DEAA) zxj0(kg(61s2mUOQ_f(?xf0G|(KkXx(eNFacW#r^NHghWimI5pdodD4ifRIBZhDUMM zHk0-R3rkYCIl}epNn@U2jG^&=Y9kN($(n^9MnB~Hk6k}5k3ggub~ai@z8L|#7LHTH zho-=|*UEZ<3z2O)vyvk2@#4I36)e#m2_d5_lsUB{)xGVS^YEL^HWhM$6J6j>8s!Rr zLvO$udU9GosGB3+NM*Y-oU_f|H7a-C;tkch^*oxZUV`+mA9;-sa?hfDyO#x$4jL=r zNEG=W3P6fD`T#&5LA}~YeY@fnimtI@>Oh=TzI8n@0p)86Yra4mo5F{%P*c@LOI-h7WZ=0>_)WphTCxVgR#;>BXK${2%i${ zfn?CRBuj_qk;Hb#=S82%-bx`DdCE;ml*y_YTLTl^R7;W_w}N!*TH6bb(lljI%4!^i z8xm+MNg6|IBy-No-e^4>TC7~&`0#3WnCxnC^KXg@Tu<~lwEgvEaqypC3M@{(_~Dwh z7JGVc;Z)NPFaD8!jT`Xd-$He(lG0&)LbyOv6J0#O3|S=xS~9{VW$iOOsI;}unQ}|u z)PUKwANlR)X>6KqY7PR)In1hA9!==D%OuffEDUWl*c_PrDB$qXet(NphKq8@z4|C@ z1OJZc7VR7U%^lNs1$1xD&RdSmX`$hdTL~Rv5ysz#fyR`mj_=^2JXZp>V_uvi0OX)u zYn3Y?Y`85G&7GIzm>|EFw(>AFH3buq{cP;%k0br-sQbW}*Y>lrhc*6uh0WRg7yRd! zf4pyeDG;A{J#k8vIo*Mg(>~k#Z%(laA?&z9Kq<<)Bago$O|0#_Ed<~1p`HJc72gEWPYs)Gc@ z+BuSNqc%5COx&gJNDQhC1p(WqhA=?7tJiLefngRCrhv9O&{E=XNt{8HlIx>SWgd0{16AZ^}>iE;)*HSDR4=xu(c}a ziz@;BKEHB?R5UC$v_SnNQuhDAI6Tjn1`bT?T{7zB`TmDqc~X&~*5TaD5Eo*^q1+F> zXO8XKjVn@k0!fB@AI4_2fkf!FY9@78J*Wm!r%72d6t6y|`*s>l9lmD`td^AO=l`+c z`x_5a9f{v)gTkF4guEvrdn7$JcNF&dwqC(Td4yd7U$il+WS zkj;w`qS6NF9=TGyqX)tXKz2m|j8xNSG=A^&UcmDv>v9Gvve9sViNgL*vAS=j_YyZ< zoBJUZ#uooM`$7E>cOC&tv>)SOZ<(@hUs)--mzNqkJ=;l&K?0co5gvg_WGk-DLc82S zbTn5cgF`<;P$Wg05X95`3|ACs6n`#gQ$Ldj}PfBg+i zV;^G`qAmG;S8s67dpYvZNm++ULuFL)6=T0#{Z#}x6i7o=4s?rO$7Q35Ne|xQc_4p) zi#pc~fp_&sDViceOq3pK)g?2+ZH+vLP8vbU&~Km*bl^}PJ-No}DCp?XiM*INC%lAq zaFoS(uLk`)MxacM{#O~B(wm>>a#+OqOCZV%85#nI6oz*R7}%AYknVuImSnFP$Z;h> zvhd|}eL$Y$KD*SK;*2_?%!$&Y817}-yEQ||95VriwL$aYiQ>qNqMRw;$b3hIKImJZ z^Y|VkS*bE7rA{G4rcY4}n2LI8?^Z`lwyw_agY?pSGOP1{VQlQySHY+o`yYUIt~tDCz>elv8w{ zI5T{FDd@p)NIh=*RWRQ}V-+GFLi7n-gIbHP)T2l%UWz#K(mZ5WQ2;9tuDMX%VzL=C zZt}Xldgv61>PQreIxHmwsMw$29Q%V){RE){TO%r(f{3LaWWT8@gXADyXxlh3QY5m& zuaTSz1!w)Z%~p~$=(pPdh3G(uBtgz>2aFoFwc4v{TWmd~cTV?ePGaUwAm zJD|Z0X1ooHARJ^9O!?tNp?Y-5COdv*A`+Gww^fWETO5O<_!5dZ)(FJo8Tvl0SSNxP01xCr?=A>b5BC2XLB)N-kkGjUM)BlZ@1Be~vxT5_H&4WG7?i2uK|k5_!Z!e=-Ajgb$!5 zbf)*BYrt`Q7RihfRqWKYGtf<()qQoHy*-`Be{d8FW%*;%w z?{Pz0a*`dJSIORU^>wnVV7*(ssc6B;Z3`CeH~aNmU~r1V&W2mJi&ToQn6_3##|}T4 z6)wJX=*PAhlM`#KopamrwCb*TmU4#@mxS$XE=ZDyDL(35($`^kWb(|ndZWKp&4WC_ zw7}2O_0bzk*OqTZw6$66%Q=0@ZM?xmG56m5c?-qaty0S>#;=~J7=3UZarqKbI<0N5tl}PA^%m;ZZ%!=)`(UfDX_JaJ-8*LC#!sy=dQ(mGi4op zBaeTtHdJ33?D)sZ-6~MlJfqZk>js~bhStCoM>zMz1Q@h)M`^vcj(Wt^$X?UoRyWw` zSOnYD&(~=p*-aUR}K<;Sl3B^ma6k`LwcF1Yq5lw?hm>&dl_I1NwE-z3zaDT>wrt~~eMv_?l-i*&jZ z4DYQLUAR`cdWmH6j{{RI?JXK@Myee(?|#wx=C(^iSs2%@~`=}@h+ zAMrs8AZHK7vAXihR$R3-s79%*pVT%7gxO|RYtJL4DL?IyJb%OfTl^Vs&iO{ongGM$s! z$1sViU`G9|>?)J8`cw9Qyhn@GC;?LE>wu_lNGX?^7qI*C#l1F=vQQ+wfknkWz#Zd9 zYWE=4+9EPgf4y!^(!vD8;HeAOmSj~|i?A$VTGQg6^ziHRW4EdShvSAWwUv^<^QTEkgbghNdOiFL_~?J!i3Z!Y_{N z#e(&i^`~h=oh7`cf79;CmByYaO^4NAzp#o+AfA&F1))ZFXvn$CDj|R6wBs#Z&DPhR zr|YV4=-2V{hWCEFaM;DBUPjvXWrpsDqnWkFmS8mce)WZrcT;mI9bcWt8@rP|y#?cgB!zv26?GlGM+Q1XbobZRn?$Zc{SW;wx;*cWzHEAjZB}%LtwN2A-Q=sG z#19QMjgz_|gY{W$&M6Z^#aUOp#q~1!R`z%6*~z9SZ$E9*-yEm;3=PDhT7Q`6Yj8f1 zVlh4(vLWBJIQgyJ#i;6MQ)-63wG@zQD^XbQWU)BaQcHA9&Qb7BOL*%jLPcsJ%rM?E5NC!E0fIshbyBrpM0+dh8P<(7Q0I}tAzFpW;QqUUWs3J*`%4V>h1bU2mN3sh z=P^)FWRh@O*>vnfYI09Oq{OS^r`N>=IvQZRugQP8YO}LRH1t-Y$qvu1>x<4vDl7u& zQ(O(J6F1M%KME+OWm-?JUf-j+yoGijy0mj9jxBaE9-H;;y~W3!)HV!lPf(Zl3UsjL zh=t4P$ht`0-C%C(8-DC|s%QAe_gR&O@Wur)UkBUtXC$7l=8HZQO_v)avxHNv!i#>J zs-Dnr&EDU`JubnYRqUIetz{HgdwG9rovBKHRfSM$nq9N4(SBE>XV>$s4u{x02x{tU zO}%WXcH6v6RZ;h#EmA7$r7!8W1901;;V!j$WfQ|gE7*(evW`x_u;k_eok4HzxMSO{ zs2IhUL6Yek)DN2ons-n6(}|@f06_J4OBK7+T!JJk&dVs2Eoyr;A0qRKAqn2$zg4j; zHCG=w{aCFcX{WDYvQE}`^7aC;$PG8J)v7WFZ`*0dPkMi=?ct$1Jx*NPa3bM_wqf$< ztS_|hS*1R{8|N9r61^g3+&*d@k`kkEE_L|f->VWX$U62Ix&=!v1dL37d`1%*<~21! z9!VG?L@jar>M#vm^>s!eDw897C*8~37v=o;gYkj1jgoP3DarS(`kh0Iin3iwQ&W@M zPstBvKD`y6cfhvvf3f%8QB|L7x2P#5CnhnXC>F3FqNqr)AVt6^h+w5y5Re2!R0INu z3Q|ll(vf0CKmiLVAfO1+vJFL1Afj}TgeuYmq)9t-E;S+F+2@S=-Eq$t_mAxu+1YA< zwSMb;pE930=Q9n2nbx$^pHI1I)8f(Jwc}W9n0}XiOvY6Oi}LW_4|R(!H=N6$t7Zy!88Y1dHoq|-NGSzeSfva!~u;@LO3yL5K`rW;!lhO2>vJ#}-pbnPB~X<;Fn zab+kQZr|SdxEhc>E`!9Q>SHqd?{4$olck|`V^&mD%&sej z_h6M;l;d1$kMR3z$x3fU2al- z<);@FUe0xG_zUzOh4O!?&vGw5W{s0r1q6vSA!tcyPp795>*-XGc-iSJg>;U6Gq~=6 z@AILXJmVu)G87nn2g3`98e;2r>`ZXjb?c0=>*m(3x0VKJ;sGnd+**!C5pt)-A{qqCImuWcH6^Q2kJxY2vw)yDN93*rj&Ejotv1@V0 z3gXfGMLe879A1Y(;017dmR<{SJn$SBjx>3amw;raD^kI|I`U5J&5Z-O-6Ix`>Tt&` zb!=WLu`t*}&Qb?rrL6PIES@hKw%dDVq+M)9?4p6!F;b1&+tQD17kxIr$amEnSI@=9 zAyJ>s+SNSM`|!}gg#$C=pD#YKJzB?UAb4>`ZT=mf{kDTPQnz=eNo@M9;qhN3`b^=P z=B@wlx#5m`#{`98u#_+Iek|BOm~c-q#dYYuo=aV&v{muqS*}y;;fk|k+OoSg0eY?n z`KkQ~$##zG!ZVadT?DsmHhHWAUUjvDM}sj+izFs-aNAU;F;<6`$D5HDwzAk zcNCodcL)3CX58!wU^-oBy%gwe<6j5Btt?{A;G5juSv&Og&jazdeyy_Hkhd7Dmbc0R z4mLv{|K=%Tw$6T=O2r;er>mL)hoWw74Dwi47PfW$wzzZsi*_{Ry^LCwc4)uk;@yEK zF={y7k8%1g%e?H<-n#vM3HsXFac|D~23P%cw+}$aw7>G+rYT;c=j&rrFP#2dL~_xN z82`rFX%)^*k8i8gyjm=}Ozu&L>zpF+Z|lKpKNzRJu;R|6DWO&Kq`zuDD~W+rwsJ{e zMw6V_(?Jo{M}sNt5B(wM>AqQ}X%@b`saQJNM@2EOx5_+Rbw+nZ_Qm8`RjU@C@EzK# zYMy^S{LrA?yni#q9dqNx$`gGUSJi`C9MwFvyYRb{V%v4yLcxU=yT(#qE_yw77Sua9 z^yh?zr+&AqM`^kAfY`l!+14fZX5*i4gH6jrdXjG|x7vjsjrD$Vut2#G|HvKKT)R0q zzW%)Vvt7SPMD8m;^plEe&`IQ`k)F7vPBo8LYFp(`jd1bXl^89JC9$*$mbH9~7s{UVKn!VOE3B>?0WkP8F2 zP8l{w8#qz~HrEJHF&9wKj;^0R0?lEq)xD;xoBB94TkU_l<;Xr9g4;2szh3+0eHWzw z$G8!aXkd+#52f~YsrytI^|zR-24PMtso8+^N-+}U^La@W(lQqI}!ijur&`KuQelHVIf$Fs2~V-UaaH8shmCFN5t^LT;be=m;Eg z8Ca?UZt5B74S=)@vE3PqXO-ARk#&C0XB0TLwLS3mgb`TUD1kCS#k_2eXaE$mBM&LO z$X9@%H+S+5LqFnPV=rP6z() zQZw3+OvRf{6ojb;yJKB9K@ z{_R>a8RbNAp@+Kt#DcK%FEgyx<4r*12}XAG-qE-3&xQCRoD04|$Sf0}ez5LT^g&{HtHP=)M9LKDMz)s9_YdOcWZF$yAG4dwD%g?&qD$E4Uh(1A*5QWL%< zb(B1MOwdWlqhX{SA_3>DKj-h}Do{;-&`17_QA;(NIEjb6#{%D%uSQc8kK@}wG()`? z)c#I2cPJqS-xeIEfl0BWp3tuimtnmDs(|9dPq!SQ>{ZSom@N{Nd>j2~Nwqy5vGM({ z-(TU4x`Hcc7G#-7j}FQo!HGKfdOGBj4^~1M3JjQPWM0z^r-a{sA!_qqXopDu!|LJI zYJr&XA=y8nLoMczV3avFE`c{1U7D6*h`)Gus=THEmjDEP#beYN1SMEMH;Wc4t8k{L zfyz(N6;?q@d8UCm-xfXe`!|H33vnMM9s+>ug3_jeY_*Z$_K^l$!!4w@QUq%}P+p=n z?H^W^S0g}lr&9hYgkWx7ww_ ztwVCffgP9i6NFVdog1D$a#-}$UvXcja}6mfbfp0CtdaUL>Tk6Ebhaem!r8ygW-u;w zG~R#5Q!^AT4Z7as+2u$7wmc}d^_S=8)^s)4i~Xe7kb2tpeKfL^azNZ*4aTpgCKaYF zt|XAB_BT-Ave|#njp)Q&@>xK~TR@T>9{RSJeSd#2V<28%1aj{6aG&xQ@dI#x??MqL zso8^|s*^eD3}5;o66F0EH7hgvH&IuBTw3%`b+&svjJI^by&mO+i%6fQhwEfJEN0eV<$^Z)qulg@vbQ2!h1uQUdAD@)3EVZde> z9sR#t22d#hh6NrNO_x)t7r;wE1#|01U57qlS|9-rU-%E3(yjN_48o;LFY3yu|919d zd$CaueT>|Rzk~?<<1hXH`CA<`kUg%0xYqE)r!yarCFN%qS}eY=GwZbpo>H)7K-kdTiwQGm#u zR%`?M8V4RrIok2eDoST=OUPGT1gV|OHYUmUzouV;AKEa;zBiLqkc>+d>Amp?j)Nfx zIs*DUmKvQ!9cqr*Ep@r9CLpG(e(*$NRxs&t3zxqCUikNy>k%&LB zzB5s{mn9h?EJ^4{<6oybT&v2&H%LApI**_~e>>UewJd+Zwmt8@-K8In)gDUwvHV$m zqG~a*ljHVL3ySVjeT1k4nxvK+eTBC zr=LT7WfqorNaMm{S%G6p_j&V>WUXn#YvGIcK${V~XuYU83K=W~u z@^wjav0!jUAH9E*X36F9r%Mm4d>(`Ddy*k8i_K~j_Zr7(xuA8esK`06)~ z>`<<4m^_L`2M6kSz0cb*vlM9eFqv-POR-11>wfASWbDjM8ujj%&z>ipiJhyfNZfOU z49^_TdjrJ9g#dp@;p;Cea5+P)9l~?{=0QBe29})kvLo*m6WzsT|2SHwa&@eK2O>eY zD^Au|I$E6^FV=}IT7k1iC8{xB8fk`(4x3}HgpK|!r`PY|z9k4>2o&<&n}&4u~_??IXxcZJ{F&Cq&{{=nD*A+fB)u#rpN7H zWmL{8IiEb%>bpN2icO-=J0Er^IGAGL z8Ix#!*sc0Lf}h>0Snz~fFz!=Ve2y0y68wcR3~!PUENc)u+<8n$wsvzn2Z0ls9BD{} z^!LP#`d)jcD`i?=Mu3mQjn=?9BUlj!TrUT2Ivm}k`qJkJyE7q#_i}SXV9kQyKMcgs zvK*Q8zM3SoN3}o)+;Pp+aXU?+!cm_%y1KgRHzDJMMNIP5(rh4nrgXI~D>w-Yyt&g{ z5-6yp55hV0Hi!>%z1A?Ej!D-Yxt5Eu^V;&_SWxjC0J!4TtQ){zPd%lkcpZ~dUBCP*H_o!N7J{kkE zar6wRY~9@4z^=%|=isqi?(gq-eB;)u5g(Q6L;ARApo2m-^C8r~N=#x^iBNZsoo-Fr zpW?Emny$c#va&D!6lNYg zKI_rNpK8qVzv)=o_`vhn#}D>A|LuYIz(?bZp#|}bFy0QhusD@A!61|UFPY)VUDlwPEX%Qr&HaxNV`Z}+RuC9R?; z?E{`N;UB5vZT=tL?$VR^Jan+&cqbIpYHA30l3cxPF@yzt=BhDRzx+dv1rq2FFV9Np zO6H<|`NQUc;qLYtUHtbYGd#{Ie1bRi(czDAGTe2Z7hEbmXSenB&8L^~GPb5ZR^YRD zu<;#p{_(M?$u_R58x{!8Rn(5ykRi;tvmA=)^p>0W_eVZPX%U`1>UPO8U)ph+4@oy4 zEF0BeeQQ2%PWsPW6*0lbJ(Re@`tMsk&W9 zFh3I<(CvFFIA+4!>=@FhX;z+Ukja>%#Uj(Wf?ZW5W+f1(%m22WYKwFA|QyvC1CYHDB! zCMwQJ$CjQBFtRnE_^yG6q!*;6IW#=cvqtW9qd6u)h3M&{p2N2qdQrHiW!oZk<`GFg ziaI3>pWYL|16*6kZGBV*Liwju0~pC0M8d#BqZQ{BlNl6Y?dG>$A~6}2?;ADz&$wrV z7M=jcx12dj-1n{dcPB>+=9c>ZqhI*XZ~yaf{yOjfISl{z4nvnxEcMCTE#CWm&L@Me9KW z^$6XQ`R#+PWrz6Lz&we-Wng30*2A#&LY6b*Kt{Bnj| zCKAqF2g9W&OH%r78fQH3{zZJR4~wbeV*h&c<&3bv-en8q8bpPan-8%Dcvk^w?5>=< zTC!{AI;+ZATl-i}NDOJ=O#mP?Qzewzbu)v7FtT|<#guhx2GL%h9y*FbEDl`sKBz=% zkrsVHg~`S%3kP~-Q00o&YMh=cBsCq#ESB9z_k^I&4m|Wfl)fyP?h#V(^}G5Hf9qGT zdSFeQY~!UD`q$RtwX#@ZmC{)=^Mj@5d%w1=9X@{lF0$fe%?#|>KKt?OJnXeb`6q4c zC0$|VjZ?kC2hV6<)jfNcTKwx=IyYr=ec(4Db%Iwqx;~yyJ?aGszDN|dLS?=h0O>8N0AZO6E~Q`A?JK0rt1j^oYZ_H- zF8u*-;RW1%o<$q16cTVyIe<*{k;&UR0sMYs@_%?=L|clf^+sf{?dzO-JSUIPDq zVg5&~{>8<802j(b&)j9RZq!>;N?*#r#t{48tUWi?dV`5Uoy$JpeYCxL0&;OFv~gjm zlRH3N$wVzG20YRi5lUSx6Z<_TW3*J>&;_XP{jXAOtXOnKuQy31W6+i{iCT?wn4Zy_ zTCbe*5?1`Oni>_4mUV^Pi{j|KZbc8V7jE69tCybh(35 zCoj}y1~{$C;{?Q~l*U;jTs5vQ$`{QGtM>JuhN(53%G5M`YO2;z8^zCQqQe*N7V z3fjW|@pJvwqoV7m&B?*=CBsdS<>>lmmWIhp&hZaEiVsyqaXxK@)tX$ajN)83@zKw+-&p_zJT^~kR7W7e`WXhM znmx7Doj{ONb&u!+>wkJ4!`VO>A-udT=OUhdh&{u6OAaOjz6jaE`@UFOf$*)R_;YYJ z1UjY={7W@`U7A0L{$ifk{qbg~+418l6zpQYXBl&`Le3%~zUHU+`VER+V$ZQrJ6x~- zaYlYaMWSrAhp&Rh!dZa^hJ|xlwx(BoP}0<3_~+8bac31Yc0W$OZr%`HzU*rH8C=t_(m` zve;Pa)>r>!JBc!FIgfL57k=;c*CQM==7brny`Mm@v!*pc?{d$X%bc?hCE+wU6jvhy z>m*h^=K%cuuTsB29@|fGY#+NAUS4xzRr8LUJ7ottNZ>Jf%yL2B9H*F_6%!T^V3jx! zn1qjt&b|hjfdsrxX9)>$eFti&u%GAtC>A7uJ>NpDveoY^?omAfQPH1jOw+%4asYzR z%e_)_T;}^#hfa~l>LpM4iKPyA@ST2lZTM(?!C#2&rM`34@8QK!;OhIIGJ9FzLcR^` z8pwA^#k*bA3@}S^z~=PAZAhp3c+Z49QiyHAS7XbH8oMJEU`ghbhX7Ajza`4 zfnPpBr*86(S81+G^nEuChPa+&DwbzBp$oNvO{F@z!!WdMaj4;r;?GR6HeN_b8|<#5 zub*fvv;D_IR^Kk}w?6RDgbGh|W@_(n#f=8$@9{)97oAGXe>eSj+x!xK{YSNXD?tj- z@xd)Xx+hwvcC1TDP=kh z2Ripn#fD0COLzxL72E;VJx`2E$G5+-)Yh!}b-cJy;i26R|An!?2l#Z?4hG?BvI9Z$ zB2u6^ChH~0Ra@v0Xan7?27*F3`e$tnAm!nlQUi{Z52lmq;(iOx>`VveEeSk?T4*rc zVTPOuXO@WwfF2y6(dR7*#D*H63l&#o+1Z1&HsGQPrWELQAMe-*OHE%W7xsh}V1jzSz3XkC%tGj>?Y7^I z+#fDh8feRHYd7{+0NG>47S*9cxwX&%3uX}nkskYfpuQj7m<>cWS>OzaeWlX(Ed{qw zBu{|4%@-W~Y;1NAr4GIZouW4d6JQbS-V;Z%&61=)eiZaypo7SQD06P5_3lA{O>uCc zK8KH!Qi2$`KJ7#NH3TU(4k_hB-(}Dl5Ypp@MIUZy=s-(N=*krEVI1lYHik}!h8;+g z*&stq2Y-!pX(=u*jqFdEbEkE`vbNzkJ7vCveX!AFXG<7p zC!G6$v8ljPVix?SwzK=?2S}VV#Ks7bnQf?>+jWfV1_2++JnMQ}M;&hs{`u(fV7=WB zU$N^1l={umRW0A*Nlk*6oEy{mjbYR~wJ0Ba0aHwIT&z0yYAxu+3+egp)Os9iM9PzJ zkZQ3b&JBo3{jrOn7%BPyY3zgP=Qv?wmm_9lj|b=V+zwf=fvG>JfQj8Y4UEFJg|%&>nXbVQVW`PO$u%c&;t2b8=%oq2RN4VqsgRXENQ zD?e7rn8hhX_CZSjpIH?9kqcXR57RMOP!0qsa z)k4&-mN|ICYZ*As9Ali*pv{Hh)GEYbB^CsA=Mh>>n_pA#mEIPOK;(_}DAen#q?4lh@%=-79NDJAS z=4j6?yc%WbN@iifFvk0!L2F=;TubJRO$ZQaTODJ;bp%nz&CR%YEBRNfT3VCeKEq#q zv=9+D@8OgB*I1GDznl!_CuTxhlX}=~xC0_zhu9=8UIW^r2dur;VRm2w$2EnQn}Jkw z4*3GKGr62DSWH}9oWXF%W}PC69(=E+{5Vr6_c3u!zt|$01VZ zN)4Z;>foPWC1-@?V^`c0 z0NjRDnBmuPv7$uloUHMUkUV(Yym=F|6s@4;6&MS&C>jjUfLmF)3qI=!)B+5`gUpk} zrf{U5)+fFs(oSYuVLWmg667mfYd(KN`=V`(nT_Q+4HkFM(Op|L%(b7Xi4H?v zgNeI9F`zL9`3NHCpMjo5Zl z%Ukjk#_0goGXSrx3&iO>P!c1?)ul|J{f@2Tw&`+_F$3tJk&i;$Fq4;~o4H_M9b=@# zJzYmZx9CznDJ~A;jptPP@4sdA(#B6I#SN_mUC0Y-Q{-}nbhcJIb#21NRte?_d3eGI z{t`AQ>HrwSFjCjXhh7SgiCXvyd(lp{1LQxe?wwiERfFZ_t`)YypPQgr<5ACK7~pOL zbXaqd2PQ!TVu8N@_D+-1-SnH0LJA)2Y^l4Ghp2aYu9PzPCYkU~6oOlYrH%va{cFQK zmuM}dn!=R&jfEq?LQUZ-CzZo@WD3Y$vz|UJArCEtgqW6)f9v#RR?Y?I2vz;O(~yiQ zkt1dKGDV(xbE(F57?}jbgJ9Dg@X`nC#>}eLj#rR~w89_QdC&J-58BV%8iWOJ6ccLr z!*EPE$}Dc6dC9`F62RN(`LV2p3aoLUgoHW}9C85ei()3_!zYCsr{mIZf|oCgs$L%U z$~An#Kw;Hz>B7NwWPicTNUXhs8(WBkZ~|j@dlu@eo>KS)ZJO4MZ+dZQG3$_K!_Qwr zzhO-~Hk(4t`RZUxhJo^=jTr{hCD+#?+X`_dK<>wRZpJDJD5j?4ELyYc!BY|Dj$XYR z+oj>5D1w*fiKw>@F-b6b#J6tO{4624tk@zX2%In_y2S0nvlC2*;!LtQ-r#JMD{Lv< zG^LqGn@mPx>Q_2x*#!IH)Rc+SHx&cN1G7?Tl9!To&0>-e_B`y!#ME)BkPs#~UKw0@oc)gfr*!kU&V9%F== z-lS0H9{m{StH&(4TjU%>XJMczCHvTbCN!VOig^nRB#kHGlx(OdK)LPDUU-tb+RJ2r ztC^T0ix}7&lkF=Dy^f*j=H7CCd}c#67c&e7tRFcf-vPuE+!JD)a@bOkgkof$;Tp<9 zE@SCqy203`-;BfK6K|}Mz{-Ce=)mRvco2rAH&SLui#9K}IZ0 zR6t#%lb8%ruWqq5ZjTNUQL`jR!JFblGO7Pr+5jW?%cUlzQ(`lQ9dr9ap~8u{$5FIy zaT<0H_`jsELmRz!#64U=tr?GSawc+~o?dOrQZy67jnmEj^*ztuz;I_#>n__3R6>Q3 zqUZ7zUt+=l>Zq3O8E1Q-O!D0k_bvUei1ADU^w96~9hxG|A=)x7e3i_Pl=YA#@Xi1W zuM;+Ty4*`rbLpq?xjT%vMd@i)LMNRZv*GiFRnEUd5SPi=#F{F%HnR8{u8C1iM8M|U zlIERw$(gjwaf-goZ`McdyoD=H%O;vHmUL-e`MRNYui&*JN%T8P!<+H+Q9vU)J-Ys` zD|c=YPRDLO8L3)q|C(!SxkI?w9hq#ghAZmZO>e9xu^I7E1er6>#t9wA3CDuBX?yk6 zMU%GvJkedv5TAPoQhj4e9*ah_0Ul5UfC`+deJOOcuiH@^o?2wR$ID_k_v0}-jAfZT zg!1q-VLe+NmDnV}UG9H2KcaHe%3zEaH9(v+1(D4Ln-+!TLuy{bwK$7{ozmO;bz{28 zS7S$meJhvFiC4(7PGw4;3eTzBT9(b^c=tb_HVpF}H_|j!0IoXae2sMg)8yZBX z7^!8FXU5ht)OA>)#F5N0T$XHZ_D;jTECK?;p2-6%c%M5 zT&2_cV}`OK4nP>fmS$)H9|3%uWi-lu{%n#?k6)BiVT7iXYFEmkCjWR}o);U$oyO$N zrG*@AGT@Rn$F9lguBY2kM1opsne0;Wqq49Eefy;(xkr{&J9MD1o5@b)xutxDj`Of( zo_P(x-Wrq&J_z{Y+yJ^@+rs{w#~v>)RHMFP1g;w>*}MJbP9eImrXpL7-L=-zn;kDZ zVRjSgFS#C+a-VjimX&Dr5!SttjAY3D<+7ExV`q?AGBIwjYqf^hkeoZiBb7_s2J>_B zhS4@>QwZ2TECL*zy>2Z!=*7QR-&D_Y4Ex>^9}hQV1m@VL?C?CG`~>PzCas>Bc6Q%5 zm8uHEo8NRB()hHe7X1km(ZwO;vafuJb11(fNt8~LFrT)1&OKlTiHdRG;&m`-XQ8M5 z{MnM&J$=&3sAx{SvFpJ*sEXRtS4mSf1hegz@OKsLS>sShuknt&g~djj`ZwDq<2!Y3 z^pdP0HlZn z<`C6K7SeP$_~vy}h42;>!n$V~d-S;#E3iKDxo=_GS5CMPaqZ~?+1Onsh6c5XiwAS(6$l-Lg8k=rB0=Z3{Me2`|P1Xe{h9=T8{)d1+J?~=^=QK8E zIx`n}SPY|u$%KHiEBK3yD-fp7)zCShJrzb*=>dPjMt2%ug@PkX2hXsv;5{y1Q9@{=X{#T;GH`?wT@BpWDxwmV0?~~S%fA@#8$Mr)A|GeETsYKsHJAPnK+W< z;WoWr(A4sA%Y-fd2%xGg<=y(sH1a{xktj+A0HG_n-ZJ^4a`d4SU9rkc$A^U$3w+V< zfBhg1%BG{Z!L?XPprs`m1eO77U^cyzUxc?#M%f6$aQS4uNJTD9>1~yohWoj?lRE&N7s%ew;un(5z>}O!+NK ze|)OGuoIP}2sj(P%c*b-NxEuqj(u5cl$ixhdO}=?*osayn;T9RHR~~e1w zm>JPBJ|eKDY5}roM)e}*XY->rM;E-^TX$z~Z}AJ**ER)B!N;*FWv#8|F+d!Jl3Kgb>q$+zup4Rk zI*w$n9PZwt$FFQl2NhSs+Z3JqRQ5lNm(-k8j|?Bt86p7@1=MtW9L9cGdWD6~vQhZ( zk`r&>0_MtG%?nj^4XPa5S^&f&nfW;W{9bK}J1T6ngch5`5=Z znO2~ZT?N39gp9@p7{aWHS$p`bx$5PY&G&U!avfq_$w~C2=5GWNxOWxix32vUQ!$;ONP44c9)LII>~D zTbo9Bka-d2GNL3b6}IJaBSug647{J*ZaX48z~i9b&-cYxF&B-h%q3Zm89f;NS z*H?lGl*B4|;%_2L4A1q59EOHMlnXF8XCqilKD-GgZ6`X16RuP?n5qf7bu5K(&P@cz z)C0hhIyyL0c(BXDCxLB4KwCeTGf@-V5}d!Gv;CLWqMpQ%2eF4NQqjsp46g9V^RQJW z%M>O%>F1rlOh*kx;+AfYJ*Z%2BhJWZ7e$Rb(}2^2VJ*?$ty${Fb?koJ-PXTZiNz1A zDnfvSNma0DByKye$Q3U@x3~aM6YEaRZYBGdf$=Wv&$IajYW#nRSX=R6yJ7Gi_&~<5@=79 zvU!oQKrZJ~8W^7kmVu7X2%w+&MQPaU@-+Jxp2$3Oe}UmH_j@sexfSTLyN2?if2oqV4|HXT66No1XtBE=~+p z5`~xFvj!e{{R~_$cXZK~b{of@KiNOTrq|MEQI%l}qHYXcti{>*D%Ah?@?=*)ZD>}_ zW!fe%zi`J@n15!mzCVx2M5v)2bWfMY#8#u4`Q)dAWhONlskHI z3tp2N;tpU(qNp99VtTPZmbAKAXn|zo0T&W)SdE&p28g$w&i3r(1s!jkjAQ!eP z)$1ti%>^`&isRWoa{xu02N3f4r=coVptDS%C9>V=_F=n#m#OLS0SHXlX}ulsiJVco zcx&e18&AP^(OQ^+tqm~S>fN{y7ZwA&B=JZmw8Es*uL_f)p@&v-a}!|Z;P_$e<@Jyc z6RXC@5Yl<}Q`=<#zjnf_S*O1p%hEVR)Jg2Goi%bUp8mm)dXrEod$PEJ_|i7;T2Mh` zo3CEVj~Z%R7*_8rVCmwrb&8p0%GfEO*w`MsTa4Yb06jbOVglZ=Cy+DqS$u7{0@qq< zOH#g0=%yPv!7-hF8bK3d(dkDoEeujiRfEmSb`OJdBdC=V@@%eLND&e;%dHzW{if-# zhB2^Lxtv2#S4AwQbc|ls&q_8HJI&!+IF!qRRe(TvC;^Khf#n_>9`zc3GS@@|z z@mvTeOuRP6`HDw9T|yjgE;xKr<^Ytu8y!OFQZvvId*q}l9B}PcRd@E`nFk#H$k#0} z!^h96m0O^%8h#369UB|iXk0m{d&+(*vThCi;<`(I0xWapb+WJ!)BLIdP(6=`a!HTR zwd-k+Zc07!eK&h8Yzr|&88a0=F}X%BVX6am4*^_`@O-hCV+Cba*HLf~`y1%`8c#Ro z-pi=Kp^-Ps;ugd>M0Kx6zNoc!9Wcm^BUP2FBy{>Q1tQ9roE<04dh51DbY5X}G>2YS z2WMPCeI;lhPJ?QphITovLdy#=X)D)v8X9pSgpZCQzcEbf)7U@Ky26F8Z zvmn3{S5oJ5wi4*-I(a2-ksYQ4n66;0QJB{0g~^H1FT2%H6V`xp>kd)wI)MFQz(+{> z6&zj^(rjnoeKSA4eS|(Geb#eXXmOk%)bXesZEdeNLhgts_r_9!N>rNBg=IWJ5f((r zA@6#|J@J(zvGWdI!AL9!CSg{HIe=eni^TkKsoVy&cEHHT*9Hpc*0vt{WNGgr$b zMLBl`rREj=P4jeLBY(tJjQW* zF62Gb$w)-~ZCG(dDdgp&PsZAlcsN$XZ~}IwXUg^7!6)BvG{nGUH|J}}2HGHof!BxI ze9<;niXUu5!7!V8j5kE+Sd?x`e+U!Qjp*nApVR25zs>v@2Pf1{6pE}%+-N&uzh;nA zaa{Sl%@-d_JGdkm&(FkOAPGU;U{CWKx#yVcNk(V6h^^hts*Ts&Hk!+JGqH+oB1=%8 zPC9SfpnVi*se!1(;TEYQ0)}HtSHI&(`4sX3(~H*=V)u~|ju_ZDQ~vuuw&%nnVJG3# z)j*Hzg<{xh+pLV?jf-0>()T>{=O8;E7hCs&m;SJ7B_@3wNH`z!@Cf(^Qo2|7*o+t8!i{(z%c?V)&_1 zPBe-8&_11zuJSqCG;xVb+N7@y%<+krX?C!Q59!|U(&e$G#xp#vdGj)_X{B!Hof*HP zU8G4+H^V*W@@h*LOY^~a0Zo*n#q^i?&}V>Nd(J!!HO$IZpKPIw)Wpb%HC zweZ$n-#_mhn5iNQ<_sefY*4bY>aFj&i9TNQvyNHLueHe~&_;19s@aCIoyS}sd2yC) zX^&nH_IK5Cki?EcD&R0T7~HNscCX+fH9_@0bwQJ}cg3Ry?vFYG;NV)NG`^GS7OmSc zTQ9E>1vDlg^CF&{ZH&n@5I7s4JeptN$cm~GSd~!6h8o8xwHi0$!@obD&b|#>-z3py zrEY6oxuN^~&T}QHUze;9QomcQg0iTy+sq?u*R>%WC6ZJEyLE} zbKkSXl5RCcZZgiz5M#(-XBE=A!B)A$f_SYH4i1BDBj==UZkPyXBjE>yx8cc)V0z~X zmy31K%Oq65hDMq0(K)Cw(~4}~9hg&Z69c9|wo;um6TSOfdRQ+vFUvVRnA=j?cK+Qh?4?2p8-{5$MkA zgvU4;zBgOEL=~2@7!T)*pTv5VvJ9%>B2*Fwubj!Vt=N==Bsq4W%zFQ3Jl3;;@-sU& ztTDpn>8dK|xr!jG`OujkAsGsIIVK1LW%Y2BaYMjShaF48q?)-dWz*7;{A5{!4vp;6 zjG5P5bmMHSBas@G*Q6B4MLH#5u$D>k3?BfU`Q4GRR=~j}sQsKrZfu5L;*}rU{L@La z$iNg_C{lIO=Uo>>dd~jtaUeRldLZt4@&06IxrUxBpZjkLHC*ylb$37d@m5IbVRYp_ zTvk(00`FpD=fG!A(&Z6ahR1z0!XZ&wfD_b@wK7;C?o-u>{h6}w z8Ivui_5|rj6t+er;td*BL>nf@uO6Cw$m>wfIJ>j?=%sRyj7+d|k-Bn9Cu@PPY9(8A z@8twE$m*UHK>^0i!~MmQ;#cC8+oM9-c6@}Q(?h{Oj9dFAPP>bOSiqRak?}vg#oMh9 zI8hi_bRFndC>lDQ7kl_ zXf|^66ppN?jVZ00+j@zK(^7N~Nj_-)``B<4$r5yvGo0Y&NIhx<%!ZO^eSf__lHIjU z*^63&?Y_cBgsF;$_DM{17f{W2-CjVS5`5#bhqhRpvpLyw5cG&Mdc~ju-8S8ZX2Za;a6v?{$@>s1LR_0vhV~*`gNS_p2h(%=W7tb3XY7Nu+RJ4&S;YM zA&b_*=e{UP3YJMJWF)RSTd6Wb@N|2>zn8poYrn$CMhEH7?>m}Q5TlpMsQevua7#f{ zs^DBLx%VQhr3cC9VTXDpvH0s1csPb&{4JZG4mhe5Vdt*mJ2`IYOij0I>mw>2THBc% zz4vhmCm&`G-yHS{LKqRnR?hvtSNyH6r9`8R6?m73Vk@4I5P_b)h9GN`o|ZCVS+sZY z!hkihigXLn&pMj&2QYPU$Bvpyt7RukBzuZ&L04`HK#ilHa}W*|b`Ymvya9;e2@G%t17S*i>2gR*%gUGA9ay!XaGpl)m9MM&^a%?X!#FfMvdNzET z)vbR^Spk?|IIwI@O&3N5_*Si|Z)^Z!dts^6En?i@Q%}~s2D| z0~$zb>srNk@);>jE~s*dLeD#mOQ1+_&Z#>tEeg4gIYt(UhG;!T2kW(7Rb1Rt zia9xTnmrxh9Kt>{!tu+e(eutq8%%-lcfFGF=cdy7TUlk|_4an)EUGAFWBjYgc9S&m zRrnmq(P-Sa3|;+E?1`#{ot$)hQYZBh?!dt!D#s3DEaZ)KDVJ9dMMDFpg0G3_#9wh} zs4IEOD$m^HJHxI+y_sZ`Vlt31PE_|^x2_G}TJFz%+>2adT%l?9=;!AQj%a9fT{_yq zZy`u?wPkLRW5V@_W70UGL8d!e2M<&xbtL|B`?uV{ZnaFAjO~6}E=NxKE;!$v8T%zr z}tFR4crxAzCs_5(0}+Gdr!0$(Nq zimNx^t!(I%XTb=|3!{C>)`lL;;OdV)2e@$*KwZF_0dQ5qT}OudXR20NIU5YTXG{Mj3bUh!@pRuuR9urc%&1 z&h?Z`cNx;ZH8&D4FR3+Bs=CiHU=RH4Aer{L=^R{M$KPGfITwWPY^B+c2QZ}cKdj>I zCg{6S_b`t|m2AY?c8sCu!Zd%L6)SX7Rk0eOskl5V5@Q6Z46j=-sbQw*s@f7|hgY(RV~of)Oz-{V z=BQK|E*|%(*rBa@i0V?`I3R$t8Tznwc6@)7=J}k9}Ps!WjTE5&I=gfZmMwI2;HF+VnBXwK*Rq>bP1>^A9?6!T zH_@F?pH4r5(8bz*WOq)?G`m4PjG2lxZ6O^TL)sAGsES)!)jR+ymGH6qwwk5o14;o! z5G%v9IOuTWBhI84CowzNpm-Q?@wJV+rXg&lAJ%~fjum-(Y{LYj+nIxM@_45oeHB1jO5v_Kjy2DkLQhVmRpq_0HABBi+X=)zpAOg=2=4 zCA7;unfsH^u(|Pt?@!$V4(G}hPQ`?RimtJJ`}V>NMr+HBbcQ6wu_OtzP*HR^Th6Dh zfGa5@V(wug{<)n5wwjh5WvheznGrX+bJ(!f(~bPqRd~fxzLWRNFK^Y4GO9#GnZ{wS z%X7>LMV0ccHv3c`w6k^EJ!ExeiF+^7jt`FTl1L|*BT5}<@^!nN@kzWiyh?{<2(dIC zGl~knGuOYwS!iB@9NmO-J7MUUgB8`Mdp2xf8N_ZTXMB58O(?QlZtx#qdKU*aj~p5| zZ~<7oa%4X4L=8SxZ{(^5#cWQ7QBiR|nyoHk6sS?BE>o1OHfWniCdaA@5bczg06!zP>z}J(h84;P5Lev5?wP|8KHZz+1)q==wp2BexM6F!!`ekJrXj4baJ+~AMm7q z`pw&SQi|RZ!hY>}JieoDVU^ESwjY7g268C*1Jew4BOckc{Ne=YJDa=11J&qeOz#Ug z*M3b1Y$yCdsP^fA81I^BGs)##GgmDOE|ZjiiH^Op6Y%)H%rH8_sXs0Tku7 zQm)7|Ae+@mm^YYW(H%X^=xVXtvV1xDbfhWVG`PEjei%$oCK$SEwz>`_)H9&N_Hdb$ zpDVgYn`hb@>L{bak+(Q1<}o7IgkOghJOaHVu_TLBWYjImK{bG-1#=~44Q(`&EuF9+ zQ3DASjM_wl3R_q6cyt_D=zGHv{iOnq@`HH+QX40*>^53D*;;}DH-Up*yJuS@CUyi> z4@#2dAP?Ja6Sx$5!Z1ugz~fxb30aFKVPL7R^9walQ>`1G!9g%+zmpmED&gNoj81#Md5uE*~b+3ui}tOMG(e5t{H83p2qBh zvqbY`_C7u*ME)j56zfnknNl2h^wl3z6>Rt9z5$11NuZI|=K~_^#yw@MLAm9L%Cunt zcQD)r6OMo;RTWOCF2EHy5>eTdjEjEa!p=dowgL+(xRHA^Rk1}N%aA}*3tX}=pY@3#F6fM zGU7w!+{)m=IU{;Ja<~hYSmOqRy=)bf&-%+7fR1wgy$;Hb;e3;giN*{HMtKrY@Yj+h z!5oT%8D#3iAI(WoA7){8uM<#Wy~kh$(V4!7xD6e!F`c}i_9yldo8X-%IZdKVOnbkD(Q0|PT-w9ZBf`5Q-%xHx(D7& z4(1Og=&5+HT4WYoz4xgMNNGtZI8AXh6fo&Ep2?qO2wu|@#5)FJQp3mx4pVxvJO)(| zN7Mns&(VqP><;XLT!{(nnS0JI&~ctKJFIWS3SF-wYIm*V z=QmI>|KMU~nNVYhd`txbqwhdJXa4vwH${~|1SOjB$5PCr z-ul7h9XsFa)Nw=H6d+UmZ%P2`M)V*G*r!)qd$R*n+)TV*o$O*9`9Mpwh2xgfi9&y0 zNi#l<>Z@I1;G5`#0#b7h19DaI%-TwK2vdr@BeT~2@pLCVJt=L+Ve`V{=0 zC>%+AUR@Ccs*T7CN5US&l5aQS3e{#JBkk@CW6N;)FN6>K z&)Z<@T%CuJG{vkXvViNJSYD_m284ALpGplZd2y^ccE$d}JwYZx=w5U}0qboVnCADg z8nl+K$rjv(c)O_mOF+CbFW#tD^$YP+TsY8mVcFxUpx^#{8w&bKYI&=^Sh&cM>T*v7TfZ~?xHWfu+Lc9v zstl9}0^i@Ls7AD)_CNZ70S*+cN($Su&*mdHoK90R-kpyo^+m5vn8f7uTzpeB!JciD)?-flE#bi?f%)=1Bl2)n{ zIO(bG#uP8tsRWan{S2SR_8U$woQ$#j!SsRrj{%l%)*nVR%0d{LP6Q2eqJ+{KR>&j; z^+y@NWOXXMuMjm-+``LMNUQ2Vl{Dq(%0WQFS)eTFUot)@wUD^v#*kGC)M>10<{G*V zaL(XGICelg0w*I2p!(#KVuyEz*kf`dMhdo0Kaii@`!ZH1q;UulljEAU)(JaJ$d23J zOefz0f&kfVtkO$P)a{K`^kMH^_F)ty-Z9h^!U^(_`r)E~OzH=98GoR%l#WyDIE=`k()K z53}k0#wF3`ANAV-%N$zpSQ4@?dE_5xkf!0|DL13Ursk1}Qhz86~nr6V{AOdxK0xxy9#|F}Of% z>kwep<0G95yTMaOiDH2jxY05Z?Cf1DwP z1x7}u_#EjJ>{Qf_8gL_KAq9J0L~EbQe1H!m2P2r7vz%UilkWfLN~Kz_Wr8jlSaIDA z7f!w~L$oQ@H?O}jOy*_$Ac;pmOlnL#0x6W{F>Xio1X`jRnTil%P*i<}$%;HIi=*D8@3)YHA zu~x%KbYsx!AT4xCT=77QfWREio&^X8tV)u5o+4kfexnSuzTHiQU`mP|>poPU{Wg}L7RIbLAaqq)hE&&c(P_xMG&TtSag2-ec{j{YKV^VsE)s@pe z+XMG{g+>_~a}glX`7XOr^5SVF9h)kcFdP>v_BnlJt)D#DjI|h{dGkIDqwC9|&Ix#g zDcNP!&Yb`AQ^_J@m#2sNK(bIMniXiio?RCRoPftW6At*NMj|79HHA088F$hGd+Rkw9w)eqTAL7Wou_6K5l! z&qQ4zQ+!UY{R?LbV;T0r!1e(*N`vYz1R*rczDZyL7rF@T*EoWtpM%MyV5~rC6R7wm zn1aVaCPtwqjn&4ZEdmayK_QiTVt6|w3ls4@e1L~$!i(1wKpS86PUvk|C{rHDu8+;~ zW^_%_>ApT_h5G2h3mn7?;~LshQeyg-Zh+7ZROkG-K7T#doqY!#YXlTou}+blR07MYCCI)sF1O)StFcufU8*~<;1+)S_*X0gylI8#SvGeO#O z=L7&Ge*Hf(8=_@Vn38qNW>KKH1e!dZrJ+x18hnbndqtX3Ij%-++%=# zbSeZjWKnY0HakW#X%SNl>Kf@J&&;tP(Y0&Q{7MXX^o$8k&$jWq@7 zNN`>-;*f8-^&5%W+W5JK1JY=GzW`3mB-V!&RfKdrP`?sjqEOPCL5Vt<>!{aA)s-4p z;A6Z;p(!Gu1_+TCAsa{}XznObO#@7zs+ave6{?K%;}u_AB9WTW72MKO78%1<>l4#% zLTiiqZ>nG-YYKpPaiFZn0VI8#$)yne%oNIae3r@)heVF9slVa)Jrn(c02}2p9WqbJ zRYtp&GYfmaaNAyP&w6LybAI+BSSM>K{*Th>0N{K$U5<3Z0q@uP5C(FWUWwsG)3zs~ zlZ0Tv(ncQ&z3pMf+?xAn#QHj68l@}jZs{Ch*i6Y#3}~8x4_|?m;Eq{%*!BDG#C_k+ z-+rt5I*sWLD^e@T3Xyzcf-r;}c#}XR^~OJ0Y{I#!gGbzKfNz&=-ec`|-|E(~Q-Ylb zbyAR!a!iODGxmVsc4G~dwwm_)7n!gK{x8f8N>^F733356b~=TZ`G@#*>nH!W-zf^W zW^9x(AZh~Uf?|VBz4rTUClj<`Ys0Tdv}U?z+{6hJVQ%L9W6Bc4sT__XGqVLS;X%W3 z$O73mDvf3)G^B93mCJ^R7m5Mpi7`ncp9SgBi?({^Ku~de2f#FU`)C z*1U>Sg;3&6Q#t|6105zKeOy*z_;m1wU$<7k*_7xJtWK76sK`+w!0A%|0aqx?Gr3YG ze6inpVR;`h&0dR;MNJd5GMil$Zj@#(+-Jh28ikTNd!vZvi;L5A4@>u?p6pE2;P@3g zco$>Fh0A|y68F1spU#QG65xUf8L561$DY&dhB_J|X~c--(Xxf3-a0YyqY91+%;$Go z4~u>~AU@~42lTf}uR#&WL8ubNP{Wf^l5h~~axBwJiC7!%whE{~zn0qwW_-0^!$|+) zcHB4x;1v`Tdt5*3ct|v}We-q2f!b~wQsAuXGLJuGNX;HN)vi*mP~ zC>UgL0aE`K;sEj()PDZCul4j_ZuZApe~q1V5e((Y0>uGJ(GCH}R_v>UgsJ#d-%YXgoOU|M3zK#0Mt4IVe%X zVAW4xkNVMnZ4v*dbz{5r*gN4vidvF?hre55>(4%5L!3p6RFq4$U{3UbalBxT%coC) zB(u$+n<|Knu37WW8v0udkq!ecFY-%QV}UQIi|O(hblYxFF>Aby`5XsYurvb-?eudRJ<*X?}*(A#Z9wM;pfxEHX>DMCUd3;E@Ia>^J5N1nhx zZ=%=79tBn@tyB16_jHk!f!bih*W#rLn(yY&W5$%`7eVI2QAbN7!LgWbl12sgOmsFu z%`K>a!GC#zGsHk1r}@V* z5PDQ*$(^CXHb0**?Mf!Qi$&Y*?hueP-TLO0-A2inHF1XUFL`#hUHOk`&P2XDGblcN zAN-Unm3QcZ(ovEzyh8DC%Tx?=DK$_BXeDqGV*gGZb8`}%MS4ofrEqhDC(~?7(FmUk z+zjA0!g&Xohh$4w3uM6y7#zfG%*7e%AW>0Dt_@I$A%@IVbPVZz{f(h^ zmJk5{#&?Lgwk}iVvzIKfxUZYd9LnB&mF?%)E0JD-TE+Y<6g9eNp-w2b zWbs|s3>W3QdZn=?_$;Z_Vad!N#@EVTi4lYyT>hQd!Scafmg#3HrG-(k2RiX}r~>J~ z5C=$A9g8b!fGIyXl2LK5t+ew_nN#R{x4KfG5n*V;G<$#V`Je2)d05ru`aMi_s*_`x zXljbq#sLHr1;Yuo)V72o$~-wR#2g?h$Y95`5~nr-0wSh>%pk}pf~ly?$tZIoLxO06 zfXwe&yNuoUJ=gWUet*3G{I0j_45_&HXLz3bx$kwaweB!1DGEL<)f+3TY>|Du0ON-F z@g?j`rj&$P$|1sHStv$A*bB~2mDi2^0Xz{!imfS&Lm%x6w977|oy2t@R(Rn?Yo|6^ zzU}5+`xLBzAHmR=N7#s+A9_=W>4-@Co$VolOMm);GgE)~fi5*-3o!QAIHTq9L(AIC z3-zn5NQDmTZs;<@y=kY`cDZ~S!uLtF)jhPKWspDnAS&IZcQX`)n}8k4yQM1q*eB}T zj`2&cH`U=R`;6LWcrj&_hbV%k(+;bO&Hbn+_TVPyZg>~)$}EuNzvK*43mL8+K4CCC zSEhoPOZ8+H?(aof&aeUyI`2E6Dku6``K7TyB__rw!McUq11byR?T=yj(eNu2OY%t% z$CW_}F`McvJgoTus-{xGrX9k*66!O!tpaDH(a4@tQ^wt6#m}UV&D|<@npC67zT(NS z-D@vb*s!wMO^(s7SPEfi_dLdDfm(Ls3=OAG3Ih0TfdK*I+7k}HrQM`X~?a22> z(iq#1-F{!JMLG2fh;DF4(7{87B@s!Wc~dyScAIdi7gBU_J%$GIM-XwGU|opA-VfGF zFHDneNE|;Q{611q2m8TD3QTGsYV~_#jyH}W?zeqtwKLV>&X2{MF!Uup5}%@2D`;Ga zy*&coM+y=I%HRVsqY=WEV>B(pE@2VM8p&`3o0k3rVxLTS;L!4JT3!ZeTeQ_-a0fsv z{Q?W6CFo8v)-|ERFTP-38l68Qfs=G9_Q_^YaM=#|k-BuEaJY4n;O) z@8U)-;|M28e^-p)J*047{Ove<#ZirRJU9SysoUfr2CRv@h-lN;<60@*_<*n@ z&SMt*5YEp($(H+@X1-`yS$S87;}j!jVH0|`oL)t`_Cfp@PN@&*-@?a9wZqI64(C&I z@cASmivd8$8fcX;jj~Y24A%N@-U|qzuhldq(~r4xsAl~_tg^Uy_zfHYXjw+tJ2gzk zys{r)Z_w6tctEo zl%Nq+$hh#iR}XSq5MgFf-07$KD-j-Kgaxqf> z(p|@L{a>eQEYB6px4E=5=uX0qKQ6r+x&4mAiQ_l)w6g>+s^+mh}}E7Qe`P?s7VBez2Vg9hMd>+&t=|$$z267=R)da6flG z+^nXun5OvHI}_+I@^D>8HfxfQETy88QhJ~ihd;` zNKELn4CU`~bTvqLAtwVrvjA&LA#D^LW~xdFR#NnWk@XC2g`k~Xfcm=`!%zyQ8QJ{=9JUQmL=lqO^FXdz(G1bVmpTIgR7_6b zn#Ym%e22ESh_Rh|Nsf|4&-|?0rugG+)wz9Xv>V7BY-`GSSHrL-xjzZ%MEXG`YlKK1 zn>_273zN`7w2OBHpQc zs@!?wUx!zDDq`}DREV!^Pl(Q(R1J`S>m#N0d_$`RC%YR*L7))(C~{ z*1}g)HIMB&%!`gSHlu+fZH*v9xPr{GAK+pV*jN76%Mi=C@-cA=Q zv|zg=ArIhjRxh7IzU}0npMQ5J?`V$#yJvjY;i<#0!}yP#(iNd!4QDd3eH!R{} zQiG3I{>f3y4_BqWOe6s6SAy%-d)DxZze!@s3s;cmUEoygKR%Xpl_v62XbU(U9;HsOW&5;3=(0>V(@@V5=S*t^T z+CUIr3otO#DSm$xt1cYDPU!FL7#r8U+koH3S`mrGbM4bVuT)3w*^|(px6MmCz8(zA zVPg;wg`yo>)nrixycj1_rl-)qLe1di+o#cDzgn2^s~`{ag3of4Iq9`X=emL4O;xYW zjqb|Wi+nNz2kY^K9B9K@VdW>Mr)QzyRwwUrA>sEm&#^DIBF_0FKpb$r^o=LbvW20i zmYYMapa;*vag9f~-MRIl*wA+vhKBJhNH-CHx}{HbBN4U&H@F1g zN0pj%>+x4MeQx{RI%(E9HMF}DOPQx`OSTs0&^*OTcS9%HS3pxCMNIi{_N=&j!`yY} ziy5yc->i#+XPU05`;%~5dvKbnuJPz)nc>J=hZ`NChGjE=)NrIxdb<}Y2nljecvS?S z12VGmQ*b98%C|okCc(|Nja@fUGg8C?jzGTjeQIecLe8Eww2uTeJ`utfBvqn~^Wspu zCDqtOPilO1cg68-VM@Nq3bPb!+g1=rW4`Hag&|U_QR9Fv zzV{B!2!-T?=p`Uh?fLB*USkHXTYiGxf;?Cwo~P-VunVMvHk`XgmW|+!|K@!Aj~Jht zl``4wSJa2CmriIzzu(zgCZ@Z4B}y~-35oObLQx0pS?F$H_HQEt_yX)k$0>8%TUFSm z2c3?MGLEtzouR7JUw1VO`G!JDe8mIkR+oW{)zpE0&{gf^qh@#oxl4x7-*ZN9K%r+1 z)r}cThMIbCmX;`oet^#D4PyirJf7dDmuQ*Rqtd-vZX1Cg>!&K%eEY#6IMa1I4_-$h z88GA6Hr<@IYS|Dp8FO18BZ<9bHr*QMqyxBV%w+N|S>2ZP{_)}`DNafTCI^*Y+xzQJ z^icV#1{JIwVtPL#7oz`sAtN^JQ(AgMz^iAkNZ@06sPn<)d&>7}kaw3MNfj(Q7AwCO z?^r>RV+8!3S#2c!UpVb(nghht<7!E4%l&mXc7dGqmob{Mva-2a^<2%=oafrJ{Ksb3 zikMig<8vfc!v%f2=>Y-L+fBRl=K9wqe8MA>&mr8Q5nt(3fIu*yJQzKWWhLqD0>p$a z-*Hx9W~EJoyOM?u^>sBo&Secfb=*<>M<-~umri+Qe_|LzYwoXoRoT5MBvSHB3RJD= zcME~)PNTjObtap}LJ?aaz}*Zi&E(rTRvu?uc6p~dhRPrRr5#t2QkdHA@r*ii`pLFb znw4>ynDQta%tKAnbM9LP+v|Gt((j8hU)|`@T(jRbv?qM87l5S4{K!PrFSUnD9DQbk zjP>N<)yLXtqrbhYzR=6y3Brj%t6KxotZ_N-YZ<8YuMS=ulraJd+7^-BwFjGWmG-# z%DMs;E^c7J2h_Bp9L@vW`q5O$v(&mJp18W7i9k6X*G@yM|7h@ zoI9gCx-rk7-`OhD)Wc!gxT84Q<36%9l}0M6-`a~E8ji@|20D*23UKu8fBwwDbAj1J@0DWW*^DI!4X*6u}QNt>juBt2$v=ih*X~nOTnvz2Xu_KiX!8gF{lwd&%2Sh zTcR^XvE5Pon8AMgt1Tgp46FziK{#<&golUY6w^yW4AObHpF=MCmVW@XD8`Cj8AZNb0pVz@h(xWH$FZ$Xf~i)$qi8Rn`C_4wQ%zI{$lz^w(CKPb#dT=@#V zTn16Y{u4s<%u%i7{?>{!IIC`n3(EOYWF=X^&H}-x!`Aah*`0O^2}Y`kr5O`9#i~1n z85#}O$r(y0&z7#gKApz`I@`Ya{yLX!x;?%WVMQvCi^OaJg^UGuv_?p0R_59SeA zfHI7y9m?5QWKe|<3?St?UI30%zgz`H}?*l zzgis}_DbW!yh~0?E;Bb|H5dUAoSoIr@pbPhXo6~w8kR083F5&HOR|Qp&r3ml^u)-9SJ*Cs*gl#_Z4btT^8A zShuRJ+V4h!XU~Py<>2R~2hEpw&Vx7|M92VIu<#1WT%=hZ!-HW&k!XD&s;SRdABW!q zHP9GwC`lkhr=tr`Q^{4gBsckPS`Fq`|<0q7(Xp4zI!(s zH%y+FApi;TNby9*xb4#j$U0z?3T`?GcKwhyhop469;Ij|x=BCg)-?CNvp8;e_Mry7P&KA)RFZy4S#J-JcI{s z-X()0P;u?3m(>l7&N`3~75&(QbKkPNL!LJS>HkVZA|s$07k{IIpk6l&v9k=X!J%?B z-FVh>!aCXur}_^hthD+*!GRttLZML9j6ybcrwnCKFS-X1j4}$B4Do;iGPLF74ucry zKOavm5}Bqh#I3ZtUL@oWPHsAl0V;v(Ue71FX%mmGtQu4)z5;BE0z4}-%Rw9sx)Zb& zE@);YA=A||e#XS8mW;ue<+>Kkf^u7OlW#R57WKogQzU5YFFv51U|N6rj)%%ncJ4GI za|Ex*aGQ&*XV0EBN8+Fd@yoY(Gwo!CrInx1B0J9(ya#1Z>s`-Jrbi&pbU_|ZP$bl`S_-x`Z9QNYDY*;+_40Z2ltUPi4J>qs$#0t^!lkS* zvb(xbwDHkXkS7xM$;fvP_0s&HuRvDE-mR^chXt=ri-HmBSPWfCm#(lIA zngE}Ifg4O~B9!HeHr}6e?iqcd}~dY)WBJUoA)9 zWAA==7@kMMmSQrc{2Uyw!_pzi8q6uMyWTwgJLHR*1WoS_qei*p zIP5lk^SrhIK(?OYeP$2Z(h&AqkPI6=8PTf%SMUrF+L5$bvbCiFn7+WvM1G_8~Cz)%MJ*vxK94cx!Dc*;<>aTJqZ_NMj?>1EATQKF* zIyW{c&Ilvsu0kdqZ=_uY7|u;RQb7BLm*dDw#5mcx9i&9tee|DN-p7g|<~2Qh;8 z4QJ4vOSJL%ZGx*!Pp?r2Qk#Pzo#du8z%PLWg;Y3xv;b(2&a~ow^Er z78fJ}vO-QHg(-V#jco_OLs2uT+ef!&9N9<>4LeMEm%4{QkiQR>zntB5Aern0SXA1l zR$J!&+Dps_Nv?mbBWO5k2yfj8zB*1l1@wy(LoS>^;!q18uI>V}5oNrUqpxrlW#>QE z>FIvjbjVvo8c>XJV}e&tOj8#CSF=Yqn$e^#Nz3kuYnrZ?jW}wFV%@CeYSm68pFCR0~)r2DK%$4k!h`$mGM2&XHKt;`oIuMvdO4W*E=sucCT|iE)kY5C% zD-c*+;Ts0rp&C268;uAH1kh@`-!k$;SLT?~)u4r!ZJP=z44UNIby%C|v3CLRzAMf<>C?(;aKyiW{g zBHEG}OEa*ge^jIVHO3GZYt$tvRjL5N*qjT5`kUq^qzYOi>E5zsi%5X1aSKjo-Siw^ z$i1Oi+`~o^8(%O_v8O`=0Ou=!$hgM94D3SPXaOLbNw&y5ZlK}^EI7`+SN2w-)emi` z3O)u;q5=4_#*3bymu}>D6CdbC>&+4`k8g+BDbsaQp0C!ALPY5Q-djq#Iwi9gm!g)U z_cZ}BG*bf6oLP(Nqg;M}a#3*fe=gqXBghL&;td94EnIa zBthUf_!49XRt`X?;`0D^m*#knBG#;;A(+wLdk5ez-3*XJ1MpNgnAR&0Kip_jj##;z z50t5}K%C&nmTb`(UcCwrl%Cgs5X!uV9?M(-%0~Z0Q5YH$$aJg5RrSR=ddib3#_XP) zKh%3WB+**BE`uH({mLQ;py^^Ugxqi<<-2u5<2ZpRrgK5Ic#=iis>Vg#?+Sz;gke`um08g(d6^g(sGUDyTcGgIVbTKKiyv$!upBz6ta`p+$Z|a&(gi zoEMdj7XYesT3VO6SlaV~ZGM{yj-{_cR;o}FKp@z;mX;%QPf`Py+rNhGgT(tdq0S+9 zZ}uK(^Hv6B9FX0D4A^4J+g-<{o;5mvDp8DY11>#NT3R}U2$@f?RN0=(3gW=VYLK2| zT0Zg@r>Ilp5&>u0ir=0^y z7=rj%k!j%4=e5*O_ML%ER0inCuie}J)Aw^eMh#hl&=_j|VB3YAE!#X5Fzt~VvKOa! z!?V8|rfG zg;*}ZAY_wYiI^ze$C5Zt1;Z0S`$Umuu59n(3}*P#UwY+E0R?nFHgR)o8biE4i%r$I zc$}na5*JBdZv`Y-gnorD3deAem-}m~vi%|VYJvZK^dD<=O+e5%$7U)O&(L4}i^ zKfv=H3#ZD}TOnyB+Cmx<>^#5Z4`hMR)Q<@6D?kwH=qIf@E2mlFMUuVi)SoKOBZ75CJ3e(#_iJmNUR(apDk z-zOvZE)ZUDYn{>fafZes5rW?Q|9N%dH(p;F4rC1X5EpW(}Aq1WTP>FU7lxt z;z2?T4Y^f3_A{1vh=!m+cTB~D7v!Py#d=@IVB+hzi2rvrI8vjy%N1m`SP$&oZ9i+> zM{3pi6!z6_=)*f^ zn=RjLq$@y>hjP~!Gh5PY?hYVC_!Iyp4q^I}HVZ?h~0GJiXC zJ;J_Y^m=82`)@s~hwomosF~1TePPB5>rKx*pWV({<6zTT7#!N$v0S53VbzsJ-H;s# zVR)a9zEg5;?3QV{u`R-)H>=;eD${y@o!ybAH%zLQ+i9uGKR#vEP-60_dgM2?Z>MGH zJJg>E?KE(xzhj-;)odHu-08GK`H4?;;F<$oLCd2!mvE3r?b5tCf9?W{NglXm@cS^76{q0){T>!l>dfL-5>WmW~erWICoeT zFDfcj3lCi4s@pJdaLXL`7mv@oH7izI`c6DE9EOAC8yk8hM(?AIAoeX^!8}qhnpX(hX(H6mJ`}{ zcUy_$sRo6f4!_txoKzjMoRhaJxjolUPdwwTK4N)nP0!Y8ii^)$uW%_IDm>q9F|w+^ zVD(75ck;bTlkOYCsf#|`vsKP3A)-W~`9Gtp1M~Wwm zPA}R)*E>Grct-2rPbejD-OwmH&225HPqr~$a4<|yM?#%lynGBi-U2}q5SfF8$0odT zZOMgU?>DXiwTgdv5KLc#s9qZ2fO~-zu5Xmp{~&DKkqnLQ3ft&QhrT*;%%ZkDX70K} zFFGyoyY8(gM`VSle*ME-)6s9V=Nxjg>#eu;m5#sRb;Di0PE{DVH~nex zbmvi*y}DoL+%uaI{3N-xrE2qY)7FRb?g=9Up>4}twm#7w%pA?=E4ykR^KoK9g#323 zm}OOUrT@G&4-sVED&%{Lnt@(uK5w`Y<5@^b*IO^3S~7K*JAyh*DOpMD`lcJh!oaGc z;N^vDSJ{hBb)K1I}?xEVy=8P2uUh`M1UcIvM_xd#oLH?mV!I}3|(tfc0o6C1` z@wcx8K&doQPv9I5KxWX2;1!ctkAk*fv64d?(3e&mHzrX_!2>O0;vp0P|6PI%$;fgr z*M9o*@#XPK0slTbKQArk>c&ZRuF2zSvsXRqzGnR(Y|Guw;gm^g-I4Vo!CR^I3m>l? z{KWB8h;f8w)kMR#1C8~cPHa~SvKZ)AvTTcbATvG9d--m|RwaG?y6n{^E57!PpLH$o z_sMJJrx({dKkg~Bz1%DPY=zyc+Lp@U_8&}=IwNNLBpj%@mX@6mK&F*zUs;0}A^&ak2&2Mz@R6Tv;<@T#Z=iR!y?-a;bp;*6lU#X$vyC zGjx$!}AD$$M8wOV+F>pf^@D;GnyXD=cL&#oe7bc{&0NN{aH0%eF z>RX^&CKM680*Tfr>gr)Q5V$5QC&(Ona`~EB+D6s(#H}@{4*n=ekJo0WK62MleU`nd zXJL1y(f6+TM_t}^$Z-n%t+PtiIPI&?4j@GIJNIVEjh~=r-dk6iHSAQKsru~N*1%m3 zqJ6a&HTOTQUi4|>D@P}BjBFH^m1UUZChuC1=e5ewa6+r%{`=2@1Kw+!pEpZJb4szv z=jpBf66bep+`pP^#jS8wK|&GBY#@U*+O_PM=e8iTPOGtf%ipm|;d-G$>>sFs7yxnEL(kooE zY*R~3p}dKfY|r8%s~uL;8iRA)_ZgyPvu0(S(fsqlf>5uudjlWUoy>7QAuM$p*5xO+ z7LDfY4PMex;I}Ba;+(BR_1_1s)XkdchOXe)erDik)3%JPLXP1x$bNHf5BT8sww$hp z-qSAemv@9B&P?G%#iO09Z-Hy1zZw9vZuEq2Cdxi$XecqHEZX3)b8Typ*_qSxzbg1t z$>Yp;2ltf=%=Vex-Fi+}_HpV+{)DogfOFEkeFHRS6+CQ88% z7+1?tj8-A8x-WWACx!s@^0)1aF&mbn{kF&)wNoBS%9zXM7pDx^Jy6pd?mB#a|0~5A z*UIMiyZ7}ZkE|@((dD=N>355kxjf3*tebZZ8)`JwB*$x`Q(aKSHK%@qYdPoC@W9PH z47@xyCV!)3J)k`hwA7`+x#Qw9=OTrWv0$KYq!QWrW~F5#n(WJHJdTDjo0~}$x<)PR z52!*m%Tk|=aJtS4gy3{LqcKtCoO@GG#YSKK`pcRn_-|smLuFlQUGJUXjGnx-;~jUs zdN-~e_Sm#CwkkpjKnd>#`8zr4OyF2f!bBBN5+8d*J zXYCr|OagIAdi~_qJ^idf+Ao<3y4TFx|6OG~mV#kL#zh%8>yo^JE&B&HbtXSTHWTgkxLkfwUYPZ=?mx!MoUOI0 z_t&)fxiq>Ap~ktx!NYuC_Bpi$3c)H{Mf>ykK_eQKTI{I3E7%SQUY>8!vB_uX0;TYN5tlFIkS(C?XN4C6)6`feB}6d&&UUP=okeUhg9< z+xc6A{qC6#e3Q}1*k_&N;4?Eaav-)g+M1!{g|F4rDon>ya8wck0R%^}U; z-qV!fO>SS8kG}2K=C61^@dHOC*AsCos^=bh{%~(b3QifpUiHCG^cuQLwQE1lsPfPK zb&2hys`F#ZdY2Hj1&XiwcREx{woR^&N-c<-(m3N+d0AQchLP{yN-LP>c_^Y`+|zQ$ zU7hlLw9;9X z1&3MJiV77x-SWYCoYCsFyi+;>>c5;MS96;hw>EIR^JUE+6RQ`byP^pK{%y zT2>)QHQ%Hp%XjH$hzLhIN|syNwN(jvN}DIw$??FPe9~S0viiAmB@rHh1}++9VcU;f zv#(zqe84sUPw4ENkV=zBD{E4%{c|p?UUGX=R*03UOu&yjtRmJAyH}@|WmY@3C0N|M z5E9+=jls$ER*4lUy=c+YaT?onuIkQ!M0`JTJV73-ufOP$d6$*E&iFVU$}Y^5_qu9U zlaTzfI{Mpum%nQeDOcf*8pYSajsGwA`P|nsEd1*5O5y*>FQeAmPnVJ^DD|#A$^Y71 zpP+A>K9-1P-kBh&-zyis`G4In$;COWvK^KC(~NH0>nBn+AdLgFO}o}YtH9Vr<=$A5 z61(+ZO>ad1f}eAF_1qz*Q@L{7iV)W?gH){b|Jx6@SL}aXIr2Se&G^6X;r;)2#5eor zKjlL4QfjG24p)#h2f-L=@KJef)u)2h)W;8p@0A_8QF!%Mf zNJ;7bh_9k*sqnm7@-2HzT`Yu)2I1n?)80dS>Ca7fJr(15V}D1==RM#!Abf;^i8K#f zY8?n8E}$EfjgG)m#X46lx6y9(QC~FcrkVi(ZBT;|jWW~`lnH@g zbwC-#@l#|0EHJbz8v(YqK*Cdu4)YlZdMrVou|_Sy@oLCv6iL598`bPlI`JX~X&M8N zasbbT^8lfn0Ze{{*I0^+rSS);@1&=rrLwc&1%B|Ebi6m(S^4%nnlo(Od9`?l%Wz3- zX9a?&2}*^2Yz~`zWMUbPPQM&~`qR_Tq@?OL2)|grv*1?7vf~M};&QBmyyLvLTsc)x zw{5f4$cYl|bLvVxGgoZ8?Rx!6-m%&U)7$NCuV?q+sZ|QzeBqY})~}8eK-gHx_3ZFlivp6YX$*SGv4IP0KQR9b>;Yxi zYp9PV8h_*^JX%^E_@Z^qqCzvzhQExU@bnH>{4}FUuN3UW5OB6+g+a@$MaXn@ggNT3 zewo~bu0u26X${aP%D{_y3RXvdWm>i$$wz3K>nKkW@UOmtF8d64yuk}r1mlusnl*Kge~$wsOCcyHU#8Ft*7x`QjT{02Eq;a!bt|+# z$^cAR4^Ugx;L%;Qr9qr2lSLu7rh(YGP#Ll>*fXT2XG2lTa1xjbCs!p!Qh7_z zjyA9yNCg5xMvPCl$0}7Lj{uD)S%U!;BmE^xi=qPVb{>lGD(E%%%+a#- zl(zyN+-Q~Mvw*gmwOjqv%>Ul3hg1HuZb*BA%-Yo)9wj109gnJjta~(<779z* zMTTwMJ{axjHtgC0l=7kMnX`w;cHNdfbum^v1C~urZn!+e%D9&Y zV@=WB&TyrsiNOeJ&))Ap;fg>~Zwa57Tvt9jQu6R5eol8lL?~u+f^EFKOD zd3jH8f@h~KTVtgEkLy?o+Y@KRU`xb;TvvMG;vFUyJ_f_;(Uy=KOWA{N#YM{{d-*7&-lwvr=>=U_@k|{k58e}uK;ME|qwv}AfKqaV<*d%2yR)-R zJ?bMn?*@*P772IncB-#?&oM$+HQRHZr5ZkThvADwcMb0}#s)neOZ(+6V@+0+Cg2gD z-(O>GVd-TW3mF2sMn!<2ckDj1--51clo+A2eFmmA;o#Jfc(07{H3*cm@%V~3ffzYw zGc@Q_PspnjV^wEqcXH$y-h-aaVqJAGg%@{cV&S8swl}}XEBm2d$uR5^+Q4y^tBOa{ zG#9ANYCr)kmd2qt6KJoRg_EcsRFGxpfZ@$27;E~Rgsd>5t)WTjWNd8ivfq^Y-K~dd zI?z%%8-)tl1A(rM7zh-bzz*xV1wuMM;DpY(BEuySQ?Ha$B*P*Qe3njnyBKCvZq47!LSZ8VbQN%LiXO zP*b(q+;4Ct;)Dw(!G-}zm3EkgAVVIdn8^TKm7D|qdyD1EV}2DNzG9|g<}G`i%ZeX5 zZxeAcDU9xWod<{cNFlphZd1N6@6tDeX!;MrNXPo9?V!zXS*dWl&aTS@iTtcl709=k zWS?00w45iO@?d>9}*wc+_{h8cJorP0N!CxecoDO-qauafzM+!>8Z@}tj|FjS9 zkFT~a6N}*rT5IcjDMD8(R&!|$gDy)W?sC{}Uujb_eJ#WF8vf88k_1~OpS)CWkz}(nrAjLlj%f8-I18#%BBYy z7o~(7!4)Ito0J;R5tcJGBX@)c4v8rvm)1iwoZmCZ7a%56r`iRw_dq$y$vIWXbj@0O zV715`!XmjyRRgcpR{-9ZEeWxpqRrA-es3p7gn9Y&B!AOTSU7vsP&a~V-6ekxm9%YOF8wfk0*vT79&&@K+d%9Xw3>Zd?66aWrH*A1QSHg#+omk zYL?~63{T{)_XRe1GL*Wp-wYz;wosI%TKCQEmQ)i!wOx>REkXkwq`zYw`-Hyns}{Q+ z;u(?A@3qMK*RPM-zNX?yN$JlDoHsHGf_Om^`Y2Rjfn*kySVX=YKZcZ7r+VRnke%`M zWGSFk4P^+FJRXpOwA7&3Qekcal3IHC3%a0ihz%C}oV}`sfO>O`MwZ zWH;OC88dU$1$Ko1?HQ5k#mP$K>?31OF7%{NB4>ha@^oa9Xvo{GvZIu2hbhveFThO0w}dG|;amhdTi* zegDj15VA}iWT+i3e<+QfrpadQ$4`??~zq-JW zvA_^OQiSN#PmMPba?-|-9WRHz=}UAeq-QT0dmA@{*3TUowdIo`@pRv>>tpKl@XW}d zua3YHzW&Pdb4wn#o~pp-k79YjB?+!R<_)}XO?Bi+Qx)Vxz|50 z6}fv#@ZY}xVfp_))MymQp7`VR23E5|s3$sKgi*vKD)`IX2Q@4D`d_1i#eb#! zoK5h&Y&i4Euld{qyOVDYaW+hAp+HC(Ds5Kz37|iQwl9<(`xiI^!>Zh)76_&6mDjRTcwWSh0pOhROiDn0|WK-;t> zW8WjCJ1fRwu9uMcHKh%*Vh+Dy{1{dU3?kP?<_DmCUN~(WA^B^DCv0S=v&4t!YsU71 zM^KGi&6n;B(Jfmf|J0=X$8{dy+4u@zx$1^ zAPGb&AsZMqihtA72%X%qnf*hkvc(qGRNyjC7m5(tuKCN4g=48*t`9C|u0&FKTF*Tc z%0??}JiMQ;__#UH5q@mw)NV;)8Gs`$^4ONK%_tS=Gq!SKoz;(Sn8)f4{sBAR){*$J z#E3|}`nRYl9!LXg%^U(>=y&(J28p^btFv^l@pO6a{fzFH05;nCOsnpY~mhRP*9(aGH%2{~)I_9%+Q zeLIhms>@>qr=z{Azr;=4oXKTQKMyzp++GZmX*@;J-!YEb878sjaDcIfP(j2?X11-2 zS$geqV+j^ICR!&fUIu+Lmsu9pGfqJ7s%unvRSe51$A64k0oxa z+?9ccLZUu7ZLJa_L z9lgLFU;)LHEZ{hAL26Qr8q&vXuEcVn70RVyrAi=S_l(H4sjK7nRm-=vz zPg58Ct>r52(U*mt7qJMOnH2a#WuYV2(mham>Das$5aL4)tDIctAq!Woi1e&q-kG&C z`ZS0-Iq5{mm0T zhlxt6k$cRh1JYoTfC6)Xp>h=^U_AO1fL=PRT&s%ek`)}iu<2WZw`BZmt>hPEA~wJh z>_vfn^1=NtQ7b8=)nkRmT#9X(2`4Pl^#uUIO&I{VAzAOjU84uWy~S{lN!U5c=`Kzg zke3wLQ-?vmeEReJcbYbz`Z%LbRpZ!84D1YrK*4Y3k<6ktO%olg%F=-plec5nj5V{@ zSrF7R9Ej^(H`Hb6v28@@{hUwiZS&zYMO}M*-2j?BhjKz_ZWfnqOyPC=X&8b8p~WE z1^Q^8nJ>MU(9*C1y63CY-iF+Ku_CHX^Ndb()C`TM8B3h2k$2Y(QDJXlQuPC1kks_m zNW|&CM#T{AtIxZ&24Y0`hzP+#4HLEc}OQ?~XnV4aBc)v%grNrlj`M80k;f0Wu~FZ;gPd#LiZIrUT7y;BAf z4{N7>=mU%$1}TM?=qKH**)sPpdyp^!JUSxO<8J=X6BhtG>XQk#-t8u&l{Jof+GRUtEIQ7G+=S!5c$T5!E zZ^lkQhf!Hb?%GQuLg%o5<0OgHYS&3UWGkn!RFTiv;ZwSjLx&f` z;T{?%>Z-7h71KtLC5n3<1Pq{+qzENCya6j+mFEWW6lT8!i7U;#hlyPSIN-Rg3&8)E zG4*u{rM>|Dgt(XIxID<@ry*)TjCYa|`dSV(4w@3wz?N;ra?Yop8n}s(+v(xkl6`vY z)kxX;APP^0Ojtd@*!xPh z8C~FAg#wo&ws7MVsxX5h9$t_FxuUBxW}sOS4f!8sd&RUL;I9Mw5`ucb=Cz-AQ7eru z)2bD9-+X?xr2QLwP8+=W3MnN8alZ|w&m|D7@f;?L9M7Q|TGnMyE^)XBlQzE-wd`PF z;G0@|MNK9=A~R;3J!x@Pl+sYT}sjW z>gaikGlhWaXK)+cCW15Qp(uMki->I!pB9-aE*6&I$6Jm)d>7DHU3NBlIRYW8hcH!% zA8pUZ7ds({`wBvlvEz%=Povl3=qCZ_O6@Arvj6rK^ERXpzNX&s4Ou+BsepUYOOkQLbwOnc(@snZ&azf}I` z?Z4fWhV!4gR$K(r;E_J>(F5UwuOJbPJI=4s$W+2XU_j zLfAIIkhamwB^S-bnd9#$Rid%PFRwF0F zUxKns*9kEeh!644C4MY%S~S+Mh1Nk5x_Gz5U(g@F(`fDp znzs4yhzLcmWswl5nlj=#ZT^S`mH`t!@0`B~==)}7q3v>ivoDz>?nDhldOrKG$QrDa z=_rKpdz{QeG#xLJV%={A*6LM_fQV@U^;R2pIgf)T|P9FO_KpC6fe4zm!CAUdv6Os5z z1e>C_q2D*FvjGMfqcU1YY#R1In(TcJL1+nW8)$JYcH?-WE%~%gRvZLm0n5T|z@wZ= zyP#1sflvy?t5KTa1CyZSKQU89jvJD>ja^dSd@=6=c7k$8H~g%sA+eV)`d8S@9f@J< zk2bb$2*vEhQ36c~8C+JmmC=!iw?Y}`3wlN&(!=$tPlF2DV| znG^4LW4-vi6Q}3(5LOWgxiaVLJ^5AbPo{ws*xH}(AkIPqd3nqb;*@DcM=U_9;_RpXC}?d zskJ9Oz|aJ$pc8v8JQ*koI68>5Y%k3d0L2;jwn%=+OnwMSLnH#>%CQhw($ggO$@gd#L+Q~t+$t-KiV?hu>M1Se=FNkKc$97Ga5m};1XlRsBOs}u2& z1+q(G#6OpY&SSunYRrW&n=)3bRz5!c6Q*z~dJ-qnI+Gm6E#H(NwHKSZCc%;_lo$M5U4B})8! z?Aw6w0890?aX=@gxG?C0-Uf$_HR8~V9c#{Ah*TPR`Q4p*66lFk>@WD+>c|F}*3;-A z7%M}O2n}$QK!hun>`Vt)1p}2_P75D825_EOh0Y|23jxN5h;Pjb1dX%+ZewhDQyM*4 z0m`9Z1&@TXZ=ShWMBC^jhKA)*`c(lCvb`jEOt)X0`5r!wk_I}tKzVG1UK6G19FM^H zN>~cCfugGpT9@G<@bp7I5Q3$(lru6kC9%<)HnDf#LeN8+N2Ctt!lyir*u4X3lh$k4 zn1L96#Q~oLw!|85JPWZn9I1p&vYo`e>b{i%_JTHQ4Hxh=E#XMOW;Fp{8U$5;efcEb zf;Mt)FeDa}vsH%P0Q+VsCK4?cMq1os6pUM|0RxKF%6w$Z#|R^Q^dBWe3mmPs8In-A z%?@#}x4W7y1@FQEe?K=V2H&Mc(g3qRz z06(C7z75%BKIP)&y*OXJiS5ymuNq(0?Hf82)_Pp>6H;^&=D&Lgg;Y01QUqeL3a=JI zAqzpET3?2>hUj;%S+|e=D#Qv&F zJm8C?467F!xN_G3K|D_uT?X3g-LUAfgbfbs9`e6pk;LmLk6_#R*G5GDlU7~+^;|Cy zRPvDLoQ2It^VuC+>2sb`=Sey*ExU2ocpCQ6ajATaFc}r6Vl>{t4zfsLAe+0?{A?T_%x$fGE~c$hcbgjSBMKc=zKoNV7BU+89HBZYUjqy=UU%C$X$! zZ?spuJWK!=+>W0}hbCq_?&vM;(H_j+0Q+t)?cNY}!y+`|ekCS{yP9Tp03g?vfqK+G zrRbv(f1wc}VQM-^l%Q{FDmyGuWyD(BcpQ6`OA^$_fJu8BPu**WE+Hg|o0x6N?Lj93*(7@DUM7_rOC#=$SZl#NG{ zO@v)ozsoqzTds#b$xCTGMY@_pEzd^;!+D%uLY6qk*m2Er-RTfpjIME-zSg)_Hg@~5 zA-j?q#cwOl+s3!2H|Wo)GOBR?>ltXxwpsJO8O|&Uv%%sc@JfWTYQM0BO*MWZs{uhZ z3|5u_v(kB%2D)NFT{clP0G#(p`axH(6~GKZ$Ry-q2`}}*>xRD&Uc^c|n2QJ@!)&Z# z%jvhY%0Q6|wA74lv)XyM%h*M0c=h+67dhQnUnFr&=uaZbw!k5#kmM$y$^r^sS-&cG zvVy0_%M(-tw9J^9kc`$cjo1i52^iJLH3MmXrHZl_Y8n^xuUfEJz7jD%@>kP6Rm>%+6JT)U4Z)uj6)31khk zsonVFW20z71`3!=@H?OqxxDG_Xf$%t=>c@mlvj}qDGo>#ypg>u5Zm{G=4~MkXv5sc z0yiM*Xh-K}&pgHf%;HN%ab2;tw^Yo}Y(cVlt+HL>^YlY_LI6`^YJiir3`AV%21k_2 zUx|=-EvV2je1MwSc*1Q*2)D(Qij)M;xSD$=K?tfVc-p<_0ScmLnYDg7m@)D5r)FV(kBK_hep877$@{9A|g$Ffz3%ZKmi@{8TD1#QgO4`^lZ4mNh z3_&tc496lr1OnT@K?Y#-+6n=_+P(_vK4<)+OQ+#c!oEujf&q-O~sU+4gv*tu{ zu1vDu@M_9;{bQ9CGao=hsRazzZA8jRG$7pRh2~rYaP?eOv^KwHi<1^zoEw4t&C>=O z^5_3o0=x{MDI!V_zV}0R%1~I1AU$ctqhpUr)C@CZhG0fX)X02f1QF4QF)a)KXKnog zAS2-zvS-O04u^jY?gKge!fr>3KQL444B7WQ0c|f!5|ecIPPDQ84Q$=k&FTZ&(I=O& zAoT^zMEssV+YJCe4G`dLV*Y`h9!8NM1LOo=k|@Sm9{F?_;Uqw7IBUL!L>hJR6ia`E z2hM$=N^1W1;yiwr%7Ncztf<=mp6%IHqu=P;2i&wNR?$l_c3FJdHr__uOgt=G5tH`| zEu{oPW)jZ&V|zfVavhR*vyZGZK?3KO!j*ICR%P=y5?2}f3vUR> zanQ#S)&5}>eY9yMe(Zk}ZvQFG9zA{I(CM_y`0m_M!>1?Tbl%b2WO7S+`xh_9MZCXg zp_|6-k3*)0{7_o%`&jMqXIopNA|5X<-gm8VyxaoYf6w}t(i@i!fBVBG)AdF2GlRzc z``uC3wfv?^R>6^5QckY<^Y(7uB_42c&4%Qp;kIJiryqRXWFEge z?_>Pwx|W5nKV#jeT>)A%{d2y%b@#sU?Vz=u?t9TWyr~<%ajyghBqe3E;qNh{_<+OF zVShV@^Ye7H|MuX;FE7efsJ1>LJqHPIFlQk7OjjyK&p{a-MwQCNhSwHih?TfXM5(d7 zGz@>|^*_s1PHO)^NYMqTs3nT)o<{kAf<=B?Y~zNBj*~IWbc7&Wd3pI%HM!S+6aTNd z)92D;4?tATa5rRSz9^ue$E>59sYi>m8cr9QV2(dOb?omPj)?tr!w?%kNQc$XFVWD{ z)Fc#4Q&OtYst%RVHBj8@qPxHJatT_nl%~ngM_-|kUIgB=;g)c1#1UU?n!$d|d<+34 z!Qqc=whi#*027~&5BkS9$K#ukW(R6ViP@HB1Em3~{phV!FA#PC9M!dvj6Ts{;hWIh zTq*3zRq}F5enw8JNC0}J{qK?#$Vly@)0zSn8;F=i7FgEnDF(lZSqp>~)4y9Wc4vf|WkGL_%#jKe2Ugw3(42<^lmub5H~=;L>Kv zt9eiU<%dnaIa3=Q0U!)LwNlqebEwx=45%8cKmZ)V0w^LK@}(pxX}Qx;?+rtnS&qQr zP~2HU%Yk0E=6GA2JF_3TCUq>VT0q$?)BjZBVgLG_is8JqLI`ex>3yt*nrW7cBn2^A z%jd3SWRyz%Zi6H^sU!U7F5YSMqb8cn`ajy^U2%6Fe=3pDuXBz1a^46jAW=JQeqB@0 z;(X+}1PHsXC;H2IYx_06oh|Wt<-$KX`U;g;;*?s63txBRxTnO07^PvzRP@!&U+bc4 z+VJpGsdc_(m##`=oKlAse*R~GiMD_GhACW$hn^kCDd2d9#b(*7nGVu?lz;^VsC!RK ze8##DE51JZ3gE<=(Sd}~AQo{qY+2xXzxv+$J}1?`xj-V2UYvPt;lND*M;2&*^atnk z2Q#0?TI?8hM?~rJOYo2-Sy4CM*x+TP#2qi+&3CGo-Q`EE;m8Z1J|W0wSMnmoL=6PN z0%}ZHXM^Y|BN8BG)if~?S6VOAB|mCA_U?1!hQt-ZwHL7sNC7uypYFcV;SI4)v#r^n zA+c!w1m(N|*)PM77o~ahcR{SBMrZ-mnLn#)L5b{n&G^V-B?v=^2NFYP)yU2zik9l8WgYG9 z6mLUIx=5@_qLP|+-%Ytb?AV}QTE-!5XXbsbj1CbI*oj?_B?5%_i8TVu66Afb`Qxhw zeFQ)Ws|^qPX<@g)`kR$gKVv6y&i{wK_kfD>?Ak_2OiW^IQ7nLp%Af*@(!~PU9SNc| zK~Pi>=?I8|)WohJG6+afP*CXz2nYyh1f&<0-Xc{U6%py}TzfFDW8U`tYn}72bI$r* zdEb~}n0cQ2x$j-BeeG*UcMX#%q`VDXb0x^~{YW$)#Z13d4$# zA|;IuK%Y;MOkpetz%)RU`g&kaQu?`$yjy!%POhJHkY_=ojEqd6UDGzU;{MrI#?(Sj z`n2dNzN`~9d5qJ21*TGj>pI{&!=L~)-s*ny<_*CDm5P~&87l``(%(czQCv9Qg@N?h zs@#EV`xdLfv{Sj4`4Jd_MWwS~4Dhli0#0{4k%CMOydR4gofH@~N{E{uh>Y z_sgH;R8Xp7$2Fm7u%gzWIr~CS71F+%KWmBdcD;`)Qa3Ng=S@ReNX7HSW3Qi4iipQq z^XDKTLO<*GZ}b)47m$T{38lag27X_%FUv=4Pdh5Qajp+hB{45o5FCpaL}!|SULA@h zZ*e-9EF{2U*ueF-q`1B-z`J&b`5L5vJqjm?#f%KKm{3bF&keytl!tP1H>9Q45JpGP zG4*g$7D!RY?bujxLc~dr^9UA+C{LmiAD0Ya?qMk*tW4@7C(@Jd{Zj0mJ+2M#e;%>u zb8~Dj4c2$HLTkSq`yqmBi(|!4l?OozpomPCJvlN^-zpM~O;HJvcjE85#hfeJ(B`mj zVvd*GGT)*N@rF|`3ft~l?tkh~{yc$}z8`s&+j+u`z>JK}%bX>WJ#Fr)g}2pAPMKlx zv0Wqqv{Lu=bu;*9iic3GMzVfKL-cUZn+l(~4&DUGjeVFlwri{(#rLM>46N^7*9>|x z$D9GlS2}dw0p*~{D8?1rGr_zjp;+PEnp8XMr|iXpP~z$KlMLf zs9@(Nxv+~#{Y|#daI%Fo%|@Z%gi>K9&13?!y~1N%MY{u%bJ;W+%S~$PEJN5=s+o$d zpZ#cQCHhnEn`rvGZv9?6@V{U90gFjp0=vcix;?C%gvR+zxIRZ3lxYq)BG|YI#Z{wx z7JwI6GEz>5DF%L7rJrI}WtmvYu1o1OiiwMfQ7QB-u+*KI)u<7LaCC6w=kHna;p_NL zHRlQ7Nrq@k{*c3xAAiu7J-Qsf!^7sempt2Jy^^|?xSuV+_%=ZQJ6NOe8FY$S3CxS3_NeruU?PLuk5^1j>|HXP_hm5^$1a zN;oG5a#@KGT)~)6miUuS)U2pYQUNlLaoP%=4R z9K`6PSR7ay)Hgzte={B*;j#naTXAl(fc`Q^)eelYJNU-J;4+k2G(_`~UgqM9{>8Z;%zFQAFW!;Y9vBzD1Z zJmmAOdbvCTPx0<8le)<7t|?@*hg{T;0_*JHbV+6LcCLiBBvrv2z_F3Dr)nQz=~m_b z)B}QgGH1XU$yLCzNI}U034s)a@qkoGlY9x2)Y^gl9K2M*b<4(=MF)TC3LG4{^Ji$Y z0}Ho1+@d(zvZUbzp6jxf!io*2k=2O+85KXLb>YTRDZ_fr?O8WX{fOaWO`Dp*;e zsk#QV1Aa&D^ey1Z#-F>W8;>-fuI{+aDfr-!@|Ui}xLrcj+DoqMj%5(mNApE}iAVwR z#PYx8EVH*P=+z(V(WgEQ67~d7+$pAG>~-*{2;3GkRd0U_o2JZAZ3W?2NFH~c)%Xv1 zJ4J+E*xRHgf;^|v-g4k$8_|D}0nnp90@g3hsQm~9sr5{ft#6-`)xX87h3k}-okr4! z1iWfo6n1Z^3ZPCgt?2EndOS7x*J0iXFh(5|gux~SGYGsqn8^JB_wfPx7^=oHq?nqW zO76F()cNJl`r0k2L*c}lXd!r@?Gsre_ z+N6BuPyg}@Zjrl<7iFS}ky$C8oCPP@6Fl08T4~aIV)wGba7O=_$#qq`R$zHL!OLc{ zNT@hKSjfcZbhA@yLg zkmJh(n`_$JSA2b~9H;N4`7D>g3}GU%Rm5SUK}=Z2JksB%0Rhxj39N&1GyK%9Q4U8O zdx!dMwp`|d|CDBG88337IW|NpK_X6R42BIj;9J$j{b(3Aru7UcR;3XqjC8j!@T9&& zVNdH<*~h`8ErR_NP`pML6rscvfz%*P;KZsV(BF{OvzHBHT+6^tD?u^i(gH5H*=sY#0B>Os=D}%VMuImk0w7&pqM4VOW#CK54bN>u92r1LT)! z0MTL8l_H{|>i!%=YER|ikk?!f!?=L|_H@OAf1D=%BK3C24(&f<&-Dh@)@9jE@Yn+& zQ{BK?*`z*voP-d@L*}+x#53*=sa?W*c^#@jhV{kXSxIi6m_Nk(BuLkK%W|p7 z3Wb^zq}+pqeF^e5>a?LIeCo&e`-6J5G@BaXbW&{&Qxy@%oW>sOw+vwH7>8Ses$3>` zVeQn?M&qGj`e_)&fah4h=i;Z_+lGTcAY=-_Ism;vlb9$p5TmTP;OfZlX6#Rkc zeWT93$){kGXJIN6lZ7PQ^m=I=T4-WA-=P>XP?t4Q2ir{X%&hBN@Soco9HdcfV^KQ!jaqht2U{Xs;k!ULhQ);}8{MFi0RY`S4)qq|0M?^!BHC2`F}3!#~$#33PP zJP#Gjp5SG!7*WqF&y60dXj^!lyH5KZv@#m5vxtl^G#UUO0e<4l%|$0Ir$&KMQ_*%- zP^IZs9DO>r?2BDLFkxrf+dTx-+H_=FC~eBP>MeodrYl z@J|u0^J)BM(#mWrPKy*$X_?-+TpufKtrE)x%tXHnD|D%-7Lpb_F0Q$qd@ z9N#KeSmXk9i$>XTGEU_A`ADJ@5za-Ue&At;YD+Orl|Pap32;qJ!%VoZ#+1%8a#BL( z3jJ29QuM%J!tk5D!>0GA)QmbdshgA9M9!3Ov09a$zqs{k0wtRrB)bBWjOR_9Hm7!} zV4coE+#f9{*<7kg|M?XXeHe(x7+}0Mfu{bg*jdDIB9H~+luuHRO;2MY{6R-}=;a+O zsF@{&@8-VjW*oDz)}@O17iWnPLzSfBK$ic6mWvzz;5Hfg$d3g68OOy3tz^kB@Mfg^)Gy$ek#{vsyuBt_3U%#U}<6S>wO>xEt#omoNd{W$N@bASR<| z0^QJ}5?nPW9?fu|+vq0pgNcP@OdM5jl559cOwIAg6tIctYHr}LA`I`R14B>&Sw|{P zf(&kTiv4ta+A5H%!>F~EU}<)jef6a5lC~_Uga#m?u!RUtZ%}_JRl@*|tvrl;I!3=6 zEvC|h*Lz{fCw5tNRxHaC33&}^7uo3 z5*wb`tVncy>cT=?e}b$k(L{98j+xYAz7VQ{G88Jp4b^LNd#bW*&9LcXSz~=;J2{DC2AfuxsG z>rP%GUMCHm;&8SzFaGh^io*HK0thn zJ?qS=*@8-j|M>H0iD`ND1gBQC=$7I(1G9$Gv&`P^{>RUGc*4I`xig)(s1l^Ui!b7L z{Htw$;1KilhC|j>15hKntQ|9YOk6jP^j1G|XMVRi+4GQgNXF5~WT(Rdi%Q zpCWRLVQZcjAGS8lfTTR5wt3`8aX?bJ431Sq7F>@VNQFr`9lM=}PpZe@?>?F`9_zn1 zUvD^7w3IEm?DE5};jb@%iQ6G4*d;6;63v!NAoYZ&^K5^m{vNvCh=_=MYUG5Ls3D+7 zR1gR^fihDdVaq_4>TcPBewU)dZbELq90Y<(CAZc9Ab%LY&22>jfUfBMuN9B%;`KjO z(7k5fMIo!t=-ZdC{|wB9Xp=NZUS{pD9vKEsi$!rnj~{7ypOo*5^AYTemD*&YGH9GY2c;Xo?_{g}T6zOTfIun#Pi9 zYisYDN4!mz>0+xSnbsuSZpzxV5Ub%W@t+<^`%Oe19|%ot1+}PnMN0j%>QHHUwC;33 zOBtoTtwG$@_O-EO@boN@5!(RWTfUVD2!o7bmHx_oY4^J(a;=Z?#(<)w+0YXObvWxi z7-TI|xZGyh>1Ek_egi~Rp%1jO(PK4teeC(}y1D1XQTafoT0mQZR4bW8IHi5^8u2Q6 z$ExNU$S~2z;zZ?`!KYNF657e-=gW@qYMo!{S2#eqB!^)|qxNVN4O_2@x_LpK9iaXJ zYCc8=6aI@p(dVy}{h?Wpqa#4WMDq0uy4dP__V@uqK8$pXIw45d?ii4u^Ab|9C6xS|IdxLrp5+3d z+7`f4T-Pv^xTtTpSiuZ(7pbpqT_VhsnvkjekXC~nv8wAJs!0AUs7AjS%lVD8-~ght zXAg6i+5+xgoYPG0$~5gyC(~(we@j2TldSP`z|4$%I{6fa6E$okjYLT3_DLxkb&31W zLfZ*+Cyk6Ag)vM$irIZdm4)rLLNwd8m2MX0m-~-pr~rfgLy5b?rBd$AU?~dV10@j} zMM`-nlZCXc_0}cv0((mn+gGlFA6DAL!dg``+)0M=NK`MtN&b2 zV@OB41ji(33ZW!qA!wYh4se!CNCFbF<11kXi$_0kU1+$`EV1lT^qewD>BYG@gNCun^l}||w~Moy0a#yUysnI?TiO@}IcC&p zFfnB&H+HSmy(@mB6Q>~<0O|XsjclB1WfmF~2mi(U;8j@+oZVr(FLb>^EtebukEqsp z!fye;5G<+sunGcqpCc}(<_xa8HC{k(MHiX(aNF|+5{X>Vcmz$3Bvr>^5D(x~5m)@C z)PqRD1vTo|FdoQCs5u(&qd{UH{%_4-KVK>{}|^G4kNjW_=;Vj;Xm6)Aq4pxRarR$mtpn+ng20# zHj~u~nFe!ukzh$x3yU1bA+Cp@;r~ZKJDOun$zf8GN7x39Cq_fj3Vi!c7NsTBmqUHz zhUkJOsiobaTw8G0nMq@>dJth+9ItUkCnGV1y%{L_y9#qRjD!^2H-7t#Y!2E$0)gbO zLOsyrtLVsh@uacS{VO!Z{RAAj_HXjHf8&A~=SDWqem#nzkQdfSW*{8sdrDTW5fQN-utH= zE{NtS7?9h^Tv6x&u{}6#l+f?6kBQi7zW%jF4`+9q|uo|@fG<~j~;w3=KG z@__O`sBQmp5kQZ7^b~AqF;EWl8_^MP-=aEfVPpoc&A(OPuu}>UgI>=}vMVAH7%}DqwhET!%@E5sIOtAsC_5tZ+Qjwx6-4c~h zYV9Gf1n{x43Qo%w8=AZTD>)k{(;#JQB>>Wbz(3=z&Ljce|Kk_5Mw3sxc0@Bk9 zQH$kFm4LKN2f&;n5=JElN$LZ)w{opj$VH8}Gn+x%xXwUg;!JM1Cw7QHA#^-}OF(%G zc$~qlD9tERPkNAbE+3PH()#neU+t_oJRYK=;VOu8f3k{bB;<1OfJtG9!jzrh(T#MI zp@(!SRp=P`r2izt4WDV=!3SX!fH0G?-k#V2C<^GhI-`d_cK6w_JeYkyifUl+RoC}u zoGRrGuIwUV24Pv4EMlT&m{GsPAe*)zd*2B(H!fikMer;iH^LZ6+YQ7URv2>e6U8Xj zX}LNh=#4wE4+#_6k04z3mict z&t#G5hs;gmpZiH6nCd-nf10LxUv?Z%l7&ovliT@Fs!msu+ni1I>JO*qjmvAD8~>>XytQPyTTVI z!5skmq?#(Z$>Piu_{D)1!7=UMBDiik{P?Vq)7&M3aNe(tBa4f>#>4Y>65W5M!v4)p z{>Ms(@R@#3SeimWCQ{HjAZk7*OMrAIa0mPYyi)V+TOiV6*ptN|Q~4unC1=kWRZT>8 z#H6#(6Jcw?YuobGRQ5s2xlWoV*TKj@TwSe?~r((5TspYRR(# zxd4`s+J1bNJk%~=;D~mwtXcq@#D2*2jss?QrZ;T|C0X6o z8L*f6Q47!p5|DlhGl|XzR?NLyiCl+T<5VqOMGxo}75bn}Ift@oaxJ6M$ddpU*Y~=)^G=5Ugj~tEVW%TjLUC%#JU21uHqgB^r&*jqD(J}ihHV{ z2V3`Q&@*nJpF2*{6$2DQo^X(jBB0gLcj%Dv!4ro<#j)2cz!Q6p$$r_WZl)pFQ6vjC z$j0bQ%cJk$U*E)?o6M&c@Big(JdqbRItdZC4h9lB8gPxo&Wo{yAH7hy9Wq|i0@6M8T5H;lF5bGUmLXuf;m}v zpo4NP@I<_`s1IS{+Rq)Lba5fI<3Sc^NP#HdNKu}~-=zQG=s^sYyG9LdB#DDU!3`?5 zd)?tm>lf}mAGZgfDvGy8fZmxbgi}Un7Jdy6Km-ObCaIB$`b{x5fkpbP05weRbwBkG zNGX>ARULo-yg>A?$m<)2T?L?T4XPqCJ%5xQP*mDGHcVtpW${!7I&1BG(H;c1Y6v3n zl;fkQ&D9@pRpjix3iGF-VUCtyGuzfBEv=Zdtdc z2h(Sg3Lc9=)JV_=D^p`B1c&CyNDFRej-=q1-{3J*Kd(xv)`M&)XP6&-hsYzPgd3Mx z<_mlPC5IRK|Au01eueKp6^bU-aD81!Y6J>J0l%Z4Cw4RqvbzHNNOl)eby=g0aIdb- zd^8Z{1r-4}rC&TtZ-`oviy8p(=!FuJ$)&zv8MDf*RZhbzCxX7P2KCC?=I(F>my8H4 z{JkJDAos?%$qP6U5#N7_h*;QIT88>(`iF13je1cY8_Hg5_vU_yV@fu99)d|NhL~v7 zcuOETTGXdv7hLB?HsURS97UlBrLUqwSC`9bX2PbbivirL4N_`MY9?b?= z?w}R-DOhUR#eYve{wsX)l4HrNt)9XR6v@Yq{)!acvsC1lF>2bUm>r{ejAA;(p-(wX z)9i?HNKLWih&kPRpxEj?1`MPCr%x_8)Pe0-dKSm6P6{Y>a4kX*iFWH@M7kPj0UMIPo7WX6~R91n1Qs zw+f47FEv5*N4uVZc9bnBrKdmcA@1gg9X!0~nmTYYz2l&!MV(}vnN~HoHV)?&^tr|W zOF4gAs8Cv@Kr`w0xvobo#UpmCKftp|R|JzuVzc(+nCEaezl2Ym)W!UF zeCWz$QF{uWDkf!~R}JF=GdREVZ{kG-OVG1>sSUjOBID#`3RuSygm>ceJ!M0K9@ zUUWOs@Q1i%J)V&4`E;?gmyjn!MNzJ#HNR*0*DoJIL27Uq3_#OB$%!|Dgi=6|_wimf zDPT|+IFm)=c2IEH-{FFy2?(9|rvM80l?YAEVo6o!{^L}VSs)E`4#$-04OH$}GJO8a z7x4sSAz=!MmP83Bwvq&eNVI{v4Pgs)9@<-zL={fdD&{kki=e_gCHU1t8l->< z&JHx2P`lz0WWdURgVd1i5$Ho>5NM`4$%`-(Kq05{;z+jQ&M#l?*1mDkXjVmKr4dvD zXxym{dYwqSv54@Fq{??7&yz@0FGxlc<@^3+p(%Ak371(*%q)%JGxIC zXv7u*?=ZYhH5<}ir}F`tM}PK_eZb!3U!I;xm^|?xI<2;Ac zUcUd~YFG#wenB#FBzXqcT?D79ZuixP%)y6PfbP+g^9d9}BhUe;h&j>?x}FO$uBd#d zgW?MHM}uClr-2*CDBi;HsNujzL5|WH>xakD4ZSw%#Zfq&cT|b>7o?|oo$IE{f|g}3 zB^ERldl<~I(Gl<#g6(2KLgoIu@{{}Y@S$6s=Kh31|; zRnp@$L%Q~T4@On$Iyjf+mM=X(YMtTT%q0aeA zK8+N>bEwi8YZVAq9_}(LmFu0Wd&Ym7!ZocrtufaU%3pBJDYKJA3iTElUbgefIxB-_ zPbf<~&ewawNBgMt0k>*S$MxbX=wq95Im8IqzY}J@7OUQ0*tpZ+}HhL16 zqo(aYJ*-{s<9H#ZTS$@apztz4O`8i>?VyCZb4<)RjG~uWOgj1wx&gj3m{cS{L}!0}2%{53QCh-3K+Bf3 zatDtwDyT}AX%G-dz3xEi{HW|m)cU`f9?&zdarC_`G0q}#9TyQJBz4#MF{}R(1ehXP z${m7e!DNE&Yw6df6(p5@ zoveU*E(FK)3_6J#Tdz7<0JztWDYfSYK<=a ziT}z!-reE+ZGeFPsb=Y(hYtIgw$JhV3XMzM3Zjta^CVBSv$La^pI{Wyvn3@p!hmTe z2dWPH4c^fxNh;mLAnZ2_##3HO0+eXP3*?CLLJPqLSn&}c`77U?`&M%UEJdJq$`n!v zB^K(Wos){ugiLjYq*Q;j+*%R;Y3nuKfFH(9OX1LW#NBMR-}h6_M}==Rrk$}r5cLA} zI_S|`W|Wo!I7YYCSP-KIuy;2koLw`_@VCCC%s!6X1aSG07OLq|;!k~r1dlUW^gBf$ zCHP^2O`LTnPzv?2b7LI&hW`w9$OZ&3f- z7%tOR?NFut1+=vCfn&~D)j zF=V}+Y6|E&C~zH00cl_iT?=W&#+%h;j<*x%%`dC_dgoQ@8T<$iMW)WsCYjIe z&JR_22s5}`<+Cd~x1>JJKA7iX`JpFv4m|v|#Eb(J2;d6^a{kgL5D_SSBjloSJ!O>C z(>B^g6w?oWM4@r-%xsI{xJUa#P1^yuVsj*mewM?HeelOVSV+ zYA0Fz^pw%I@h)ua4~{$7zh`4Su1zIy<2ogw)yR-RUWGQZ%nj1}ku~vzG^qk91h86( z$_O4izon9~5w>9B&4(jN#684t0&7-lC%()U^nMK+X=C5|4a_e3s^E+>kOHW;1aY@)z` z6lAv@XsCnSC318kvg91${EYh5P0uS9eiOL!UoV>}R1#q7Cd`vwH&}shqEBe@E655D<+v9!86u?(UWii9(;ZX?Ad% zo66Gh|8T$-#F3g8Cw5aCkLONIGBk0FFVi3hYLn>kgd_=Pax#r|pjN6%*D=xz1t^oQ z<4B+-wJvK#YkM(>W=ob@Ca->144!^Bvs)JBHKO0`WYH7~nyVqEAf+Mo`N^?deJ50Y zK0ANAjyJcbPuukk!o_?r3}qj*qo=-*PMgk9takijn#Ro$SI&o=%KkCAmV;*am>^YF z3If1Z96V)CLnyHVw-}vn-#q@zs-Z@jL2U~HO5*RfkN<;KsnzBJm zaT*~VnDn&3N6Z$}0E~cGFJAboYBgosK=YRJIan(u5BM;ALa7}BT`7qk)1UMC_w zZCvU=YtyjK)aqlObq}!fX-jpb&A8a+{B8IG?9o|AX)rWVBu~8~cjN@6!xEHMIf_sP zAWe1#SNt|c_qd|C4zJtbGzv3a`kx3I!NXYmiE#6?iQykCwRv>*UvHz6NMxn0$u0w$ zu|@fDDU%{i>JlT>A0gF=#ok354$~aYajHGI(#JE$Mmpnc8#q-|Dp_`bM5xRn1~_$d ziz+UYp2@i{2(-07;rN~jA@Uy-zL8cqWAC=)ZK&dK69{BX7A0TcL6DBx=Pzmlp3LCP zRPBVSK;W@bfYDyK5TShiQ3v_4A9y~WYGg;*yKTxHBYo4PYTw%7B z42+E}d8jzN>2TgWPH<(OF}}HYctmf~X*!@ny}7xbm>yRHt8R<_;CAfW##hehq3cLM zdySU;8ea;Zw3B{`cpSbt&93iYnSIjyRlt za>ow-Jic#Mf40+w>>WJGvsdsta>tC-GdLO7+eKV29+H$b84mDlJK_0`| zyE6mSyUH|IFo4*4YY}v=SaSr^PkZ{P16@FG2(3)5U<{~I_6{pblI`^Po+Ff^q3|Ss zVN)zqWe6w2ZfF6}UP4k8sG1xGQYP+*>=YUgbL+!`*9Oe-*R8|prYRU5`lD_7He|m@aND34^vA>AM*4mL~KrH5qwnok@2lRBO*jq)ODvHTBQx;Wemn||=<_{=}DaLCu6ZTnRE6{Ey zJVm9w68}>9%{;aaw|9+hOiX`Z2l;CvyP`7mE5-5XAn9-&i5v4#9DX@1@RNa?r*|qW zvgn4!N|nX~Kl^+W5Y8}jvcOWeCbdx8z3&MX95MFpd-eQo>B%FaeuZslNlL4S zF;J;x1F~M8MN#F*K*(Tz{a^#&tTyES&Y_O`Fl$$!fgOk5N4vmb+il&8ZOsr5-IYDr z388P<*Bu=#MD+ZK4vTx`85o9ouf|b*pP{A7eqA*a$rKm->nF*SC2=bv7!JDLRX!S6 zvtaz|cPu1(N5G^Sy*fl>oKR9WZ5zDK07xw>T(LTVpSr(qODKxDRJ2>7wF(Sk%iC;^WoM~=}!SVGp%E0oh45qmyL~i zW=-GqolgCU$9lH;&ZP}{l7&5z#d?J+EXoTCcBm`IDEMgUpD2IfY0|#YKea8^_`OTL zoW?Uf>j~fdHZbzo^QTtxN8d1Rc(R+*+JjO%*7r^BzF{lYnW_rV!3Pk@nKNg|WA05b zykOas8G{OL6d)TSI8T}-dUbgTDkZ#pd?HNI{SO~NHQ+Y>7qRQ+#*6Om8b^=b1Nag? z)YoYSoW!&;+W+@cr&i?tku^=;PvgLW5Y)_P<>ch#=j3cb?nRYS4WwqN4<5XNJBOp! zk2Eq(wePyM)q`J{`SIgNHG6yeiv|nXLLG)(&M1&;Mr-dzGxBp(iqtByu3VXp20S&q z;LU&>7o_*?)N(G)YJ2xC23Z&z4`L8U)ec~h#=8kR)8zZ6<@)CR*s7=IzQU_9PhQ6< zI9~YKgn`~^xjy4Bi+z6CT3Avbm5)@jOdB7jmIN!VWPa@J)xx4^qQ$|%(UA?Z_TE&R z+qk<~0s>L65BU`pn&_(A-fg|maZyLJmn}X~ciqP++e3>Yp8UWw^l2>ESPTocJJCpG z)5YK|c{gC|O6q$_^SE5yZU6rL?H@4?#Qgo!Ms>Fh8#c5LVqf#)Hd6RH@nyNV8IQw- zzB+oYUCmpBmM?59lKJd58Hj$rFTe%%)ybZ)yCZHUjziq;#Ib9Kfn9t#_y%H zpm<~cUO-1s-mf0xMm*-p+stLTtKHtghw67yt?ZH|x&6qC%-?tKMm{US-~)G15-#p= zU;WFbGeOF40O@YUYIvZTaV|XJsz07Qnet@SWLH;LTSXCn3lUM#f^dv3UzCr0=Uf-2 zX7Ec$NNCmRKeR7@S)CVv?Ow``S*oe_XdH|5ehG?-f^+nzX)Qajb27w?ikN58ry^Ba>*8`il! z)oR|L+WcD)%&)nn-zud@^}ClPpv%WQ<>ZW%xP9r##c@9Ia<9o_GRvEuV0&tZWA26j;p;D8tOqg70Vr`cP{GBOxvPt00L&) zCF|?!-2q!Lj5HvL2# zT%7O4#@w~m&8JiAk-cd=Wk%q91r~zpn35xl^I*G73(r^sn}eygdGl$=!FcyhU97yY zBMcJ+YaX^^*vn0@s>9@sen)U>4ETl>1`0$ZHpR|M9?=(qY-4|zw!j$BJuA%3jfWI* z{Og6*S?0+C1N?RiH*Oe_tTL@auo3S7ea+Cu|~FQ z5C-F}LsOnhp1jRezH?4OO>XyZEnJ}Fdj%wWySLT5S)_@xW!pAgbH7y!KYsX7MoOBI z&JY~hEiNw33PV+P7q(o)MQ{6bnB_a7TTL0G6Lxtm-tuf->C>lA^)Rt;LqN+SvxOb3 z(Fttq(4Dpqvq7b~g%cKqCSHCiBX@NJA+yCDL1tK=Nk1bcL~UBR!0VK2??Mrq5^MJr zFDJLoGf*~MZc~yvSND(EAB(Ns#ptz`9pkU<`9mj9g)CwFN0>QUPb$BK=>n@uL8H% zyCHe_;^ZgWHG{4`a2?A$J5A6s^F08_fM%@9Qa+`Dp`p7X3leiK6}V!`^<65NXj7%nD@;5qXV z#0^EiDVb)@SQ-x$ObjQL#TlWwc=6&Ki5U}jvhLfo?T|5ju^p21WpEMCUdV}zirGND zH3I)-N>6(7-fv zCQFOF7Z}aJipbZ&yA&U`va*t~sMUavt-=^7J6OCeTegTuOKU*mWKUJ>zKUC0evTUR zcXM57^Ii~#>K9tf43XE!=K37H^Y%s{h^Fe$QDHF8O`ZP? zRic~dha)o03q0{2@W7%_*ymqoe~%G?lw>d5aAfcI+gs1Cx9Fqh}UHJ0Ia z$%}JVT4u@_70unmdimy!>FKj)mzY3zPT^E*!bCXb7wuhpz_H6mIpl=Qcq_KB_ghlz^elb=_(+PuhmKiUl#mYhJk(x$qW{FAj(IJe zomRQ)T}K~os^b+=%dWqJuQ{nfPz2OzaC+WJ3JLXnET0V zCdOz+`ToMoYnYvzyM}k>cBK;GZ6%8`u2|(`_22BCKJCx=!zN`jXUsVC;K2jNli&($kSDmh2Yn(t+u}91tS`D z>gwq+;|2H~Ym^etm3YJ`ieqAXPCqc^`4FL%l|M>or1Rj88vMyl4;_g2L{ot!gBq9b{)L?2@Zxt6VeAJx0 zbZ5}2i?dglCI@2B>BXUQ(Wm}bmdd1to;5nrvX9!@Ci2IjpE4JGo_D$5&JgNYF4S>l zL7rS;p4>t^n;&@+rXvpNX0MqrVM1|VM{(fY2M<;wq6|l$z|NXIH#eSB$bhwB}>J{chV6?jg!GdGg5rN%;vlW31%gTb%h=F zC7bKuh^atmF28oFpn@LaR2E~sQ~xd)9OcR-%a{L(db$r|J=CM#09?Q5Ii+UP@nZQS zj?(8kiu^;}+}sS`iE4z0AUO<-L(&$|xbKO<*`Q0XxqQOu?c|hv-v-#0phS4zM1*pBm`*i3G9rVMoMiHRE27625N?;&8F6$GI#ih@=X&| z_;vTg>_*oyXDKbvjMVO1z(RE4iUR8VL^TUu`measgOmQ_q)9uTGTzT&e=ZkRv~();jOm%mH8Cfr(*Lq~N4%Ak4E|5l!e_s^41HBy53K@?!LK zHZ?^IA&aO)HolCBWa$NP)Swx1syEh|J`1~`F!)Mn{`!;B`7_Vb5v1nm`Hp~lMfb!8 zF-*FebJif5+f>JQ_9hGn5-S~6*bLzvub?G)Z}r;m6a3y@vxQK!%EvUcg2}ePbJXc6 zIQE*e?7(9Sghb>*W&ljVj;qVOA`O&!z2@7N)XeC(moaQ-ZM_?fu*FrL?W;yd^5pAa zz-kf}xwwB}OMzDkB191q8N72QzITOM_rWnRUp(pa;U4Yn=sIFCsO? zYzGj-n_NV8YaxE_%$XYVzz^qW>?@N6Z85l4*$Bk3CT}K;?n}@unYC<}5>jC){iamg zr$*7)Y{3yv^Cznq%$F};4x;m>r+k~B&anK^KiyJ`Ar@BL2QPB{E8`3#Y%f6qghl+dDy9Fap)BsbEiRxxw`t3U`J?1s&*g7CL zmo<3TkY1%1yX)rfGf$p-D~^XLb(`Tj6tWvJ<`|rOOwPAtp z7}sTs?|&E$8u_(20VIJ*?@(|oQd~YuL~S!|Wdx_{8CJvTSao(qlwaz+xpTu`y)ur9 zhYH?7aR@w%=cgfvEm$=ki0nEz!YZycSMxBL|ZF*4MTqyLyJmmQ0;o#gJhj7myBY2>{pq(lMW94h2UZBtW1W7--;9Ofks0C0uKzfZrQ?zitS~- z!s6oM)z;0MS+aX#^&}t;Y*zo!;knV+&Kv7YG-CB?+$$bB*hX1^s%n%hrWUq6!Fln} zq7IfBEJlqb;Nz|4=H>~qDB;G`WZ9xE7C`tfKQtd(!~o0{pK|E1J>v}8*!L{?W(Pa{`M>m^$N^y z5xyS*?71^xk*AawmAqJ!dOjcYPm828m6x`uZz%rLP z_{VqjcHlyqu!XHHEuq*m2z6+7@ZR*}F_`VVjDy^PMQjmo1w(riOsuu=RjV_>wpimb z{NS>nqGk6r_!!ExY804mGY@N(2wzihPUDv~4~GzeE<9p6fPx4pZ0d}Tj;2|+oCU=z z7DHdJy`zW)K0y9u-fu--HS%)seuk=PIG%NSqv#G6(${Zdu$TUN?SKS1dtP4NFhSSh z-HR42A~^RGD*WMKkx5S8^!V}P0Fd61$uXEn5z0UYH(wz2C;Sng?qY+(y&be`<@~P` z)Np-OtRW(8pPFE#lNx zx6eB%skf!2#ehK=_6sOos!N~~3{{;+Ia>Q$;0lkIPDzmcM~qTP5ZaY- z2yIc*=;H@7_6vvb=}mw;#Tzb8(v+u`wz=#!uTk6%CXY;CK|^9q1^tw zlvdnS&oxYcQG10lo%*g_SKwqUk^|Q}CPRlJ8i&se3ax9L`tOz8SRZxo-YTFAVu(;g z(U=^m6%z^8_*UdmiEEUVm6Y5dfgFZsZVcZ_M4yeDHrXM$NKH>Ktg70BNLd|kt;ldZ zQ4ymnw|?sUJYIh<+Y6czYFqF)UI5VDCrn5}hyT7zCmohXw6=tdWxX~;F)w0gUb{K+?F_&UhS5c% zmfg;9$-+}t;1exu`k>EPV0Mklx!i{phix2pwSHS@WO zaN~bnD6@>44?q5*Aro-M-`|?Rn8dB*GX7(o8Q*j3Aah=HTYdq7)ZhLn`EK5@?r*R0 zEK~-L_V*XB2*YRn{lz@i|JN>4?I3gkP-5}poj!fxv}x0j^yR_hqPXY7ICJ_m!*SZ{ zP$YrvsZ)RYDgW*Juyj00dIRLMy$;UxZ@8|pNRfqD@P@-qU@v~Jt=*!}4((8b$gWK3 zQ@kRTzV@gRwz;039=_p9%|2!`icvn7E?uHz?A5DRMeoODMclSn`Zs*;q8J`JeeRqY zd`ImA>(J5RZq#T7aUn^$IhqIgd# zU2JRzS$-UUVJ1fM+!-DomP}Ig^hTXgY}Kk=tWRL8;U|kslL#h_L-SSW$*LEC7T}eZC#GZ1OXlICd(fJ){bI0+Lg=Jsw80Y= zWrA!6=ZysxPBSJ5LMBP*W$eyC91{uCP33dV5T$RIp#!zl-|>Lyz4jgI?`pCMMrUq{Sj zHedpPqK(Td5gpwl#|n^T3n1$JS~$DCSm!eM%m@I4vZ3jLOQ0Mk=?5;i|AqA%j&}o- zQ@-e0eG4YgzaD+1SN64HW()qS0_q$PqB!P(k8psYd}}$rGS_SjkALp$y3#`mHbxRo z@Y%fc=0&4_bno)Bs=Kf*7>musw2a+&GE$LICbMVFIxOyJZ*O==^~&|2)z%vU1YBj{ zIV#KBe9%wl;aS*%=|S7nhEq0Tu*WjaGT=V1@Vd&3f2&e}9;`0{TptEb99oOEV3^@i z`ym7?Y#8*Uv{5|YK}>J{1o6#q{Jg+l>h4F7*D-PW6}l8(U~^gI2IJh}V6`?k-$LHX zWTAL>8yheRPyXYlPgjG3gAopepy0G@&6?UnL8s@UpFye#);V|}^U@_{RtpZb1gU+; z6)Q9%;evG8Z;=`iw@|QIhQ_(b-JxgEt`!0o##DouFS7SV*He|A;EWlpW?R;^+nM^f z2}B<8;OpK*Jgd7d77MWqM+~v_ZY&4D_@yUKoWRAZf_qs-8vs-R-FJKbkU*W(dEyx#AMb38!SoL%s8@v61Hg?1GGz*xq^W|o zhY+z>K#Z{m8bAfe+7~QXAgYrPMpuE5PUq}F?FQMTszhGbsnE#_15_%qaG?^)lLg*O zJ<0B&b668)L%aUD5O>^0NYa@}vshFVfNBUZZ8KOlk@fRMk|B%)&_f?^*lrLq=!8-Q z0iok=z&kk>(4jE8kurbn@zwW(VQd5IstfGQmhZDgyjV_)jaqYFVWIMti@dvm^(5g! z4C|mGtd7!%21T|nU*5#Ah{D}aW%`NpYRBGag7qpQ)VHBbp@D#bVrLXrHbO*jH>z9> zb1~$u43bxOU`7aMz(zv?k*Wb@M11~UysaE4-uySEFwzyF+ot)TFBf-i6oho`4GOiK zvs+eHW>{B~Ub?hBuR{~=fmNCs9=^;OA|h1o2IMIU(-;*zVv^sK9ptPZza+7f4@>eK zNKH7BoCs|60>72P^$iW!VRr$5y#|A`0$i-1lSP-ICUqlopj`kkT*a9Q05ff=&OBY`?fqSSeR6fdimr`LStufC4W%Fm6hb<#Wiau(@8)j1JWnD( z*1|ON2$QN0JsqXNDGvJu*Dv)vdA}38M+$zyFWTyYCmTZCX9EKR8*E{C4xxfx@?N5P z&;*6y$ji>~KgWl(QQ&Vuy<5wGF}i-8{f9@dYLlXh?Qt|xWAx8F!97}SmxNPx2g;oO zt~5yAfO1fyCaMbqwbAv62Q-oR4%nVMcA<9Bb&RY5of>>=oKgAc_sH90b5S0gSKtZT z)Pf$LL`M~AlV{lv5CrGUjYz$L$~xr=h;GZ@XHr(1ot=$j;{5p^LlN3&rJS&{lR>lf z&*=`Y!?(3KVg3ps+7%;m=GA2nuq+9yVaTNGKpm-j@1h#C+N-tCi{EpDr2#(!7g3;1UVfDF|123kB~%9(a~_O#TG2pitVDgI@hoWvha`vwjM4 z(2rj#)ycPe_wKW>6PvR7upvE)7pR)I(0r9M$lq`?x73aHI1sj zqhb)~tO&c%klFXD{E0iXs9m)!`oOOWv|g#7X)Y(u*{Vugtv`2)T$zaMHa)feuBzbO z902t95E;Y*lw%?h_W6XuJXsAA(4hh%Qf!gMi#H+{L@2?(>VR8ArZ`Ad*wZI5tx)6c z&iU}T%9QHBpW1bl%qPJjc>ZJ(lr*fvdTwVCF5?UWF)NIGA8>Ue0aevf-uW$vFFrw$ zI6&%qMXdju=XX8Hyb^GU+?DcpI8%n8U@&jaabMK*!m+E!c90bZ089=j4ZwSS{pdhv z4kDqP?CfwvXjHnOG!BN7YNeESfx*&4X&Cuz;DSDit}as`d7qz2HA^bJ!_ij&U1Ts( zrA2M~4C}raqKb>2EtsSD48&JPPHArw%6Vi#8yg$fID8C4l16wL;A{;qRYeLYba;GK>1M=AaW04 z8V>x}5JoeiT5z~(r?^EkvZC*JFUA6N7yy7fXshT|UkT9kbm zOv-5~B*uvl#rn}47y!@FQ1~~)W#ZBJU^&}fzC0*%C$y>?vPLiNo2U~*kVO%2l2y}w zT=>OHm!9Kk!x^sImvK@P9I)$1{n3aw`O?r^EDJ>IS1>7EfNXOd@I)qySVjQ<(3!Fg z5JZCGPPI3PBFh2$UIkri%G^~-xIz|;0Q}lr;2Lib!V?7m{D8YC@hp}90g)VpNoO7V z*vcG1)*S#c0c=?Gd$F$dwir5S@_q;pNd}ed)U-5H48V$n92WqNB!_OJ?2uF-b`^*#0Vb(;~;iTQ6L^NVJNctg#VQ zKpRv~*Sxj)C!i9_K-*>N?6aT z?(SC)&L!}P>X(G{Jx-p54%= zCnAcdX5`h_{TY!sKPK2~M79GBmO==c@~SiWi2J@7o40Ivjj<{cS-YPU7pvg=wGcIE z0yp2XdGjkUNB07G#TM!KASblFtFHxjSUvBQf+^IaK^+R5{G<$%R;7WoMp;EpzS5gJ2tHANj%cm{u|Dw7|`qH}h-fE4tJNU8rtoXy`}cx}vwZT<64m zf9n)b>Wl&r#O9yBe0gW35#+hFj*R`fJj3X`pP{{j1D}Ab`8_^lj)A!dZOxMb%|DcX zo*Ea2PN>EVX~9cE?@paMRm^^Y5~Mch8ws*~olSDMO|#tiUW*?t9KpvxG~ti)9v&){ zmhk*Wfk3mW*H6@?s8*^HQj&%#Bf!Yal3hV^_4x4NgP{dTxpG2?+RrjD-gaItax7L7 zbDeGqBNT=)7j?Pl_k9R3+KX~g>dl4j#MNkw`Z`ghWA?7?DZ(}5WIhz&bNGD7&4Nx9 z@Q#EiqE&Xpb-)g`Kznvor&y)Y>@4S@%&r@O z<7_v723wO*4fJ?@uDi&|=22BpO-N1U{*mua6C3-uX zg5QpO3RtA@lI9A?8mtj>z3!}Uz1gTy>d>J>?ttctn>%*=j<97eFojj6Ob{1yj+X>1 za)AwLx9em7e*2?o;IbZY4Fh*SHty@LiI$_&?xnxlE zoxaP!ZN5HK1n=|7bvd7eZIt~y!P@X2;M^5c*nWvihfrK@yZ-=OO{02fMi(5qss1~> zXrKj&wHmT#&Y6~)nyRy0X$K_qmLw`oc_9k&gd-Xo^s#cKmj)#YVcyIgS^8Zbo6qH0i)fFguokOu8s% z&P&~sAy0M_{iZJ!JhDIw#8u|Wu=n9P-#+@zdx?r<9mLrsY6iDm^4=4zebBZE+Ty-7 zy+C)8Yu_U-KAVz~QarpGIaF$++T#zQ#7CShrm)xj`ZKlz9-~nm;@E0)M^KZVfJuj1 zKiK0ngLvDuc$bl$JyF*9|EgnR=To$T%;uhrcO7#{T$PSK_pKTl8mu1doU?cD-qo?U zEJpKF(Fl?a$qHxJPXTrD2Inf0FI*aknU&6e_MjfPqE1I65d!5`Zri$*AAgZy@R1gA zg6)TJ=mb5gw>oMdQe6LYv%dYy?*qGn{Ig0P z#;q1bzOZr(LQUoNRX%=vd%TZKiX2j_ifF9C(&GC;Je@IYxCz zm%`HYdvnF-tlTe+6b)$$md6^_OXJ;B|L?m{1VB!dS>uAf<)H4p2X^v9n96?S&4cH{gtfcgsl| zT>~YZD0Z+zJD)A|{D*==8&rHy;tME5GUf%ZTak>1r%|-iL^9YW1d0 zoqBw)p4qKix9*?pF=eVjl3E(1;3x}cXJ_x|loRXGlc?LC)VB<|DY;)32;32%ecZXv zzJudrD-Se8wrJumfRwN!#4YVK$12`aMmaJ7{ zG_vOZtGhG*%Q^r0xG}r$%=iw5v76f!qHLiuDaKNlC0wFKiAqI6Qk1dHVn(uESu1-+ z3x!sU%#g}dqJ2>)rPWqQoBMgn%=doW|G|C#G>>VTKG*emzu)Ja*K*F=-G-C?MIXHZ zN=c6Sx7GZ|YM8;OOA%SY78g!}>=%1%B;cer74xiU@QhGU|CwHa`9azweX6T;h!dGPl@4a*nDJ3k|n{mx$PoIvETS+Z0z9fBS@>UgX{3>eu zvPfGBmr(~P4_qS8X@C0qI(VY5;rDvC#?PKT+ue(-7E)~NE!VfXhOcvM9bXr(?$y&P z-DJBkkQ`2R#_Py>LUp;vBEra3G}y|R5t?Q8KhZe4P=AfZ+j&kU)qv%tl{pb`G8I&R z>LB4!W*a=868gGqCLexGdO8M7t(bgh9jZ|cWmU%M9T3KgbsS6FOiG@&&$5{4=$=y3 zPC9oOP(Avyfi@+BhW{#3XIc!UWhn0q^)8%-f+{lXP>BLDgw*pr!+>+;Yb-1*I(Y}K z;V$$umjdVLxz0k~EUT^#Bd!Ly9McMcGy@@)u5JQSRNzy&gB!_%no(Sko%KbJ76=}}qP(#VZOrFy{CjElCu2j}JrPmx+iIaatLKz;A|ix(LHjB_ju zRGt+S%z5vbl0&STM<{vanbcg0n?08F0AH5Ja}Q1ko(4;sYam&r&`Of+F6`02fIyRC zZ3sAoBeuJ6dBPR_DEqbmv54Q&K*EJ$hm1 zzNe=^Ji3O@9`ezFAdp{yYi|sBqdIpDAFGtn-|AoXYbl3#@7L;)H{cAl#cvFy+WGs~^eG)Wc216Au5}PIjWli^*GrFA4fX+irZl~Bf z-8WYa9Qd^7c2T;-yb6{Djg6u=1JHTh?)Um;ts)|reh$d*2$3qY+f&Z2RovcYTM(whpcEaefVZGMC4?p-I7z89`IyvY# zCsnYnvOqC>U;i=Adx{RhF_r7jGB9kf`;u4utzQku?$YCNG4k#*x2}%WFJHYfRno^H+{nQ}+)$cxi=8%nGMC3i(Uczs>ArIp1 zqdC)xb-KU{O&&PS_qakEQJPq@4lzwBmD`yNh5&eV@2;-6b|)oR-kXOLDS^g*hJC=TF6#Z9BNcC||` z8VeeO)$#H1Q7`#=T>nGu8f5a2|E#7@H_elX$3w6nfTKHo{`suA??sPgC;KU`M0yQ2 zG1%a8J?4Og|5sxoQMv+BsOdcmzx5s-s7=)WBD#stT;C?{7^Open-e3keYyy?4XN=fyfd)+H4q7_T;z1$b~EO*h?#om1*u2w$)h?ZZ}34a`Au()gA zzOEieu9liq|2Sd$>$U~ll`!*OV!_yv4$1#+0r7i3d$1LA+i~P0rO$|w5QOn0 zM==^2Z}uOsURxe5q6`~df|k1EU!HW!9Op1f;1-k(%+*jbh~}60mRAdv2cG^+pn|pk zMZSbHC>KLUF2FO2c;s4ic|H}Dil*_GP|cMxkKugN-hszHy*>WS?#W*)eKcXR8uDWB z=lu-wzArjH>3#8bvkebKM66P${|F&aMVhE=wDfP#8LV-?dtES{`ZpL=S5#z!k~JS> zDgf0BTZ~S+Eu)pbKc86arr+4KslMOAxB(;HkbXNwx8IT4W zmiqC}suvUIUDxf-KHp*2t`I-V`$wJ%IWTy3KjC)UZegv>yD(V-g1fauhWL%&D(}W= zZ{JZ~lXko%&cyooS6_a)Hz+x+i*fqT0|RXgdQdUYk)!mvHb_DjvCwKob!zGAww2sy zv@2It(sQ+I+NR+eMvFG&?vLE1>-OI9fswrmn`|hzlLBwO2CRiVF`uqsxC?%wn%L(# zUCSxZFKe~Jv1D9M_*Qhaxd1CO5H}RrvhE4^VbjZ>o`Km@`WQ$0BwRZK(-H>F60$4W z;Z~Q(uj2{sRwydj?v0q+^LIXe`ZNN}z8pTc0IE_VA^e96F!3@?69S*PIb;UhrE2lw z#i32LHR&HXl|3!pGx?QX-O+E9q9G}yk&6L?bNH>ZF-)Q|&x{&j+|bZ)C}q3R2vrHa zSpGO+6d~)Mm+%{StZ-u1yZ-1@d`TzDl(-@WEVh1bQ`+MRl8f)OFH_lU8z1($lK zPMw5}&`=WX)e5im>w}r<^u^5`yLOF>_}SR0uwBVZp1C`HpGAA!n5^ar_QMCKb_-lU zrrO`l$nP;_P;HEnN7cSd)^KWP>c6?F@#HZDEk7jhrM=w#uVb!hd$g4t{PS0??aO0E zjb)#)&UXfceYjNJQo1_kCl|!FV}`D=Ev4(V_Yb!zu#PeR_c6<)fc1rB{XExI5wouA z&br5b$ewyAWG;OH?r-?graQ`f380!q5z#IKIO#&aDJnspl+Onh%fRPq1vJ~|ee_1| zyqeZ$`?YMvxodTBq=!?$Lp#%cnWY<`M!8hBuuGA;td<$#52gVVlN`iHe$=)$f zjRvC1C0WZ~PK%+PDHq_sBIXuR`_hV4&q6w_|Fn+rNTbBFXltjUqOO83D!A+shLC?JWuK;BVzJ|{pIm0GscChX zG5yKU@Z=jpUzZ}n5!6IdD&K%|J|kn0j#1aZ{nu{CM+ANBRIH;~({dKNNNe=$*^sZ} z>NZhudV|1y?DaPI?m2XI!#$|FwHU^gDyT9)-w}cN5nSDTD24vNi}~ZURo@zlbj9rA zp_;9Mm^z#pu2$X%%F^#M%D|jQqU3`r!{Z z3c0V$8k}tUVi6i^Y8e*-8u{$p8GX5h@RWA)ctYWXCxUO>y*PBxYf%NS1e5n3abm#i z=~fRKgJBh_DCphKAKXPvJNlq&+MV9ryZ?qsBXQ;L$j*mS@YXJ?S_r)?_cbWhPwrof z5T!1%00Qv}`jr2hMz>2xu`#V04ooU^EKJyVl%HCD;(QCyB?8Vxsg`LM4$~^L5PAKA z8vZMf!o9f`eetm9 z_WGr-s#JJCin{EV1hmorn!bbdE|76~A^SxX2Xzn$stR^$WbWypLoeJi!)?C(O3Px7 zdJLz0Oxoxt6<1e-pT>LFJBPmZ8}K~V7(Gg1`MARAI|l~`b<=RP#O=#YdfqKl zjE!9|SBDhOp56@;vMg&i+lfz-of=N{nGjB$eYzoFTYO{Huz;T*3flq@`S;O`P5Prf zDeBXf%|f z-7vK}(8HuM92uqD_oT4NG=|%`m1k3mEhF=LmQJWk)q%a`YgMcWiDQ$rs@PFlM^mDx zpkwrZtbqd*!qdT3o-DEJ$V2n#R*x=WKN<7-Dc1>(a+=>#acZ~LZHqo%etCi;AZSfp z67eTI($KUf&u!TMLy;)dLEQe6>9|Ndb0o5Z3a)0G3T}zg8`dfT;(8jD8n0yTvL}x* z52J~HqZx1FhWqGN0nm3g={?pmc*M>0@M0`@$_#+#ONJ;^P=qWf?7i&->U#zF-t@+o z3Cx?wO|=Yx*#ZSl%MbiYAFxJ|p3TlYw!)^QmAU5=={=dAQ%;~m?waV|u!eJFDSVoZvG5@!8Oqcle*!XM_ubSIVjh!kK1-EOqNrpWgXKS5#Rh`M$zwHI{BXPL1t1nSU(c|TG*8C zPAPA(SAi@?E1%yVx98b|?>ai|vZ2FbN%uhR*|6nNnPACtF-CH9UJPeH%faij5Yc$v z-Me?ys@93~zMJ$r4iZsPPP(=us|>$|j_cas*1v!M+WXh;YZZH&;YrQ{U%_*m z6EEgc!H%Ui7h^vdBLmqy1m1ZVIw^ZGmu(&B*@wjt6)kFQrAkl}6^$IRlhk#$kS)Zc z0Z#IunT6X}%sHEpJtO$IyaH{YYW4gMQO{fo)7P2fwnORYck}k zpTo2VSQ#0LWo5KQRxt(^xnqU0 zGXu8E41r2|kPPl)tfuMJ3`2~dqCNEK%lIh8tjH&Tr0TE&%fqEN3L$nyYd=Y@w~LSEarbR%egU?(5a689JR-MIM!m#lU<*Z;Ds<3-m}&#%Msn7s`ce)^O-%NN3p$q`{%F*PVTal2ZdzlyGU#RPRXCfgile$2Z zav5s@2be`yTu5G?#fKk%EVsjfJl8qQ_#TB1e6?4N?BxLsZSQ+91W45=O zZQKO+@|nC1MPcz_`-6m?YQ9GnIE$pGiJUoKW@CsN7(C}2VGfAlVo(*Q7&>1V@`+}n z4T~_JP@N-G>*|N!Ne>Yf`a$UX{NGr;1+o1{JN)`)v7bT+4sJBOc%y0e&YjsjMdRk? zmEE+h#!PK#M=>Vm@9qtIprDx}kx{|Eoax@`n%20>mwU0KqZz8PYE&_)&liJRp6^4Z zFK~Te;c{_!r^Ntk$2DQ^@rrcC(38u*a4ihZrWavGpnxi7wz!>&obJxf>Ha&fu+4H`hJHt{hW=|uSJgi8(eTcR{A>Kr}ia$}@w z-GtuTmQc(mIvQ%Uy3r#-DQCaKjiv`*d}=x68=vvZGw)vg>i|w4x{=>GeVp(75l83z zjpto$b1B#p`PQnUT;tP^{AVaP>8oS;ONmsqpF>~&vgz@Mow|(-x4m{$>o)EaM;(!4 zikhC0Df(9Zs5phtnuy)(Y`vx2Y?Yfwt}(}SbQ z7nR^kC7vlM1pSmcVVqbHS3zgt~=t{M_dw6;M^A<#^9@|^p&YG{KCY^ z=pH^&DC;F26=_0|CUCnzcBBgdvyq1Y4XreZ4V26W<*%bbL1o=Ik)J=Wn%jZr@P`@E z83t#O!h7cp%p>lN#X{zKKfb7ZW7%ASsOd#(tWxzAp4;3>P*PZ+f{9U@rB_|BySZb+2(U1-S6)8^fLD<) z#ebhg<;|;A5~z^;_TxViPX&Q*=#U{M1n;>-`g4hb4#iZ)xeCP`B9>{f?GjpDNOg@2 z1w6N@&cy1SV`aqcIHyNkBFLzd99AzYNZGZ{JrqWYF0XTP3WzMVQqUF zU{%-J$oOVwrmBr!^GL38aq-yTwA;pD!i4C>e|Dq`(>&tpD}QPzEzSv+BJmjut;zw2 zb}OCcQE}&+sY(8eY0eW%ikQ|ovikRQ+omi*Nu=d6xMKRO#19ykIoa^_V5s zf13H^^NVe!w`Vsxt4o|5O=~|(>afLE*Xed2TCHZN<#wl&uHdZD#M$U!X!hg~fIjn757PNv+u+#cJ8)=e|(o5=Z#>~*Da`7-X^G=5z zQwKqoQ)p{T2g&LL4nsXW_6#);?hKqmJ=h%9{!zBf z97CBeF^{nnXT?KLrL2idWfzIZoJIIO>z+prLz^a45FSe9s zUnISu3hxFLBQso3)+@7Z+CT0Xw^5(nNUkw`w^Q=xb4b_LSIiA?MKBY$o8Y*fS;IQ) zAuOGt+}H4;xtzFuP6f-R6Jp-sVkAZxo4D1Jq%LjiA~MM7a(liV-kvBBY=Z9h?N8d= zw>9=%J6G})p%X*|q(fJ}m~=-nFiUh+Nk!D6GId2T*Ojmi7cY=^!U*w9yWX9}J?zp` z^v5JIJl^;He%#XKy3X@>9mjJW=d-KIvU@4$DM%#JUU@kwRT62J9Er5!3;9lb zhkRDX9seg_Cw<*c&FZ$Dqk)YvNy)(O-W@BuJ7$K=4#qaNW>$C4^PD}ybLk|rsh!i{YS+uamRPvtzeO35qo}c^&mxVqBOUdjsj=m^<6Wn6x)4{eJGY>*%c|~ zy+7P(v$}Y+;ML;P-ThBT2su@QvdJQ>`8}!|MjwQjt2$h{@+)nZW4TcG~s_< z_S?ye7Z0C%xd;FI^y!nQ?h!w{NP^8ix^oHdo=Zwesi%DY{5hea;^ldhgAUf-6g}N@ zN5iVAUh!K>tp1(;H9Wk-!op&lzQ5Gv0;xrBX~(i_VuMLod3kxrQ*ZCFa&O`nrBoMM z5AMAEE#l$G_gC?XFVDTp$$9+h6+@av`qPKo$PB+c-79zbGWV3>K{FKK_ z)!$yH#%q1~kz-v+4;~E5vKca-a!WbQvRye^z*8&dcC4h=J`SDylT%YuSMfpq{rmSu z_66hhg1bkXTUz$kUO0Dde@l{*DTrSR{DL(&?!*7wowvxN@wRw z#hsTgUoLT7Zhm9`UBz)P-8m_X*G~GGnVIJE@`{R5*RLOTUl~8KFx$_hk*5C8)6+9o zPfF^E_$pU;bhMvkcYdsXxqFIw>i#1~kN%pP^2KM2H>&cF5cLpY6uKXfRl>(fvDn~I zfjdTZ_%JyW6I17pAD`;#c7K0$q3W~e?!c(1J3*F11w0)E8)Y`b6u8%3l%i`?!*$`Z z*@hK2O-#I+6EDZBB`aTle`^=+!5%JK@wP1Wf4A$So}s+v$ZoSbpSV%#tdZ7{saLAq zS-~QVGM5|uuHN9I&bvNWcRJ#dExY^jaQJ5)pHJVk%6%Dy>9_6N^HFqj?J=r>g`2xB zVYz(zdcgsd_!TN@YFg@s7|}!Pt4oK@y;LMMu5f8)Ms=1r*%}-wtd110PpdcL@K~Js zp?2)}aoQi#i~@FY_nn-~zR9u&KdT8oVey@fmsfq(re^-gg}YJJ_4WKU)-@L2Bl)Q{PEscdwyZrURJfd6dprLhvmzP(6`06d6)vHVeR{aMP z6+)XQwcn~H`t(&9udghkDpid1@NxsAF&_LyYZG=;ohl?MrC0oX_jPSsb;ZTScMgva z48#S=y1JIsH%$Egeedzo(%f)eocsEUKpn5a=k1T@+mqf*#I&VrUS56Yyl9?dS#|x_ zuV1Q{9zS}d7-XWY9XZyK71KViooy&>J>A6@Ul7_~bVG9;zurOx-z%MY6?FWv&to#m ziM-xNY5J+AXa2hEq^1?BXb4?%FZa+aJ=Je1_cFGFw_(1!uBu9wL?t33;;BpX_~m3# zvE6u3NLF%ka)hu;KJG(KXP!kk@z4E@wZfVLEB{e$rG>(B+U}^Zzp;LJ)ij#l=K%AY*<*Bx4(bovjZoHzjw>n*kH6i`u_R~pFg88 z#qQmMk1t2s6o>QZO5hhKq)6jvNN4Gl{u~?go*!*6YD?3QQ&z5gA-X}XqoeaJT2Rwu z*}VO2Y7o!)^9N9GRi7S{QSaZsm1XfqMTl;Q100J#rl0RAuwr`j=utp)^xl_>Py0$;;@wvN zzRSfecehvI)~#DVa~eb*2L~SzSsn^@4HwNC8oG5v*qMgZm}j9GKX*>Hm+x-R zMKpQ^3wn%^(4{9-R1$KV;Wf92K4fTUNOkB?@+c#xp#21mfc?aiuV1A|g@LVI1y(O? zhHF1HH}6MV{$Mx0|L<&nOx9YS!Mc$U%qT(lkpWp&!j$Z;PK*Q$KcxX z{QZBwSp@9H9@vi7$KLtz-t&f`1}7&c@!bbKv#DFQY%y($+kq=j#=yYPQ|w4-(f(Ha z{{4%3CHEf^y^L(%5iv2b2bDs*-@6}Yja&P(w4}MxC%K@1{#G@Wa_Sb9O^Mw_e+pd)6yPS2;pGyQ#NZ(c=o|%DYGm0$B!R|O>xqn zySlU$Hos7c(%gIYybOmoy6GhvA(xP_aN?k;kN1tFp1aD*{-%v@9){cn_%NThad0?D zdV)^=!F|1`t7OkJ_SnO{HIv-QF6-@mEG$H%ohR+mR(5?x|(O`W4PbrqeMCI-JgJMg)q zLv!s<6%#$%^4E$A&ZF@dQs&=IU%I5l$*OVh-aCazkBw#7>sng7m@iCdtE;P*)x~u9 z9~Dfn>@Tvf`?-i2#wim}J8RmQkZ`OiOJ6km=BKUJ3u7u~IzFQn0pzN<_n*Ifk)*Fk zpN{-qTN@;^7%3s%E^1XLcSG&PJ=9}Ch<>5Xu+*(v3C(j;Q+LkVja}tbn(Zs~>v~SNscwXlz@G$x9T=Dod`aQ zu!T*Gdb-eK%NfVfMB}Rmtfz%;ey8ZhKO@}oaG`T5m~>a;W%dRSr~4O9@<`)Sbmp2JV88O(_rU|whYuePhxR|& zb>KH<5XuE@Yde4&eT%lXb{s~4jO7*TNv8Q%mu!<3mVhu+GH+Z;96osPAYjI`g^j<{ z-PZFX-yfZf(<><{`90Re=DM*wiq@Y}xTW!LAG*8sgzQVaT_ zuV3@5TUuD?cpqnBWsUp%d7mnQhLv9nJ;U_dO2Q|7x|f0fBS0UM~E9l9S?f4Yp?7^`fn;Z0X3nSkd z)_`qHyYo*egmUe`O!-h;JXpHa_vJ#rD}igwdQW(Gc%+`Rd3C``raRwKB~EHrXIIzf zswy(cr+YqOR^*K&o~~e;8jfzIcqGytdZl99kn*;~RLRBw7RjGeQ+t>Tax`pg-Zu@W zU(1MGaFmdE)U;f)izPw%Saag;oqOqzooWzVe4VZ6=KkIO#!$2z%F60$)$6x! zPu_QNkz^14)ZNX1MkUZuh+bnCvpCVl-EYjp0q`d~TpPM&O`nXzKJ!N5Ba4pAEf3_s zd_1(^Ha0ev;-FA~%U;s>-uUwP=7qaGS6?Y-uN`1lY>bE*D6{M>(&!&B4-F1xvnQ?p z{z9n~l*}_yB>SBcfES;18)*+cFS#E*A6Mh_Xu*bx2@dZ+{Mdh20N@kHW*gZ)-jp0| zb8{|WY|ksA>ok507oPqPXBZfGgsL?JXJ{_Xtix?PW7d3(AOomY3@YNs(ynFr089L; zJ6--2sA#d4o1J|pfV&idwTFhbpEz-X*R+v@BT&jK9F=sgxfq)EP;&_u1J^lY(i{(MOBy##z_(V^&h%l#h;aA3wfh_aPoBOpOKIR)CLF+PNQ> z`aQC8UcV$N_9b^rtHr#$R-krd#7sf=MJ$v@4px^yDrN}=LrUrq1 z(8&FX{{#}LLOtYW&Q4D52K3k0*C&Z-Yty^0FKXtRHf`wEX^-PQ4wW!$K@HwyPN zxh+tex25d>@wu*}b6JhiWB&UE?Q)CZb$^ek&a>mf72!#oS7v$&AMZNA{sx!ip0)ME z(o$hpSJ%b@EB&diyvw6!>l#ER*{sTdDF|8sNVOg8>f*(V#LxemNYA2TU`Xf7Drt~F zrI7F2_g}RZ*_ol<<-1iv`X_ls`*(TZ9hyaBNLna5j(npdKs=v$>x1-kZi4;b@S+D# z90}Qf@F2UTdK0WUvUb5W6Z<@<2H{z zEG`zL=QVvGAIvV_;$SE4lwiV%iy9CZOilVFA08g}%mENsS3! z`=z%m=gysbXWqtTH{C^RTo>kv(!r-wNmf2ed|t42^NwQ2S;iwrj-ZeH1g^I}vx9_D z8hfMA2Cw!mH}~tG!K${T^ZBK+O}L(C-Pi78fM~8{j?T>ZSNbs+H7CelbexH*6_^lF z<7*1J=6;G>c~j7${bXK@tk2M>4tp@m#f#S`^Noy*&YIL8A;uM6#|veq5O(qzzh&3W zKLZt@<}F1V=p+f2H4U@)z)httj^o9Wsj7*M$yb>2s%mKFY~5*IuACPXUCa=AQC%%Z zDr~qhQJ<4EuNATM`qD(Ww>P<&aD_uCt|a$xYG;^XwY|q+co*XVcjsCr?+R5^k5Fj!zd{jI8&Kgx7&Yj6IX-F^&ymjs+uxwfL99lR9 zDNe?Z-n^sSb@_d+B`FSqgL%vJj~rQroI5|x<(UCko<{o$nX6m1v9Z~Y+6}Kyhw1cr9+-o0waf3w5~Hf|n${cml)A#tJDVOmxD8clMi%bx+y2tmh8 z5;Gvbl$~8}m;F%Hr-};cgoFfdylCVtA7QmVucV}Z%dvOU>M@PX1TG&sbSPTwdiz_g zmXz_|a{|@_Pnt$Q9=5Ck_XmBQ7{ zEzdnXpuFtg!0rrVP)E;qXofGZ^cLA`w0{@e@zrzpA*u1^M7|LMS^mKo&l|p8az8ud zE{>sgo$Ig9J2GPh8RO&71%IOVYs1 zEOp-9=H5LjL5HcU#Y%277D_RP2t-vsy}G(;Q9?PN)3BoY?E)*S#ENqIc*%_q_r#6# zd^&!E07*YTcuIY)T2MM5yTdG6zK$4z6I+kr{)~*bZ@<=spVkN~cbb7y z%F58nkulb-nC0!#t5lq{$MsYxb9KsC?(XdL2F#M2UcoyIZ|1Y?QU}{jRYG5};_TJb zcT42=OJ7JTOQK<9H z;^j~-&-M(RLDUMB#7{I_)dX4+IXU^qFJ6>HbC;5p{nFj7W47Mdcr|eo$B`a9hKnVP zwuRF_H@G|7h-~Y&ZGqsmAf-fi2HgDp_pgkhA*+O>WVZd}H3&wrAXgXd#;COO%oFYV zF@&B1h{YRxc?toA2_wcUBI59!&UfNmqq0nHPh6H}%%Vo5!@&jy2->yLk6z?DY~(*J zBQ5<4~@sQ&=fR70HZ}k!1ak1jZQ*nK-A5Ki>JP zL*HWu>8*Y_<559})(C#!edxJc0~kKlMNB3z?`0GsxW-;$q`9vGm)lzrw-o*&^RB;;r5@jELsiy8^aLsSXtvDiEQ1g}fmfZw4Cy7Byqo$&IU_DsL zDP40eEPL@q4_H%TU^RpsfG!?Z1p7WZ$< zkj!<4GdD>@#iXQg-03Z%O;(QnjIPs^c=-@oyQn}sB`_m7pup4QTs@5sdmU@6Q30`!($ zxu4LHbK8ee^ zw@Ki-;Cej&r#w1&*NLV1k>2O;XI3F1>EfAEIV+UKrcL>{xWmzRAL(Ee&M$ zblJbHqHDjmJ?P<-S%o|YwD_s7j}dg>V~oefGyF1$7%a@pBxvBMzavsocXOghOR3y2JIH+kOi|?U}lQgjQ^v_Ps=8WsEIK$Z5a*WV-u*avZlLznIxuZUJ ze{q7A{=(gp(g&@rt;KNx{@`4F`td_+<<5^{$E?aRj-*UV-#<9?@@j$I-QDpXn;WLh z3A^B$1crq@O}-K{Fi`Ov?>C?(-|UXdCvaD&QM)-O2wwgl3`UO|5{P$<6gFG5fo(n$$;$qpt+OZGk?B;}Q=D!=YQm zi?X){@_(ySun%QSVCy@UmKt$m`K5OdA5n5$ zU+f?~`1nz5VX~uPtYm4n{{;$NUL{&5|IRa<6$}IhqPpO@-0k`B5GDSlp@DL=A*N%w z)O}@PoKPgv)Keb;0LQB)D#$4)e8!-M(EAAI&zJ7}PW0>xD|DY|Mypz5epI#=VC2dw zC{W1w9)5`18H+-@Ugla@%MW_;W??h#-t5bG_dj!UL1oX^S0=Mm1!vzI*ZmwGKAzaJ zxPB}PQ=Wh~&6~h0-Z-G2+S+L0F;&1bTkxUZIV7WR(UqG@)0T}tBhE0FM%wn>yLb1L zI2{1lxc_&G4u{MW>^(X#V~_Dzb^+8U9N9m^;rgsRJo^X~wDPvv0>IS!`SWcg0*VWC zo5B?QmP93JLTHvf)7=G$O(%eF>1E4J&)beDa=zHIZ70VmopSfmTt!{GKsaD>Y>`9b za0u0#+p@qwsS+=TU(@;v$_5ANq@O#X6?ECW7IajGibGC{gQRS}q+z|Ca$}P`4c9G^ z7erBkO?veEtK`Lh`{!2_?u z^5l&Cc22c^ucU;-(Wxi`bJ@g9z_l!uNE-rvhPcDSlnGZ6VFj6axLp6`h`lBh#p9cI z?CkAnF{2KG8n#_jZEJ2;bp?E4=g#7sh=G}TdDzz8UPJKFDo|7$M1O_VxbaS}Cr`Z5 zIQ_Z->73A_MSE zxy;aW_M|;Wt2vSAF*o$Aeqn;fVS6gnXwJ{6KyKd8o~g(i`LDF8Qzu|JgvRYwiZ~-a zJqD%HvLv^wYB1wm;-o)!uCzPW&PGQJ5w6zb@(a{4TrPhTH-yA2Pk;sDNES^TJH8&h%mqsLc4r zJ+MufNt&-JTl={W95|3T`jxsK@i zLa;(4>gKHLzk8fYKyIyfxvkXO@q5xUuF$f{zhD&B9u*lgoLW9RYLb6;F~g9TOV_8l z^u!aVi9;+QS!XR9-WKIE9XfOZ63dQ&)jef~6%?Vokgo7fCAF?k{WIs^8 zHut=XS6*H|4zoa259p7}%=Z~{;%phWkB?7~h=sjdx#f@diFQguOIk0WrPKVEfS~ve zkC#pO9_Y>~eWyEJx$?Dx#4Bt6o%(U!><3_6Xc%sd{9awm>tdKkL0w)|ot@WHRj&5F z=)su}W}JaI8JG7K3TRMmj=`6mp~F6uDUnI52UrWvO2pXNz|zogKCI~qZ(RPQ)_W-v z#$|(V4x1~Jr#UU*{@LBVtm!N%QKO@+O#wAF+i_Me{;y~j4ySzg)eD}!123xK*|rPn zSIA_StTvUoxn%sB^t@_m+VPi5>wUsuILLBxa%b*uwgDRiCRv_KQc)Axag>zpFs#zE=1NZx4daoz}hME?S zs2-dGZ4a7`V^(CGmWvYvdE|>O3BE)EQr{ch&CgpZ@U-Vbq~3Byxvh;&h`Zg!LdzA1 zQ?l#kY5P=ty3H}Z+b!g*Cq(Tv${wzKq87DF(bLq zrc7&+>2~c~0bYnqYXylRks<1>39o0RL_uC& zMf)!dP&4h8*TOF5-z2?zlvU?S5Eyv-UdZaG!`eSq&JjHqgmjC_yM(cvX#a~qnr}8lA?!ul5VB_B~)?wIyW~rjZ711 zpXkgsB~cUqeS3(57cz=pn46?=gfu^0(4VGs0J>-1I1(VHhtc={rf6iCX!0qz&S%-!ecWr*IXzh}?0%fgA$&NI`7qD%p?DvBi{@ zOTDp>vMXX>En@O~cf`uIv!~qzF?Zbx;-@UJm?!DHL%TM?P9}t94ATly|xrDj8t}c3jd1)N&`Dm}g z{8hLp<5rd|Rl_vNG?x_=&ZW7+0fN|~@zT8MdSp^Zuv(PiT!ZK)ud@pR38o#H(Vtj8 z=VikL2o4UmyK;CEM~*;Eq`{6nvcP9*gW-A)ySRA9Vw&7@ zaM(#oJTq}xzw9fh!4?uMKla|+27O*#z=AT7hRKkl4~x2&blG`oZIo}U>=v7`v%vj1 z!*$y2W%|syWLR+Hvj7`X($cYD(I1@V*$Mq@VY>SeCUPYd(06v@s+`lgW-Y%Fe2CAt z^jSFp7J-&YvPnL-j`zr$G~H8pK20<0b=LGy;&l&jRAu8r*V@N0Oyggufvlaq`H|#T zN1Lp3btyU_rJZf$oa4-mBNy+H#AaFpCX!0FezUy+^u}yM_9<1--q0Jbpx^;pc2qn0 zQWU>sWY=P=S$~-@G0Ux&;psv~+ePA13j_s`SK9+|lnKMKz35bhi%N&%3_5|C?p^W< zmw7ntGQ)&}=2qr%|NhUx7ozPE?eME$Ymf!FSkexOaIZ||H1O)3M$AebbT8*OgA#Ik(G9U<(2oyWntbB|Y)-Vmo!}djKo<*}rQM^TauAzbZZ4=c2>m z9{!ToEU41%_~o$H!k9Uu)4?^lBRfcdB+Yi+U@EdW>t)jXuT(g7p&0R@O8|I)Ioo>5J(vJ6!Wn;l`TUtg*h^yQ8dQ*8Xt}Tb z1TB?QQVONtMJuQX6vjR52I85j7KSW`!)#whjxRh~?-wtsP%ybk7DQ_wMkQ2&)Fu>m z=h5+*hb+8{3=DjD=(HX)FE0(@R^fnULsLg!=8(SYAJ6)dJu26`+Y)i82)Vl1<Z=!dafxoyHey6C?1ATOc4c&tuH}(;xU9f&eCuYU>Futy~T%4TzUP~ zvER*$NQ!_dsgiwN>jL9_(6h>FMJ6RlWbz;cU$h?B?tkRcBWN|yBP5J<=9B|LgTHyi z?tTWhY~}CtbEFInk=FY+o*Z-2($WvF$HvXQ8|_FTda6%j_ra4NLE=ax%u@tao>$7u ze?C^8zrMDX;BQ&^KarS3Ce7X2ef43`bO^bl%T~a=mE(wrD`bKxvD>%V!+3OkX3JK2 z`n&n1*KLv08Yim*v0IPot&4lUMBRdq_%cW@-x%+xQkZ^ zml0)xRK(*hlLHm|{!O?K3Bl%ASVAAL*NgXA97ZYsoSaPOj}~coGBNj!o0WAdk^Vt! za;C5JIoh>f^e#KyVnjZP93~%6L3Z{-2y$_8ajjwF=t?A;hUiSDpKWI~-ER8gS`2_g zdHeZ!#>6lsDM$N2EO<2~_(q^I_YjA}&pQ}E?XHCA2T%4F{tQVV9t`X6Mf$sT?P?Yy zhs0XieVsHt>WIs5(PoGoKa;$mr4L=7@IK8u&KkSCSSup)WjL%(11n#kx25F(bm7KG zy+4sHi@oCg0$0f&;$4UGlicWB+`lcpw7lbH;T>l;Te|lNm8$G%e1$sXmLSrV!Ta;X z>E$JIO-;>Gfgg2Q1q%yyLe2}-x3*c&aYK~@B>sfXoqhW|J#l&wsB4}>D|xGxvr}y5 z<_O-|p=&D7yuC^9hzv6=YXy@ZO}yM;32R~dz4p+~p?h2+{UpHTb~z&Kr)o#I0_50aeywkx8EZ8+H2G)K0#w5S;s z5)K1Vz=Q^h5+&IG8+0HA(I>FSOpgwaj*mZs6v|PW%>#_~hlpSI-d1W4(?itYw6zOY3SO*7u3khyPouh~o zosMH_3*Kt!q_u6eOVqY)+Y-&-e6hAMx4dUcmpn~7&`Jggf{2=bL2u~M0@vUNay^ia4!#xtifg+No zUn?tF+1Mlpm$tgi27(w!kUvC^9jDh=t1kr*$`Cy>oH#95;N|6|M_uxSDuUmV-hJ(l zY-zk{TJeRwETWWRcaM?j(zI}7z6vi}y~BZELiEwCv{_p!R@9E_syqbf7Z4Co=>KE| zwHp`r4Urch%9xPc#b(aO7R}cBbfhFP74bDW_|fZ-ZTpDgdITCVzTT5E*|qrlcO!lF z33=a>7W+^9(q*3~wwd~?0<_G9sC6wYbfLzy+OoF&b=#qqlOOV09$UVRr z;_egSm}7GGIhO?g3!tT$T#33E+q!)9kmYDcZ7l^dol>w+66Ad6mUyRM08G#bG9Gl0 z-Kln2Q8A>;cY#g}zj_D!+GEF#nXZ>?^-bTl*R1Z`|3wT`(&K$D4ERIlOPA;#n7!UM zyr^{J#t}&J23NIwJ~PvzQR5cgoExekGO)Ltx>5 zD+#pp6-@JNtgHbqU#9YYmSTc%8oqJu!rZm&ErOWrM63ixQG%D76Im*v!EqR?L@$F; zMo8+os|su>tw#nnrB8l0weV3JMCYAv4IKUnT@s{S2Z9zVv(wX5BriP1N&lASHX&xR!>HfC_2I z!wMh7b5vHSm}0Nx(aeW9=asW}nz6V_wP z%HWgiepw$?VM87wn+R?UYm89Gh<=XRYc5h&5#gC*%36s2tB{?esj9kPNJywP>HGqt zN=f;*#b}-WdWwFxrK2MaF_g>~3}Jm(He%|BaBJ?vSMz)w(OB_Lwq5nGqL;Adl6I?? z?wfz}j8o8*@FCsYmt;NIUUEkqJ1iPmXn z#|wGV8Z7okk=-s3Fz%?Rs590BQi?Gmy~moa_+6s(iKbcLd|tNrob#})dz&6ZCNA}qH;cu|oLPL!_Eu(QW?{-{o;8gP+{{an3UYFdPQrAy(G1OD#?Zq$OdP~> zch`bT&syG`82uRc@5k*FBB+v{RVLx=ECA^?A2fK?Q9u_ zUKZdU;q?3G!}h0qx-fF6XlZva7yJT2wI}&CbAq6|ggXyC^HxXap_kh)j2L zzt_H_wT>clW2b=D*mq@3PfT#xa}*;pqZ+Y`KwMIQU|Upd z{GLUX&ZXF%B&Anp5WXd6xoKfRWZNhCj%yo)>bqNZq-#=wq*;ZaLQdz9J|llo;xK+IJ~7l3FW>TSn+RrgW8W&X6s3#J>v_aF2yQQJuea8iH< z6RC7Wp%KkAz@>Cn8q`*w(6k?Fx^JhOW#c1CW%hi?XiPZnz6a;VI$-P?3Hnnit z>h+$sQLn{EZ}n*!X3;eE3X~W5PhuM-MC8gW;5h)qq{cy`ZkM4BHwTCOracfXkgDh#oFk--)w-SH1#OeHl zp41!PsHx-L%}SCw^DP+&Xa`-Zv$OLid@Mq3(wl5gQIVv#cY$pjElLv+w+8dQnI>un zov?zkPOpY~TdJDuijHRTiHhMBMatx7sS`{UW|!gk=tqy|CDv3|2gK>{o41l(=kk84X}i?Ho3N41>tkW$hLK5e=Uks>oMItNGC6gQCYl zuK>IERAm2xpCDrKrR}FE7Sq|GFKiF!eyFQSwv*)v&G+v}*IOLhFF8Z~Qjy*M+7^l> z!6k&u6PWU_l#+Qk z8S@gInuwnT@-rJ7tEh`6EsNHaSOjY*a5H;%G7ohMJkz>7KvESNB$x)i;($) z%35B{&({XzpEPP{=>GdMLs_M0sc$DJhN?P;QOvf7n}QoC;#RuAbdF%z_4=BcPtbFA zF&CWADLHi;8F0^Q{7^!r|J%^?G++Dor@0^Plt0DV0bY+n%N!BVv@9xVnW_;K8TJ!n zy%LVC9NZIyqq-=`HsyZJa(6eggda#VQP_pNPUYCRNfM*~rA?F1{X+fNlK_DuDZxQO ztm`RvtgI+88RP51gM;H+W99m<{&A}K`c+Oe4dnV}r7xX;KA|+_y1NT%)^PHFj+p4- zQl1a)9}I#`jSwleq1YoSjtj<*c?ceO5T zOW`4mJol7SzWY>@(uyg91}O831R6Pb8mwrJswF zf=U>dc_X&CvD+@Eodg3VaNoivE)#ud48M;GnSH2^2rbu}BU`dL1YG`}#BR_r`>17L zj>PfQ1&xTsbbdGkW$HW?56LH@khV;lADT3ao0NS=Pzu=^cq$zBkFfuf*lM8Moqf&= z#WZfE*7Is^y^*i+4ai#ai(du?1_B-@)r5G8>OBj`1gd- zgW)8avUbGw(qc_ZqU_vg2ib3~gUcqFu|#HDY`h3S+P zt74i`KZ!#Nu1d0DdoUUurfE%aMop@6Zeby$%qs@QY~;$f!{~p9&)?pI)uc*-A>t@O zfPIF>vWHRV`Z%3teaN)%+$7__a{ra5s!@iT9n9XC0nlJ;=h=g65(L(fX#e26nAW}^ z{cE1Se+3f)K@R(q>TEu!pTnddFVLHy^Q{kP_OHP5t3!e?g?#J3kBD{@)C zzalSt2wD}_x_t%_@)STdP3eS*nHjvt{d9ECfQ2Sb zI;!L#cW6d`0ZK@;-{_7)-3mET;JEZkuQ_b7rT$66M`a(Ux(DBLTp%|)TO3yGTxbK5 zcnOu+B5o@;A@T|6V;+VlJuh)z7vRw^i<(^r8ZKOL%(?x2F8T$8imO8{$LP7R2q(ak zG$j>xn1u_n5AIB>g?326_s(o1!q6p_Z?FR#TT`DQ;AzhU4T}l0wYs+U5&9ed-5zh; zMuw;thRcUeX(ycRMVKbhZ4w>nQ+HtSg%m6jYFmU*+n&|v)d;{PX7>~5hwj(Tq3UC= z?Jn3D@qQyaFHah&FVfHkRVhMt8QpFCth(-dNF4Ue_1v7CyRbm;P1Y=w(_t;l@!L5s z$VQi2{DtrTCcz9aZ%Ir{G{Avr-t;S^{63A24t-%8_G?Kbn5O3|{f}DQ6Y*RYmVHI! zT|74b4ZeVn_ws_(YpBqd?)`kw?(!4apiluin5k-JFT=yb?HXc)oZgEGN{_i%>6N+C z0Ti~qjB%x))m6*D=81yZgUYX8nGqc~>cX|U39(ak!{_LM-9h<9oUkYe4|c)BD0wG; zm3nlG#+?LqTr0=m#iz<>qSnxO0&8;MG+^1JWmZgk!VTmK&g&*?fJTqinE3C@b*|G1QxDewy_pIJhJxfoQM0M@jwPt5*n2F(Rl< z2nOE=i7@PsE#7qJZ;$n4@+bA({l6Q9&>6YR@K@Py+_d#oXp%loR`qB4D4=0y}yc`C(|3E!os@P zW{|J_f~~}!QWs52?FtJ{tW{eS5^4EFEX7@QIXY?d@xhj@4^l91%Gx_dko!}mgMGn* zycH{n9AXEWQIfr`Waa@SJVn4@K{)ZgGFCDJvxEpBvgFiO8L|{hB~eTEfSi(2^VJAG zHEr8HjA)1%* zGZYjQLCtUVO1q8?hp3e}@I~v}mu;+An3ZBIJOhsi<@H$m;}uegGzymO&*>ZU+OHGg zi_{zNV+hfg%M2ajG&UMi?bg!;|Dq1V!iYSOv6HH5kZFHesjA3C=JN)TRmG@FwlWrl zxOQrsFOZ+%KDC>YGCXX@jvZ#-Wn*%Y_zY_%j9?fPiNgp=83LxNMuOW}^d|8vXVfjG zmS^F9A;*_!2Z_GU@sg5mIai-YWe*Y{KjNU$sp?@k@5g^|jT2c!P9>ykrOnOL6zD6A zD8yPbb#Fr=)Nct za$8a7Do;;OzrQdRs1ON?vK#A1iEeFndkGYqR;8+{ntOK@v;(9{f2h-TJfQR!M+HAb z!{j;w5O=}!(RA^stEj4a4)5q07HT%Ju6JaKqG`PoFxa#Pu8>iwLmWOYKjZf8+dfD(dH1`oDjB~N z>1VTZlR0cVH8?mZjp_;cOZTAC!JXKZy!l@y{~Gobj|kY2BM|o)X?G&++M1@Z8zwk* zs7b`ePe=$2v1AQJ^ed5cC&*DEjl4oI64pJ1wzBjYTCh8C0#cNgur0yqH_=KXPC>$& z++#>g@!&dgSFXrjy-G!7$B3Lhj@nb0aQpY~zlVNG#Cb7QB|u`^h5Qyl8x^AkXz}a? zF(}$i2-^Sus?sj_LMUWv!YbnoI!NNR>N|po#;w{NT62pK&$JN`#XBWz1LEZ%nEm?s z^8s$H5jbzNX%dlchFsXLRgcL3R9~sixG>B1fq{X05T6iLBBHY-5+x<&w-`}T{VHOw zA08Gx=_CAb=n<_hy+~Ew#D~HYCMsbz{DE7pz|sve;)BP65Rpk@H54J4ZE$SBD4F^B z4|ZhfSD{she4Zi3d5U&kTK)8EAtzNcXOP^l9=C2CgKxmC^ap#OFSL+1E?tu6;pV>V z!))l!Sslr5=>xg{7qnmLd-u*DG)Eee`5nY0=>`kh2vJw(#Ob#5a2girCZuc@8Cfux zZzY-?@S(8n^@>F3nijw>Smr&Zz-}Znxt8wefpSV>GN|{R#;R7>X?=h6N*1V`DENvp zx79|F3hV&>!h9ey@^-He^4x&KHR~F>4!H7P>|*{#_YMumaSM2PT=6YmzI;h6_rTLi z5;@<46k4~vgxrsQM6mfa!2A@A^n=981qCe^Gt#W+l-DBNksCAS&@1`y$(Ubuy#)zo zB9#j)B8Yp@2)%aujvcW$g;Xu@wz{VyC&>T^K8^Pmya{L%xc9@7@X^9+f0TI_$I%PE9k7 z3j8@-MlLw`F-G3sJd%Ag81I~S&Y9Ki9Fa&;8~ z8h8%Y8i)Rh#|wM{$|tTE^tADk3tQfR>ER&-0tnOoL7ak>gM<9_>(|X82H}Lbyg|jx+UoHyaW(U@6iC8mK-H;^JbhTvv_LQO#^)z+P5FTB66tOTN z-C;ifd&+Fs8^-4iOnPl?Yx@Ougm2&L!ps(gX$GLjKBC$Rw>O-1bnJlV0_IQ^nx{1a zyicA!jYFVdtS$WzaW;V8+1S{&5s#L@IYqkb30NcHWY5mNdWtpQPly}-LG+jn&-=gu z{`$_~3$e-a36PT5DM1IPW@5_f$g@zuTls|F^8)GFBQWm~tdJ^qU;poifBc|BUmz|C z;qW2D5=w61sljCW9?8kK*R!|sS+q-`OA;Hqkfc$lgE)(jnsQp7B9>2!sA%`e;WJ+m zsA>YjE^%I@K($uleo@oVJi+N^5fNd;GeAD!^d&4n?~z8GY0RbS7V~4B@i$a-V*Q6$ zECN8d2`PcuL+y*e#fvv_1b)xYOG2tiQhfOc+AOh)Ky17sTZbv*4L<)1=^nxm#Ciz! zYLY`5@T{Cay^j4`sx1pte5EKzK&wD1+i)Sg5O5+k{vXH0#5CLHz|JBbL4@79Er)n+ ze8lBf$ulROxe$j%m3xkkvhW(vIXqzp5TOOoUxfxQ1kjP5JVHb^;MVWbd{3()`Ujon!0EXjNMbZyNL~dIM^zX<;^Ja zaC+krGbZ`8m!3HNDs}RZnvK1E1?r7>G7`&ncxXNlL5K+jo94d<)S%lCzd+b|;UgM% z?#L1ZWMZl01D=dU@Au`ejDYIXg6~rD18W^$S{K~i=tO8mT`JO$H zvBzg;iCyMbiEsZ~c=e9XL3}@i0A~QuvXIh41KFMQfpZN%J=&oiKgQ8jWS`!=f+u9D zz2aeKmoZyKLAG*lV%yytjdbi}LCMqvD=ETUY7B#Pfr~7F!S5y{Eh5X|Je|xx(lvvD zB!yTR-^lUcny(4+!MS?7eb`bK+}bcr`>BJTP#N}ouvXP0@;#p>xn*|!z;9gGz_S|` z`p7W7vJvdb=vFBSTX53S)=u-=_a+OfoPtv-gN7Le-iA8o63h*|%eI9_2>JAesbml{|?#= zMg5VKVnteUg|kd<`dvWks>XJfec`30rCl61SI%j!2Zx3phfbT2-GjbrxUse@Aj>j3 zG~|tDVJ^HABkllzLGxytv|KI2C9xLbVMaH~T(uCYmJQg?dSOBwM>#hv8CIN(+f40C z?nmnGqHK0YCDq!plJI^V0e3>HO`P=ch=l?4=>M~{st{P$I-M*McnFRwTS;l@qQGBb ztrcg1%jqz^5+`m9fy*a_MIiZ?%1nvkeoii5TY3reQhzs`^o-RBIdPW@K3}YSyaO+d#G3;7Cl%2J&hvFrTJG3Z(qV5LWUl1sX% z>`U!50G5wy>bOScjZ1{R=jTLoZ96B6uc)X5&6-JmW-g$ss=B59Y5UO}h+Nn|C5qrK z+MhxW)sP#^wabXU>93UMn>ZKVLGeO`h-^Ihp#OTI?Py3dVL@6r1J6FH zBqyHzaf_HY=mr(&5X1;2TKWqnQ5i)=Uz!~|tB+cHK(OSsXjhyqSzlkr<3AFbk|S9! zBCbn3bVYTD3AypoIV)^lK`-*ck%+2M!C@4te+KB!|1WI5?Be5tx z`<@{UJ&~}yh*1R5>lgkykvaj(f}ghg!i5WHh}($gN)S*0NSw8_7trJhzyZ3WQa&AM26-}p$SPuNtv1qMVUg#kTR6v_gcB` z=Xj2L??3lG_WmBvCg93}WlR&21llQ-#5ySPR_;-1hExGC7b?X_H&X*Qa(E)5Z92Q^nfKk8(1!8MgY2w$j7LaPtxu$xff4PreQg@R{Rz$k3t3&=tJNiD*Ix zna%x=hGQ*EPt67f_5wql0}1;5U3BtNnwjx>x<)pENNXfeW54;g+H49|`TezE$(!7$ z7Qes0Xd(nSLJCT|>(J>Q*n7N*I|82B8J1L1wD;hd(@xJEiP{MI5re^vJeXqy6nLwS zJQnf7>auH8A4(W0sf0_9M~zG3TqE6k^?iha!pI{BG@xj`bBWelZ@Fopv+D#D9Nacw z`{iG6s2A9jH77FjbMVbX$W|MyPBd2L&J9d!i8gK8>P7%iH7zo(>4_wbfj66V?>~7x==fk$BaY0b7u_! zF^fPa_+xf;V7l{l}AT5E-aQu1O}3^pDo7hLl2v zSnYriP9_Om_vva~NzzF|BFS}5kO%B|u+-y>((;(!=v{H`jyWI<`=~ z3Enu9Cr|FYZ(FX-uhHgb!~&BPxP}7RR7I!N`b(MssPUTcc#4>;^N2mv%rwjS`zITY z##V}c7&(e%l&2M{?%-j)loAzWfR%w@WZKED6#sI3khZsUj@mYEHp=-Fo^cHY7(9lE zO%LoE)_)Jg1-5WA)5AKuO#dTa&*#7cit(^yfLGIM&nvUt!s@&^bq>`wUJtbtOF8Xh zmlSnEq3StPQ!RVzr54-qZs7q> zzC^eV_;Lf+0EGhjVGPh`g9|J8@#81|K8Y>o;^I;fjw>a$ss5V3fBZ3eTcmQci z`M>-szW4t#!(Vxpe?M=6Amsk*VsSqEfAw;M|M(G3uoJUEThZ-y0w*z6Mi>c18hz`TQ>Y@whHs3I?a*)!sCiEC2tCW(OS z(tLoTr@^F2ljOG%3nVJi*Bj#UgvJzcA{~niMDP+hh*0W)m8IZn0qgUtW_|6V z(Fcfd5=jhI0l5eM74K5|wEMg6er-JvQ+k|+R6L}5NI*fV@ly8)hf|!B1uTaY&w!TV zD|-F<_5Ekc&6~cgmBx2TF$sAx(2Y|8TIqKFtf)!4Q9mpCtPHr|TA?X{M+^tg?7%~` zeBvD06qR?y%1cU6q8R>mS3}nfcF6vRf>sQaCH)Ec5)&UEpyL#|_CjLf!K1ma1YBM5 z=3Wh|*fg*v@Q)^S>(&*WFQ|mDaA)vtjC)7_T@8SsRO-{utaIRZUGLK$%F(m~IMDKmvQpNu(4L`7snBo#d8$+jQP* z_4w>mi2MqDI7O|DI$d(&|7F*JpE5 zNJ1W2nL>kBaj&+FR&S?J>{jl`&cc?FkG1!z18HMD%hk}A?C;$eepHQ-J zT2N5L3W=}ywzF_0PfYP?;x0`NaX!Hb(?-Tx*SLaF&p!%5{5Lo4cDvyA?MAcRDq4~$ zC@BQsl{#wg;!U@rIawi?sx#Q?ZdCfzyg{IY*Zn9XbFizi&xf1q;aQ8l&QTvD;{Z`;_l08X(5=${DJ{oLrQa`Rz>LZY=GYJWX zXce-^?@l6)F161Bh!ZWi*x`sj9|BRnDg-p#b#^At4k(l*uq7sD0Q%i~q!)2HGl{G% z@T*5-SwlgRWc+&4XE)VVFGD7D+iNiD6%F?Q|MR8eh~KsqLm#mY?qay>P2A;fTV_sS zcTLArL^`#|T&|&hr3Vokx~t^x5)Zr!_NX+VR7buTLyokQTJ9P~q0|mHP)L^z&-jL} zYJ+r*&TM(D)bV(9*@v~Ljk`_QuWJ2{N?HuG?>G(ziB_S6_+OrmLWV#`oH*C$@2U$P zRTSTI><_0dVrfabzL~<#-tG6Gw4)<8g(m046}*h_9(8uODtlg%UU6IGwFZ^Wt)x{x zh0BtsDEfefFC@(rXQ0OyUB=r5@kqr(5>f90^B=U#BWS(KwgDK?%tZHIUH(&BO6Zm4 z2S2o~B5#^FI~-h4xZo!g-%)Z!)Cn6sJ;z?oui3Fel7JH!AIoW!sy=Bwd)l`rue$@oK8> zRX(fQzqMZ&&6zAGn<8)J$xEe|)-P<)M+D98|TK!TtG>FJT@y+ukb4np9WPkGa3 zNA(&U+xR%BI7j9Bs?x#!c2rP%?ihgyTLdhaKi{}?URk^}HIKpA_f5xDzbf(RQ#3`Y zdT;+76c)w{Y;C&?8#atx>REnx$BrHJ2X4Oa1n7+VfW`1b^Tefwa2c-AUHZ%WMTP z_9BkeQ=}9Y&Z5W8IFp?`X56@K85gNA%>p@X#=Vn57(-5xJ!+H!+=@AbU)c(0XW!pT z936Y&q_eoJzf$Kh#>QT6o+KvtwenzR2kaWI$bun*d^#9y3+pk%X2F68=a1ss`O*4< z^H_%(+eP}NtYWMmlg_IsATz_>Ofws2$Z z2AlC!@w)@+2RAc58Xfa^hdAqRn2C$)V)jbjgC{Zqg({YeKvfQxKeiuzI*QHPqvHr* zErLPMr0yWiG;HK9HoSZq7C{bB77I@*>d+>5;ywQUVdvIQp6sJgMSX4_AWwd}l~n`9 zTo_Z$%z=5+ys9o^Xuzb3uBD03*ALaz-NAD|{v;YIsOOVuUf;HZ17KZzBDUACVKZN> zp`_xYX_KMKi5nFk%U6;mj74vg=7jUr^y2;Lyq`%? z9nVcaJ2wS+Or<*@WvW>0EiPbbE$q4X=DZY)@8~8v{$!HL%TgZ9t)`UGEfizWx|x5~ z7&_FFUyMS-W-z)$BNjmeHFoDHkjg0~w7GYt{z* z9Ju<^oVeo5jEv?8;EbIn67I7r^5Nc2IgQQ7gJ3%>sHXol+$KvN);vDK&M|Q}5pS)7 ziaUEN%<}Kty#fZmeLrWYKDf%xpUm4C*K#MBlZ{du<)z^wN}v zuAK3U-(BZ*Z$8%Nb?@FqORE_P!dPfEA1Yg69yw*Y^?kv~ zdWKdbyv1uEOV%GXswSWV;NeE_v<4V7T;ZVATJ*5rrribLoK0czfP_VhfJcqbhaey{ z9I9)QDPqs+o$V@T3%utXTiZ<{b^|(*=4tBUFz273L;@6PFJ8-n5~dE8pIA)0y9f31 zQ+gi%QAi^vW{E!8DAd}LRZE++$Ky(JY-87(MGv|{(9+N*N51$-(L_MY|1&l)iDX}n zIuGu#?TQY0Su0Iy=;H+2&o-^$%uB1AkyuIU3PvQz8&IP{p;z^O0ENrg7lRx#Gy75$>JXK{udizgTpt-pXpra}r98?>t=Da7|Sn z^wNZwT5Eb3Ts1w^=bo~Y^7w#WR|U_)#zEsw*zI%rE@<0SZ^)LFz}(-Kp%$J!cnkM&bEZ z;=UATpkYdhTU$w+B3;kp6iux!uE3lN@skJ;YI;zU2i=b?GhNGGjX;RIdp%wsQ`@#w zcejW#8|%=~ep`BQpQ4RkssqW#qh=C67v_i@&o~Dvb5mInOndAHs>+5lrtKNl1Oz-T zq>h@g==5#n!dd3gW+9!U9XE3z&uU$8geGldIL z7h(?VX{yy_c3?ef;~J3$uG!sCmDmTa{iZ;MD;&s ztd(6r&TVm@Ua=Oe%jSyfRACkY?*=$e%5iZAXm%)g`*s7FALf}ym#d%ZRyk`*rO)nq z5)3Q?Gz^YpYu7}Cb-1R7J<_*~bC*JDV$*Y7LSTu`6Ye(6+2MG!^0y=u7iwL*jR-oJJBnUaE)b`#of2xe&-nfjKy9p0DI zeWTyc1Iiu`8Sc_F<@+F|)UV<`*~Wan%a51c)l5jatm)E# zO=3o4NwjCv8C}M?yty}0(Ud*YoGql8NyA33u_b0Zx2bn88qBlzb_y%o7yiz;#hUib z6j`_fc$e&aRM%E*TuI{Prj=uTvRB^Ch>jMyaLF%`@|m8@jnmRtY<2e3xY=cv^@NYP z=ep+O3&fy88EMq?zqr!b_U%z5c^Y%?qofJ_mXn>m8LFAAb^m@6C55W!9>RtdJ9$EL zYul@AXZ^O3*it71Ob13)kGHw@lHL0V`BDQ6@R)hN2#OL1~Etmf{MLwL)|6WwL@g zicb|4qE)g8ppj<|Dzk2n0^%KwRj^&Cb2RBPkZsz2iWMr?s2wH7qI=(rina=(6>sL1FKDEy*4F<3jz04acx0%f zB3-dfReMYrdZjru<_I^cI4T?)UIL1oI8zS!DPEnd@5K&x%seu56_%Fh z`NHf7{i3>33-OZPya#} zWm`I>`Vs zQcS<#+*J;LAZDpyX&k?SRO$@LI0<^Kbf@cChJcb%V)IPIT1OtkT5BgF5TqW!X;|Zt zYY5LXORMnaGrKm5zt>(Y4!;;6bwtr|?qZ+ zwG~I{1VmA@r|=!!1I_dohj@f9pT^#R$e*F%Hmj^ek*fhEu0T>b^`mpi==FtJULOMGM!G(-s-hza z3Bs!*EeLJAilco?)0&W$IYX@KZGSLGk3v>(<1*(}C0G5&qD`)g_3T24_&7i^`Dq+TZ@( zXR@NFuU~KF&^M1VtBoz~{UJ}n&${?rT5R!N(e+Ov8I!n95L}jt+&-p|%9F-`I)Dz^ z4%uTq{^D7F(fa+MJ&2xKANVBw#j(mx;CGrX8*4g;7+|k%)NH)u?W;S!JN+1Q z`N_3XYG%&u>3TINu$Z)hF0rryi<=4H=xB`0!A10&uFjAKCSS| z_V@J-9^idw-8{)83g4w?*0nEEQT?_b-|7i3KH~wIeg5>N&w06x%*qUG*M#g5H4SI4 z_QIF#kNybsMakW|^4rJV97H9Og2qk z?{>=W{=Q?cr+(JUS-E(H_42<2kmp#isG+t6TFN0`+vTh;#=W)J4zn-5R9*&}?JJxF zU=gZe11;KnhQBTbiHLL?0RjuUpM7Dt;yW!nLQH}ZDHJ?7konLrzK{r<u17Hzug+6^-G{_p3yzquFL2eX%9v+_)F}Xgs`XdxJutOSQQQX<8JDT zmuEw%{m7m}i^0EgbSw2BOIvqguqI;$^@^?0&8KP8h~TSZ=#AuF;Um=x0^_B0Mu_`!z<{h$juC!x_g1eM+WFmg+c*%w-H^ zgvdM?RrQR+jUJi6>WN|PMmsTaAnj#U{g;Dkwi;VDu?s5CyrCHr>umrrO8RnN`}zCi zqpDx63kG61?kJWrQX&AeQZ`E6$)PWG1_fWB^PNLK@t?v9TJOj8c{%;tyhvVW-CPdQ ztb*}J;(Jap+u7W9LRFp^POg+JQrt2`f6~vj+&=7%sYgkE5AET79Tzqbdj!YXBq^Ji zt|x<}+$zgO$|2o^ln4L;Nc9zg1$DMIYG237%8Hht)--C@RY-48pzjjohMMovcaNVl z=jf~(UDec5j=q?C5_IV*hZhkcjq+b)BXU%~czQj-)YR!gPvcvU3d^XRz~MT`s2wD% zpgo@axTv>9{6I>i-FV}B&Us<+xfFDa3p@UFSky5W^@=FdL1IMpL);SxfDYPm&0@<8 z4*34bEUAF0Xk5)ytU7#A<3L$HC63}Kwg}?>4(Ae9w)EtZF>QXVd|5DX@S34<-u-hD zw9<1)2}G1Eyf-JG;p){xM81Xkx(ieXCwVO&T`z;Za42f@x59X^cu-M}wsnR1_ihp| zYRJSpJZtS&5$A8+%UTTvn*AedP~L&({Y?teC$%jO>E7so&ZOBtZd=XLvz}s+vG1Vc zKl9e7e{^sX*Z}Ri ztUs{#xf`B>_KB!o5;YRs+=cO8bHq-MRY8aJDPtU z`yeKsL`&EU1%Q@kb~4-8T({AI3%hrmC1!^?)F?K6#4b7h;G)-qBYLHCITT$d4X~WN z^GV$`^TLB4E<13RO)}FI@5sO?y zM_*G5E%2EaG( zOG^b6k8_|bl%h*{sN@3O`tBI^_-wo#uWRA8Y)9;QyWC#uj>z1*0t*P@7B|n z{Gx^e3Dv;J$CW;-7SdXmLKRRkE1FYZ;hfp4iRF<8lty7&{DZaCOxG{Q6Fpq(Hmq3e z#PJK1%GM_RWjpuhYnM4@JjSrZ10fy_EzeQ?efsvTug@JzFD$b{T|4*cHNjTLDf7SH zD2;M;6(Z9E%&p_6{pZ#r0~@8X1*OEJg@+cK99d9Gxmhc{G*<=|nixg&aaI20exn$W z0@T9uYH^=>WhLdvrRIEAfAO*jtfd+loGUO5`@p;*$;0x`=!282)av+BZ1ytg4%HNO~#u#mK0eR@%R`x*qfA-^)S-Jq80 zPys97tX;c>0e4IlXHb9AxHPHw7ZpM;tN4ccizC1Ml))b!kGGs>L&$uKR92cK!CDk%Cuv1q{<7ph4$P=adP*qGGoPOL&Lr(vEEMQF&6+Lsy-y8)ZajZR~TXpWK=2nukGAv!dv8@e%tZcAx zXhDw&BY&TC!_+q}$P}SfdArrpOCz0Ie^F6!C4)8!>oH+X1p^vbKk2t_U9WbE?J@yJ zLCS!Ne3dB+5tOnYJSk9Yo~3xY8dj}zaS<6r4Fy7}$-2KDK?H{my3O?;Jbka*-1mCv z1x}t#1u@9DC^gslrN`1rL62(bA3L;4As!m$bxfRQ5#e)n;(`WUe5hQK zyMd)rV0);v^a(LQv!Q-z%A}41a^XxVaf8b{`RHzoCMghG%U#n*T0+OorPlfWZ>_@t zu+`tcB`bO<2^$eOUyTeBSQZWdTJ`?EbV;pTTW$_soWx44X z`ZMB!%wtAG-hcpOO6DhUW_GKF?6DclbZ!Uh}DjCQ0@hWJ5Znz4uRK#ITH$cKyz;$=2HpSt^>?V_L`eM#VZlXpHb)^aP*+VfI$kcrrl}w7f32 z7!%Mrn%Z1i#)RBN_gQzuh~%0<*)lUqhVkIbE02eEOrviyay1J}ofyNfe^+%zU$^|z z%54jFbzYJzW0{S9RM^}Zq%_4Rp;6&Qmj*nFDD6fzp!viH8IJC1+Uypy4GsOKc@x4CT?^Qnb18~9y&9Q z2L8snMbvVng9@L-*>uSo5)EVsFO|RXjZ=|Y^Hrpk5xc>^AbAJ9b<-Di$Ly{-0C`Bo zjA)wN6sqtEfFc8Ld~hhH>yQJWW_nN`Xa2?Pq&+YYlIGGHv#@URTWMvJA?9paKb;M! zX=x%UGRr+puJ-QM309iaHzGim3BKenX1YV?1nNZc4QFsq;CTu@53MYnV#W?HJBGik zI63UCI65{H@^i#duA`>McZ7#|eeR~GH&t2pX@f#lA%FlXBVj-lsxoU9N8CP&dZZfO z1rBt$f_)*uS?8Yvr}cVJSq9`0*@mVaQ+{FbwhPhp;|{~E?A*vjYUoSn^-;Tvy=I=D zthtv;JNxrLs#N6~f&s65f=JS+go$`UEZ%=WzjC8KN1BexPXkcVkg>XoZ3u%zf$ngC zruc`9Klzs&1-&{|z!>^13?Pa1D`c4pRp}%Pw_D$krlTm9cL?HV+(2KTvO`T2&snf_ zRIZ`bkvY#{=T@&?U5`OQB}oM69;aHUsPwhotJv)C@4w=9M$y4)Hq;?8pqK{rW1sbA zAcaw=GCNWGbad%Ou^EuW-FzZ^+cV6w6dY&`#KKXAmDbP)GcbBFC|B!cpiCvtzOa1R zvStw1H9e^NFCj(T6x*w>(+$Hu^v%Dh#7Ll~2h@X^C5XTP>At9;Z+>#RX>p<7>$dIM z1xC9)Sh-ks*^1dc3cIt0=W>tP=V;use_W>;7&EoGm$$O(=Fd+5{O#ai@*IQ0NJZPt zoS6E0@aTdaI~_vqMGV(h$uBEG&0yA+xrBmxC{zW9Wvund7?Y9pIzwP>W+tQGUACV~ zfFmTB(3oLvV}GCQ>wao>@+^npZihctbaj6N@n#K5U&N zu9RNf659lm_iIO;-{lz0^Kv`yT0*h4xSVI`YRdi(yk0 zVhKN%#$YTktWS`AHwUt*>9HESrLk0VvA2{6hP@y18g-)5n>^YbAMC%mJBGAj- zT+%_i5xLTJ0M*xzd5q>KrZz&Y1hFledkO`06T4~m&=l^VG4&W2q3EIsSL7gBsBPsq zM=XZuZ(8E%b$%Uubj|we=*)OGUaw3uSvpf$qEGSD9xL;j_Ux%ae?mE4-H80N9Q}L; ziTaQbFU=Mr%OMY!K5Aq+S|ZLxLK3KUE7uN7m8(Q5Nw1`Ml;%aee}GS<51Gv31u2S9 zXCiCd^vwzL3z07}b7LDyl`E9c9O978BPquiQlle66^D*^IY_QXO}jBUc?6Okar2VF zP8bi0ULSjpIZG{-Ms08273orKacmTsAH|d;nlL+31Gj;0W9=!foZM3&&L1d=T1g)% z8M#dPqI#$v=ZaWyGF_rEPF4pIfRYC{6b%pO=`&C!^q@5#tt1Utiln1qM;&Dl00dfa&gwD-6SRkt^%yB1lbc&DF1!?#z0z-=Ug@T?bc3-@Dx#9% z2$B&|uukIf47~dpbmKXHqZ9IqF2jz0s*3)|T4l<$8bT%3NbNzThsl4Wv4iJkTsj#h zO_~v>o}Rg#M9tVvD>liF<+OkBl1`$h0<&P1(}pa-Eoz}u3KU6;gQmRKI8M7LAj4#O zC54X#*BA+nsL073>DBC*a>RqZxPR@@?tKhq4@1j2>75y{JYa8q>!Mh^gbHhbjzpS; zDTx}5)BPAdsNIUvfGU~c*-+k;FnTw?bk9K^q3{4Llq`3#&6^-`H5af-Ax%Pl^jG8 z-}9(mXBkIJL!~n9I;j+HN{M*ajcKQ~kwfuUcY*my4>53x-amuQNnC75EI?0s?d6Xa zSq0Ak-0J%1&hQQj@!AEMFOq$Q2b)`YvCZFVEVth(6sm&NlbnwsH-0oUV<)X!o9XYD zo*w+Rdc1$$^gtd}*(c=4^4ew6ShX_!RCVV1djQ>ZTP51y1q6g~$>_^(kT_vv!2Yk@ zxN#$NO$*a3k$J8wyV+evbE9+cD^x*kH8pxxBLzmfGi+LZ9^pu6-mmAT!+Mh9S)tEn zkns%A|IbKa&(JIbpRaTwR36`$F>lc$Ymy0pP;5aZwkPQRZ)@qvkR1y=e>-Q@487ps zlkR_hn~@44}v}r=O zS_&G=anVR#+FLM{|@2h6J(didp-A$l9)<2LoMb=GSXq8hX4+B0ge!d1Eme1V85vDc+?5dP?$Fh;u1|w|IgsT2`PQhe8|UigIv!u zEyF55&+Qo)YmF&C5^(5HL)BIP)K|e|ciS5$Jn&G8ck|BJ>wZ^mjZbA?n}7c;?qZMS z^{}&_b04qXxqX{g0Enhy&T4CZ$hA9luyB;$lRplvj&>kQT^D`pt~oP&@e#Ut`d_#< z>BG^R@oR?sd^ypkb#YLt?SwJrD?WFY;8NJ~=d8bn%$V!1ZG%(hWmM5fl^)u7#Dz`I z3?FnE+OyWR-KQ7S?KI%2mVhvoOU6yQtNC8--!3eR<5rwT)f26j(1&pTk_}NJ%Ctjxu+)en>%Cy&M%6QY2vXYO{L_nQLp)dgKW^6;4+*~7O~t5+KXAGMG- zZ_jtR(Zle^4c={eQ2@0zAcW)l4*n0&NXS@DR_uWb^8#<;S-cPY1Q!#i_T7LT1{c(_ zpH;@S-nPS}wnF`S#Iy3KZ7IEdn(^RZJ6oORB(x%Kr;A0nF5X+`?uK8qz2H0y02!Ws z_#jPEkZpC5dNw#06E~v0@>mg(OA9XluH>9J{?-xZ^I>ZB3za{Hx5-}oZIF=E{Dr{k z0uVzv@~%dapi4KWkOc}+V6sJoMqkkW33W26*U3~-8z0uruGM9n>E)7wWrn*Dik%+jx^#DS=(F4r>8Mv|-1N@tp&9 zMZVc<*d@mh6($npTQmQC+`+4^F;pXqg@lr;E^atNyvt>feNhy*$7QZAg9@h!gG7QY zb419ow@?ZnBOPt`#ro{r+6wXKs2$fcw3aSUy-hw(z2oX~dFo?eO0spG&1cnZ+~Eo+ zVHOU@E%q6^ujvve*Dby^+{Z4^*^*yfRI=P}{z{Dg^n6Wl_F4^%14CL>HtndRmA^{| zRozvBgUoCP|LD=X0ACkKvU{{#OOw>cVTh;2Kp4H@Q=XJ>I~_vxJw#EUbhUtLWq!|! zR~D2SB=B|7`fTw`{Zo6{Z|2Z_f{-S^@A&Y-a$$|S_GZGtbyyv-IEBlQB1j>l0eHzu z7{C2-fB$6F3iG@NX`dHl*Y6xVxrOQqpFSq8{^8}vl}l7rE%Ya^R}EHf7B%XPiN1c| zxW4sOlQxa(yQfe$^iEaXB?ep6=lM?PZ}ffr{fz8K-wKz14bmOZdBNA=hA2QyN&9-~ zb+<}nTxoW=iR07`U5#ktk|z8LiV~w}rNi*6+W6FsYY>Y3@+39QO$KnnO?rL#;$b&^ zZj-E5IVnN?dev?{@UX37oDOCK#UDSagdBkeOFRBi{){YobLZULC_qsSroi-X=MDTh z`Ftr=!xX4#KP_7W3MUQj=52P^4rHL678kJ`qY&z9nuTu54mIjR@s04qiWG3buRS%d zZ}st+{3Q9$)Uw@jZr)9q{4n-}3AmpKl#Wk78~jLhD+mr^$u36V-GO7r zZ=sm5K!Y@}=Put+w*GMzvadRXXN(ilhzBf>T&RXA%lyiZ>r;d^^9=>>oUx*hO=btc zM0!u(8=7kVV&DSy3;7bCyQFRznyW6W3i zO)t)jOQ0jbB4)k`K4GaRPw|BrbNZqE8h;V6$wBAq8N1fA@7^`b`r&4?9Sz@cfJEa1 z{lj3K!;u#|aFlV!^%9~%9>ul=?5*Ns(QE)of1V3(B@DvMsyy)f%e|B4^=tF;_3PHp zzQVa$*V!}@*Pm@!KU5tu_G<~a+OlL4vK@6Je#=w@mWvKmFQ%u~tzZ$M|Dq^Xu@mw~ zYAS;W_$h<)e~1xzbVnDb?-|Yd+RNV&Dk2a>8sy&qCjda|9X4*yD7doYvyv`?47Mnp znapqiYltCn5?znI-O^N3Uq3xJR};D!R>X)gra#3WNKl{b&yi3RCB#Tfe%lU+=&pK8 z)zf~v)3b^DkKSwQ^6;X5$HJZw8yyB%X(Wa|dzthu(B*Ak&o;JSQ!U!e_pRx%`59HH z=8V`#lgi&H&&%7)44yn)`4XEf=V%-Gv~_npJTPmmUAB9qg)GyHuF z!)Sj_`jSvFo80!o#fu9;xmX@&><^4Ey11gm7kpm4T%v#gk%Vs%ZY2*_AL5J|b@sK6 zF<`$=_{#vTOoiok;cPU@2AUFA8`g#~r;<4kr3+O%)ijwr3iEoNY}jK1(Ri{R~6mG%8}UlA3iZKzq> zoF|5I%7`bHL(Q8@gG~^^YbJ^WJ^C$9ezbYC#FYnlOasIc@;*|=aB`6|4!z?QOc-o! zFz|4fo7#M}vIg zf-d9m@WkIZe4t`9{qe=>>qqI)dz0BWITtGxd#?({qly-dg=7JaAwAh0V}M*`XU|-Z zp&?5rBA0XysNuGDOs=SKX#gdJO_BG;E|8hz+@n2&k#9=hAAIP9Yp=qNk{28L{-9 zF7><;eS*_KNx8>SiHDH1+C2x+8y<}6KNz&<_?)X7Ta6gJIsu^22)(r*wP^4R*A}w6 ze9Fb<5DIS!O2#-`NOHV4aH*fF?AufkGf9|8!=b7}9`&#s> zLKZJVEzRj?jy_tG@|APC)!CKHl%F@rgvMTE2Q|a#xSxgRC^}Vw+P*CNTqsL_jVax7 z)^kY8Zlhgh&R4qxb#iSJ)^?645aO)6S%NQLsz{$`@)@oO#}(#G+X~ET}ryn)77+l0ZPZM}F_*TKPT} zWtS-%3JTV7@DLgMwcO=-fJ9-ZL5M@0K3AICtVo@Mx1XOoJ&XT_304hOOgBK?9v zj;<*82F^=wYNRx{uz6OaZCEb6Elf%zD~fY@H+9N7Vu#N1S35z&?a9@WF1GEbGk8GZ z`DLShUlteJeb%xmdwn;WU?fvr=l?M3?`_(Dt$E6|hGyHcH|@Kd`(%|?s_FLWlkK-% ztm<>WFuaz3e%A}9a%w0pbrGB7m>Zngbo80gCwA+(&AW|NKYo<3X`ba@aI1=^G+05C zM$5jlW@eu#_J-q&W#p0sZ73cy`#0{HIkOnHk{?k^Ke|o>!d5u2;XAgVD>gFfNi(v) zAbEI8YuWkY=g&7qL`0C5gmB0&Oxaw2#>Ev;WCXr*>UNglhj~vH<@$Ixf^}pmluespB+pzu^b&kN^E=?6+^JT*Hd; zH>NReUafi`XPb}nZLFfVDC398nzEOB5dNaR2}O)>pyi3M2?xxO>ng39fIGLk5e;J7 z%4P~19eugdm`i3j#V_@w$|r-!xWXm|&0tmSIM+zumur@WCL^!kWv`ePIkdH*bE!VRyDu$WeVIQ{z(KnNu{~n@0>7 z7v?;%+k&d}eX6Tl|Fij@5}PYKB3JFKVW_*-BG6}e3x{ZAXG_87R&8( z+!zwX%@JtX=1!fiHR0ieiA$7rMDV2Yoru_$GW|=`d>kAG)8{F-{rr zA^p_V-zFcN5<2b}nLlCxhkQn?;jt4;jD5o)`IZ*snKS2p(@RaXB^tWIlcTR8E~q@x z=_sq`if*q(C%Xn~buCHO+W+9ju|BgdMUrcmKp$wrjqCmVG@9cslZYtfCgsSjw5n^n zyceD9dG32b*#+vk)Ek#hL!u%QIs(WOLV}UOSQ{xfh?nKiVw!|Ut%a?ZZLO&>J=8Ve zXP{}Ef8LJJ4yEr0hAix9ao4wKp;Nc{%g*|BzN@8p7_e#6=$!$pd_zf397vM1J|vQ| zhd;7jq`_{I+*>Tyl5o_J0mmy|-D*pAXxzW=KajP%&n2NhUPyf+KgW3KC4;O?XbPu7 zYB9Kb%`qtGCpu)GPdo1YM7OWiOiqT@eIGyARCU*Xz&^v1mEKY49y6Xiu@(swou@}3 z4;_fKQr>wfYs}qyNIeRMlns>&pCBG3$nkrm#aI^KJ~+*nnDwY$!VyM9XP>a#5^PJ- zL=f7o)T&n`uUiuyWY1SIFpN3yC54WCiJatylNqwv>q8Y;Rot-CEs zKJ%hbdCvqTPT@UQ3f{OfYsGc)xkP8)fZ9_7fBl1F{POOOvsG>plI9KEBl9zpgDF zbO|PDvm3>eVfKlS^MiH1HDO3JRIJqNggxcbvw9G&5aPzv(AkJzWSqL}6W?4?^D$L{ zM)q98yd_Kewr;&|`fk&QmKxnJ)k^V_<#CFQsQc`5hgynR56CeNc3R>@JvJ5mO`iQP z8YsiRqk~d4%D=EYVr(f?=qz7Y@T8@wO{hnj4F1)}$7i^Er^uwWrzw^y948t6NL@M8 zD4^E?WJTfRRF1f50`KkN*mKfZ>!<#xpx^}Xk6b$^C7Q1`FXB_?UjUOCeclP!ELW5Y zAUpOgdw6#mK7&_kO0piuzcOw|Nbp;Cz!ulvKQ9nF9Rr8nZ4MQ6pR_3@mAUj?jiCQy zB7^XR&H$L;CwD9jh$NbhcZ~}@62~br);#6T(Dy@EKOwhw;r{G+MFs;rEra3d$wftG({~ryk#S+ZTJCY5xRWVHl?o=z%qA zha(O{N2>Qy9@lxsNsB9^pG|zZdRGmF4ML~WAQGu>9?G9L_vM_ez#KOH(IZQ8_dNEY zx)Ei$Pe@1zxkig;i+a=1K7aNWVAJtRe>_2N67oQNhGcf!)?n@Li@{es-rVi<*1h=H zZ@p%p8|r=T{C@Fio`fU5v9e9LYo~xo`$?8$d*@SU#(M25q`2r>eD}y~`o`|@PA7ck z>}k|t$mHx3iC1PFvC-(!V;pC|eO66M9P+}V609H0AlXTB?)Uh(Vj*+235SwibR4$& zp0YbVB{J~OD0;q%*Dg+M&-QPIuKvVStcq!Ja+8=!tYB!+@Ub&)UG%|;UdqnMNq%7p__iq7ee>WsMKl7C*zR1Kx|?wR zmZ^pJs9q+dnvr37O>MkwM6a`3?RS(quT}mq#`|>}g=*4_`&eKa%Ist6_g;FZQJ)iQ zwl8&1xB5bI!6F$*UkaTW1}OZ@Q6p>Pk`=e4XjW|{<;g&Be%&W^V&=w8<{i)|V@~=0 z7w03oOF4dO`q@$7brvLIyq4agYvNWc2suvc0zB+C)*Kz{cq6QhO0vz9w#9vYMt=SJ z*}Z5ySrh{?`>-1)KujFBO88*4V9Z6L(Rqz?*Kk68OMAQ4jc4pW;azo#Ued}! zayG_)y&poBfgqpYgBIALn+m))&~d8MlZzI##hCNexd_i=Y)N#&$)I}T`EZeLI1N2o znSbH7y_e2YL5@HTx4gSsP*Cvx(32-m1|pvrNS_q)j;VxRLD5cG#u$)2$}h-e=i3KO zAqw!uCh%Itdp-xIr5q|<2Dacn*{Zi?>-l)iN<8Z}>F`M^O9|}U)oBcWj+4;Y!z)m% zu8l2=8N;D4CJd!gyMA*#q+y!3c^tUe`0%g`9DUKOfa4eq@z#^`?!IuLG_{~j0~_yw z80KO)Zcx4IL>*{$NLjt6thT0mBGXDQ|129X?|?I4Do~I)?ltiEGuePST=hU~mdFqb zS4`lf(B>HqbjmlEXUOYv!I)7n7n0T-2!{Fhw_crd4@;kN4C#-PA}mCeA~+s-!9ub* zvb(99W6l|NqkI?VY^jkYE2UPf#IVPoH$uVm7UkET=;&zX`~p-TuUGKq@r4*H-A_Xs^zz-6#HX7qU=5D(t?%c~XSQ@8 zg_7j|08>5r?R>942Yj%U10dl7dio7p<7`LTJhQV0FIy5DG^hq#bq>>pVB4bDX>So| zWs%M;xE52Z*Qz;xl3l-Q_e;A`bh*NOYm-?PH?u8I?e{O14$Yi~qfCEg{Q=UXROy@I z1k}-Rnrh2OmwmO@|NIVSeKHEeT^*P_nGVdWDnT90F>+&MH^98pN+SHHn^Yg>H;I&WQxANN1^*4 zH(AD0cu^sFQvu;W2NqSF#+KTm< zcws`e_zhSv5$n`ZWJwq{P3aJHv!~4fXK$4ev*;rAzJG4}>I7Np>;Z_=AYO#syA`t1 zo(2_9c{ar=qG@!mwh0B&9S;BrLh3-iA>p2d$CFv@hN5Qv4dp&Fr^{;**P9W4eoleW zl$1;+zj6S0_Vn?c_2d#j=R`NOF*6u=8#>K)4Ly_9r-V|dx(1G&MTb0i@=4iL`j-OcbPyO~UC-&NBmzn%$ z`b;?rZ;g39G$1s}e}u)8_nt3?^i*6qNR=4;`tFfTPI)r!7}qL;5L3wnfS%&$%!`RT zO+Q;MDTPExD5CWn4-HlFxByr1>-(nY4@K^sPOLNwch2&-YC-mvSbSj53Ttf9Nj)A& z4*>U>uhVgi^i|8j&k_wG`zp}sI3VD6RkyR=cIve`cJBd1n4?@Jqz#-{`ShY}@6``d zE6$Jbh_qb3&Z})`)1Co$oX0FPzcqGMH;q5tzrE<(Q=z|ZK%0h|vo4*Nvz=Id1Rns4k--Jlo2mMw3+2v#o>FqDtI9nkW9WD}LK1Vf;vi zV$luqZ-P>+OFPBf&Iea5bSY)thJ)hF9uBFufMNnfk};MeLdw_as;(wtH4xDVdiqbW z>fs|%Fnp3(YcxBnK>KOqfir4cE&6lBtNTAY~OVCdwlKhO+WUsY^~L~ zpzPy_aLZjic6qg5qIVaZNBc)Y1_vtXNEiOnxwT!`K0EefRLAsCn_7J?iqaKMU$Az5r@Yf z-(R}?U22QoLs#81Xj>hntFME)zk~9)t8b4lxEY;fQrE0Q%dLg$b~(;>o3v(%UB7|_ z8OvtZ-uzF2+VFpWTK>c1pG`G2u5ZunbJHnkl<&97<%<@anG&sLaXY?f_3v?+cT~JS z#{RD#S2R~v3iVKb-sR({j063HEEmaldTx2xqYFQDJq?^^QPBTfkWtOrX8O%-mMnh~{0m0s;n_5SnQ_MQ2Ee#f}-9zHdFed-ML>S5&- zJO8ZF4Y!uf+b#V2?^w&f6BOS&H|D86<9vP8H2(9i&D~y8L$&_0GcgZZd-gn-+GqIX z69f0{D(Kdk9~d)!`Z9CoR*yW^S=p#MgOcArUU^&%qX{o&e$345?Qgj#`?HhM!2Uaw zdH18f&R=HUMnLPopKTrMT6I@kxk&Yqmo(X~QGQH@vToI<`1{sBtW?&vK4(2w;bEWn z%TcBFca3T*V6CmoHI7_6&`QueHszodcn^{K?ausbyuBcokx(vM-oI^ zbIh0v*B1Xj|=H~?HQhB|M25kD)j>GCe^n;=YY?$j*Eg)tR8&} zvU{gI$F{!9%v-Hq%;|NuuKLJ3UmI6mn%p(!VN})N*aLQ!3dQF)VqQVtkU%SF4R5k_ zX^b)>Sp^&(Xce(w3Yzz+oDD9_5s+{v$TF$XqM%lbu4F%t>{s9B!thO3KixkfQLWeD z{a-37yvtmZezX{*D*eZ;5@KSefM%G-FS3MF8%yAiAAVuXabn-&*uP7!dC-F!+xpBr z-?U^shL1)~Uu>5CGO9O{?RhaBd_MKtr0g@sEil5bn@Z7X-M)td=f2pzIB%+76CB2#DAXZ)gDG}XF0w3yFAAHDUNhUT%NvgpA zM?H$CND(?_P)5Vg8W&SdZJK9HFylVG`^^Fp`fl>ip0clB8?Ooi2#&;%rw2R&pv(*W~@Dt0QoWXW?qYZ@NE938ij8EM`% zr%Yl%4w#nB#VX^h8nHREK zX&C#W%{s>6eVpuOP-cz#EylmEVKn{Sx-kFzM|r&lJ$o~?t+zpYyMGiPJK2sqv+={s zU0J)e-q^jbYU0}V*ZR?O*DtsMFexR}O=g=wp4JzFfHeF*ez_-U@p2P5y-}mM$Wls+ zNGL|hr^!k}C>`F=uaI>|YtfnzphLGWw+9PvgcGViS&$o+F-i>^He9vy>KtLnU9ho| z)B>{WTtdR&gKDd3-;l!HEPO%~n?*?Ic0-**$!jN6ZSK9Q<)DzvcR7u!Dk}qv7es+y zIFhGk92t=;{5YriagJ5+y(zIr+Lz>6>yeo30oq33?}ycLoI{TEPc=yRNK}|`0YG>r zw52cBe{$tCKuwSJdrpdF*RKukYl-y|>i+XjIw&39aP#lljcV4RU9kI+$Gh*}eLkQn zZ`r$zZCV?8)>mp|dgD}s*zdP~+gGS`$XwrB=AOg2#i117>05w9`9pS-gCCgn4jx$~ z2v|>Q;N~Fe0eRto^4Huj%I&LG6@?P~(QkIJ(0b)p>#C#^Eoh#BV%Sfu)8TaKgXLI2 zoCA;P483~c{P{>I6c(jZ?49AgjtB!rb!dD$PYHZ!(}j0@M!*-=~uBNu|?J@>X61i)s=ri~kNBT~9VN$UC90X^FsB&fF=ym(Rkp+!`evGmo%#HY+!fN`a-a>={F@AEHf zsH*Bq5<(jKk_IMh@GT-IkA&WCln@;^JEwY!C~W8q2&GwWOZ(Y0C}orQfq7dd;f~R) z)7m%pI>UaA=T=Nf>0ds79?ajLpi9B^)6)%)E{ho*kA&N})_ZSj8g1KI%|0_{&b4_lmaMIv-n#S7uWo6Go&{(VF05av0FAiljiYGa3#CDw(ynAxoBw-p zOi7)R{f86F7v9oXbRdQN{{#VBo(~GX3ZUl_O1ka+BHno2`BnoV{sS~gf&=6XrA)At zpbE&^(nj)TYS^hT+T4p8KF~tLl9`_~{+*TDkP|iFah9dE($ssAquZOD9leUVb>0m$ zj||<{Xh1EGRfeB_ow%4=v@&MorAz1g=Koso=kvx6Icsi2CUdYtYfLAAPLZ7dNi=ws z+%c(i7g`{DEZFlewcZD$M`%ZhN$eVPhQPJ#GjoW+g+bU$Bm0_6S)e-(qVwuEnojg^TC{TANZO1j--Nl_{|D!p88-EFTx5oHVDaUWOnD z6DnMT;CPUZGiaC-KA7FJMfU^XNESgHv~;)}o4iJ;*u!}eCG`I|uUxq+_AXc6gEu0S zSsH8!F(}Z9{;eVxrWA15A5WD&%!2+Ih7e z)yOWmbGK;^{w!9Ko;-dkjx8_CQdSCf6S0^+y5Z%7QS$7~wYo!7m{J=1bunx#zXxlG z2XbdHKZ5pV1(tpgY6D7}^EV8#w}f~hw@3?Q6pu5?{m<|DRk&#;2=Oe@69vF@DAxzb zL4=S8%by5uuQhy$4OxN(HP#cR9YKiuz%WBbEnc?kY_pZuJiDeIV#iG;2g89!ORkrV zz$p2>Br8WHw-Z+5ZZ@Ypvu_-Zn}vB%@Au^W|Kbq`4qYBDGzX~UWDbzC8Eyy~$SgnDvb#iJ)+HPl@^kzJznr5 z!~X5|j;_kzQLs?by`qbWQxiXQAvr^q&z~|cNJMf zlr8(TM@B9DxyzeZ($@6@d`#$$M*SW6i zB!jifP{C$yeRKy}Q0ndBWa&oJ)wwq)71g;q>H-*=MF9_s#ZeV$g|q0W%i`Eo3VrYuuukUBd=E&Pt{6pN#tyx2fG4e|P|&u({b*2<05pos!g9v~_1gys> zc&kubIVMbA67g#>&_LlS+@f;8!W8@Woji#2 zJj`Sd0;`?>DDrE4^paEwgNNBbN`(*l9Rj?axm;!RXrY(4H36&zGjJi2 zi#_m4arw-bre(-pYGJfM#Ti1voMEmEu(ChAOd+-Ey!sZvkk2-GFGIHiB`H*e%79?v zAa_(exME~vqz4H-B5E%r4NP1heFyr^!S1sctlmi*z6uO101|6oHz;{_Q$9hMQAGl`hxoeV4Fy2 zC$)KqPlSbs2Al(dkf_4y@jH2g1JKER8yTTh*EZtIi=H`HI#pX|A=%iC?Eero?y!;U zzJIl?Uf_&}p$gD$D}p{n&Brm@Yv{k zh*<~yM=yR@T4|S(q>GaswBaN$g=*p@r$o7p45OaxS1`^L`2duS~ zFebt>)4ulE&=@>W6bh89;?~~Beq(?(7nxQMMN+a6(ZN7oE%SJK1Gfcvj-6zHz zy&IddT{19`AwkTzICxBymjPr#lR7tO#H%iD?!~_6vVMO5en^~<_SraWM0~w3=#ARn z2f3aj5iqJe!!u5w*ye6<#6g8AcKpRrjhlCw$$w*(ZB;(S3HGPrR4mPNa!Jb$fIt2W)SVT_a;oC;LNbM~uh{MW82NA{ z1T!%6)S!l(!DFYp_@DMmG*$INLy|x$&if@)^#GZx;nW2s zRJ1w$9rj2NPA?Nqq~bZCd$K?!qAAR6*Sm4(o(E+(j{{fWe#8!Y8TAobAt&NbBN+d~ z)*6!Bhy}Ys?D%EYn&64KI2(%fIY1CbDk zI64mhTIdc7yCjs4M5mx6g4`ZJ5vCZtc!)wfnsXtWl7=5)9fw?dN)%}9l6st(L=j6d zi~`CY1A7bVLm@*m@dCSA9j_#G^Rc~|FU#Y=z{1YxMxyjL5_UuTzN$E`Bmc{`3_%mbJ$@R&$*7`;C~%!wn(KY#B*S7>3~{wkT*7Vg zJ}m&GAN-&)6WbU$Sv01B=UIM~5uAJbrC(|UsvgUP}feW583%yo`UIyGCv7j7+dCe)sW}oPlax0%rCcSOaB{cE?66LL4il~l$EG9;{$cKq1 z@b|U1FR{K-iS0VL6~#4ZT6Ff^;75T7xrNwn3j`S{T#@mjT@I*J@;EFQNr~5RI)Lc1q;L2SHiB)457h)yf^Xnr_l zS{<8t7$yF;G6#0?#C9aX=)<0ue1l|j?wmQ5SObby&pVsfm2BT>)bD$xdw3f4AgITN z^r#2X9rR*x$v*r8Na}vtYfg>|ug~F&nI1#h6`Tfsm|`J@FGE}Y6&RH$B>zTUiMggi zfP2d<6Am2A%*r}2GHYH|rp(^J;+N_lJ_XgXI+_%dSJzi`e_ZX6eMi{2e>R)U7YWvY z%Fr5-{C5p2Rd?5xc#j&^<{rFWRs7{!xY_EC8_xxwj~SN+D^Gm?>;to$){Vmpnl`-m z@p%CcH#z27{q=o=+F{}A!iV7ZLRk-XrEc>fEqC9stj(&C3ZI6DDF!6y@KD9as6vEE zLwiha*k$BP3`=vS3PPY&=7(b#*mU#K7YncU?b$neuklt3ZAe?XGga)9P?}9Q1c1mNyKWWL?Y=4KwFfN6u9y+lyO%-IqBOamW4I7*bTJH zQnQq*Ju1y%Dob1u5TuARK+v(Lr{}LeY5r-u!|nxVM^1Gm4C-T4HxNM_z*ao~Pc`th zoOcLEfaMJ4nyA z=eJwQd)Zu-wiFt*YBIRP#cw(P6Jp2J_iB70K6{>jX-*Iw;`R-`xbZ zstO|&)Y}Sp)wL*)(lBTZpL7Vj8{8eEn_tWqEOZzm?4m|;JonJ2K$V^K>^-SgchBxM z8lP>s94ytc=tlY4lujReUE4OR{xGC(f*Ut(Y#wxL8vO{TeXo z;bstTGfQ-7?c3Vk{s^R}0!U#HzI`;-ESAbZ%)Dk3DWN-O<&)&&ocE}5ZqL6$ba?pU zuC{*Ehl)X$FY29M*k<4C>xF2N6DJW!6}YINCTRd;wdpef>yA=^KJRb?``3V1EXc`C39g~5J@rp@=#rArtd7Ke9hf#j-f zs0oPF?cE-U=z~ZJNuOg6|i{f`8qeiHTTet#CVQKh zGfKj_88`j!t+hCbVhDly$|t3u`U@XcD(@LXVek*5t}0}gAvjM4V+O4t*{C%4!M9EW zU}n=q7yw3>+9ah>9!;VIWPo3w9@*Dm4K|d&fV#kkS6TJ%;dVyMYakVuyf0d8>m3iv?) zTw20zS{`{cNXUbhoL74qR;EWVcj@w%Ou!+6*x8X?Eu`K8kbl0(5qlD_pazf-Lnn>E zD}4>F3KdeqvcsdEVEp!=>IbG~nb{sqoa?}9msd?WO>_P^_%ez>B4lZDIH56-YAfz( zWygOQ{lqtG9xc6Ym7*-p=E>R!8Q2P_tB{)$D2cBTXqF~L3HY2~hb95SC@$~&Fu@CI zQjSWLQUZY>cycC9NXj|ePRRv+h2*ug?j6#xduY}0GN_I&_yB`A1Lju4{9j)^JYC5C z8_sW(kuk_!z#0F2|Bf6*7$K>F7*8ZLEXC_KcR=Oam#Lt=4~bJ0PAq(kKH}ITH?4+yEpAygMVpa-o%w;QIHA7+Kzb>uw(fYzr-|Yo=c6(Td za`g-jmmwl-1&fNnbRd%Rw%=Iv9An6jxDH&P-WE3Ceu+#Qyw+cLhsE>r^6o=?+A5g; z5TZ9wmNw3wJsWJ=V)lNpR9KqTOyK)cdb;3>RwIw=*xH)O6xQDfF8*HF;%#>0&gsKC zj;#22-}rmWs#Rxja#ITd{lmCU096ldW;$Rs1R^~H!h8>2nOe;nTIDAf@{*nn8PU~` z!S{`qVA}eoz9?XI5XBOo?B(gx(9OAlQ5lIr<@D2D4Mub-cIjl%v*6@OFUwV3NSGih zaXDWIf2CNwFNjWJ5UV#J@d52t4wGsKyRw4~Rdo&uA8J72_~A4bPOgRgc?VqL0P*a^ zhChEM*I*Zi-)Wx{m1C2o1P-fOsGv#%7tRMJj3r_R!HSLB`LayPbHd7QPmDN<;^^9x zJS^@pHWgkCk${ZpIgq3Eij?c}%^(B$NM(hxO*|=#2 ze|qJFztXGRyTeuc9_MUiW2+?Ci$Hh08dW03LFvk|@vsaEp+=#rBHhKRK zC<)EMe5HJaxPrKGN+qy=gJ7Q1d|UoEr_q>6D#a4feS`o(JJlzKO@U zeUYeJz*3$T7!k?Qgx-jqOB^Tq5D0MMfLB+RS1chs9aRgS%51_JN%=|$2YAG*fkJ^f zSc8Q{+1JJTUq@&_t9XCOujS@<@pP2D%&}Wahm9mNssX9#EysO9_%~ zo#x&}8oV5@x`vjxkcfD>HGoM%&q;fRT^C#zn~QZy2rj{VLnlG3Lzq;DQC0Q(CrO`& zhQ!C7N};HPyiNh*R41icc<05)_c0Br4q)CFRD<0xcB)73Vwve%%}kqP#2~{-M^v;g zKz2y42?16O{1r+AJK|ychcoIP$&ld)5UUz>9VmPv!VsZTgk&IdNdwR(QL)FQk5>9t$Zr@HmMTj6!iz8j4zlkacQGLY^slTlAwnHlYf{ zI;lt^IQUief$>$3cS>D1q#g%)w%Vfjyx$(( ztvYlCNM2c+sNWP>!Ya5V&vn||qTtY%Z`TVrIT$+{U!K6fsqq^Lhwa{evUm4`-ib#@ zu;})Qx`Yy(H+zL%@aAJ@LG(6s&}&dQ>{}M7L2$^~Sn#|j9|!_(Du5>`w-Gfobo|hl zCb$h7Bc#zspg!6lWibQE3ej${uPNW9?g8BxBG@~>L(+5VHP8`LM?z0qnnFCf*a?(f zW0ChXBu5~kFFrOQdS=06_p5w`x+5=FaDkVAu7x@BYXD#&WTFZ!!kIH?VsIc7-;&IT z8N-Rhk1U3cdkh8+2lCSmWmg^gRkheZHaiXsz5JS9Q3=9;8G5$VAOv$k2=E71v>9C2 z1TaOf;Eruo+}KQf^!vwh1rFnNG?XJP0{&8rZrQMAfxL3t-l4IS`M z)czG9s0p7SB^(bvL7&b4D??fvjWsg^4o%qI z%W+bmL-iB|0beoku~j2$AY6~8wgev4;vrF}2)c^=4&az1$e8dltO^ph6Cb#}V{6&+ ztH;(EFvJh%M+_H)hU(MDkALI1=t5H*ap48JPsovgddC-7 z>FDqbj(igGOmp*aO$pTBVd89g~{{B^ww@s z=44#pIH3NycxAWreJQkdsSk?>lWZ>_6Hk2)WNm&C5godw9ZHCwtO>8A8(Ys`J7%j- z7{TMX|HUM>-+B3brI|m-7s%E4U<0BXHA67vi)P25pzP0?>ec?Y^cra~8 zqA?5Sl3<-xV^dR@Nf7nG-yxWAD?K9=e@f>V*Q%oH2uBAfltHSb)u0G(oJbO z0zt!0zOXrh%u}f;LHqvwI<~rxO6+nBCSLvAWx_%X`t$0I^L533myl=>_%Lmts7Wee zMdwCC&e5^&$HqoGbFPU^hb{Eik&#o|8Pm5QaRn=munv$xG?8Uk-eM|V;9&4M+Js98 zEJjw3HhUTv*Hb^9^^dpARH^c~piqY~t;ogrx^L-?&tN=Tb|LNs&$({f`=V;9N?%`Y zCL*w0k{4@8e6`&FT&%{&a(1~}5aS`!^5;bgwYBa8-d9nL0g^?dSKLSEJyf};3BG)( z;lYa4HG`%;mZfoR{>6>g=_zMI*N>geTV%Ov zk}m1#IJ0Bi(@NXLRiLPphge|XFe>592mrcF{W%;IKE|)6US6e?<^W=( zCja>OI4vZ6F!H%bWWDcjugpuA4m96lUv$WsL-dZU01vO0^ajUeU|93q!NKgl;X+`npa`Lys|9i5TBbXkY$tabhEzwpKP_?MjhTIW!r zXZ3A3wc8ea)DKoW51Z>ZTLyEm@p7Z`dt}pg*4*3xM;wa2QwQeYe%12&O6RR~eEEHQ$Y+BRhxB>pTtR>55-8m|FpznL!qZ3iV1ldqYdWL>;6BVb`%u+ZJ^*Q^8`HE{Ph=Ii-)#> z-xKd_k#aa~S9hDt3trY<)TZ}>lN^is9Ws0*k;(Qu@u`(2NE0DG5JZ}OGJA{2d?PN# z()!8N+~8RAn#&%xqSCT|s!f0Pda0%PL_�s&%M#JfcsJ=KDXWrHdcAcgsuULNgQT zpV@el?;l_FOgM)*?Z(G|;@1gcT&IHzOzw;v^qFy_aguvK`N!}V&xX4)b3H>^WMf06 z+}XdXoSt}T6=C_$f2F@0LUYY#QfyKjv!pvEq`>5R_Q8s?)2FuEKYvc9RGDYy6Jo9Q zhu4-G{x<#giSh1i@#0_q`2+Yn&l@oH7b)EU!I&~^W!5tSw`rGTQuCiD`rx<5$#8)tF*xufIUsG&`xTiQIbti z4C&2}Zl=DgZ=VOqG$ggfzo-NDhG#4biElIvyQ!E+Q<_H?xkP9#i)~*tNcRFa*86E& z_7tsl=2~_9-*3)8i4vL=;S0HUMWa(0gM47*U4)MvI(VQ7=+r!?gyPJneesgMw^%OGW0lkfAKJo zzAN11vCVtzaO9tjoug0ob4_CYOIAD=W3EpWT(q4m?18cMrc_8*qv6SeOA(U)46sJ?GZxim7Cx7dvt#R4;P>H?MoAwllNGw6Xq0m*qVd%Xv{ zh+M=dNdeU|tdm2-zw7mr`pM_a6drHi6OV|9nH_Qf>grJ3b1YhPv=LtS@u>eX6w!f{ zO9hTwe{YNK)z|8{ZdqAS=%|=nRG1NGo;cif)b!-D(y4Fv%4K?wA9)RZ!({qj zkk9wv4~Q4DiyHqUhS5KB>PVb0cbXy&efp7EYlnGbj4CwDHzLsoi-mq=r~CIz`UF5D z(I`X)0uMo|N329}&2gkC00F(8ku!B2$~+gAyL{@rLXt`RH?7nOB+4`}#fxRwjkmJX zw*5Rhq!CDC-r-+uTTIX0lnvM3;9n-y=}e!L0Bo_7;I_jX7~n4x4K%Lj0z zsAi|b6TBQO!D=8!?Yn}&jVIzD)mnt_U;>L+`RHW$(hZlHD{2TMAh>6(hKIUNA^y4# zMY?aL+0}?r=6X~}XpKUnYfXg!&TSTvI3W_5m}4!m19CPDhusrw(mz1iQqEIIl$}? zEdV>5g}dMf7*4^6G!95$s6&XDhJJ)8%L;t4SVR-rI-mhAZhkd`O_Qf9zP zuUJ2x%&^xQ6-msRsWL~waJefjWvm$X!RvEMdoQTf3FXWkO&(IKA)z(^K||N?Kvvb^ z3MOdsrMNWL(Y#?i%_R9@?QYdUp>GK-lTV7L?I4cj%hD8k6AiLs@i=?H?n5ZARm1Rb ztW*YC4NSB@!K)$^h*1TcC_u`^fnpjs*WxEk0j}Z(19-dfb;Q;t%@go)IVKDQh&on1 zj=aAbjGiuZ(gDB7I;9w)XQB@3m&A^f^u8`147_S|KT{(710eZyrQi2cdkwf@FEDc3 zsvNW+Gs9!waq4D|JE87RE;hvuwpHP&6+6kVDBTyAB{Jc%`s(gSinWhFiIRAYem-(y zkCK5O#sDSaG&KXi032&gRDavd{^r3JD-L(K&MieKcSr%5yTIU0o{0xwqQOI|EJhW| zf^G1AkZZapGZ&vIqXKR{o_waHNOC#PV4|&qLml24x$7c5jdC zZMyXC&Aa&WnZ+kc+Y}=k{%q1r9epDkP~yKuX=E~-{rKq1bLO%ZtpoP#qyDEFD(Qsa zG7oHc|FyDnVX?7Rsc}1!UnpWyy_3R2XZ8*{fqzo_$sSr!9o>@_VY%_HN_~GvJPC?y zI3JesI7a3;2Vmbs-#75%$syP2M!$(yQsqKDENbE{;DPceOgxUYl== zQ=xY5kDHqOTjep+-d}D=fOija#y*#j$Ix_>Kz`P_->-`u{Ejk~1n$)MjlG4(3P+q* z^4hZT&s@6vo1Yd5iH|p=OMIAgkNfedJUc1+k?{z1>;TycHRj?DuwT?~Zt@7K$#>w& z%hY-P(LUyueyzBNc1mz-TFTB-Z(Ak$CnJleVWHvTPy*^U+;LrtL^9- zRj02EKG@~{T?^#|^|5eD!ZM;x;>ep#{QsQ9fLOI$kx2iIho*)qTcWhI`Qz9LP}1~G zlCKRq$I3L84g^jfX-~u<0^V;}`6H-P07mvhIk$-Dfyf76*I0s7BwB{){TO^fb%0Y*??}o4fn3+C)BpHGA$jDbx0!s`q4s+o z^y{8hn*$eTAkX?8+9;9{P@aR13KWSEL>Vh9Q>1}ez~MGRKNdGsVlQh5L%M!WlgYjJ zac5w|!Sl;q_bO0wf(Ncb1jIM!*WeH&inVvps+`7bm3bBKw@8V<;A!3O7O=(mkyG~? z`tn5+%S$fnf_tG3&L?cSce-KZ)<{oWQl3;j^3^J2CLla>csDJ`vQ|b8t(5v{rX}C; z?&Z2so=(SGEj<%d>0@qqvJMW%uK$y7$kS0K*|30QjwuZw1ORN81cI>D;M6;VXy&T< z6K?U#jcZs&O&#|p&kiy4JfR%Pt>PmebRt{JSBpudZ*MR^(BWPj zxYp|C@FW&;+2g8legdQdL^NWuKvJ6Y;vk!wQ9N*3tTb{FsX#RVw}dK@dX6g#4!+OU zg^MG;{YkG7SK-lFkZOyf;fNj&ECeh#C!pI0^FG|E=SL?rriO?_j<-sNhu&hGu=OUWEWtjUbRleHe;PvnH-sZ(AyckXLPqAAqQ zw_)nB=U;?C{R%PVQ9BxOO1cciV+ix2w{CBh0u0uiS+>=CleCM>_S;fkpC|KqHaU4> zm%;CZQH85m*ZdZsEt6M7B+02{vfjdwIvjIy;gB_xEsT)%gWv)rrSbT~g=Z$DP;R@d zTD~IOe1l{DnW;$6yA9dAA20Gjk!EqE-&m6JOgI;=u$G*>MmAF3ry{8~M*#@Kjm_TZ z_8mdf^m=tp>p?%pwv-%GPF*LHJAIFxmPCzB#={#mm~nLk6&~5skZ`zK#TnoYv5i2_ zUyn$Pag~M7Lw*EL1$sw%pjVIvJ*5FFhKxdCC9_Luv~Tdakjk{*pMKi$aEsDZ5y2C~ zfVLAgK85s2cM`N<;1y&}3`Wv)tcR;l`79fAs1=U0u2Nema5wc%^e1V5mjfN;@a0=>1O#vS_ zj&cDRtE^+Z-VI2DnQ#1Onv~N=8DvydP^FZ$6U_(x(vJAnKQhMDCn}DV8cNO6tn_T( z))9Y*p0UjLV?yi9+y8s~Mw0aLy4Lfi$Exwcg&`@_`-i=Jyo7Wzy-V%E}~-J_g&>gJb!uY1u- zG>XWp`B}lM=H~F)ax}lcpe_kc0fp@U(gB@_t^VY9cw3;l zNjmGuhgLz;+8>28EzJ&8Vfp zzvDTBXcL6|^d36$g`b3QR_4!Fc_Jx-J_Sf2fk5=j+ELh6qOqOOa^zZkKfg!MRg>f+ zN?-YTcpG}${@on;`*2jG!ZVkSLSkXT6RZX}h(0WP;J^pkV_~>u2Y3@q7Lr+Igc4Bh zYao*#@e%bUU|n|@Nm$wSISUrlf{VN!NE+dbO63@W=cvpSxIz3H>JCBaL<}O#e^9~6 zht0%KT4Jyht|@21)fwsYE7+EA!tqJaD(-~L-{W0pd0M%`tbGKDahW0hv^r{;K}^Sr z8zMJ=I63u{hO)OlY44kVj~n^{w~%EES#_Z14fy?!Suofn3u)5#6Db-e59aF~Q%=XU zW&v_*BI`ZazMTQ%iChh+B?8?94Ejyen_I9Rgn(3%F*hMtan&>M`u$jQS+tHwHVQjO zGAbbdEodM}07yJQ$Z-TbJSNQ2&VZNn4>V8Uw&hCF+3tQa-$9@Bb3X%JpGw?2d zKw{vJ8Jn6)I~D*V{V+sLy)-V@%Rwf6qxJ8&z#xp36-R=j| z;%LA(I(DTn+7f2a9Riv%PcOrn57tKjFLM7<=PT32H2^r1CArWLrU2qBZD}9A&bR9Q z`)6fq?Y|XCB|Nliw)jDH#P#m0r)0BjzB0-2K|JPVpym}TIETU2a|nqE zi6~Irl%``pGLXWf{T@T=eTE9DQ9xGt)g#X&Vg$@lN^VG)6dHxufXrd^P~m2Fgvo-4 zO#r{lxq7L_G&s+E`!(ZI0kj6uz+(lE&*7aPGW9=d5_GqdjFRm8AWE_apP1UKsO;{UPk`LSk7OA+ zqfTZzv0;#PMq{)k^-)Mq22M&est~AR>(Lf{rdWe-h!-KoWhAbQge?gARhWcFX%=Z& z!;NOlm@zqr!OKoN`Mwj-Z7TjM)k0^Tdm28D)pDZJT}JO49)()2yW6WAwmB6a(I$tcNX9q1}2&kC% z=tMyYL+3<6>o@dPx=Dc!#|PpU0d6NJ68v}QV9(L(k&p%`!Nd4b``3v>V`a1D6>?=+ zs9}8U#G|)^OMCzrUeFhd6=nDNMhr-UK`zA3+q~7l|BSG#uksKg!g2VRXJ#tQMu@U- zH>vC$xe5}ajf~;s*ZleW)7UJlKt0vGAZ}IZaSty!a zo%j5HW}=+X-skrB?dM)TFyv_Jk&Sk!i<&0CaNa+}h2Gv}cf*^;mgN-JUCgEeO zh6Eg>#3jZn!M)x(mJ5@<^P`x|95z0I>VdP1v-U5Q2HDRaKi-d);t{6TPs&^H)lPa5 zJV$E*mDkZB?#tJ>Wn;}=t!3FDAH9gml{pWk|cb+=Q_) znJlQ;bT%F0`MtL(gZee-TEIUMs&tEhp{6$?5o+a$CdmT_GPLhm{rGVD+ed48v6(9H zw4rd;iRnt#O*mu8K$K+jDEduias%rK!Yx3m|7>+VQbuA>lX41DkT19sh6>h%^Otvb zk?sm0OyRYu-n_N!#tm@VO7Pye#Ex(F#zN7@@U8>#O^&DSGzy1{LkfmoJz`017kU2Z z|6UJ1{cmtE6Y!RcKr;r8imiN)#`GXuFwjv|MNlQ`dP>(v6brF1t02v8TVrZU3JE3s zX@n1$L}&~XzYmoc zwmn31o<*v+de-EJ^cwyDi`@L_cV6~$N~cU?8+gjJy~?}g9J}XNFlt3W*1C7-PKyv6 zthSB;*-5sLOgnfgtw{5^Q2JOS*Xk^nZ5khbx=K&S-{6B`CuRT z7Au{yYv4QDFjVg&d4f^1_3+++PBYV!&X)eZ|M7*M1*qSSZo~3b1_`#z5M+2YOjlyZ zPlWiEta~cedIe>MaU!d$3l2CM-JC+DrY_jY5!d=87CtD^SlXcl#E9t685MqR`8R@?4!ETy69^Ci2n?q?0G zpP>?_1TBORGT%UNLDYM-gpCYZe{NSBd$luTUv^;n*4U{kk>}rj`aVyV4=&S|R5`?# zgFR1KJ38s;z=q-bT57n|fTX81w5QCr>;2ZD)%73WCrsbR4^sVq++k`{uX?^KXN>2K zqu*{eFYfsM@1q4!RD~9`r-UXrrGtI1nuHuM{1Gq)0}ba&i#YTQVtx5?wJo29xIG$e zH@B?*d7U#B;k~pMfXPi<=oE~C9)*X?3$u*cIJ0djg(7FqZG|8{x%lT(npY91+2v)R zV^a1Gajg;O8u%TEpKMu`knZK=6!XB8JEh^V$I!c{C8o*cKR0@KH_@+dN2wFK)7~P} z2_+{CrPIvPYO^kX6H=)-J7e4Dnj`<~>!j#+G$DSzi-gRa6LPWJRL+7G>fOZe`+Hci zaj`jP*yjpssfDiQ0lR*B5i(UWcrs4I$mQY?v`vzc>!M|>Txk)d=^Ja}vu1Wq&8mN! z_LGk^iT3W}(~#02D-FO!rhJ9ekP38snV;w-dE<}b-enSlLAPhk)LQrRw!iTUZH@If z&q=y>elq@{jh_*{;M~(i)|@eyrd4d4_+k*(`7<$U$!@&v1_j`UR1~H(-~tKs!8xj1 zv%jG9p}LShXG+5tKgq$h5`SBq`FZQ-dt;G_6SxzwQ8`a2o%|-zdbjh&a(%JUhxraY>a1Ap_vjeuIE>#{KmGH*xN#u+&)9~$&R_VI}oiX zKoKXPlO-cD6lF1}o8JMnuiN<>f*;C?ZD5n71jD!=cqdAW1i9er+Ly&iCP0xR#i~}ya~}_g@u9oUJkTx$VIOcM z+JPFZLa0sRT<#W|Byaicr1@AtZpYgH*s`84<7F_7^B^ey_tl5lmRupTHr$H1H_As{ zG&I$rC}$OJE79M}rC(-xC)CC=46EX1=YOp3^|%(w@X$I{28FRCEYBFj(6=n8QE(RndR(~4#TmoNw?HF zk{%EEPPteo*G*Xp_a}@JLl*hxzz=31IIeNi`HviFHCy8cxCi+9K>#h+6zkbq_TXO# ztlNq)2deK;bWwvVP~?cPWY$N&_OxX_B=VQYMGa875hGf_yp2bleXVa1jsF= zZNXmV+V5e@r&d8yeymJ!)!Ge6d8u9_#0kl4z^*f&8tmr1UM-H+C=JC!Cwxl*=1h~_ z51CoE4ISzu`i6knm$qw>4Nr-_@qA`L$xT6nQ&Zoy0$8@Cc#7E6*9a{Y!4Y|M6IcbI z@YI1qf+)Pf?WBi-p8@%(UyzKn8W#(7s5*oW1o;D7MNCp*eymncN4C=^XUN%+tdN9e z1wf=|+o8bU&^*q6Q?I5i`;$pB$`1?hdu{(SYOuwt@YrMO&)|n9O#r~j(&Jx^&?Y+o zT|>tHYPLl*ks=z8*d!U?P&9IU6s-#~JVu9$i_0oslA&##MA;N;^91RVi17!m1j%XX z-6O9(-kUmgp-XmAI{!TKPgJ60`4Yf;VH+S`UIvXvL}$}5s7-SH`-k3L#?x9RXNKX_ zvTx6ylkyd)5e8n|u^IJ|PF421esa5a!ne$@mP0|?ls;2b^<-U5zM5G-wP-NDfB-?1 zkHE1K%+k0VUWG5IR49|#%l^RbYJ#fFt_LT@pVaZQU7jhO(z?mjWCmONA3#NUv)x<; zG`7eAcT+^E?uXzA$`{##^+xzetKgrFx2_;#Uh`fJ6SHZHw!_`Ul0G1IstB(#ke;-Q=w&2jDoOp#xYf)u@#QQArZshFrW3000pqX}(LH`P4zm ztAzuHMrDwNB6xgv=_CcQfyn~29gV0#h&%Ptk^bv$YGYl`68`g@F8(aI1!kNj;HHpD zq^<23=S1I=oyg~oaq1o!p2zX&0IZP0Fp6p&yX0J|RGW|INonF23fwt(#0wj9a^dN51Y;GkQuMp}cm-8ik%vom-77U* z`F)~p_wdQ*XV6Q+kCrZgYhDJffdp;WQ1a1_Trkc{NEz9W!6O>LK24It#e2&B{8|rO zUB1|QeO3kPD!*IVaAvpt5ej*w!5fU6FkLm3DDW623*5jITwai(Y3ya0AtECQ+Q3OR z^)i;ft+MHp|JJ5{pMx9h#@nn?8emS}F03urzT^6dUDOemsT#%&g^jnCl$2ZynY=(Y zUcH6MRL#o&p37i~CT0hc!g9v8@Na5pIcDCFnK95dB->J*LZk*8X5smd;~jMZ{9pOyfW{0g_a)Y>stgb z0u^}nURZ}gqC-lfoxgV_cIlkz`>Q)S`>9gSwUM5@2+1k4l0Q&1l3j)JQA1m*38VUXHxCJA&Kk zVf^^9yaX0?ARYnBh4{l2fRqwt1s!iyOSWrzg|?PqXlpV_J8_RTH^x-dqj@~ev9o1(9~9CC94E7&Q17@ zHaZMm-D`YRU2<39;*>X^OHR4`vmV&kn2!(v$R-3mZJcpFc59I3VmX!x)x~Lwj6qS1 z6U?u6E-U;Ksuu=&yeCq920`tGo$nypaip!l)=)$LQa#m7Sr1XK0sMF@pihr?&akbw$0&kQw&bJ1{sxQFzv#f2q3$GCCj1I>m?whKwA`zcUcfQ zYW59GGhPN-$Crna$>9OXfPxRou{iL9$d_Ly~M;M5hy`A&=gG2M9 zzmA!v>n2Fdl@rJ_>FEj1N)2^droQuSC+EaZ@Tuo{VH}|-8E693vhE>D%0LTVY)zM; zbm#gVu07P409WWB6YQ^}1GXTV(t<%T5f2I;4ar2hK`(~X*4U6Qz~jZWUyG}mrd9Op z$+D52zjbD={KM+j+#5U>ile*V4q1#m%$D!yjCJL&7Cch?r8rD1`_n_y!-1VK#}8Eg zV1(n@cIUCUscu}-E&yehGS2@!Tn_# z@4HO(Wo5nM1-&^Iv9>SfmtQ)~x5Z^ex0Q*z$I-QaOlLKR6qUNRBxxGlc-!Jq?&tC{ zvoI-6$d2&kW+6LrlKy^AN? zALcbRv~cxxv%lUSsJv7P$(H=l(jk$nFWJ}}HCUsmcOF?>CVwcz2 z7`7DIj;z+$+k>H8b&=aUxU&KU=ZzH$KCx7;w%Att!1=LGetw+s;7$u}Ro%8}du}Vy z+?6dX zVv&(*h4fYM^YO`&jvL4XRVIpeq+eltG~L>QTGXxywBk+EU}Gc`;74 zA3Fl32!^<9un08p42cp#&W03Kz+oz}yY{T@Z!mz2#X@S5j^q~Gd1!QZcgrg%5PnK3 z7K+747B3E~s@bS(X_+}SvFSxcKj|zlxD$7O$({3Qp$hDMH)jqvq+NZSyVSW^!_{NU zyicjyx-N@vSX1}L|5+o`RjaA$dO+5q6DRjS7&77Qy_6m1Wt6setycyHM%LdMTxaMN z_b^-C!a6Yg+_PP`E*`?0;2!=_c(Sp5eI(BzM;^uS5)G%RV4$;^1O0%gb4Twt6daE* zQGwqXZg9B`X`jZe$qd}mh`Dfwb;q&_sFag@EWkB4Nc(8Hy>zz`) z+f>7>v`6k8yP=?Q@L`f8d!yYzxM*r}ZNKZoTfyOx{VBmMzIF6jlov zU3q^rnD}@vUc5LpXztjMCbCXecU;HZ0?)&Wu_J!zj*u?J-19u%>dQ3Z?!6IB!|uAJ zDyURYqhTg<^j?#ARrm~D?~EPCSIX7nNz>au%Kk>ob!i1h%k!afn?q4-+aFvFeb>#- zrZ(Lu-Q!s4EVj%)wBLvd=%f1zVn(G^#;ZtwfB)0g0Ub6HY6&{qXfzDmF+7a3> zp^7XfCWev$*jAiusCe4B^ffm+pei_?_yUrioX@-E)yAErVcbW~)Arr>g5W;ApD06A z`JF%ii*`cSk>j7V97c}BDXiufzS$V$tJmjHew@iz5Mk)88DjCRK2}wfyK%|;FX7C! z>_N-DUEVu48?kx%2A6zP+@qH~IVlTv zy&YyjUbt`}2G}%_5fW*ED`<#co3rA2;+JhN`* zoK^EZ%@KS!(`Y_!>S4$D{{8zomnq&d*hR{d3l{>cWTSTJ-c7&9U5j8_a6#3BwMyCH znESm`eyr#xi_$K~xW(GUI7gsn6JGXJKe0F*CMC#YCPEqYK;YlcFZaJ1nhrLS7<|=Bu z`TlXEn!JI=z0Y$WXxS^<=?&DmZi+38)3&?0H~vp!>j)uPKG}`JG0Xdsdu@HDu_fmH z*HcbdM6oa1Rv(Q^c-!2#@NFS>hBSh4QqK2JB})dX=P8t{Yn>|DlwpQ& z-6vOj!q!JW?!j;d4n^|&bLeX(A+R~xq7Rw`a|d6Ac%{D=Y>e#x(jL>$(2!z%<*ki` z-B!u$Os&{xn;sbxi+iWV!W1l4sYf`u zO|I|~|Bw;vI=$Hxy)x{2$CyHdId;1fj$Llao$+yfLVB5pQIp#ndl~q+yw};^+KR|Z zCj7*r!I|zoFN*FAX-a0N>Bhzf{iOj>0Ns`zcv5#g^V`i zERQy0$=9!zr;N=r5@&m|TF-z-y_2ZC=%V4o6Cxin@i4{pevE;Efk$sbPt?GW1-7u~ z622CD7HxRqwrb&}i}Sg{aWV$D?=`#Oe@nCHZhrngSfqEw3%0&yr!DY#9QMu+GLMqF ziI_SAnjq=OdYnHkK)2K$QwZuNfH5pZ$r-}rihlXga@vvZ`jqCFs5ODNlZ2E`$wgh>fa{bv~dTOzi?6TWoLV> z_unsk9WGo@NC;~G65;}meSG#C&4R1eaFvZasPEi;FH!z!@8)m6WFFbS|01gSGehc5 zN`Aq?(TK6inW{?7M)k3=!`nqf4xzf)l;%{dt~(?vE4w2>TwQIK{nMGtHVO$TEWGr` zr4%KE&);!p3r%m?F{C|3l0!>ZG!1eNX1z8%6g8bK+VH<$WXDVu(-EKyUK#D9U{)N@ zXhNPx6dz#qt@`F_=7=%3+iWDr7Y{HC16Z1>K}%~Blc+`HWC;8Ub>XwXlN|X{4x3J9 z<(iwD^La&jjh7Y`*)r~6c@+p!kW^1Hkh-_5)8cgTv9gB2u$-2M1`B6sG*5F?S37z+ zR3k2CWN~f4qQwy-zO}VHYxfVXi;Z)((F#~@6dPL8EF?So(wUvuo?Jgp{EI;!#xDeY z@N_i#rvivRMe9UfesbCz9ybgY4sSU0;MBSN&ZUPrSW!_Yuz8D1OVym5GEpY5b8&Td zr@GEXR{Hqcc;g2LVujH51eTDBs;#{}1{*%hvLpvZjm)5kMeLEp=!EGhZTf8P>vr+H zQ>&~wGE;TI+SoYv&Y0Z7OR_~4=}tRkGj#2a#`TTb#y)q*y0B^Pw68+{{er7#eQX3@ zO2#v)I(i8L%KxM6P2jOy*Y;u6QhRCI6^&A>B10%8GFFmM#>YG*Ly-)bXVua`DI!tG zJd=4I8YC2%lQEK+WF~yaWms!}-}|op|9{VKt-XIMdhYwa?rS*D<2;Vzq70WlF(sa_IikuuFd}DN>pIlB#M_$PkU6XG3){B#JF@vShsB&^JL+$`|;lU&S zc+fPi4MV)>gM$S%eJJzzc9WJAq`xv~Go%NE9qcUg^tsnsU;6nwmCH|l^RoPqfoaxc zy&yKXFy{NFR(6$lWiyM*)v=q$>z{kuDk|^kyiJ<>?tR(wk0!1)D}6cI*2+SqRrHT1 zh%x6QJ5WBy`co=;Nu?M6YS9OwCTX1cb#=pl)d~6yWLrktb_Aq~@C%S_p7{*{2-V?o ztmXg}0m}sTnRV=5$qlu}V(F@P>SVB0)9u-#^-BSM-YSRYGCH5?8m-tUdNBOxG($m) znM?24Ggq~ceS}9|aXwobU2yq8pRQBFiXA}#8P}Sv6YMJrl&JFW{^JJQq`8+E9}ve0 z?1u8^&!0ET1wKfmo5tBaOoab|a5|cR@AaDY7o-J-|6IOh(}&d={ro95tSl@gXp1Tq ze8Kkdw$gk{=jh^i#n|@t_UCR9<-dSOJgG1iH9CV%yG&i*6YQV+`^G!=lzg#Pt*mJo zm3J2Q577LYUlwB1m;b)kP$u_UYir}At8#OWrIyOziFxA{lRN)$E}Gikpn%?_hdmN{ ztM<=uy|4i5EfxqAkr*B@K*mhkAz`J5B9^R;8gyLwSQ-hVrmCtxv1*%{n^kpn4=Y!q zS5U;Ef`F0l@+Ji`=CrhwXeWViE@}Zu zhMDZqQ43NT^7Hef*|<>&gGslr!%!8x`Nv5PFyQ6RW&`a0)f0qAS)yCzqomnGn+OY3 zcm?QKOdz+refR$T8X}Rx?#ycP^EKBt@c>PF(icg?w$q0gK2^_dpb*{2(tP0U$=y*Gvh2mU1lDsCK zHGXVjWSb`!5GwwPzpHvLB>HGU`5``Q-yc7IY-W=Vn4;UVMdwQ8DGYJ~T5MhWaDih) z(3nnL1H3mG42`z!B$rlh@nz%YjEPjbD9lX7E0V8%M$v5!#z{nXG5tzyWplDpdqJy; zcuYSR7Z+)%1*Y*(ZWj|16V`;8!g%Z9Hp%4^R&oC@XWZG9{}E7Tm?TZWXQX$2ZfG8& z=&Q|U#k)!6n&1@AjI+f7W3vl!a&s#rB_)v(7(uxInzWX@^czsk37ZuCmE{&ze#nF1 zF2oj7{hT-UXB54kr`%=b=1yxDR%iT;n%YN|*V0Z~0hM6A=sZ|aD))Eu9r^dVXiVOeS9`X&MT`&-RAAdL440#dLjO z50K*IV7XQl5R)1%55|buCgM;1+%3{gF^aUF@3+cnG(Usif7%#jxIAb3^hgITXAOPINt@Y~2{@V{UZgF%W`=~gwsx=w2!Q22 zQDh6ed2?mPcRk}LG7b+V*_uvu`iwD%i6UA_E5$!X$II})my))RJ65)`ZKMq3hp^@!%r52`W z!$GtnsHdUZMhYoHqwu8#c&323LS0SFBPnJ*z%@lbqHSxVKF{2Si%yaq@qLDtH=>NXu&yY>3Y{~no+sAZYD!pfghmIHPY&P+0(}-9JXSb z@KB{Q-VY@zv1CAtW8?{ddYPYnjClR}_2$+_qx5!8B}eB3`qJz1mG_m!OSMzjhEk|$ z#j;*8`l^QzH6K7mlG4Z3NjU2Nt59osxc=vc>1&c?&4CmT?!(5zrE)mb7}MMPaLZV{ zU2cPUTS??m7}SJWI1O8EHiudEjHVC>Gj~4q9Qw*I|M9Eo^Awku)R4px#~?qyXwRWs z=Cb%^HuVFYFB>dE1*LpfEAmVp9A6J$c)Gq|F8uZD(;|%z+}x@Zlr+N8K{6Unp~OTZ zcwp7spZ^%~380S-UruuZ{KT#R1X}GGY>y_NLAxUpl30a!{ic@DddQi=EjddK}nRf|{byso|GR?5hFANEl;cL2F3;yLt^ zC{BH63bp=mQj7So;AInRA)_7C7SZ8@89*VS;o!txO$bAzMCEwx(xniTZ{>cqN;B^U z7UJgisWhcI@Xc4|v3#M*hzy3zZ>)n;U_o|YKK`sMt>_X0xf+DEq@V%1wzDZl| zA>9QYhcf{RaezHB@CgOVfm@Q=v}u#^`w!Np4>j2K?-#*e&4X^rwA?3*-t~Cr1x<33 z*{K(HaXafj)$gClduvum??W>Yb<}&1WjaXM>0{U0f_iIYSLhwaTSG8Vlqa0 zW|9i-Rm*6&>_*mxBM2=UGygle(B|Ge|Jv}tf8@5NGh6LZ+PDiW6%95A z7j~!u>`o;%4O21@H z>yAh>e(6(euXlo@p^_D`-gu#DrJCvbg^%>FY<}hI}49dh!1K zIqAqwc_21lDSxJSD*4^9cwA!W8TyH-c;2bT<@;_4_R}*q)m(YWR>t?nRqLZ7E9^HY zSeQQLcfQa1(Z|&ZM*(p6Do}UHy~6@0T|HK^=7iW(}`M%Ud>$utuwuqANrX24LHY>=>+ijG0u;Jczx!+M; zqubZtKL(1XD^MM(!hsM=NA~+wz;q<>THZu4sXpkZXlJcZ_^)9P>xmO8$VimHB_#r1 z@XLI7FGI_l4T^;iiloTne57jv7veOk!i=!*`aZVQQLZ!dpPE5(vQ0l7X*2MNc{Z`a zlsj6nnvNx89fBUhH9_o7m&l44UDFvtem##ct$4`zv>AWbZNYrynL3CsXTUc=_AYj$5k4)I)D4tr728mQN? z=eDTS4e#pWD~hFd?H$n+sf{>$^A6XW9yvKV;?YmYO>jR`#DDrFazFc66ifUgByq-G zvShL0{;$a{Y@}&-?TSSN{CuJb5ilDD3!w;4jD1ic**2+MgylTCgVr+jC`;Xz7kIn# z#c=a{w6UeX9rID|cg7igFa)j!Ws-D$Cmn3m3#tjD%fK=^<(k+=io$;#2cpVChX@!g z2n}U{ECL^h9F(nfmZN5HF;a#1i{9#zM!c&VnOUn_XRp)}*0)Qu*YB;VW=Tm2*;dIu zf~f-j_tdX#vqeV7Nh1_8w?W26&^S{n<&}%|~UXu<|FdV8?seqGxE2-#` zyCa&hzei)A_zA!1uGbZ*EK@S~bU4+m9^B>`>c0^u!vGX{*KwmyWGt}a(77yFy-Jw>=igZP*(Y9S+bJuzt+VMkLpC>pz?h4Ri?mc7MbLzT!kM5EeUz!s)*BwGs^ox z!bL2Me0%%PzP`7cdTHxlzD>e`5I(O~;Ige=;d#*|SzvghHS*F1;Cho#kQ8~)$okxo zBS*UO=HQDCp-aqQ?uZnrMJZ`HrjdZ1uwJZkn%4_2QXO$h35zH_f*}cUfJnPkM1$)W z6&Lq~xO2!6k}a$rFxD5K(elFGf9%THvi!@v)s$bXah{eiANOf0 zcQ{wQ!7i<1Ecxa7XZ1-9Mr6xIOcw^pE*bsK?0g*agiJ!-U2HxoQ>p8hi zasP_{s01};e105;>{t+;0EzJ1+s8+t-9}mY>0nkDL*9?qYXOZPNxr%~m8(AYTM6b& z+dC!=_--yvNN7F1M}2A|wc09yM?3Cc2V)lt=?t#{=J$bELeR=Cx5`vk=GcRnq@rvR7y0a-5FmzNV z6CMLbmL6OE7)dK2sT}T=(_=RxryzcVoPr2k*%nSgunP%TNYZUYTi3$ZzHHzrxsuqb2py5Q3>;7I}-l9yM3eHY9&%s0BYjCHUitRazY;%nHss-#4iM> zcJzOiLK>G!A*VcP@#wPud!bl|T8?ka8$-pM+qCqO7d!-~fCB6Co(jt~Z*C zb7Y8F_gMh$@;~|beqW9Ffws*!b?NGO$;1Ch!g^- z7X8+ir^O}|SvkTxZC!%XQMFlrsX6y4HQ$;|=ujz3APU>B)-oBsniIxd2)r^(>qV|o zZ(bo72~OQYZ(X8RK-O2M$HprSEfC|fO4dbo9s47@zW9s-;)JKcVAHk&#p}cxy>Yzg zc<4fiv+r4}1qq;2w0N4of8_)N_q~`N+KLo90`!m8dZaOwt_$1>Oa&c}uMlM?8rhKI zG5mp)2|Y3HAps|@stJ=j$H$Zh&r15vfNa`}*EY&qiGhh+y={ z%eG3;R6QzYb*FiiUoflN!-q+@NAuOEmC<5*94QFFCz{M3f_SCap4n_noJdBo|FgnYc%$&Imy6>@HFxQ_A9sX^ zvtVty^gSK-4ZosajT~fU=WHb7c5e|*X03e(P#OD$fy`N|az=iBHByyFv0)U`)Shqm z;)NH6kVyH_i;viOXAi&IMG7cHjNw&-7LY~n-Wh&-?^#+=QIGoz!y?<>u&^fEj*)xS zy6)4`?{Mn_)8pzS?&1l9w&!L*)dz$7Wfa!!I{qQFeid_$^K75immKTF7Td1fC~w;S zeQoaSS8Et-PHxk%@CIZwTe4Qe(j}MB+%M!5YyuuDx@5 zTC5d{%Ji*Itj^b0%$=L>6I5RxYG`v<#71s0!6bYypP4Txa*&Sl9Peog{gB(ezyL@i zX!!&*L9S=~{AJqmzzc{i!)I1Dz1phYT+LGW;L!aGE=o%!N0*q_pXkEeMiFviUBU}* zW)z&lZqhasR$-_iZXgZ0X~jrP;qpvFqab{)v2TOxn%_T{ zBy~K-&vAlj3|<8TH5zGF_FgusNF015G}cX0M!{{*-y@^bO(Ta_K)PN*z7PjzW^54B z2!4+yyQh$|5ROfx(+*HNb;1=QXbniMU)Re5PbTF0OP)Wr-7t@ zLPPqpG%t#=al&9`#|Dx!B0}K!@b4cc1Tcc=A{Lh;{&2TbdrW!DZF8@U^@fpb@TqeNWVkWg+7XXh($n zX^XVL*K*2=g6pJB5`=&NNUDxj2z2yVEH0p2uV899a;hK(^1SfG58)!TQe&^y?CC}z zL3U*_Hb5ftaL9nuKp5|klyz`r7@!u{@qhfVNlte?L}=Zx^=lA)M7s*C^-5_q&00DoN?miN#`&b85#Pu=mv^Bq)BKGRRuzeG(zhIq)M8!GkWOh>$9@4N#YzY z8c~GToszDuZum^+|AAC|JDBa9;7orCL2?pu2KkWRym}u|!N$e)Gp68CLIWKH^@@o?Y4LCAl+eZwcRo5D`I2Nwvv zUekmgp+sZ{yi=YYGwE5NP5bW;cJtTW@>`Lwha&N1!Q=WLwv!Zuzzjq+ozc0d<8HgQ zF5oNS3!P_@ANE@Cdq%`nVVBRpmK>pM;&LGMms+6Kk`D>jL2=>>5Zy&vL?Q?TU1-+% zBX{75rM+8NSQuBp8gk2LZF*lP(3n5vHt0%}8uqhysvo}p*e-I`k|u4n9Wy^gfruvn zM4~lpaQkP_96|qOYc_qep5enf^7rwXL-bLDI*`7u`E5v&Jo2GS6tBXnmW{ZIj+`pF z2W%}YETX%fGe}>!5U;!78`O3JgS+u~j$Gx9V7r-zdjdS-rM^UNip{UPv)fq4LZ(5f z({%_^N`Eq$n^F5=J#^@hM*$g=Z(a*_^c(dT?jP5W+zu2OY4@&%=YIg933r|w3Gxa< zWGI6Uk*w90qG~_I$ydL$$HM1RpRyMdTx7iVj?GiIEW4R;0OUDim%CU#(Ua>I|OT^TiFQlkQx<8w83 zSq5TCof5glG@QsVkgx!GCkJc`Vo@%wUG((oza}7*l&dHyD`T6phZ4JJ3z^z3thzEC z$Ng(Mj#I0xeBPcT75K?AMYw7_RR=++LM=%3vA|R#KO>lphQ)W4$c`0r+8>@C!9hg? z;*l_rB%i1%hxC&%QOt(eTzi|cQXrB{fYb!j1zj#bf+#^ZQ_msyIV`Wnd@y~0S0U(+EvjMrasDT#7On><|9YYsP$>WSGw**EW%BW)7mPrlhY zE7m&jr48{m9VRrP=UcfU;_UI@ACv)?{Z8_t7?!GS$RXw=tFW*ZF;Rv?LM=>P2vmmh za#z8RL$Hcr7AG!VVTrUc!N%ce_mDO!ts$%hcAce^@u_WF4yzH!^PEx*>;Q z8iVm!Zu*WyTM!~aA!2L7k`4@15Jw2Za%Ulzf)T%!EGP-V&gCk!mpD#7LApfRZ!Qw4 zV?abV{Pg0~JsqAV(_2KSTT?@6wradq`u_c+1LTCBn;ELd>L+d=Nr2+ zueuhahm7y!J4TNOQ)6rL@{I$Le67Lg>l}YJhTw=rNT`Knu62G+3488?zXzitE>NnF zned;9!|{S073Aa&b>`BoT$}(hP%>j16G&-@gR%Kp;<^rkqyp$~M3@iSox=6&$&iPS zedK8c+wQ~XJqt93m=`}>_Ffn~LD@Jznx;Q6+uZZ{PQiRs*z@ywEe0-e`ssTotN}QY zKD#J;az6X0NWX{VoOz>i+y+%|Z#67UjvqWi@_T)JeU*Mq5SA;__}i(CaVke=F%G19 zMC8&W)W&yZkJYJR;aGo2aqktwdL^w%>b{!U)`bezI5? z<&BYo$Iz;4;5~-g6m-M9Ndi{QjScn8P_C9pF4hBRnJ~`5Xnm zu3HYZH1|wGM^2p4d*C6O(9J+@we1r2-cZ$v8a7|8gOATk6)Wy(4kg*Xjn(Zt$s!8c zm<>>%$6_%_4bT7T5}w9<74Tl{tcyC2j?E3yt(BC9NF#8}#v3R-NK&qlB}KFis2klM zT)foT44%HIhuO^`R4GL)b7G~N);}Fd8jm<=&wP|Zhk&~tjaBuMc;|Q7e)AL~y#_Yw zEu8dtH9^?c$>B#_KuP*SYWH}Bz2F&3bF3c{+xw_z^0IUn|erv{d$Ec z?oTZFP)4l!?hwabatC-yj2i+n^$MYxIQc`EuXum*h_e5l;G6&pi1ZC5f&97Yt0++P z#&x;gxkF@eUvh0zmU2RYFiD3kjsd)($H+8uc# z)TkI~BFRYOQ+Va})^lY3Az+h1C)09!aD{hI>K z*%!UO4>Y#>y_I2n`cdEU*KjfPP3iK#NM|ez?SH(9i%8Cq?PGBv?MDR&hU@p2zE=A| zh~bs91@;!&X{u7g;M5PQPP;}JYju#l?=3P;v{T`F77)3g!D&)^pyea0&f@)2slP#< zjAWXyR8ZXQ65xc6iewQm<1$=CXsFu`sw`}a>#D=1A%lO=myu_$Qmc5ygTUZxbSH! zBd`xfjE;^je4Cbo>_(x<7oeq3enq!|K91~H6Gw5Z&~*2HB4gSp@WW;4-Nron_th6n zqmz-Vj~i;Vf1YF)?)%!=3fbMrN&O$ku(wGIO`QQ=Dm4rxL#X+u`8^$_ z+AJ9oG=gcmQFpa8F;yU8a>IagBGXm|h6I9j-CWD@*D|8N-zYphoJBxD-P6-Ep}+Ltj~)T? zCi45s2Ck02Ej36E8s?XV&TmX~=&RdC_n$KQKWK5h;S@6fA}%$IBdYrWPsz1HWC26#*pKc8d(KK*tFePlL{l=iUZ4=(`AZ{dmbiO($9crq?&9I-Z;#7NQ>Zc zg<8KLO%vIAzUtod^N}`D+%j9T=jD~^4CErL(@kk$t;(w) zWFS&-Yw-1q15jlE7OuPih{`h>##L$S)dEJQL16C9-xFzQl%ZCWqIpy)f`;d1gAr*| z;56NHFmh7Ex>+H_lty9A^7a1*4a{4Q;f-qLJH%R3Zkh{|-{nBG(;3T;-PS-|4vRyL zM_atM|9Epozhw9~y{lJJJi;||x(gK_DF1Y&=|4{X4>C)ieJq@p7vGR2>=a2|+`}2^ zG%08h1;t;1eW99y4K+*VYQkH@sr9gG#GKniA}OinPCWbwwF zb7?s3>AlwG(%LsVmc#;y)u@; z`IlLwvYlNv={kkICgc0ko>VxKm3xvE#}2Wk#3mh^v+x6(B!n05G`P>)dS=CJ);}Vw z)R(U(&u!iMT@LgDnFD&yLr$m0y>@umRE_rJ>IkdM+{pe#QrxQxM+a^;H%O8u3|h+& zLS4ft@8;$vhw?Pp{Rqa5_S@w6V#{8bwl*i+8{&BQ)_=O?tFF-gd80i~Pl7qE7+vtW zn4jh9;bmJUYAz)a(`yLJZp;5d-JlZvO`4u8=UGFSaqy!sQH$#4Y?ttSCCDBN`wF4R z;cz{V;#4itu8S7+3nRJbBA4IjnMR?V)xpm9bS}4^YS|$wN2lbIe)zt-do}ng$S%GIO1C?EJ`82E0r1lqY2>Af7ko?!iF=0Nn)VTnOb6#5*)rQJF57pAiNLBM&g{5=#B!2&N39;zrhu zZegW^4n?w}0%p>%3=y{+Y-(TuU7as)vfbUdY&o5o;5tF?kvu44s))ZBx)R~@oHD@62e6jg2Ia4XuLv8M45wJ&zN;439 zi8t6Iuzp@bp9{sdtI#tBsE%Ff4w;W~I&;7M^nN;{^^_>?lYi8mbX8qNcWFKI+|h5N zL+mSP*8hugH{4sNj*41!-E7*I{j7G0?uuU5l{_g}8)N8r-i_SI!oNEcF>R_*0~bQS zip~=Y(*%Prb-VmqIt-Fg1eZZP6IYkQqk&QbN=-4CQ_G{g;_NoD0AnrCsrACP*9F$< za&SE5Q>#??VL+`AzD;M=vDvTdH?@_lX+>>QlyS|RoO3bAK1iVo_343jWzeF@+Jvf) z4WjkGuP%gFPm-|S>d1#E(y`bZ>>8^gmvEHG=I7)S~^%RO=2v=C+ZW?jejDdp18p1v=ds7`P}2`MQn zH)%xFpaz)P&YQklBG&&`QvlbsVITBIRD7qbdFl1OtSkfp^5x5y`?$p9ElF@{yyc#;3or(t({9f5C^(FCG(^qfVtHX3zv2|8tN9tniJz$qAT~PC8CsA6waqj76wh%a~@Ubez~je&cVSn;HadPJ+y`3V$l}Q!z3sc zMKZ#2X}rGhm4GWO{R9IRS7VnPVL2;W|pbQl`@<BC-aupX_0muXlp;ywOQ3M(F5#yw<}3Zl)t2ygRpxH^+{1NG5(@ffRccN-p6Mxf2%6K70Ms{lLl+c%}GBU7H28LWj1n~ z?X1bJUq2^gTrsWQ5Y8|UP`Ji%nxBAQo(6%WF(pa3eebrSv`y3~Y72-NzI$N^7V`}0 z6Dcq5v`3Y)4Q;(mXp%hqtuGR%n;`i+!dTA#+)^%s{nz=8v!gzUG1Yyneg~(lZq-JC zHd-R%MlWY@5xW06V=22~drIy-S5Een^i%Ff?aa3*x@+H{B9+!XeTO?)berFWh|Tp5 z8CP#*UvL&n8UMv`=grsTCTWyTE241}OC^Dj)B_V*YsvYgL5SJ<_)TF$37EhzG+^W9 zRfeGjz=cH3c%)3E=t8K=4349+V5sk`O{+LO%k9^nAij`z{;-P9QgPd`VZ>smkY44D zc$9h9(tW;Gl$0D-H3pXxJEJ1&>7!#;qKjt^9II#Bov5T9)z`I(xOlK7=E^}YS#$(b z6>CQo$OvL}SM)P}D4^^dBZW5>GuXU3HKG{X~4b34!P=@pvYJ!>+P z$ibN%1?LNrWJW%P?}y+-Qz%3xnW-jA1Et`Of)?B$9yJ)%!ICMUU0QI=+vxZ+^0zcgxPz6&cl#;epw`$|gV zSVNtUozAQS_Azf>2BY6UbzHOSDgUnY{?hT$qv}NvbJS#*XrcOA4jZ-5hrJD#gDk~^ z=58+s-#JY4udOcpe~7s_&q(Ye4?lXKxkwJ_ib>%KrSL0=_m%ygPrD{-hfarXST2T>a*T}A6bX?k1qEy$- z+vxwVda>)~<_{KeD=Ny7-!~nZ)*AY1H{*87jb3|@m5eC(am`I(0Z%sAP#pU*4w>bC(ypJ`<;y%g<9xMw*zxv6E5@}MJA_nHSl9tl zgidZSx;?QJoxkWMbP6Ek_(LL15Q0lRDbq+mlmJ=A{@7Yo2ThpckPg_e*IbHdsG`~p zZ=db19?URoQV^l+Kal%&c=y4FebWs=-BVinEU|klKIJ3fvqiIxcP!(mbM^tnJ(jwF zabq?w%w#fqi3bOOo^EXBRd6IzfHO$mXqBW?mm>-@K!u3c#}d*)#aWs7sz{aZqPMYe zRr=c_cj)~mi)qH54+lE&rM`J7gz5peS^JsAw`bSK4yw(EmrhGdhr*6v4SO(Q{Mq5F zF$&f_kWkfuh(MYYM~)DC7E;G1@c3U>lw)V6m}r@W&ehvz_PUDs@;dE3s&H%D`5KT_ zLQ`{%M5g0Fj38sSuwqe(w}jF}WP0hUGHFkNl)tGKZ!-aOS`}TQvhagnqKc;i=S*f- zUihUGPR}~UZw70oJclXE-o_WZNUG_f6>fcGyY-qey+9!vfuWL>lUE7@YZSC%g@+b>u&fFh2QIsE`ok1iq3dV%e zUp*fb+$9%Tm=Y3hSexZ`Qm|%;&7P2OfGsmVZ9_keT&LS6!lz9;l_|4@wC=u$^RG?fRI z1h{)y(6^9Awi>3dsKcF5h5XdLKtiQ*@m-s&Om423Whh+EvVQNsULu^C>){5s`2;>q_p7rr4bCC^Gwwz3P=|6=I7$dW>opO z^?{LX_*RYimwZJ&>i75%T*1>1#QyJTel?XP4M1`2^$%I zxQi+Q)h6AuMW&rV;Tr?+x~r0C5FpL-+r&Ck4;|rT2}2SSI#PadLPU81{4~UoL5dO}VjU32 zdi9bxBt8#1b2MJmH1N-Lef-CdFX+U;Zy{Yk4WZY*ow^GJGi-HPHHk1VYJ4QL@Gyl_ zgf?yb<&@;=`*K5eA0_BHUp9`8-xkfb`@Uv+;O&j;X^wD_=tj{jL;eEjfma z-y*QqPyrY&aDS#Z(@x+f6J&z0WbWO&2YMY(+_i=Kl(DSY;9m8XA)}|m_?ChoZ`|AA zYX-#@1N{$2s}fUu%(A%|Jhpy`#SC!M_~It1f+lVd*&fI8| zA?^nPREO_pEt+z>RP9jRTMOzDIYOtVB837P;2epa0DkvpNzm6tP8lMEh5XMzcY=dB z`QR)jLOFsPG1z|F1UrGIYJXYI>p^uA^C8@M1c`}y z?t_aefOok9e7(IX304cY{p0esHA?8?*%Glaoul5@p2wSau3(_R@avrLXn&DlEqlW7 ziAFbY5>e%f13NX7S3jc>+w5V|N)UC=vymIU-9PFRn28vblXD5%jvbJQ{mY3II23x?w8%Hs5 zs;&Zre?@oVHolfPrX9mW5g|V@;>fel(6sAeBoa~7G;5)b80RuK&1;ITlubDwt!6 zu?j%ws*&fsK#}DtEOoQL-rgk;u!xVI^wC4;vBBEH0=|Xt9#eurvSDS2Yn-CUwnSEK zFj}T7oaaJOxFM+Ok<%`owq3`5$QS73He=u7JP z<6AB|6WR8CbG77pvc%aR6gv8bmQLtX^P> z5Oz1|!zDaFB~9Kv&4vxf<2WJtc`yKakJJid-5uPO)htv|zb!i|5iU1*Az5#Wih3}y zlt7IEa?ojr@F#$D_ZBCua_&GRL_U{k(pPtvq&t%Q#L{exkI^HtQxD}Vt-heO!qu-n z)KoS8YeOBZ28e!u)*0XtkOxEGZ|k3z5q~#qf+_{Se!6=w2*!ZsM{L$e7y~(6ThHji z1*l>U`EET23lY)@5t1KkJst*G%HA|mzrDN|wv9bz=)GZSHYA>`K4IM zOI>a+H#+HpJfQ}0p_{uqDdq-v6#Oh@Bl84Wudi-{&HHeWd69(oE805ZQ<$avaci@^ znG$>M;2#=_eK$83r_hXyHT4d0o0jtqoafhUpTu0}MpNd<&sZfE6@nA*jX6~))S2=~ z&MiwAeb0HJ{yg;6XzBTx-mDD^wEez=SlFYgOVTW49TNS+uq2NpjdeiiJe+Nxfqce2 zO#ZRV!e4vQdv&{lNUOTw=|sz1O;}wrdL*96m(DA2*4ksF9s|;J&u3(_A;8CYiwH;t zKJusV6RY%OM^^|5yT7h?4P=Pj0oE;Md=>gBJp@2Pn5*{l_Ph$d zRnu`y0S-(BZ<7e09_!8r6D{0VGX@ch-CmJ}SK<;=fpRw>8&W+XXCUhHIH!T8>ngWG zB#siU(QrL>8y0cGnGI;ipE2j7AMHXN0(@nOL#GIxx*Hs0d01gA=$6sGm+bu5A|nWO z60{8e6Xp?dim}?li{O~)9$*zK9&pq?KV@&QPNKN^w$N z?2%B|ibIsB6$#|m0}`YSIiH}@cQ={KBiN<-(i@HCiN_&! zP2}-X4Gj&1OST~L`uy?~=(${noEJu~ZAa~`H)GoM&R4EifN-=Exk=#Yf%X0~(Sa)5 zB0HwC>;}^6YjSPPBh86Z2dJ~zYVh%li^VM{FtujI(3y7@{YL^%ZDcIMdt8lz}mfZ1Y@sqi6afd*qC%*jC zusV9s+j8*fc_wkM8xI#_LvL9-rvyzkex>>5amnc7t9Q=!j7hDca2Cg*VFUu5*vY3w zmxx0e@#wG)S7Th(iT~FM3Mkg1TW+DE&fjGVc5XTb(*iWPT}zSB$?!9)SNEk!ts4DU zYaM;+Oo>ABL*1a70e7apR0z zO3$tg$7 zO=MwaJ_myp1pqXEU5N>5&dZHj@hY=J3L{Uy;Dq_6XK>-xUIphXFV4$q7Y$=6_aHD} zH5Uf3*Co6}|2XN2smLh4YR!fk#6`z07E+;aF*A5}i!|NGz^cilK#)aF{NyoE5m34JSO(BWo@g&~tq_~gPwi?Xwd*j=x~mS6p) zV@5*IxY1I1N^Aarm|Qlg0Fq|a;9Q3X{zp-p`YHR))>*}|xcU!1JvqNE70-3-FK0%| zyX;aqdtuaqhKlvZvZE-`_S0#eVr61Fi><-hjo7D>Rtv)u=#oQHWvuBiB0OzpUR+_d zJC$Oip&9B8FG~Qk>E&B9$xs_=;p6A(VpV$zgKuRc0hJ7RdLj?%tHl~SHJkY2qM_oD z`MNS1_Dx9L`HRq~GKOe1tCoIBQP6*^Vts5VjByo%Y(3uvNqQb?WxCJvT-3rgdEaNp z-HGZmvg7hrQXpQUx|puND$2PImXvT7xyM^(yo$<=KIrG&pW`NRlwGHOkEfWw&snYa zO)awkHy^0LBcQ4^nk~{gpqhB3pj;8ojMZl5c{^#GzOBy7G>YK3RyizQAc9+d!GsJdcN^V5^v1EMCBHL>~aL7!6=S=557f*=2{?9#->m72dn{$?#(Vd zfOaZ#nvE56UMLGDeYf}9cbctf>+%{Ng?|9J%QWKnxwSV{Hk`I{$+uSd58N`*>X z|NIlALR_jmSh3kgWqzGcyw`xt@>5Ve09B)k%AJg2t#^*OkCpKIx_JEd4~`{ga!U4p z-0R6Azvjvkh@6NjCOedE@d+NXvU;$-+@-)^DIx1w0}JBy*Om z7X1ddaQv8+jgR-dPR;`9ka1uQR%qkER=6Mj$6r=iD0g*jwm8LK)O@0(+JGW1c5t11 zOJJJM)whZ-Qm2MkuFcug+&4*hFO|U>(!ov>6?pNy7rxvn>T;ewS@Lf(8rB|GJC;8a zPyEBRms?oQxuAs|6nPug$;$+6B_qtWW@nD!$+@q>-lGFYC8o!BxT@D&@Z7zIEq8XW z%IVN`0|@L2_t4UbH)l;2uv&a(YesE5f_G`nMeWTnN-BN=L70v2b7AM2q>90+!8KR= zetdrBlcoKH-gPQaYAwDXb~LX3dAsHo_^Jy{uz5}njz zrX<150B#>JvPyk_l#=-GPUa8t-~CN2CujT8`o3oKjkm7xPU+j`7KGJQ8s(bi4EPZ7B#YinPte*FA)@fA=Bw?APY-@Ii@b0zj^=E%a#wF+I8CrZCe-l|e^3XW^- zl1P6oSCMv^sqb6+LyPOfVl+{;XFLOLGlz6sq^>ji_B{^Q$Q6T`WWU-3o5|19)awTr zAJ@u#zv}K5O8y98sH@7-D$W1M&W4(mOH3L4_PD#_w1pGo>RG+|IM%y$!58+c}gofMHffdycQjP5C0oR~}mi>L>|tasSRR`CE8dIby2I`VOGI||H2 zOm8O`V}wLKMw(AXMdP-CGJvt6U0PaNg6c^DJ#6_hbtb~^WuIENT#ZXw zPupPjav%Q&*^9qKjo;%IUl^M4((mNby^GvNiT)^$B(v-R1ywW3k|mFTmJHqh4ev#^ zXVFLlvCm_qF1xLpf{8vE#|?*B%kA~MM!_mFgyPpLjI<4_mKKgbUoUhIek=L==08gf zIL7#Wd%3MRX3Xgv_nLx^S8*2)s44n$)jP$G`pnDYIMT~z>b_~^*3WxplkdDp-FttI z@_$n`bxeATd1pB5gn#*$#FcP=krr;|{ZtCF%dUEGec-jS^Q0IQk*K>4`bXMLo4(%j zF1x=}FZf>_iO1k9MO4wmq6CFMevp>Ba_c`JaYm3uw1fn1EaS+D3u&G?0J_Gco6dZe z(m`L38UBk&y*GKRF^6t;dEG3dJuo2ut~XjjYs&TKMiaq#YJK<9YU6Uck!o};(|mrn zPHO}mpvruIqdwa=OJ>yQG)I4ooiyhW@(2oe1fb(8*BPVLa@dC485tElfEcLzaC&$_ zGOzHzO6Ex<^u5&LFly?6SV=Og$LirFb>Rmo8UVKO=k=Ty`Hf18V)OH9m9Mmwo2dHu zMz>H0<&#E^MWfCVV_FSie;&C?iJuTjK2ygxLv8ge**PdOG^i;?V`FVy<*+sL%)_GV zpRWx{?!@-^ksQO&un@1qjHc`EK(2LLv$aknp7(*vK8$#YCb1HpH{AH8L}rhrT_^Vx z48RP(q(&2t8@#5`u`BaKJ)lljK*(UD!Vk}0G-pB_)f5-bbJNDDa{76_w6XJ5CNweD zNq4wz9^^H0Ep16Y*TYsI^L={9^>1CzCw5-7*e$B=f zNO3&dYj5Q*e<*vXBT#&`!tE0WPJ0HJ$Tt^v#eU_Vp|u~GXNl5`HP?LX+-A+_IQHlG z{g}jC^Yv;6o#!6*pV(ns`d2#&Dwo74$_)x~4q{D(r1iRnFH%_0)0l;X^0^?WkJ&D! zeM7FRppSN~v5ntpj$DYMf?1Q#7PEctm5rINva5CO;Srs1RIf9S|97V;vwhj6-Omo6 z&tZs(-ygJ2wrg7NmONRx;TsNf$wH;f71a*;FXCoj*0kq4>Zqkp9L!En?gKvg5dOKU( z`8hwG($;O&U@m3Hx3m$1p31-^#yI|74LT^I@jw&#E9er4?4;f{Q3X9G$Z>)UpFl1^ zU=y!Q2t{{ya;R-v`QtcpteLY~F~4p-bJtms<{$Oys+umBn!wW?xeo+6)%Rt2aY`E5y$MkYs)U6vYpR+)81^3QOA#p09Vr@3csh>ifdm18cR=dCOxW*~_tB>Z7 zZB`0-k-nL^d#FsQ|8Mx|tZ6qXDA8CVr6nqaY{2Z?$bSjQ0Sg_4d*(baNFp*@v|+|Z zuv5r^w@8+%O(fYVsEz`P9beBNpiDHVW}=D|iZ{HvqhNFk(U<{zZD4Hp*rGT5Edk1L zT$f(@8b0MWX~95cV%e4!>zy(X**?VXnP%}yEk|#|y77yOKNzRP(pj+H!V~>NUWes% z3AXC4r+f~drEBO>bk(??cE0Xp{lVW~NHDi-9{Oz6xyOv-&zKms_N1G%R3^)0z5iKn z&2*q8VAi>+)riR%ubjR_GwW53f>^l6{w`RTdZk1+2$w2fc)V2r_*)xXZrP<6%ke_%}7r zpz^Oh;#9dxl}~J_#pYRWPRDL9^-*j0TPZ@T92>pTkVoNSd<0EgGi8O@15IkauG&DzsORda~9Xs6a-jxC5iaOIlA}0bi62-BCO5sdz zQ^xar9_l-6y)m8)rK|JHb&s=!_0Z86*tr_6vz1oab(xLK-nrv5ag90+DRLIqOAhj7 zecM7^V-$ARLxcnpD?9wj9@Z&)NitW5Uut&t^M=knjXnPC!{_y>YcfybX5HLh;I~-! z4xUztEh?jw4jpAm(JoL#?6&yXcG!Zc6-WsV+8gHceGrt&jrHQJP`R=mUcl39PLd#*PrF%cAXVV{APCBXpBaO^nL$&>Y zWea~*kA0TvN^*(@O8(2KY%@csq0$3NTI^ZY`AK8FxalCWecjaIFSomZaI&;HO|>{u zDCDL4mc*Yo+>)C%f-NOwkbTaVwAE!GoVEF~N*^NPN{+`c1+%83T zin`8lpQLiCP!FzV9Nx^6GIk z_YIW)kXFHJU9nRyZu0bw0w+ebvvMS!^%-m2SUNqh#!qA)U5nGBef-U=C9=~~R1ylQ zj4jdoqOM7)wbRkJGKy*E4+r>vpt*bPhxH~3rfr&k=}wKv8usC(TcC1rCUiv^?dY|_ z%)`n2)4jz4-s@#xv?6!z%?28S&!@PJZPiqsb+PH-tTcYgB6i9+y8g3ymCs!Df$;fb z;>pP$x616a*L*_bF+iECyjChajVsMn4kTZW%nFm1JhL7z=##BvZe|2cRKwlBFI;#+ zb}uzm)W6lFvZXkt-m+}9GH|A@H5kOhn;%tKMeAhX?5g!eEB(2JUc#(Y4_hR2SH+%y zjC%qa+JS23zs1D7uDQiGKIJ&rxS2EI-pURRfiy}10$&iq&J&`oBO=FN%UA6b`KXb!fFeG!hB zHQ03L=4RFCx`91eOh=Z7dG9@v9aHntjQH@)3q~rNc1Y0}R5cv^eTC1zW%2?qO$!Eh z=RagKnLn!+=N~d!E3YTXf9c%JiALutk4@UQQ#;-1SI1Qqq2tB=K{HY1eTKA&bCl7} zA+>gDo-3h0((BpGoA0-OE(u~{i5S{~g;1ztyS#ea`j{d7@m8CTVMz|(ZRJ)p>i9vV zf$AOnAZZeuqi^Gwb&54JT->KM>y>-oaphpXKlwoSWXv zH)DF$+HGM;QOs;UJK=0HV8ABx^MfxPkHgK&TNV+193Xdqy2SiHjJEJ_k}4^J#1Ilgm$-m{g2VvQ-Q6+3d)mOcjzE0_P)&Em11Xq8JXZeipatGjU_w#Ok zrS_YHkA!zeD6Xvx22PJ&^84b<4T=WKtBlGHeJ-N^$WUW2VhK1JT!hoYTSYra(XUsA zZdk2!;CQw7F}pwFJQ_Jdm)MFEFtu-o!igG3p)+=Q$0gN--4oL}hb??`6XEf=)9U|< zNm8^xYtET)cK}$96l; zlaV9o&y-c0*;O5cUvcvoE^LHtM9+4yM1<^HJXGeHW13N?x+b62Do4Y#hrQ;}mFYz% z>j}Dl-n2wcD(iksHGoaBw-Z}K%>z^B% z(lX`oMN~wng_t}6iGGzxzOvfUyF#T=$+;RqKREXqZU?Pe&zuFdw~3DdgEy>y+5cD0 z4(pW&>jgA^?t57qfu*I!z8dwzJo$1FX&!1*LYsQNS>FdHShCPoXgiJeL1#$fV~>fo zNcXfYnBr9$yu5-zlSr5zd@Hx@;t}+Po0qFu0{(FWU_kIG+M=%xDXuJF5eFn-M@axL zgwz~wf*G$eu7I~h7q zXi^;A6?AVOLW>PZjBX)TX>jWCkGJhO(!znk3uGo<2-6g<<~rE8qK?crur5p-2>6kJ z&FHS8s>da-M6HUx&tMe~JjqWK47xCGM|QA$za#yLcR0+G~{$L_J^dD>V z7=`&z%Cyo?tMj3HXPqLZ6fU|tkeOzZW0j`tfa127Mb!xk772~B&N0=_9|VKatzV{P z@hRU+Cqkw`dHki@mv{V=(l>}&ONvyuBFkvlt7}>==(u-GIH+&4p<&O^LsdO=Da0rf zJPQD;3LHQI_G3_63j}ux97!+r0f>Q^4Is$pt}pOMbx9yysRLSjpe~LB)d4IC@+)B^ ziUEisR zTGu2mRQgT#tVhnku$8uI@4zMYdgJ$?CBZ&HaxT!Q2b!lC7Jd&_h^s+-_o0S1ri*V+o)(u?r;P_q zqw)JsGRX_TTZUQ691`1G#dL6-UY~(08`n~2!7Z3s&!w`KTbmX-Jvl|qXAOei%cUm1 zN)KEs-~ZtkZV-dHcALk_0^H6#9Ia8PKGg%QBoUZdp*@R61AwUx2ZD$6fBmJ+$+xHlZ7cQP58iuzK%B&suk)Zol2klcf>AUG_fHg^g zL2bz+inK8_gf#PL_huxJ9^gt46EphlS&3f6?7og<$SvjG0sg=DHLp zC9aS^&@(emB2PHQn3@KaHEA7DZKhP_j;JmtajvMfV|xc^ba3W+DL5CFNvO5dsgY80 z3%(IqWc-}E?u{?c)BTiV;q~tS?xICXIl^sq=Y{U_uT3jq2wny2hky`>BNto+f`dU# z3&Es-JT|*w*5A+X0wC1@@|J#3L74`qrBHRn*?C_+UL6%57kds`8J0GWP(YR&@|j2o!R!D%C1#);ff zJ}rlFQn(zVGludr7BbfWHeiMr9SAYwtIq*G7<<6HdkrCaGV;)XTfFG)zx{_ z&|q|4+nT4RU97h7P#$&m`F4CCmFHzcZx5|9e7&_^WoOZ)e%AGChlxdagV^IhsZ@B( zXfdVo_F9b6>8LNaXpJALxwUwXox{h|Z6r|(H(cWR$zOTgD~_n8<8rnN8dmtRBZyD5 z5|7U%U>pWOW&;q|3jpZ}vYv>k1o#arwT5<5AE1l>Tew_g0xD>-AqN=4BK~OV4)DK9 z4sM0|W!P#^{o<+nz6ta8#neTPh|2BYx-Y90dwXV)ze1O1PT|HB4Qz2~9kB^|$W9m> zdWj6C^#+-7W#6lbJ}p+~11|Tv(Pr%`{{PF;rVnBe)LT<9R6J&VFj&iGq&^)mUsmzu zo;^r<2ke22=nhQ&tt&HIVp76K=soj$Y}7swAV$PsDE>lcZl7(s+)K|Rn0alOX9IG^ zF1m9~H44=>gPj({`Fx!%0(^1!JYB{`#d}rSa;cE$sCING1rivGxxkeE0I*v;ST7xT zlcg3Q<%-BXZQHF0Ayx9?YYj@9=C?}`$*Ld{Q8XL`L%Yz?Kz$INhDL-J$+I;>&}4)z zgd#qg;sHaPAln49Ct>iL@Y4bU#BP?Z_Z8p~p)Mc5hS>c;Z-dLi5h`HRiI*G!evElc z2F;{QY>KSzIWA#~fWiG>JbWNv{cyb-3ooEb@eQ2QuJ6z+!LF=u#l8_boqPSbwj?=w z`PT4UQo&f}{m$5!hZfX8ZCzwGuq9Fz1j&t-U#%AV7gY?;($3uXlvjjWT@Vf+}lJ$l?;P64^kvW`a;7< z_bmXNH%xz_f*D|hgV_KeSO5~0h-e#_N#-E@wL4*rITOmuqikGNCQ@3`+7VfteQNVi z+)ev~d!S`7RL@P?!P;`!2>x%!X_MhtZovV{OVUV_cM$vO7T<-ZBU({k`D_JC)0|hY z{L~0OQYYq`UWH5^$XVG7Zl2T2y(kg}L>x4vw7YqSfI>R~ct2!-*rg6Y9Pfc?Y6$#O z6LMF8X{`cQ9-WPqCbblCi=JmUybiq&F)LI9+!wXtCE2k~lW5wE86M+qLJk-lqKCG2a{|p(-3Kbf)>~+s`4* zX>|dVHBY%z*LSZ_*YWwn0g)~m!^CwQM3g3-%@d)9h4lmQQsunJI3Vr; zXA2^o_kn97_C8WgKqe0$xJD*cP_hFknmte*T9-4&ib=ik^DCJoE5E4urf`s+9+=c~ zmVi529{HVKdVWq$POA{0O+jSch#>(O!0$u%046gU;02@rymaJah^ZynXB)=Uu%VfhL0?D73Wo)wpX+FyEhrs$U+9+vGfiKrq@|9nvc8KNW~XTFM(C>Zi1<#VHkjFK>OqE9QXqn zo#~th!{uRmi7Yf23G!bSKfhy_yRG|KTX0M=Z|C1dB{T6^{ng6V-9{@n#;drKwJqDf zq_w@vZ(T3h8PDND!Dss4^wbBJ6+5-?)L_q_Aou861n`sK_2n8DtRIWx)D^^)tl1P{ zCYMMh{A~7M-`$wGOhXw%2 zX+h$YpbhlDkR|{}A5cVTuRuo`n~?)M7X}oqkU8^ok>9&>r!O05Jxn=RGxVHC4R`J` z-7GFB6hhI$KA3Z^M~r%0L#x5&uguqol)}>oWcKMpG;8KJ*Pd=kKOj;{p!9B3=95(H z-ah@p;P8b`F0~}!DOO{uP-bNtg$w~T3iAU3=n;(9tYALq7rwW6RFpFH1tFP%6hQ#y zd{9?#1w1WEdyzCb&^EQ=t#Q$7tqFxY?Ll&> z0j%1p-G_OCw#L1Qib4YM36S-^6LYek=GdgzlPE=;xd^R~097#7!3Bh1h(Hrut$@K- z-~FwvK+l;6;|On2(+Jc__8PuQi-OuB1!h4NDwzi5k7y7%IKjJ97c!vC;39Fw^{m`k zhr4MwM(3`#&lLJ0oo9C%zO1XC_D4e5tCXC4Gbj=rxg=tyWLEQCVaAJ5tK(AFC`=fk zb%e|nsaCxMIkc$k{bJkpGI(?ZQ}rU$5PE=c;zuYW+@c_C-{zV0lCy6F`usn{q^MlxDn6|UQjjv zvPUkp!OpBIC~yGoI0k_Vgj$19Vnubi<}w@qakJ)vRb zdH@<;<}j?@5OOkujs?D{8>*QYOsFTMBJ)SJf4t91=JWoUULTO#C0ZJunoCa!`EW)D zDp-|VclUUzd=se27VtT8sVLq_qeW^@dAw`;#{)Xz@xpsuCL1qfbbKQsQnqaSpmyIRS z;ez_;1JT4|x2|m8PIEn`xi;lkrd8q#8akFwx;IlJKD>a;Tkoa^C-Z4nP7f^It|zV% zM~)Xba5lS8rX$^z-oXHG-YV zxbt}{oKRU%L4bkkS8d*#tD*|V9*@KM0qPC5>rnh6(5iq-iK1d$fT%|SP#B_B0O*JX zZ@>-Xu`s3Q#|G3XYB0@hpW{;tOfTad1sX~pWfkF|g^3=+A2|?>iS!nxj3}zyd7<}5 z#~2P;{0fJI47r^De%dnqH%2~*jbFytDng||hQ;daVS;DR*q<@Z3%*GQZ;LCQ-L^iA z4PSb_mwR+|(U8{iGO1v)J?XB0VQ~N8Aof&otK-z?9Y_*_O&aC~VX_tPHky1pHLApr z8bvK7pvchbXda|AGJc3S77Zi$A@UTM8d3L~gB&6O5PAsB3B<{lSJ6-tDttMC7%R#u zP=EoaaRhu*6L5bV*Z11=jI)x;W~JLt#; z086tSZ=;Ghl%8cw#C%-jpQUKj?`F5#5pel^lE-;l2Ld$d$UW1Y{fzV@>U?LJY<7R<<-o^A1!nuLuV(gus3 zZu8g}(oN`O#SRN1!`5YzPhFx+4c9K39`kZPxbyFHwP!(=zZ5hC}>~@_~x9@4D zx?+os;ps;mG1sXJhQW*+IVH74g4sWHis9`OCJo+&wmeNN1ascXB|BH29Z(8L$@uA$ z6Yz&=ReLU0t#hqL#dF9a^{Ph10PaLyoR}(4oHU09@3?0;L5c2UlXRga)rwg$w?Z)zg2UfkD=HtE+GcJtLRX`uR+C^I1Mav1q6Xbezl{uJC|W zNXm364dGt8?soS4z)YqjYke+8g-MyXe*^$81 zK5=f!{q&@RE}yr}_`I;xulml&|%CS>&`rr{LBg@|IGex7f7?=aACkm6sler{sh8E6Y*7 z?l7cr`$=%e6F)}F7+Chupml1OB|9j}7>S-Ys848Z&inVH#*ygS%R z+0e(6(yKjZ1DT{qg$*iCD}Sx8>oKJ-f}!k_#epbL^d~R+5VaWhzzLt8Zs;&gNYELjAI=Noj?v z5lIJY*G{-)inYgFFF%!AL#8FP)vCzAGBJ#Ij-oZBUWFdJaG0(yw2P;B?vqbz@UDDI zg%@ti1ywY2T-)+u1?|pEGEO|a4|hcH^Pvv~lbt5~D?7Nmd&=8(J!Cq)fg$J3Noyd- z5g#E+ed+)G*t!AvK-r2;8gZA>J-O`#642mNe4P4;G46Ro3K*k6sEV#c$unH}-9>jf=Ej?a@0 z{_NdP(|EsI?ilR2lReMnd>QEP5-k|GI`4SUk`|vjJP;_Fx?LLP5+U=_JHk+HN6HC9 zH5W!15BR%X6K-(A(A8!220F@O-1^-z=V)>=)!TAx!3=>-32+_y!{hKH|D>SV)~o#w zZi7w12Yaz^D0WZW%2bC}+RwUg5EnJFuWgK8GVE-NO4YhWrlqq^54s=fCFYLS_$hWK zJ#cznCR6=(tQ9@{GSxr5qrcQAr*?c_rus;^zk!zp*FJAm3%QRtaiRj=JFjavnU#rqw?XNJbe|dpGOb$Y!gupM;!{ zAfCadl=fZEdV>~9d%1;t2}E|y%aYEIKl8CCXrp%_vb0BoSG<3Rq0^%Hzx$Y-`my>R zw1HwS2}uXr(n7AbE^cm%Yh#hE{fr0oi80O;{^S$=ZpiVLJu%Kyml~~SfK3;?U+CmS zTV5Y4xmcxjvG#f(lTGWrU-uiwP2I{B>Y_4|@vRR?*~e`w(@s;r_tKJkIj7CkBMn}& zQEtPO5b$nHL?_l;0bG??s(+~0J)p(M0=+CdM2xNm%F?s_F2N2 zvU`)O3xo3OkFnX0JVYK7T>Dug?SsX1iGh6+sv|~wbAH<){ySiYmV7h5B(wChD(irG zwL!-^ZnxeaZJZl%f54{T@HfPIXknjKR{lOBf96YHnLnU~#h)*cJCgXxd)oujx6%!j z&&jX&*#Ftg%JP4lDT?u4$d-;;a8PWrB%2O?`nz8t`kvXFK>KM{<4n=HEF1ad*Ly#C!`>NOz%V z3LOpl*C<~?MtRBgNm1rd@jD+A(vWi#`R(2xU@~4W)3|X?P*xw1O4>s*Hd<|AnNM<6 zSKwhWTL_Qm)wxy@Aq@G-JbjYY&$^kcuvlmtEGzx;w`+aGgYJ2+RD`(lId?~eK2|s! zrdIsWI*qfAs$n9!K0BDH-eR=3ik>0&z)lA2YL5lgr6XTbRD36Rf6mD>nW=g^I(3Qt zv6RTf$@RhC2DF1}K>Iz;ZPcyzo3#*Li}EYoaXfDeN+z!AU_EgZn#ERvn}dH~j^1-{ z+~FlotpTZb-X$IMd-U!^E^?H%rMq*K))jX@Ai-|OOJ7O;_r_168xO8pT9JGk<*31H z`ZbI%23!eA4L?bwZ;m7-1uZPo2htCyexf0?KoAxrltr}tsg!oQD(3pc4Baa{uawdLkZ9IPB7^^n`*FYl-lNTniSsJ|?-SFCPE7u72(*;_{2^G~Q_vcA2Hqpj z^>CZFPvnN~5M#cFM?t6-y#I=y*V&fU4&6ZubX@w*+HNJEfbSpCT6vbF;+Lmm93eXP z?@~WUb~FYepSmK)M)1dErS~Q)+2}Yu+E!JgGyP>;^)+{OTauS#3hXPtfVKJu;8h4{ z`~NyJdEixUgI8%L9F|%CU#}8Ey=y#V5b>NQ?nwt7Cc>!82sf^A{b8QMm0R+ei$)F;erVS106{A*2PrbL+IYIJ?)I2SnI%N|2!B9*Fa<(wuomYipK9?%{vKx$#0U zct2V0_;(SWghk~{^#z{@9-}XHw3)v?+x00r?4S!fXvf{0+VMXYDr?uI<>MzBHtFQL z0nwzMP+9n-ThthA?1~2OCv6$Yz1g)f*GU=_x4l)gv?TIEp-a~P9Gz`w2Qp$K&~e`w zdhRhY??T;z9beN_HBJyvjbr${S;JBhCF@&5Zw(V8b15$IkS)h)>3#wtwq7^ary{f*LMm~4&a%|5w zV_0wvTBQwzGTj4pSNF>+U0?KU-_^(7^Sq4<*~yl|y_2faSHgcEoCX`}4)+(EG;W=A zKK|Nc!sV$9ByFyj(%~JQ+?(ftpKy+;aVd+rZe3Nlm!p!tzm+i0=YS0|%nA4^vgCGy z+qus`BZ+fcBb?tj>mxYnTJ2{w=NMPIW5qQ`ul+wSS~Ermr% zLF;5X?v|o*`0#7mGe-=aR=pt_8^7un-b{SLYaTZwrqA&3>W+W%n3R8S`!4n4{lMoM zeCwpykSP5#_d7Lg;$U)S;PQ`1niG=G4JcbY3F8uuR+Z8B;plB)ox?&mMxXyA=(n0blElt@)8x-M9F}&wkP=Rpd0SS&w9H&#P z%u7d&1^M*AJd60GgXg<$&DO!Y^Dv-EA*)KtkeWtR#0TDMkp7WrIo;m48& zaZPMr2>da*qK0)y>Al(%l7B$mte0mHND4Q)}#W>p2nQwD8H{ zvEy}JF!q9ubq>UEk0Iw&k;jg*5Xx4FCln@SPJUTc#`*qx1W7MzZuzPs{QF)OL=^<+ zjNxd1xjKCx(T_0d!9C6aEcQW$%+r81nXnk*#B?1~!bQ#5$+r@&ylUa11k6(7FGnr& zUT=>vySGW18zCDBcbaYl?_YlI#47hIa$CsRe%3;s@9%}Jcjq( zvE+Jf<{>tF1k6JetL|ap_3YXx~dM^CGk-1N6MHjpLCxh z{V+3-3IXa9v%tkx0JS7Rd95wNh0@EqhiMmv!fP_2l-aqaFQ?EqteQ>DT^O$HZskl^ z-1tcV)0;wJ9@WD36;M5H`$CjUB*Zt6rLf*z-oXhc@d3%NZ4KLh@HK{zGm?AcB^#Rj z27Cm07e{C$jK{#rU4^Qn5@>7Yi~?iFD6kPILbntYm9N3hLFIHbm4IfeSZZ{)xXPpI zI8fl7Lc=*J@Je5ea{xmV0>nYHodm!j&~I9J!+I2g$K;u}yKqQI(CaL9vH|o)nFC)SafsjRlM!KB~!huM^*WggFl#rt|7#BgqWt$Y%n0o zdi6Wxp~43}pM=Bb?;`bXq$dRv7Ci8Lx1d&;3oJxc56%-BIJNBh1F_Q5wTSqxiZ?3` zrXCSxg`DU-9%zk*9_`tUag+m|BWviSu`DK~5pyFzFc0K}uLB!|aB?|RdgH1b^`78C zq79fHj!EYNbfWUKIA}m1ws1t$JzFE5GW-Z*E?YMsX0qDOe*s&&mSRgy67D0b^pJHm z%Wp4b>g7!4+7K9ONZ3e=Yr8Vv>D>0};m_^_LsJ8OT5~1(;n0p-rIIbHpG6_%b^Q>R zOnlw;sP`3G*KnXJL=!rQDhOWkiFP166lOene) zvPYY{2pAe@cXDp{!T_(ysEP_O*Q1L3Bj<=qX5nBYg|r^lN+Qk^1{N0Wm^I~vSKd(6 zRyHoeI95HDXC*OGo*Lw8RWyTrAl!cOL~d2QKjl%oQib-Vn|Us8Qd|f6U4&?MphR&% z*S~_&^DaKSDkij^j44{rk|ExecFPHGN#5v4j?);Zn3;V!>9NK2J18BoOAm%xc~kZr z9Yio8KvwYH{v-h~$FM*C6nw-}=glmF^+RnJm~lc4|IB3=G-pGE5{WOVidGoGYK{&D zS$Q~YKn9|cEjOH2XYF$e2!T|!tOHS(0s8t}fHi~3E;2%V!9I3c>UIYWhq-C@9j;#j z3?^`&_p-JSKDYA)>EO4a#M?FZ^ju&7PeP(=3vn=|1bj4EnFn^8$(Xl zpG9vD(AO=pW~xswMBqP7C_&cb`sJeOsh`_W&WI_VsOG3}zFNML~Ar=uQP`Okubj$%9tkQ5|*_)Gm`~@X=fM3Kc?kdh2(^?9^ zYcAP-PHVWgv_`9^32)C-O3GKD>jnxTmAG3n@VfCuJc%a#8@%{n`j6}H=lVBNAqJRl z*tM;UrD@z>L4ci64mXovi_drQi479pun-t=2&6o@sww*ihK%H({6Nf0YhtF(9>zkG zKfhpnCL|nq8AD)tSm(Wt#-78nFxQ+G=4$AnA<7&$R zOrt9dTxIOifZ3R~nB0rOG7K7UTTK_4kX|UcbHyoYP^%(Rse>KtJZvQ@GHXjZH`lgG zX^|#B>@c96eL%7g7BB6KJ2Jp{Bz|e1zX>U-!K89&g$7=Z1h2kVEg(n}fQVcgupdPr ztZaj(x}dd}c3K8R)N6rKvCl>pRtUCCUAxwBLQtW;U_Cb3OCT> zLS*Bwf_Tv~7{!a2J+v(nl&9_0Lfp>-kJ`SMGm6mtnGlD4BFDJQi^x?OF33uLytphjw2 zgcEl_4y(&6(`2u&OqlD8MF12NOgVYK>z9!(=~!Xgn1QP3%95aNp86U~Ne- zbiw$}e{Y547;4htRpbv4Z;W@bvJjg;XYtJYSA?r;*V$IC+l8Ny6KX2BLy}sM*WtAN zvR{LDAn_TNA1WN}#xz}v$ zY-~gDcR_53;Rtp9kaAr>Kmhn&e|KtKAu{oem8sIolR;&_FbMISFH1uV@!C+0@03*p7k5L)B1bd98BoZq7a@!0 zG*k=P&*y}5k)1dds6CySEwy^Lop``>`D?p5P!&UrgxZH7i~-I=bRLum!>p=>Q233g z{*YW_xFNhN97yUR{|MkAhY?_;s)cUn`*@|;krF!oxwen;epH4o2+g3uj|WscVW10B zOh6x?6SMGK?0Y5N&?g%G^&r&u6dfH#P4mZqWy-J_S4R|; z>+9<%2)ba|_w~r|muc>B`&X^|6rNJbuYX51+I>IbrfzxX?TuF54_ci^k3@PFksZ z?+^N{oq{?iqkp%8o3Tj1x%Wx3y5X@<^v`=II&~||XK6mWpSaF$X)RZ#%&v?p(P{Xj&CM~`aUT|-6;uvjaQ#vp26h-%|Nz<_#a zVU{@o0&KGN8}HtyA-Sa?;@MkZDVzT9rECy!1tRDy^bP1#1DFscL@gP{ACNRTP@$4; zc?-XTNoZBH4dpIRUq{dfamv(FOH0cJ2!})=q(;~tqLU@PYc7{RhF~c2c79 zB$P!ldys&o*I9;jCgFWZN7#_yjzGeQ>=`gl@elz6jBTFlgGg?_*!0ArRV5e=IDAqC^ zumG>dY42^B{=QdxoQL!8QiG0Hi{~5<)bQl*?tGAs@@Ows8jmQWI$$0R3kj@YKvff|`(>eH(0T z9gr@yd#m2J5b(?GH_U<}9$W;0_J%`eUX}J7l;BEzat`baq-NB9)Qnm-#(5oznyw!px_Fli*c>W6 z4_uqFrskLp8}%D8DJ9TCHmetqQD-~ z=1onaFkQ;1NX_^4_FRH&#}*A<=nrz=Mw$PD$7QF0Cr3!>WSo(B0=1Ve#BcR;l)IVq zS;aH2%Ng26aMsFCMB6b+&ga&th)jKnt^AqY_8TS;iWVm z56YAJM$5~=IUab6GCo5-NTFVaDueNhkGOn~NW+$)@Xa*@dP%+eT)I!DZN2sYL7Lc= za6P{}4stuqMWE6ou&P>(2RjN~VP6p~1XnSqK~N)O;MKr%yw*{WqS1)FNauAws`+zo;6(?ISSW?7s}aQ%8jOQ;-u!7EZau_% z%I6MfiSOaCkkdTgxjRW_!Ofh#JooQT&+=0NKHyYzza?uNO5ZZ6^6iaQ?9$4c?=S}5 z0L8e(xn(62QshStp0VgF)MBKdb5KOY{u}79Elw&zKOyJ9*W+&St%2+P>CmA?RY42r zj43epR2OKhE6$scoYE(h{#!f8Uqbs|V5oR+MBG#gFZH2$*1l_@YA-jXL6_FFR(Y9A z{1E}5Vz@cA4PTdg0iG2J@+0MYztXpGd~bk#QQe;(iVnTId|i0fdf_cS#gU5u%!%r$ z?#HG~X;CvQ=2qCBgcIH;>$c`--PQ8*4IZ)C-f+x?pCv>=tZXdTjr!bwyYSyPL-%hQdY@mGa<;M4Ld+MU!A zx5PZHxZMkAI6(hoP@LE^_7LzYdjT>84MuTSR&Ysacs4dSAkN+Z3X*NRWgd{; zq_4+EKIec|r1ws7!?>%h5r$BBx0j^JyM(+B^^&mGMHm)W%D6bsU@*T4>CNLy75svm zr=oi2=oyLkCaW!kVY-j;_cpYH_cOC9X+}Fu%6}qLu6|$;AKMbV|1(#RVSI<7JT_?F zHkBHyj{fdfk)J>XUSS(loeWpoY5!ED9u!#YuF#p}6NUg}aBg1q&HAdjk_$%v6S;>k zTW5XWG#~pD|Euq874a!NCZuiRj!|v5c&L#cy5&e=WO0~Ovdu!$3SBx?JV&fZTcbv( zO~+z2U6WN#DlAIBwu-y5c^ns|uS7kmEgTMTQtw3N`FOG}e@ZO=OT%Q+!L}kGfNGD? z2NzH4!BPmGUaK06_UkF43yRR&;bY4SiJio)7p%*^)}_(4G|FvKeZh@kuZvj)z!BU}85q?cRDSNgtY0%Y+02*c_6jzPT6*+Ce34;ep|G_gto@vP#5=f*q^ZPBJa9gn2drW1%QKovp93JG?|y&9Q=k!3ocY=Ji5~@Q|Jpormj8gywA`O z7lyjiA7GKFFg)rblvCS9zH2;p69n%AL`mcs5h5SgTz2RS4F4eVF)(t0pVO?ex=?m# z+-bwK;!Tk!%Ac3cE?)i20G(mTuS!&SD!xPSpR_m+6n^N6bVMDU9dDosQWHTS(p;L1 z0$2nU!qMg3bt;Vh=PXXc1PsDv(bxO5L%U}GdSSDQr_g8wE#Xm=%*(EiQ%cXd!F6cx zq8->=0MM_^565`ZK=Y+|%Z^6J=#l~r6A}PKD~0DqkRP?aPWj{_-FSS1a+*s8fPy)U zaa%vaQ4)J|2RdChc&MGOC#{>nywg?;8DMMwZF6xW!~_1kz85O)5&^;#|8;_K^E_yQ zrQ%L#f-*Psqq#O}bf6qk2wRN^;SCsMz-2+30m7c$0CH3xn0J8u3Q&e_zk6JG2GlTm z*H6h*SZ-}>0h?AL8HaOS?@pD%coyxKg07@z{XlFtvw$>>* zle{N!S<0@}*^=XjF8o?d&zi#n(lbvM=gPj-F-p6Nr^trb^378XUwgmgxn&3{zb;75K%28U zP($+9Eei@m9i-TzQuE?9_Md=XYx|K$-5FiXl#@O5R@H(4MXS10x!f;P&YH(|2_TLT zR&DJG;eoB`Eq>AQW(h(~{Fw@<30u`yw`O2d!F4ke=p~@FrNR+v{se?E$e?QhobGbN z!^7qIHBbVr=lsL-ACECQLe6-wO-bZA3z@+2NT~&1_nlbR;HN_LLCGmalt)mM5B04>yV^_2-+Cb zPmQK0yIBt=0obpFuv~E;0kOSB@9MUATb|0cRWNG(IE&E_Xm`NJ}mz`uD1|p=R zP(H7R=G;7#8L2u)koYC*xU4Bd;fKtd)fNSH6{ec^TLcjxz*XPCOK}B7mI#?Vbg_)8qwo|fP!uN352 zsf#QH^X5-z=#Z)j8b8l8ghW5GOUx z=i~c4L>EmplJ2b52!9&*KyXRSHxVanTrz8g1kW8B6cGfM(+$kEM%u*48{-d1fVEFp zk|6VwH_?6Go&V`2t>QFc(ODpLSV8gS01-~458CE7DBdT&VMAd#-ra)zEN z)(?r)$FG6dlr=XOS1mE6(7wU?!2bPg(iIgIhCxsUB?s-5_>LkPrld@&*BZDJqHFd?0D&T8A|vx+;Vp~T3k7CH zuOf87Be{#3x2^ucR<&EWaj9g1Y&-vOyd49^edep|g#YIPioJcNTjowDVzD zI}RG~2!((e&Zx0lT3RZNd)vCcCGIdKVh2VUKh<=Q`SxsT(m|BfBHSie3Z zwyqlZl@w*(gK)~*q4RCk$N$D8-Pwz~189wFe_PwzIU%alrG~&GH-3&*F$NfXHH}42 zof>$tdRW9J@#?i}NH9`$CG{w%;Q@#JD2%q^pd*g@UTCTm1U2G8T$WurL8r($W^qU4 z3e|yeI<4a?0w`KjdlklndNr$1a>5f;ldTST&=!E8ik?uyn}1t9UGW_Uhyau?4giX~ z>g?o!IH})edwr7cu>SAuRiA01AzkA|qdj|z&(fkw!Xy8v zZ$%j{%ORQSko@zUDKuV&8VF4GB)UqJ?skxHDG+N^;=#6m{AO5>c7l&_1WK~0r3WZ2 zRq}I2KjC%NXFSv?3lSX`jKPSK@{>A8i$~Nv#uRg8rl}stwYMH-)1{I1fe|vjycjK5 zPQ+*n6pUduEyg$7YG7C}CNwnk+&u7h+=oIrnp#JY2c%KQ-K1g0oR3?9;WTasPE5i+ z6iU*9^Hu!F32ECegcrt9zF}CE6K+8{dv(%mDdxMgtfG;oR=@Nu6q~L0<287*4Q(YC zvNP1vw)Ved7QXUUCYC68KP5KPtkH>f@|-LJp`W7Oaipn~tZakOVCV;s&l&=?kviTi ziWB=Z3@UPoh-e^HLa>~o^VFF}=!^}I#8fHstBSY zN$X%}5#0?c|5s{S&`NwqaBDnHx?_vs>5NW&K}z3oT2p&JYS8TH13COs_eR}>YRX)L zN8JW~z;Acpqtstz>xN8c7$9YrE_>5!-v$;I+`fcS zgi|K;hG3u$P4yw=CuR5@m<~MM%8-OSj{?akR0@TEw+o0E%6I;-6#1o&x0VEkNUCL8(@xnpTnbBMJDDa@fqg!@Ju zFY3UnDks>z{U!@zajgXnRO7azkcB^jgnHokdm+%TfskuT3rbT^2X0Jep;Slo#5 za^vTLN}zXc0kwoGBRqH1WqnJ6ZAGu#-!zq6m5GAjC_^6V9 zoVv=}2%SCbs_yOGnO_rtZMpe7!b%8Tt`s0H9{!uc`qo4}7WhwI!}=40Q>y zGnvkl@i8?HHvr!h;PU7}POAZ*%QYGimvnMZb&))jzP<0#R34~{BA$>hx~kS$*c%TP z{WXG50WMJsUwD-@$Dts)c-vjQR-!eOqvUA%IV+r7EFqi!0|Y;PqvY}f{So&B8|jIf-N z`n+#>S<>|8l-*pj9sH^C){1t{(cqSmRHII)!~7MDTRx~=UL93b{h|E#PdNuvgxoTR zVba7!;1=}yFBb(b2m;EDyJI8Rq#csV{mw)q^+fony<(P3IHFv2xh@76Fll0;BU;y-Q3hqnGW+{`t`q z-9wp*rQL~L39Q%|2|8A16^?^_Et%&_-C)31LOL{ynMeBz`Q&`B z^bDP+ibbQgd=g}@ne9!|MQ)+xMU!Y#z~#Q&wmlw9F>2hY7ub_PqYVwsgQnTAx5$J7 zT!D*Opk8?E?Kw$Sr@7aeW}V3}v>ygAuUx<z|POv>5fuRE_CAEB+# zS>a13zFVKp-HPwJdUf)qPnhakmXzJhF@Hbl4`*SFnN(0*@qXt$V&&l}lkw2PkIncq z<<>}BOzPIB6fL=ak)f2?q11nVEaodV^utVG*t8;mlVjLn?JsLmAAeG<0;z#9)2jCg z?Wr#xEFbMwji87(#zP(3aMVF*?i!wB-RalhqTsh!)6_+oaHP|9PZ74l0&dQQE14vr0TmP@i=jDaVG%GA*{`O?M+sphin}@LZ7ns zEM4+tGhAgK%IZ4K$1x)c`yGD7Q(9x@GM?&N-n|hHBp5xW$5^kqguw#*{wcQ$Nq@9k z#HR@+CD5gC-Vsm>NLB$DR3OMh1PTaj2vJ2GxIqLqM9u4mFk7r*904#*Wk4<3FO0_g)n-R5BIpFZCBsZ%@y@JtDbhI zLPdizf3SbxAPm!F{rK-;&>ytu7&xu{rMg$(*BERAQ^eb2^Y8^#hdw~ytqx=Q3k?Sj z91w6?c_>|n1lb^C!xLOk7xtGh9q>+UC;@_tT^icyNWWxdaWVzD5LiO8%NcFLO$o*r znv68^%v#i()z_VXB4(lP^!!^#7YI$&&~%fNdq23qD=0SCgjV(UPiU5qXVa(IN4x(y zdK_@{$j5wEusG7D@Yomp_{JRr0p=SxTt2onh)dzagdiA2I;^r zA|H()$tX%Mx7%K*tMyf~ztLsa;g5ISi1dVWCi5jdoG~%|4c9NEWE2Tv?VO>9LV6z@ zoX00Bc5$wmnXa%6Z|bz{mo)Mt_3?=lPwxx9Hhcd+hK=kwoB6K%em0Q)3gnCRZMFUy zK|JHHeV}&p)w>FRK38@q>l#xE+*CYJO910yBY>6Y|5C8&K)Tc)Kq2Q^lcv=xZ9w>Z zn3zpWudaj44ZV%~fNC=gf`6k0OH9_48?v?R zd7>mJN2qUTZG(O4}Az#9n}@;ZUtO}AUO zI^;5M?u>{#iI}9_%>6dMc5$)M(%%fxz~I9s4ZtLZS>rl5u$*#1Kp*!<+U{ z0n@ycA1y5{lu!Vqv1#hvkB$1SZF_}6fAzqooW#yv zNjR5PL;Pm4w5QWZPEO8k^+Flv$&(+jt8+ZE$7Qk8tR?OGZA9P?|F<2Pn1+svR_;SZ1qI)k*;x|gp_WZlRoi%= zCJAgY8n%yVRn8~^?k6JTD$1|M4mkfG;@&)->V175UYe8!wM$WkiYQ|!nKiSFB}3*) z2qBbtYS3uNl6fkb$0W08q!5{sAu3a3C`1U)^=6;*-RJi^ujhIGcm}U?c5R!r*5~ti zzwh^bU)ObC_exo9Z>>ffq5OUK({q-PoyG5TimykNTmC)L;etOTlw?S4vssJ)l?(=~;5UMY3yEPwOTI-cO?|rf$BUIa zsKh`7a+k2~cz^raBb(SP&u40#T{LF-!@0b8Cj330=9x4L(qf71KUM=~mIW4|%;bnnm2f2UGy-RsxKaRIix1*s>X!}n0q%KX9;FtC5O zmln6hI{(10?@bf@F!C8Ds234jV+8o4K{5}5$N?aH+5_4AU~{%S4TWi}JiGJZJDY^G z;8}wXj_S?gg)O~#ZGIJ_cO-fSMgK%t95TGozw%!vlvZX-#=zpGxR~hDHRlGy+2oC~ z{Qfgx{NHb1NR9Gc$-yV}V^AOApb|DNOM*_Pzb^g`(N_xS#P1O+e`q0mw;Rk8)+lTq26;rI8$X_IRf&Q!KmQKt1>f&B@seqK7jyeENQHdD4N| z&cq@%x<;RWD;L}uaW=ewO3rCC)F5m^*sbSpwf5h8n{q*XUXu8)e?_sVLpZ&Ee0|O) zY|B5so|%#Q>mOfUH3aGV=T{6tBLDd%dk4JsKfl~I0^RuM*Kg9_yzmXfQ_rs~B?}iW zq8PNA5U46i^_LkBMLciwTi6uGQvdxKlDf%gTTE|7VMU0-w~(tZNZYU$SZ&+w1rp_-pECaOWAJOW&Tm%!|%YuG&d_A)y zLay#?{>s6>d7BH(0MfXqIR3Tw@1L~k-oee?@*&-)qGaNgtHzIwbGA|xiw|rz`B9W- zwx_BxjG=CUC7VI{@86;CnErkwUb?&>9%w{I_(d|lQn6{_`FpWx-%T;`5w?~Ww~p`N zAM-X&PXAY3ko~^|v2R)P-x2oXCiZ$8Hf+%NB8h!OLn}m^;&4b4tJ@0WT@)uq*xvkL z_i6sISN~sYDk&cNe=aJnn<*nO20CHVNEWc5Y^Su7R^<7w{Q{a{t-YE(Vr~oneu?E@ z3(8A31poyt4hea3Qc_aeB{ZU(|GuyQp85UH?~`$?c$dFb+`Am3nE135NGh)r_;*5B zyzGBolNT<^1SjI^%V{#?%XR~Jha@Tx?Rs#8eH8AuP~3cG`uApOID-jH_e^w%-4}%M zeaUF*>kGHc2;d?*UVy44=&PFl>=t|Horvf<&gcYl-3ke||G&Dayx-w!)Bvm0h38DB zpk^9n#>Q^{HPC#540t|yGPH6Ro#XD{MP3u9s+LiPpNFnCdik$C`D~>d9Jc4Zx-m%` zHf71!3WXRad=Zvvm>KnP&}reG=KSxQRPJxF-og(6Tt4yd3tsrv|Lf6V;cfmIrksM% zV}rJ6K=1i%-T82BvxHA6nVk+7FKEM&C1A94Q)UdB`)E6q&bKN@OX~nchNd=m3g1BI zBm5P$Ih*MktW^Fp9~c?PO*x~Ie_^iz>9EK+x!;YUz1{Ke9S<~P=kGa+_4N| z1r94pt)oYe<_y*|KLmJa@#5MFIJ$>m8tn#V<|I%N-*Cj%ld0`i{@ttbWFx=X9zg}P z7YtqC+=bNTGnd$99HLLiAocqI&3wZ5j3U*+5nKuz%$!Y}3tg2mA(0a~Dzs`;>BYThyZk zAjQhF-DlyX8x;fCV9QPRpI%z_dP+e%uuoun5gVlc)9ZyuiM4wcM~adl3g{4 zI-Jkgr|7}4U6@X;MWx;Xo^f)BUYxi}ik<&>AETY~@2=UexqIVcuZb-nX|&Q#9v`Z~ zWPmn5$0`h|OGDy$?#l~rYx(4L|Gu6CNV$nF8;jlQlBcI9;kA zg2s_aI+1yF;xuAQOB*9bjew)xqs1@>T`w4KP>#W};W%fh>c}Z8sT5YBvNMPcExvpT z&wFr^A`FuJjw>03b*^iwpw8pNO2Pc-=ODY1q^B5Tmt)gm_~5|t11(s(H)K!3$S6GO?D-xI#u7$Gs+9Ya!oDO=fR0Stu`1 zSRl2!WBURn23^u>;(uq=UaSLaU^wO7>%Hv}&(WYA2T?My)JCqKe(l;4j5bKcLLzo? zzi^*iKHzk`jGuN6jrH6NXl;H&o?6R8{SZaW0jh!2J5PK--_ z>N@cO=u8TZo|L^^%aPJ@qK6u<2ELT4t#9j1KK1+e$KaMq#r`+}d%Cbe#qwz+%EYY> zCx-?sicz+vfYoaKFoO|~7hf}JlfG<{4Pe%uVE*St$fK)r+aGvxaGCej#pkvMrzF`> zEH;&WP1yyLL+rT*45q4a{z($quCA_!)^VILt8hnK zf&C>6e;}tM+AT=j*eGP^*3gNXzEQMB`XQNGE-9&t2i*TL>mi|K)6&u&S~K47VRcar zL+(YeEVKz8$9N1p=UyElknd!q2h4$E(a4Rd3`uA>m^a4_>|#W=w!QK(sdlLJr8hsI ztph(mZai8WqDe1Ft5}e;QEah0M+P1)MCgT6 zOiWBWDj(kq2(WFCWuMw-jyr@$6pDSm0U0Qg`|l7v2s#!)-cb3NUj+vU0tT_6ZG3E^ zge-zEl0lN~7>S5#!krAc2f@n3#+FXlsE)Z=$AUR_@jp+ai0LAU?x58wpT=*q5Q646 zg0!Tg@s-dysCEg8i#6yPEH5w52d2C3xt^B3(--flKch7S4PL`0c@AhE$ty^QXm@2u zDy}HVW1zTLnu??UqLpdce^oKkL;!BQ)@a@`q)%VC^ z($J>c2g0opCe;ry9zYRB)nC@h{m4wg;y#CCoOEoU3v~n9afpNjZF%(-L43au#6qT> z3JMB{d%=ecgH(k1ltVYxphg(s_r!-KQ>=sT-6M;RIHzM95rw=voS=ytw2XF!kDs3w z8A22mmW=Y552YMW)J63iHcqeb*C+Lz%a{$&dNl=TZ(e7iuB_}s&`IRgYUpM@;oS<8 z&4Z*{3C6ZefBhAUwzBT7E+;!dwBrH9whoIVLqZ7Oi^oDj3NG|-Z&6aOV7Z<{{t^AF zDNX0$fvd}Q2?(fQQvvr6M4vn91}A+=q@qbsi!tQxSSeI|AMV4NG>uXZcW{Qv1m9h@ zhBb*N+9~!$C#vcYtfDyl!V!SHN2Bg0#Jis;kJIPai!|DG9qAk)qhAjvfXtTkkMol;}L`A~^)Gn_T^T`Q)PSzS8 zKG}AQ7Wv7}n4p4_{GOKm#Hv*c?(#`6kB=_hL2U75XU0yr4391T~(mVnS?8bbfv>^yA^ttjk&_= zRi)Au{UtCIg7Ymp*ayVxUFor{cwCEt6?6^{+hD}y=VS!7w+bSF>NAE8B?m3+Knn|g z1jQ!-XK@cTeEXg*oIr{A>QxcJPDS}~k$x)gPt{}a{ZPG6L!k2>x z{O3|_+KZcsIlxgGNlNrgJ>@sM@~Kbd@$TOJe(ZZw%^%!}XcK_=yV1Xkcf->;-->hM zrGf~S7q_r*Ev)i|qD?R(K6+4~Qlz%Fwgd^#T*%I6{cyNpB9{v zs?D?MuoKm8$h#dK_;8RB@*Dn)mXqW~wotjZZ+}55l4zH{0^3ukP8IYP;_wr}2zb2n z=gaiU=*%>0MwB_r{&&J4}^=sFj;tGSRt!=}mOi91pS-}T6e6jkv=z&jlY85P z#=24d$~SLzBc$^=N*VjfvWE;!{}6t|Y@Ct*m4Wjuc}%%!&CSg{uwdCoQ-*M`;!{j% zY3Yj9tM?KT&`ig}qr_~LG_5Y0e%Z2cCBcGMVzTJ6ZH*SbgoH15P;riZYI-^Xt1T2~ zLCxd3S?!C(cv2*BJW7)Q3-k`8m2@S^j!wY5obzoPdiwBtHCZz=Gb%_n28SK`j9*AW z0MwmQ6;VX1LSj+L-$9Vyv!g!(2W|tBEYGe%{ljkNGQ1zffQJ8_+qad0S}5ZgH6uJE z8aBzs{VPAq3T5=^%L+s_1hA9jnVr}0x5b}>k$16B6L4ncGUg1!T?Uu$I#ghii|_mP zUcE_nu)2R|*NqW*toM!Ba+~)gRXy$u+seesdfoxjO4G>`YBDBVVK|psxW$81@6yf! z(lZz;!HTlr3L{#;wr*rxqO>XyI>xD@2N-^M_`D9{xTB<~C>`h@?b4qtQpqT@Pxg=g z1d`m{v?`>}Cx=SV)Gu$W6HHZBe$t5b<;M9|A2W-BD#XFNalU<3ULHY4W$4HvljVH- zym&Fpq+IeqKbD&?O&Ld~)%t@SzSm$hJo=6l@6rOwSb$xY8 zLOGvQzgF;NVl2tYNB64)I7mpR){(ifn4!_6{Qlva>S{8Od=H8%0t=E}ZWgo12V^uB z4aY{Jg?llkn2~9RCjBT_pAxTcv?}5p3({$aDi7IIy7CN?&-ZY~xu2geZh-8oTej!N zmzO)Js9Pj51}GLBWnU{`26Pp+)TGb{8eCE}a;V0aaRU9WO6mozRCQ8+BSXbbGsh`Q zThqQdXN+|kg?6$=83uVM(QwCf(QY21Im;L0IPQP_x62=va8s|uwun=Q_}=w8~$IuRq(Q16@+t>oy|lqR8YwD zjnrTFX2pB*_%&jmoES=tQYB(|07T#u;%1n`IFL5s-xgwy4eu$w`1AN+j=ArUT$+9!7EdS-OOlfU@>PnJO zY8?um(|QgSyAsLCPkO)6jmd_>r#*WB+GM!iV;St-G``$pvU?L5)qvQZc_FHQijt&p zByb%mi$FW<;Nh{v-mw5a5uZDa0lHjNSP4j?rMc3eoSBn4g}daxSIkTIW>&t1VvQpL zehTnLyQW=;9vV2akUi{Ri@bkUN2{Q92-C#so@eEPts*E2($o#>);++GJ565Wz9r2g zb_(hjFFqk9K0-9^C6$zDup}}2OaDl#riReWmJu=`4@pyZMmDmV8e6&xA3kOo?{yyO zQcTyYE%;Jfwyy8pwFyjTr_~q{ACm zS_h|s%CjRA6PY8OvEvZ<`RR}YO#cH$>`);i)dDB%u`>%n8mm(;#%<&lc3G-S;y%C$ zsTTf3GtMS{VLvh#UPm#}frXUF&yl~Zc0mTTfT8f0t$+Yx+5JKie89F~9$&1g)Zi15 z%DLgWSv4|N5=pVi^f3?EJ;5&IhTz_0)O!$xW)W!&@@j%c5x1EQ>(@W@@}i(3APFU| zu5l_~eoG*}Drbtc+k*~s?ZPCjy63jV#Wvv_yh1{mBM0ajTzp<@+JAmVM-7s~BkdFC zqr%4F-Ej^c6sL7rzY6vkJ4>v2hoLfGC-sgO`fB{l0x4N%JdNzhKRkUbpVF{~DQb(B z18YEqXJ<j3RyJH?A3($yui$zXM2fv6Cgg7Iym{kgw3yVe*Z@nLs8DVn3(0CEgaF zY zMVir+{4WfXoY0T8#hJ(Cu|7d?;Ru$B&SDN)phy2B$a?i_xhN&Z#lzFgv425^NpYW@DfOZ=TT^%S#9kT*^N+3Hige!Kb|Ik>f#rDe!E*V`{P z`(40B$*B)e?NSOdEXkBEotrI+yBZQErLU^$S1mOxjded?5aRaf@RjU1UQZy|{w(36 z#Iz+Wrp{Bpr@PzP0ztdnc|*>o1#~vhSa?nBJj3{(IIIeJ(u5TR3h_n1R;};aXUwZ{ z$%TX4LoQO*rjK=Mnnv=?h+R6u;bPPLI9@q@%z=uMif_WgSye@hm(7te1g$7+QVt3! zEgetmqA_PKt0mK6>HYCD{27IHdLcu2trvU^!TXMew0}@dX$uNgjB@&V3lW5kV zXDgpSDOWu&-!HCn11M?auNWC%LezHGSpusS=RzJGJ`;ra28VvcXdUDiJ!- zfl8Ak_*7ndqv|VH7Wpy@AFK=!(k7f4ZU*1BZ4WjFx-ya)$R!dejj?ip9KCveoGME+{)RHh3>4r+M+tBtq$iL(8mcXbN^JzqbPjF;{*(qO22zwKOVu225 z@N#;!SV8p$A|Iv*ypFU!fJKRjy_GdJYOoSh1(t3yd-R@J*P8%ZoFh)Ho7xlhbr^vY z6cb~VU^q1~3JNeOSFbh5uaN{*L^(s#$uynpCzefu1S12C)Ho8FWDvU5+YZ3LBJIh= zzB-1c6__D~;~IJCb~P)&T?GBa1vWgCX9W0BlbwdBi76=ajWh6aL?6^Kc;(p*Fk48? zKS`g`|Ild8%hG^xllWX~Oqmb5NZ0+i`chIe5YaXnTnvC>hbvvn??7^K5m{64TP*`bQKD~X9hAmk-ER^@MMH(D6&)`;VFE)W-L2L|D#Sex z*L%Lw5zZgP+!EZa%_jTz9+H4jwPc^iEhw;j5EPbU+?LZ~;$44Vpco!lQsNz#C-m%s1xrU=DM1uiN%p21#mQArDj z*?Bob0)lgl=uEgjZZ;mn;qX0X?oT2{sOSplFopyNCjf|TacCUG z!fdi{C-sJQtMRYl4kIEf#PSv_8W3sir~n&eK5wU~XI)~NMO;TXxCy3NF=FWw)1 zWM1?0pNCrb9?w}6Y)eob<0{Y_1VJqewFltCL<6WD6<9#X7;EE%R6*c)2iwh4{FNd% zO06Qp!p!H+rGypdMzz`VVDgc@1$yITML4a8|>V)T$JLrnk zBbpV^k1*oBC~AFFaH6OdYaYKO0E2=EaDbqpPq2RS<+4YULImJxa!bgqQB$RwA_gSn z=YP!oM5Ok6hf2mO#l|2fWf-aB?=%<{=UoZjT{&GjX^JDu?U~JW)>6QTZW~|EhhG%&2w&s$*-WpbMC46`}>yz&LY86 zL7k$4wg2uGPxLvf( zMO*11b|SyqGykf8{q;4ux2H#|u!fS7fSTA%dZ${SL^rt``et}!3CIkN(teJPiUPQ$ zNg&CSHf}^lWPxJhd@C0ZkG43{igRr;b<=;zw>lyrXMuT^*)%HD2oQhako>$peE6_a zKXL@upvUYg<~1DfBqfgk1}gK|1(t-2zjLcqUQlyJ?C9r1Y`cu0_D%Fhr^Eacz)=xh z&WifwZB0#{)k}%oVPwL*doXy{9>FI91W%3EBLJ_hlw&~Mh~p{gbt3No;>+f!KfBd@ z0QZcy;nL_y%z%g~b%>pVYVhaomHAqy<`#R1wd<3ZmmNMMn*WW_+7-|781j+HAl=fj zLx1pofni~N$(uHA42R10T_lvZ+}zw{OX%q%0J}z@B`%zJ^Hq5O6R|=dJ_DLsnqz=f zb`6T12y~XTiQ1xz*3T5uh6H4RFBrR#T~#5Box^mqQ&{TC`udR)c%bjlo;vOgLtNjY=U+aDt38N?JLyFSpVqR;o~^9 z$K1Mkb2lR39=HPE-o?+)pDo+_vE-#l!WB!+MNB8bxf8tHMdv zbwT|-|4n9!g+Ca}bAbNC!AA9dnAb{K7?yg>Ani~mV`EVi+DonJU;h7yZp4Je*r#>d zzoaMge?D9BZ{g6w_y2E-(JmO^EK5+0N+DzDK@c4S%83e96Xh~-2b`Lx>XpL=q0fh( zBDZqreSnf+RaLR|Ef6kADd7%>K0#KMLG5G5?&D-@ug}H?BV>3a=uyA)CY??8T$gjL^AemA;crZaWAjaS%wG|{#ILm z)JZ5-yXCbR^yUK`GE{Yp-Xu^18GarYx07hSNGBg@t3Z-8ay}0<0kH|DK^cRxfY6v= z<93p+TLP*PC#}c)sn`8e97t0J;hg}?svtK5R+5e_mIIoZ-~{jjeE`zzCFc+)3T6yb zuY+V@2Ng_MdL6w(37J-jQ_~=93X9l(>T;&=Q9|zlRuQ5SKR3dW!`Y75VS{0(7NOuF3A{*dUg@Py}iGT7f(AOniei`rvOBbL!}*ua2}ZK;}=ztF~a2 zCLTs2GA|3%fc>=(Z=mRY2r=>V!6d_5Tc|rPkfLb$$oV~ zm;FAhnFQ2hP}I-tU0+Qvtg69CWHa;YJcZu)$h$L;o)CxDfU|&(%`j=Tl79{Wt1p+a z&PI3HWh6BHE!p15p?!B`U9gF>^`r>!EvQoD;U`Pr6eP!KU0r8u3bEca5t|4Wl$5j+ zK&3KtID*+VfZX7M0TG!34Iy8KsOXTc-M)3J7BjIEalRI`NdqEmB7O+CX?)`1Cy|2@ zff*@g2->7r4h~w1N}s890q5OLbR=lyf4YC*Di?JN_E}x0=Bo(T$dHgkZ6kM0&en@ae)xd=3PDURJlLsFTu@rxi4UxC{CEKF<(eZ$k7~7gRCZR%m_{aT>AcqU zNk*34hslR2C%(LWx0aK=r@y?4-+KAZcQprKTf~(NrUY8)o!OcAph}WX-3351Q}!ip zp3j>X*8DU20_g)Q8xPY0&KkP?JOYm8!iyCopW)uKr$Ir4WUN@*!ZmRBMB`z zxKU@VU}Vg!%QA53i$6*$=ub&adGe`fxC9F)KR=(>N_(Bn&`;bV@Ha@%HXHj)!uzud z$g6d= zdz95=P$ga*ZvUhpLKG>W>T(1+P%`wjx3idZZMb=td$SsfHCbMELK5z35-(KGclRq;S3in4LHGJ>SF9!c_NHa z)N!z-TViPLPp=#|_ITCkUr1HkAvk_0YiROY*Pn%_zg_&akK;tXG+Be~AB=67`LJZW zlYKj1)jCbv@5b?f^#pjGuuR;1e3gV1Ek>9eY zn^Upl+0*V|y$XK5nd|3{RDwLgvVbo~vzCziVr?23{a$%~0w4ejm0MD>MT-MdjRVfQ zzRY_Jiv85|)^b!=;J`RKV^# z0uqo2Lb7I`#~D!`8GkQxEY$WzhM@2IGABp>%9kf?*BZa>uyFN%dOGb#4_-)&?9aiP z11Q3r2eQhgwYw$95^$MGrCUJ&OM{L`S7{PNvUl^&%b zcYD;x=)QO#dV|`^SFczhOshhwCY=M(=m!N6=r)Cs1!zk8VFQGL35kH@B?7UCtdZx? zf+5wwAsz}k3;Zsy3QdD_a zs3QPez!zhrVSKp5h7h&*s)7#5M6V;6QS@mRR({ouFp|`Dl?>*#Rv4He7sj*l@!_9P zWYzp)DI<2>Hde!cg~#XT5PQgttgkjcO!ST8?5;0Qet9<|pk~F&<5w0Da}~J?DH*p= zO(Cr129le79>;QYYpTJ8=S$@Wub%kd_DhB9;&G>tkdn)-D54qi_-3AQOGq?p8Rb|6 zNkF|6#ZLsEP1F9+`%+^f5G(16)OU`K$5#!Ah`;ysk*(9{~Y4lp=6MfdP3l|>EkFW=k7PIN?Y}@Ze07QwdIqQY9 z4^y>q+683p((z>}-}sG? zi$7jIf=W85D5h>-BeZ(zrp-82$G~_ud*H?hQ6WbgYd}hsI{iamn=SLpZS22P;4ri} z6be=VO)r!2(tj%$r=V~7K0q(4{#wQw z{a*-x+*Hz;Q+YlQ^@KZ4e_C92wPfW9A`Y*5u4fxO8HTEpi*)Kn>*Bm%lYHFr<)qO| zc{>>IAeBS%W+#{CiQJWo3OK;jTf6_q4ss0;S1nva`N=PUuqQOfSR+BH(AJcgR(>~z z;J=sQV!Za31AiB}^QTl21}HyhA*~xzkaLm|0U!J2=Kf^M{{9TY6P;=@zqAdVkC5g8 zln_?mfx%cH78K%0M$>Fcp8kZBe~8+HPeMWm!pZDjy|yxDV$DJz zWRQPo4N2+#X|eLZuEr@=edxJp^EDfBc-Q6zbF(Oj_QN1x z2}9M&`4)SrBTX+a^LzWundG8-fv zy5ynQBLEH55Cqb#>697&Xw1};$zU@i+_|Ll@GhHSVdEo+rp9UFCEY$uo}=TlE-%Gx z_q5NlN<&B)-TjZ#YE=K)jT;e2GN<0CTv$tp!oM3f-54jd_UE|b9umnZ4qaumV;%!= zh$TQ8G7KO)C`1hnTH~9}95^OT)Z^%YNI{W8Ov(WmfP7KMsSmNEw)waB4@}t6#@sR5 zu|OH81azjZPHE{>Cu$}Fm&zdC6Y((8p9fGA&Uc*Pdne2?f^GMd*L9%R9~EkFbJK&4 zj~NOff(B1KW?YDGPq>t0268nXR2dd{8={IH+<2tq%75GI#?EU<$R%}^(94A?2f}{;-EH(?7q&z5msk_4jxG z&sqc*e&nA`K=a@E|Gro8|IZKd+jHQPqsE4&j%7d0`kwNATS5@hv z_KqJ5+Mr2M|NM$|yF&*6MprpXl8M245tezkrJxhzLZ)Po zpgY;?;Dh~uvPKH|4zqkc`A!Pw=;ebCo&mL&Jy#UdA~7*Ca^>IyuaFRJL&xqwns(>+ z6Hpw{dmp6Rw=b`FhyjK7I`^}7lAb+1JIGIGdXOHSUU7$WEk$(zjBp5a{^ z7q@S)?lau}PChzC$FO0i7XZy~@zpqeVi9zgGcdfsq50TA z4dX-pEES#c4+vPt%EtDkukRL^e%zshLNnEjaa*3HwXf}Z5;BX~WTRWEdu5}lL;%Dvp^;eP9y{M>A6p-1ub2+e=GNen#lGS%Gv#|8JMfu;k zbG5Wo4s`l;^mx5!Y1s_Q;DW8~0puv3dwXvnvC{v@M?0yb^97~)A!X%Nz{J9uK5dJ2 z^CAmk^B>3q6%-Vn;X|A@H&4!UiRU?sezdd@q3ZGR@e2+P*R!)l)6&z|F)?`qfqCBA zx&_?bE1dfFZOgjbaHmC-=YXGHfP{dtA)=-gu=$tH&cHs7`l!grV;TAq`GtiZp?=4s zJ0*Cg#=g0z@Q{1h7N9q3Mnk&*U0GRKsEn%^Qu8rGId^d&v_F3Sd~55a;Zhfwg!je* zjh1%4g2si1KvvH+Wv~I%)EtrX_`{4`jC}4d!^8eXj>8tK;PS|(LAiXZb0})Thd;2fSoCXAypWrp20ChCf`?cO9LI-EY9qkA8#iv0(+}-k z*H;fN=q`yQnX zfl=ITUdP3}dGl9bz~4TkZ&@s89}pD81Q`4Vjzd{4`NI-rlwI3nCw@;&ZQr|>38{rQ zhDx@7AMPlXqnyI@8JEn~4<9xHWKEZ*`||BTE1gDH!x%J5dF}$`5Uor-SH3I>ZY&nkz>c!0w}x&h2x74AH*^Xl@Cjd^}O41zP*^`nP(~AlkRCOWikv3 z-2kfu1qCJZzF-sKUVeoPTplZ+dl26Z6*XG}JZ<&X!Mz?xl-~73-xZ9rFGjycvHP@` zzw4pHF(S(l%i=k&J%iAljsnd5-JUrloL8a!dI(L&qeqXte0^8*YNg$TEPyP!&6Ntj zqQz|&F|tbo?5jsz<@?9RD%1J|Z_vkLXHhiZtu!+;!^*k8|MH@9jVVkZ{ZlU0^+AdX z>SqN6%GVf}c%R{3BJ}PwDBOcJlQ5zE8r0y+w{KUFm=B`#x-;a!j96(OKYn!EV??88 zWM8#^+HxG?Yj1E1d(7*W>`=V-#@1CwCKDOVReVP5Khn{cRlU!q{l0RT2%f+>l-;^) zNM}x#xJkhOmVt+dXHsSW6u}FSlp=%nTC(!U-tdzw8MzSckdwgs;ANm|&P4|w+^vyE zJ7}c*RDj{-6>(43h-N$18Wq?rMN)qF=hxS#O-+}%y1F{KxST-`t3_qL<@Y(TEbozV zZs65eo{*5RQ|B=kq#OR~-$Ty5-b zVd2ZTb&A3Kw~+UqhK~Ox%rTC;BFmKr%8FN|C@Cp5BpqK20JhP&c^LhYu@IMDxNyPC z+k2_Bv^3$JcLpi?PCk72Fbml)AguM!-CW1!abrh&@EvS(vybg$Nx_Hy!Qoqf?G_ci z8Xv#o_9h`eaHhu4nmYaYomFNh@zn4vcV<~gNXRBU%HrbUt=FERJLhkcQLm9}mhzrV2A6`0iaV+~|jAUfb7*lS%6F3|Jedk%-{7^|3Emw4ty@SJ1Je@4Ccyc6Z*k#LeCPc%mvBx#u{gvDRLq=UHUHQFW*svWuTU3SyyQ->H|&PhQ)^#2c4# z1|l8Z;Ts*pu6A}JJvA}@;d?D#R9COTZpJzJ?NiY{JW|q|ds+^vQy%}oG<_R8NuK+a zvT-3%!nF1q(RmavEIMOo`sZJnpP$?a+}Z~?so;abp&^@Eh!@xbDZw#7O&aOCnwGMM z4!z`W5(udF6~o~XH0O?8c|PyqTf9&f=A^A+6<^EF&c5^5eTuS*%JKXQrvW2MD2<*& zTJsWg3UV}tXKih)LqkL4V8_!4IbG29+kF!on=c4}hv4`hWn~5QE=$!&z5*$~#f-5qvCcv&AlOrab&eE2aAs+E*QUQXi!VISPUm3@VaOu*=(wZXx`FWTD; z5y5{o(tXwP<#${0kN2Sn=Cc8#RT#P|AcNjkkRb~3bGf;> zymRP&kq-dNsO11bpB+0VCnq-LRv#aqE-MlNu-`XNcMlE{4mMxWYM)8VFKQWLK zh1@jeu9_D3q^GCv-o5*7Wl>Sl9mMc+pX9CTCR`zV94RUBR7I||>z-n~w3e@r&$XPK zoJealf!p#g-}b7r-2X_tVo+BxGBn;_TbP+|LTzMV+>FHL7Op6@Ddgs*E@?`x#A8EF z4iTEy_yO|uuRskfy5sHlp<#S-^DefUu^IFtpSnJ%)Z2=u9U2~<;uefso|csc5mLqP zeS7x?$?4VAAPH@#=cP)#;Yf1%^@9Dm!QX^eSt>z zSx-w#OWUqD>vHU-@-xy5inMlQlDmYohH@An`D>&^BE{qEQVz-{11wv%tVFP$oKsO& zeuc!Qu_As>)qbRlI%#iWWtBeiDHaCaUr~banq=F}C%4;Y>MqUlW4&plYN=FPo;}Ou zX```o!x_ACtkci0y2IPJz(Q=Iyh84p-PSZ_zl0@cTfh*@lXvv`1$IX^2rLEIBK8i~F5aw=|A1~)fq`mT-vcT##om&eTj>4fH>+Yd+IV@9WpGW*%(%{AA6uqLB~Emzhwo`; zV`FO^I(`*iD!Bqdgy;S-Hg<1X@RL9Nk|l49>z_T_52(db<|qowzbJ=cy^_;Y%av%q zsBqxwT8S;3&Js4ZW8aPk*E{$tFUPA~1Z&p=X=OlzzgT z-sL?vOd%7nYa<|r@2`XyKFBD--24Uy1~hu~Pz7-#Bi{2NU`y#Rn&!Cl4GbE~E=(2V z`BYb;pKB?l7MVo4^Ua$gFG*&45zq}U76c39^QP!qQWFG=!2o=2=CA)QY+Q8{O)=zkJz75Ltj$-fIuOdj0wqWCC}vgcv9! zep_6s3E!(kVi5l25QJ?jx7E!ixsCRy)Iz017q_%P+Ii$~`gw=Nadq@=GsrVHkK%q+ z8jV4jnoJ80p;JHjXX5qD%$BDRs605f^BO|HO@IHI<6au7HQ%9UT0;5K)fFeQE*H4a z-$B(DkLIa$fLTJ*xeNMk3x-K$rxZo-}Pjpu;jlXC&O5zUjV-CX+>n???vmDWNT zE7a5hIp9y^;M)$jaJ@$U*tO*dvZMg?_h-4UT)FbJg#|CZ%grw9w)5KB?Xt3l{4lqv z3@dmeY=@De54(&mv~^~wGUlG;uy`;maodHg1YK!d z49f5;&O$p~923jetl5afWG(6I*%ea%-CJthirHW5X1m!%O>ab}2fr3+ktemxTOxUaMxS@9io zlvWJ#aC0-jk;J)pHS_NM=~f1reW210#~Bq?$6V^GJ2J~Tg*@^Iq*rUV56Y$@lYM~_ zV%FH^=yN5)MpCxokp^~`^o#EA$MYX)Q$!(m1f&b=%^zQ1uM~#Paa-oj!_S+f4@zWuyBqb$vZ=U^Ke7Tdj@f8I~UjCk`7q=Yy>#(e-0%XrH(x^ zU@9(o7(G%1lmUpXTD@uAx@{e;+DXR_P+G1QWBHM@nX(lKf8sB33tGJdox#L59%C+{ zUV*q9uk~^)c0e4y%|=Z@VJ*(%4YIPbk_Isy+V}U<(e1}!B&O6o#t-Ted4Z1ozIct8MD~yhd;S&)P+lYkqZtrb$-tobBW*dDv zc}D4O)-xr6v#MWvdbs^K=Q9Mo8)}9$^E3l&+)H2zym1+HYPDu54W=?$`7~t=?sT9|Er6NQ~ zh+ef=2-+Gf;L~T$Y&R@+3W%R2h5(ec=o3>{s(gD^p}?+t$=&e{!QA9nb5ri{tk{vc zI1VW_r{_<#j(z6dF<)NZSh?-6-=a7>7emZ==0pjP91OSe4meL+D9a^I9XR1thK}X# znTD~mw6zBZAXTH7U)HYcieB41$LhX{PQC7$B506C?p5B$o(iKU^hVjZXt>)J;Nz67 zc)_UC9HRaED3HmV&l$s#4bn6~-&|a9FWH_`(`p#po2#N)KX&*erpvPK*96!W^n^z( z`pv%TmMf)5oH&~@cin%FNEaFraR(@&1^?XCFU!fd+$#HeS!x}s(RaP{?yp4pOIB)n zpKq+4TRH%+E_rEGg3&v*RxWkW14}5v@EzlyP}2<}&^>Q!yq6}DjG&r}rgnBu-ybpb z&m8MLq&Hn>;kIVY8s5?fut`V4L`;@YvLMt>pGNPXJl%dkIa(&Js+oL5VJu;*!~%AD>y+hr;8#34AeR7MR@Li zm+iXf)74mvLSLxMVQ;!>hyl*KknivM58at!xQh-HTDd8T#3$8`CR{vCNBeA)k)UZS zDap;dTKG1-K%?^;XF>h^`?O%a(&4y{L(qWMTJdz)BWM;_?|Kt*b7$(R{U}%x)nXP= z07T#4TZ3Hjfd((ZBse%?&gXP?buFQwkkQGlP=f0P!=_D}Ae(yD-p=me;P4w$7Xt3x zTY|h47$86i<@l<3T)~!nJ763vipRo^`KXML2Lqnku znCRMCi#}-9mQY{=A@p+MO0PFQ#Y40kFJHMb&_8kQBE)}s$|@ESu5zocF$~nyIgK!pjn9?a+)Nq^v&QuALhJ6V*3(L&NzaU{lGK;-d^ad+u>X!D=Y8(1+V@a zZGv`#yjXiWfL&i*%yO+??mw->`ymVYpp}fMa?ydqhQF=G%5QrpibNu5VS>`+^WflWO1|?bHwm4IgKY)t zQ4IngD95{JW_1vhy-*F9lY)xM5#;*E&{F2>Ob}0yl$2+vGM{`1V#1B4cp>C&Md~7S zsl*0bd2^mx=x#bdW(rj5bD%eSdUk20F%wa&a5^Npl!lszf=>)8<2&(muDyAq5cR$C z?XQEII4XZvB)RuV%O5%N1$%L19Nl%DD8Vn-*}Zsl`sI^xTMT$&$z6SRrVwZ~XA_Ta z-UJy#U1D7<1ds*6PJA6WA=AD4L9;2^hX?S$|%ldIzTQTtKW#w^|F7v-ZZr z%0JD(KqzsavOzhV6!`T1{aOfS>XT*6*#&m&SV~s9g`)|9WU<0U{!n2?cXAxX)W^pr zD=DX-Khaw7zl_ewMgROz`;Mw< zmH87#H!DG|7%2y|@DSWL29-S<=;&c_&zYL96p|FgTRC8TLhX6BAK;GDAOQ-JQ)2^H{ZZ zoryQ79j}|;jJK;i78e&k_ihyHX)%T9vNyWL@g>g!#YW!z#%NVG^clLH4vdV9=one+ z&A_DI9338h8!+5rg(Be@M4_AyY)ni`Da#odwUyBRxEaWwH;l=);(237SsYOE=a8$e zD>+O04MN4NHY1I`iEZ-$xSw4M5Y8?td*7YOa#Nt-p&}*>WG*~xdsgwNM(%fV66e)O z@{FK)EqZN zzPaq~#^uDKCw&qZcMG=1Zzel>0eR4Gdzy&?cBB+QBl3cPZVzM+>wvFpu8)k0qN5=4 zHEdS?HZb6WhNWZ!U~ ze~#aOUwFwbrt9%DcNyff_Y@(bB&FSHD%|9q3Er6 zG&rEIEofB2A2SM4#oVe0l3a1ep~x18_FZVa#sCjDcfmGtCewv*7_&f@;sik z|KWKk-0|r2A?GT~a8a0m27ho>L~09+YCRz(az+Dd-ScxLu1B`}E1~!25m00T-(nSh z_*806noflq`(9m>dCA#ODU(2fZMH&DS7Jja}j!s z%TdFu`|GbOI81U*C4to<$+_{V7*MjLRUpfH@#4jVe;`N=i%`MjbOtIw+<8{8kv_=N zA41p#V4*>hyJKQ@f^K{4z$vc{>yAs-ZIVNBlxIVYcBAK}FyUi{YvUh!P0$s3dU}(g zaNPQgo8)3-{<^V|31a)Br%r7pP4j-6Pr0P{Y^V+rU#X#18dJd0kj?&8A&RrpqZw4{<{tXdsITH z#)3H062B`sXm_~8L`AWo=(`Qb;wyl<-%w`JQBa$oYe-t>GCpt%hpx%=uBDpfCE6Y z#NwHZoCu>u8VWsG%8y!NCC;B^KYscM8C3V;VAHHlY;Zslt*X|BdwH%cI_kUPl*icy z$G-4!S+bwhdlQ%Dut{0XG?niSAlqk5DP&UAV3-4#lIVNCi)$!202m;ri~A z=F_UPhn#!O0Fy-Jp2W?E`*CiXcX9!ab@hmH_4uCDKnaE zX-W47{dMd)|@oB4?R7<2j7Uj8Xcy#)HrZ8>O=tf8Y58#Nj9c;s zL}Hyac8YUFTOV5LT9f{)VumJUK6ZnH%YiQ+LgKX!xq(ggPhe%a|z?x+C=}% zkDS{T%WG@b?JqmPJ#|u;YbH=kJwl=zJQnsUQb$U4b@j(Q8x91wcpA@4{ZhyeQNY*7 zj45rMLxD-C@_ULqfGJ$YA@LbW#7#fHG8}V1|4iERpL|52>?v6$lo;3e08QSzI?ka5 z=}o1FQs$RwW+p*>W+Vtl4<1}Z7?m7mQ z(yEv%e?A^Qdh|tQBU@oX|JTp)K|K)0c&klLvwtLrpen}=tHE}9+-jgDXC?IX z`+;cR1;d2JX*~9=cBfHEaYgDPJC~>L$NTRfG(BYt1ct2MiUXuJd9dyGOt^u12XOaG zLq!o0dhHJ3WRcn_W0AU7_&PsHQ^exDd-s05V@Ko?-EYJn^t()8?hU??mF9vY;AW8x zl;dnC1ikcXzt}Wr$wFCKi7*go;9`7q5QDUuVJyg%TRSIve6&)73Hc6nfIQGPw56=v zI+!gW^G4F`_m9XSRljeK8J)K!j&1~>o$od&vW7#}2cpGgkR^JlC%*!3MtorJ2^wrh zidm%|Q#U90`&j86um6L+_m1ni|KI*!S=q8zlo@4arj)I$M8nKTHc>{=a@sSa%#u-( zB%^Xhk*z{xq-+%uqN$>O_eaLLuJ3ibu77^Ne}3=V?epP8@AvEddOe?yaU93vc!sXM zak1b{V>@hUU?WfURKp#xI8Vvbob%lAW$8yi7NcIqkAuHErq9XhY=gju!+p58^} zM|+(@{FUYE9-7!J%DJEx*Yu8B`$Sl9jfB}|6v6#jZCD>|tU4*pFUWhpRSdP~y#P}U zfY!s97{Nh5#;*gc=Rwwy%%w|`Z+&`eoE*PtzuLVScdr)tX+_Y`m|H$~@(oQr+rc!% zWr*MRn3*QokQ~iG@LRbRy3MF(?ALyxM&fhtlcAwkS%9LeLB?o+0tFqF&aGRw%q@4u zYESf7Yj|UO*TAXpf}&PI$**lXS!K_pIGBUGfZB{8RPHhVX*6l*BtR(j=1X#7112H7 zdr+Rt+tD#SWVmz|cM5LBo$|S$(J;67nUs1s4dr{a5PtMsihta{}`?k=3(?Jgs)<(vIwjySe45cS=lp1E-5Ml2ac= z0zazA53;vfQ1FperQZZxuL-S3ril%CcYk2vi}DS>cSs0mHFW3AY&Ue}b(!1SK4%_X zL3L)XI(6-O74i4#ju*c6o42~hL_?6Xnhl#KZo1QB>v9JR66$IXj~XMqd#+%HSKY`} z9i*h+?Aga6V?A4ITp{Nd=5+3}VO<*^sL?V?Qv=5X-8=#Uw8)*~PoLj3W9G~@$^l5; zByHb4C_E0C2abZ$xJpZ#_K|gS6lhJEO#L+WL_YUjNw^T*sKLzQy7u^#&dnLq z@190sB=7muXdizcxu60WDPA zQIa-d(OAS}0JU|zw9lvenIgT@XV0o`4gU}lUMUJa%HTOpvz|`R58Mm*UB-C%hPN1! zjLSChT}cIH8|35R6c(wgvwST6`z~vR_Hq^6|#W zRrcGO5A4Kd7n27i;kx}2gJn4tfi)6$=b>nsaBfkhXtx`t8h&$}ND$`Mt9MyjhNNxT zvgIpdmclldQBF*VXl`2gCI{Zk!`9jMK{4j9@MgZIfgj(-`^JUz_&$>3J3CCOU$#t{ z&^Pz@3MK(@zdEp20`5S*Iq0Yz)EUn`WnHR=NM;{_HB# z7~b2K*)Z3|)xNoKOuopqj&z<_aa+n4Uuz)VDK<7MDUa3l?8{xaa6xgWZ-3)BUr!%O zDK@=9jERPW&2y2$Q9ycGWi9*idUfpVR2eqgq~2inz0Bq=V@bxbv1YW4w;$&D#SetC z4!uy%Y*)m2+FEAFXAP^}@8-oC3BH+6zqtggr$nV!-hx0S`oV)b{VE0*wRX_}i|+>_ zXCGxZz|-i*vK+Lzk&42_9O?E|5);ktI!M*CXP1U2Z<}unF5?}wWjYLr8cbLj1yrUy zW$@1XsOVKR=at{alA1myzQ}YEjCL6XpyS||K_##`%UnKgZJjVDZ%fnO8_>DM>X?{3 zF`ACfZ%m|PXCc9cdH1Zz6nx_NXL@=e%1LM-d2f1RVxb5qO2q{%vUMmZeGOe6?C-v9 zrw?dAdse`9e_5!<5Usg~HvW^Gebis8Wy@)QhMofKQI1BQ@2zvpJg~WGIpzn)+r~_Z zfl^J2|9a!Xv9XqzgEj7xJbD$ANv%81g{_BPQ zr}u(9mBVkJSax&KEwHRvJ|AL^ZQ6glyksEt4a(P^f&O&<6wsv;Op&~b} zrUZ8-Z7Nbozq!?LbM|cc-YB#``8?DN$ZnY6e2cu*twM67bHS+UUGJqj6n@%&Z@0jgaDWXd zn8uEv2WyN3;q#|w`URH`=YPjCngF&6RYbD-ibbwl)b$Q6T>Jvj;cJn%JP+6erZQpg z;k)5eZ_zyGHd0yyt5x@NspscpK5KqAEsE6xLEc-odFOQ2%C5I3?O}Hh&xSfWI_4qP zO%#fN@;-64g9~4ZX-+RIt1a7&eXFE<1!jW8c6KfKz7$72oP!UTYd;L9r?IYnb-QZ~ zm58FE4u_ov(s4QE_MB(axA6Pu!-J}UK)iq2q2{*AqS>H|nnG)=tfOe0bGwIA)-;tq ziW*9ix;#Ve#d87I5sZwrNn)=dIc=+~lZ7AQK}T=@g#R0tjceeB<^wr~j?DqgiFd0(??6r-uj&d+4 zHwKbUQKN;sljRzad67vDMH8Rpwn{}><}cjvU|T9pE*kh@J)dg(H5*7j$}8@@>j^j> z!wX4z9g=#75aH`W{giQU0GTi(nN64Wr4TEMK))!b>_GxGC~9Y@n!G5c+!W0gGLssjzj=?!p$!*ou)l-)Nt31rJ7?Pcz zpSaAROQRs_-d!1_rZbM8U#{TaBBB z@~nkUNSxDy?$6=jR>5{En&9Nx;DYT|85c#N^6dz)t~n|0#l@xt1qGG(=E5g0EGc$z zTFzbU>HPI%n{lB>;T<0a1&rz-h)-(T*X?)K0-%p80wmpDRH08h%&~$$lD6Ys7zyPp zte9Ik7X#-kq}U-Uby;WHwhqfCMmT;sMA1@589;A$Ri3s>UVeUkx<*u3Bjym%pKf(q zDmLGz6Lilz+De`xA!Sa)@m z`t_6n93>$vz4@D!@~osC>$UV#JL0^*rS+5iWS)^347f@k3e9y?V6oYig!2`#PFiN_ zp4+O^TSBX1!Xl!SGVX1xPI&*x_Z_M2g;Li7@MIl=(xbO-L}mi`lyR5)Ah?oc$?C6} z*1gWVC6lGi_QYrBvKJpAsc*$k?GTX+Q=ZU5-g(1 zDJU4EBpZlfCcLq$6A!^Y2@EvQE6U;R7H8iVfWl72uesTmu~h?4y!m$U{l|~G*YLC+ zwc^d3-Ds`aZ#G{3XIw@!TBx=3z5p4+hpAiT122pL|(H_Nl%Y>s`) z>V2LS*5P{|qqt$GsTaIFHXbR6@J`5!kQ(J}D#x$4gc@zJ>Z@IGFa@W!!_v(7Tw{lU zv`RvUi+JQh2Uf;9a!;i$UAn0BS;VOk!IL6#i|!;Pjg`Y}y!DrkNXyid*diQmNDC~O z5*#C`wxK?`q}lp@0|xAw3fgsVxhQaybx;L;=j-JBIsqoHQ20eN@kFdC&k z6R^8msh%zqs3L9QBE(gtoas8Cu-L^qaKgN|=`j-{nqkYJ9_M`z>sXyQ`oNA|@S;;M zef^CTS2LsZ>hu$?H!hu8Tvnz0PrFCSp3M*WLPym{bHTzbm$+f8VNvEmOW$Sgo@%o$ zRYz)QuR)h(-g?fkk#Pn1~qktdw&B|nLY2#Kh$Ym!)x?G zMmchSRp%F=;HA2|MBb=puU?f&tzO}5wBRq^sj25qL-%^QCbOYF?j+0rz-V6SE`+&i z$Du`^_zn=c7KJA=c}|PYmzFKtiqm!7Xd0UxvkX3#)x_$Yn`qBaGuYqV9RAXuNzR-3 z`#Vm$aLe8(SDxUw=u!No#itcb$URQ$bMoyoxA$7?&|C9-A?tK%B&jtLtZ>v3jE?w;?PL5 zmCuV?uBW>!ZTqk{{@Mkv3?Lxrcm+&Tppy?B~vO~g~b^a0ixY5A{i+oZNyT094{J5vS{*12-s=k=! zq=CRJm|0(cH*$Pdx5Stj6LA|D_jXxAeBJhFou1dATJXwn1XHqTI(b;f97?4>&zop_AwRgl!r%90q4>m(@wt+==0JDe% zix&?>*t<$iW-HcIZ?XYLvRUoUquNxWp{=IOo$IZy|D{Cw0~GcZo0(fToJQ-zqe~5BRYn18Br0WS5`t|dzJypR{kbgnSmdBRM?~noe-ylt#3!RBZ6^GA@=mVH zkPUU({z2s=up~)S{t@z*jim7@aBQOUj{J~Q1U|Fi-Q8yGtRovBTNY`)S{ZbYSJ4@D z=G#<#g63+soi)GDLLhX1HP@Hc_D&OguNt5yWjFWW3|@<%dCpWi*1J zFcES?nak7*<}sI;Kt8?LSyy-J)rzuSM0me)Wm&?2S)}7aWx=lV{G9+Tj3_x2$|sm-{!m`OxvLJ3Ru}6y)z4_AbnL^+lgJW%6VN z?X^JJq8^7O>@sig-oY6O{#8Uoak0tGBfQZ|QWqw%bt!GivxXvFzc=bvQ9w@8r92CU z>GIk=9DjP6$%Lq7b*b$JGd-la${i*HS=bg7FO9KJc`&rZLD%;llpr1cL4HIg4FOqY zWPqk31@^7s^ZJb&HhcyDjmEFR5YIL%0OJeJpAWa%?RJcF)tDfu^o>PLt)$M~Kt;Au z2r<~7-QDKwd}Pl!?c)dsE%3-Hg!0!Qsz66qMH5c}WP?O;+}j;4&dw)gG$nX^eOOk& zPdwyR1uy5H#D$!QtwxE{qkH%3EJYO6r4xaVA2;Y%QD9+|UC~I|_vlbg9##btT_zU! zcDd7+j_6ZguSpBNk2Xu*v=;(`^2r9O3l4OR^KB#-EBTPtP+;Kh^;!J4vc7!&{J9T{ zXd7TSpFATXvO_|N>kjLs0Ux!H--OI#oTV1}L>UL2%rpYUhETe|cijZx$l7~nlkDZ5 z_Nf)&`QYx#HfYe`fOA2XY$eP~RC&;w)9h37pYcUH6ZZ+*YSh<)*D42CW(M&IG>^-#o#qoNA@{=&yC=AYw$xj zKI!8A8nsOfS6RrO{@Bl`xLOIw(M@EoXkf?J!n|3-WpDRs!Y6DiR{t(eM=hO(pPMih|X@UoeFc> z3Yk=^)sO?=@ymkutur(Nc21^Knfi5=YnP!jWSuKs-qJW z`KeaA*@Ow|68@3t9RR}S)-;b*6~@6&HV@-cIhM>lc*$67)Wg9)?)rcMp1X2-yR~jR zF0_qR*=Eh7K+pxqkxr{_| z$3B7_id?S(Pg+K!NcXE~g;2)Tq;n|^A?fLXmLqt^eAUP2mv)3@eERH})C?SiO&om1 z(O=x}gN93lkpAb({*qSMAD##`aGLU9Uivj_jUmtRM{Iu*gq-ZQ=)>ch5T$(=EaEc6r_$B2^wzaH7| zU;S$Mn!7)sK0M`C;>u|})i_mQg_oDb5nE*7D`~t%I7go?-&&Cphu&-;bu49j7iLY5 zd7|aG?5*dZD4QjKW`uZWv~50)z}Bog~F6k+0E$* zc~mQ+c=rl|j3A*rh^nBTwvy07yANd)X-4_zA^$&8AH| zsP^U{wrOKV3|S5;5ypX3C_%yiYB6rx#|c@}Ch zh3uEOTkCrsjv8TZb98nYws+C{wRM00TIiUXQP+>nu2nMr{+uki)8@rXk$jIEJ1;F^ z*YHUly6ovaylrov{6M3{V;4P3jr>%mW?Fv_)%ci<4$hwaBCBSft1`f|hF9*;+XnY% ztV^8GJhqx|VYgj6)za*I3)O$E@HDq`w@lOcD2laIH@L}Jw|4B_-KOLT(>lS&!|l`K zj&%IJuYY~X0U938O+*4-rR>Vki4XD`Mdm!Ye{qEAuEe+LS&a;Te~DkeiPbOY)hG7q z)pBrHstOrpl1HVjq^9%kyY#8tK!~`R+ChXRr(-tQxvMjd%HW#3mJQ+ zHIedR?8HA8^i7$QCkr2t?`W!Ee*btK@)&xd##SO%V`VZEJ4_&3m>_gt;p?lR8~`=f zo<(|LAiAu`nAq}R%Q#T3tRf@iv7T*rROv0V^8a`p@{Y|{8AXs>&5$O3ww-bAp!cbu zAZ1y{xPWHMmgSgGCIJ08M30ps)PV9Cgz^>TjeRneCpE9S5)A@lba(%MqL`Vbj6j6w z9}Hr)`w;51tdbkouF1SN5=n-Zx1P;>a#l>|ca@Y7#0P?K)oj?X60%j#_T%dTUkTxX zm;tEz5Af8ZVY|@CF_0j%8&vFqrAr6XEr9PSic}H;Ge>$i*R zuWO^R%)oJk_0m^|>%6wz?bN#Yp?2CCyY?nd{MzQr-k}2`UX8x^=FP5%s-v!D5Ba^| zC%);t$|d`oaabjOhHB&Vs21IF@4Ko|t~fwK zRz%1>X8iaoH;s)B4)=Xfd6e(UZ&{0)T5ZMkAiAz!*Ob|<2Y5C%%zZe_zDh_$>2K%E zcij`GX<-l|-zKN634RP@D%8W-7;y~n8_cE9E7IRMttrTQRvF)hf4sI+oO!*(TB}6^ zMgZN~Y2kygQLPL^`>0x1pbOy5%wTcULt0*w;^FVF!j^(uSKnff(?e8L@0Ev1@(cS_ z958rr3flxQ&@uZ>HM$!I^MDom3kUt*d;G z&r0h5_p6}Z4>kK=Pl(^Od7&HopJbi3^T#^V&2jwf9r>x>5r(3LG-GP>^6lI0t&$N* zR1otGH8mOjP94(NQ=fy6%#I9em;>X5mTlMB88e<57szDegMb>Hr#r~-Hsix}R;f8X z^Q!TNy>^r!fm}yq{xzN(^6>xpD!f|ifwCd;Qw5hbkx@}nhBH6?X0cV9CLKF=q-qGh zG#w3L!GXk0)B^_qK}0jkSeN0L^R@nVYt!5OuSyP%oN96ZG_DgjXsKS|#RBqJhwdGe z?JCuNmv#Z>+Mqj?5nNOOYdd-7On)ltmzkL%0Z~6MGO*62Aw7=mFsu#PAL>- zp7Eo12WRAT-it&P$=w#B!1tABZ(Du%{RGozGA|4@xo~IPGqw>O3c~nuXtB*G&$!1^ zDbXKCE$;!HSs4sJ%pO@KJl_>I8Q8Ftx)@ARnl!`jgaYvfwQ%mKIX@rw*l++boWY~x z;)Zt2g=!qqG579`Kss5ZJO(1UrC7fb{qjn1#?`u|(+#ETVG#%DuA^2Wpo!uKrp0XU)U6vBVhawLUPr0-#!-V$>;#!6`p)u|>y!?=GwWZo z#USy&85h^2_p&-P>V2rV7}VPU2y&44-P84(V|Z3!K|x6}?FY!IJ@3;nuasqqmk!;UFndk&wv>K|Vf}>I_B6qL z5qGFHZoH8-IhE2AUZ%O|PES1*ybK=sFaYtp2Z6OE&r(IpR-rgqfitIfKAsH%1Tq9G z^5;b#lcmU#BAi1U+#&d-gGze7mM~xgj6M^p9J3o>^*?A4rcfzAzUXEI`ex>5T+>VU z`Sa(}(6cn|GD#t598l^tHvMKQmxFwCQtZBol-M`;qJjW&dV+bYnKOVMB-ayDMjjp6 z458j@080rZa3w2IR0lq&+o;h7R#kHY19jOP02MIJq7Me*n~EY_v{wA#g5qyZ)bm0U zj>@>t5@T&2yT4&1OY|}&%3a!=s@cOSat3cv?~9c2E~j|YKu0Gg*2ll(3U|57ytjN+ zQ3#^y-UK#s5ag9g?3$_3_f<4OQDHvMI`6yZbZZzAf`!cUVEo$zxlpM{^49JZRO|u* zQ@z+fCy$Eu)Of^*L420Xs)z_oS}pWOk}kNwvc`=s5&*6tau+|{$4N;l3gefPXUtHA zhXsge##iRH+BZ&LxyL@kQ9MeN1s@)FMlUh$t&P{{zqxnC4wPF-HHHMK}JH*`_IWmcfxrogogng**{B;4iUdoS-q^XR$o!9SI8aWH(3LjY89x>PM z?D%=I;DI0}eJN`n#A5nc56XKWVOvGv+{Y@?ugmh_VM)?aG+VVQ&%zl%7Y-P4mnnjX zEriKKgxdza7PTP_;^?tA*hL<&%<6_^QQKmJ?Ysg(q755S`w6QPOm2uKI-mBA+TV|j z9vic-HX6mN9JGU=vs+jw^vN-w!*JWqobW6{LGE!H>>@`NkLp0QrcGB8 zJ`G30Ay%+u*+ZRl3p>tjNqdIG;N6q69Z^W@ek`G9Rr;i@Xgd6kuaSpTeW?T|H@9j5YT-?d zdyJS@Hvn^WO|BW>n~G!2xQq7Lj*^g-1q>z~Z*pAywNBpe&|`lOHK~e1HoZ#=QlonH zL2QI^5kNp!NSDDAkP}GE_WIcZo^v$%TtUGnB6TeXhv^+U^^_%(_OeajEg~VPt2;NY z|8rNoRs~rTN7f;e%PUeMkl?Sfp%-G7GdHt8aj0trE6$>-497OR=;XARTFR`TCy^ST z%PH*s+m_G08*u)|Bh6aF-`=FP63riVvH6x~5Pb{q1#xtAlmVpKX*`nDhf(x*omhY3 ziA6&+I_50tT#4p5x4_XQ3@=vH{8-$z1p#hXSx-c(t1IA7r2GkgdE_IgTT|-buUMs; zTbkpX)BrFcU{|<NeE0BT2p zWsp4mQSv?0E$RJ@BJMX-^<}2*ni^?T{#%U}+lnz+$OzQ~PWJlW^SHq>k>A&p)zxt4 zjsy~t=G%uBg_Y7avAL!NT1&}vsK!V3jNrMW0taxumGx4s@j1`zCcfXeY136u7U9gO zeq?OE)psya?Ob%_%B+y<%Ptm6vZnQ5Nbd^bP(p=PNV__X8fk}Udr=kTD}KSw_c^Iv z*{aci3%fTFlkRt6M&0cMM^S}B&Of;Gh5zrGmkD^RNRK9FQ%bRZ1=^TK zMOwLiM~|DYQA1T#HIig_-8c~UT$iIVNWK{6 zHPmgmep+W5II=+oDw#<673cbDsB2vbA2;J+d=lP4X@OiUuX_e2KN#!XCA?TyQ^a1BGw`|xj%Bj5O*6R6Y+9*fw5M#tu(sNkwxoJ!N}q z)rH0(y*YUmA4Uw@7!?Ks{o8t!EeLqqRXnI)J@vd3$^i%8~zXEvTE4Z?d8nEzzThVU|hE z@3KsB@Tx>wdywF7bt%n!zQ@>kU47yqxHW0^X%BXswGD6^6}>&&;+*b~_}eLBtx~bs z%U7=s=AY|w`LFK&(1fXYG8ml-{MgMg{cM2mx<_fbPUj-mc$8+qHoa*Ols?#a#^xa$ zc}c>EIr#x5*ur`ut!(Ehm&*}NWjOZvb4*M;wJj>s)LcQU+U?O|vXG1(4=&OyzulM) zx)Ws*3+bp=LFr`F3zz&RI7}Yex!-lXWfrtRZPEMg}(^JC<~a!}>#G4)dv3Yb0qt{6b6p#<)UsC8$o?L)Xi za2+AUyy|Y#AOMp^)BQI;ZF(kIY)8^Al#oSa{!=;Xd_>Y8nipCdX**|2#t8e;JorNH z=U4qV&zs@7f8UM%jXF|^-?mTke);!mfyt6eVg*N(lLXIb*626~lyY8?+o9W!`ORtW zZ{qX=(0cCHYWy-8(32syrJp8HoK#1K2|rdgn*ND!6JDBcqG$ z-nKVCsz&r#OXww(mgUp0kQNtye%TktuZmVDCd6bFbAV|h=|ed$LPkp@Tr&xNmCYc2 zK%3t((KwtFt}D5IeWfb1LiEV!1khw%BU;D+k>2Max^o|Qy5+4`TL=$Yr*6;BOuq^v z56wH6N*pxS;o~bG1Y3e6KE_l{gkip!?4`pHZFFEE-xiEol9spo;Sn?j(jU>HAoAD* z7ySzPp?I4z$*=j=0n(T(c#jkMgeKdhBBdST4>d*LcgMRg z$i>41|3nplhSAo~t+iDZ*#17iw(u3^vAtai&0XiM@+{2->mKxOYM>Qz{&U}86vA_o z8LJ(M#RK4t42H`yk_J^w1mXGtItH#$83he=m1Q8!G0itLxiq-+m`7rDuCEA93}LNR zL#G$b&RR|3a49-kVUnO9CFn~``n0qH=4QU7>NTU-#VWxD&PSZ+1SX-}pY7bO$=2|3 zrkr5&8QoQi(7FIQ=ugCPXevYbNp~`spd&lZeRi%cF=8cQ?gFDx!>|DWIf&1?fZs=X z*8gsX+665JTg4TxE=?ZXoX>R!^FsqFO}OCP@Z$Hp#)1h^=Tbpdoj-s6SM-e|PfqR| zrpt_-IJ3%M!-(m|eJ_{rM}*{Zp2-w?UcORh&*|N^{%);K-w?oR**`;uQ$sSEMyuk7W!I z$Ziuq6u7yFC~m`0?&sXxYdoN>7#Xk|<5a{dzCXkgv#D!P+*Cs#%uw1X_1#{LZ?s_C zPEagrbTdvrP*;yQK6+hFhKJey9TmO?xQ^R!VxM>7M9o9a?;OsrxM#6$*OWI*Kc7B& z-1p7W?QWkOZ?`)1^mE`OhwQisgO)fsIZdJNfmsDh@6z=RN5^W>Vyd8}637+PqY*-i zu*3mPm`>{_?a>GbynfVHYWNN#SU75+S6LyeFip`zgl&1#7L9QE^wvwVgJBaoD955a zEcDu$TTW|r=vG@MHj#&5b`T9naJud^PiP1v^0Z3j2nS|W-p9-nnwDryg}djYX2-$> zOe*~$qjVhbG!wS&NWtQ$>lwNp+s<^mUYdgB|JwBbKWAPF>eiLaB5q6p5QPmK-uU$% zpo_SzMyS>gFw@ib13nhaIl=``4!4@DOCb!GBMqXM+Qj$%7EdSiFub?(SFzCfW81cx zoa4B4&b%G26mgOR`%(-ns=Otk$Azgs@+8-2JQ!?fAX}Ua>-(byi_HgPC_^o2jt2tab#+kCoGlsOII}qCZWnP`k?LOjh{2Cm<8urGxsvzz zeCeR+0D@E};;+G+NzHVZsc5(?I2T#}%jl8+drnkH(!Xz(3W2Ykghf1LI*^?Sh0fEc zQU{aCkn{Zq3|Quu(an07Q`NFj`9I*o-wTu9ylZjkuWIU7P5FPvm59Xn->?6joBXfm z^7B_uI~jxe_baRY-xAHQcQ?CW>zUXD{~04=kl$yfkPGaP~e3AW6M9)Ou;h_=t#zOK?DmPuK@L;!zybrB^=O*;lO$`jYM9 zf-*Zen{L_u0y58x2TKMSL3;Mej3-?$EXnV4^huV3aUzA=8WN{5B#bvZF+z#RSMEm09jI2$LYygip4=2M?A&78f_Tuw@ z{yB5PrFRqvQksCqi3>BJ7-o8v)-lNurxP{*3-vu}e*@&an237&A2=<>8w5geSN8XB zB@6=n;OyHMQJDhu*TGQ<-|7Am`*2f(l!8>aRNg?b9y^bmk^GC5yZ8f7vifOa|- zO%clsh?`|;=oq=vQ0w>3+1s%7f479jjxvq>UT!R<^P-jikrQdNi+kE$g1O)%U@F)y zlwGAJbN5UoOR0}Wu$J-}&y?4aF~wJ&f*;)PHA_v16@+Np()#S)|18K%JF zD-wHvO=Xo}^n3{8b^N;;N;0Hox(vlY1}(d_-Z#-{(#EN{#|FKn=Sg(~$!tKSAmVN^ zwmXfH>WLgqG8s#A3zKlwpcxG@;*ke2~;Ywvh7|pmDzn(MQ6@V z6tyDiVNBsS581<35Jvz&F~O+>{&2PB1j;&en*HgGz5vt84Hg3dAp#iMJi0lXwsa-{|SEc^rNCKf;qtQ{lM0Nyu%LEhx#Rc+VB7ybeUdN)}vC^aD98=!Xyw)YYE%UBBmAF@@9+*xWxdVHDR-T+>z^V6u3^p9@rY zZbF2L@Y%`8--ggR+)m`h>66f;lEl2Z{*E!5nAD=$-kt06>kbayv5P!529CpoiI>BH$0 z;@29oz^f)(UKu&o#CTWYhikyg{CrA0KehOMJ}9q>bi4^|jg$|Ft=M5qdh8mqppGJ= z{<5NYAgSkW3>#E5 z*0*E;LhkbQVt4|h@j(lcl9Q)GHfcX$7IwW*ZLM2ZU;tp-Sz4zC6djoR8$B@W&kkM5 zJkz6M+rjgj*1)+?DcQW)wAz2ls#q-^l;RgPA|5b8W-O=)WQs%2l!0J5JcKp!ntewt zV-=V}XwSL`(7%aU-^mO@2uX$){YDb=l`B{Jw^zfY&Q%V#ki`UKaENkH7Tw%=^DMHM z0BuI(vK@~;JvDKi2@@2-i;e7_om&l~_?k5*p&IfeV6ld-0|=lJO%>VoL@cVXGKzQ> zJ}h%o_dYcd1>hw@xhP|VUKa1{y?^Elhk=@3gfX1BmHJVJ+JU)*GZeo$N&+*T@r2m0 zlzkSc&6EVoj0CUH%a(<8_qXJ}iPe&DeGGAM{7tMAx(HP1>u3gFrpD|5|CY0|?_tR4 zZQxW(*-CdRq&}sJOtwtO)gJVtubN{Jm2Un(ZV(b<<15sE2-=H~X_@zkIxR-(%3vyGnOqVrc%^)pT) z`@52Z?S=$NT2e8~;PmT?WC3EbEDP!>OQ(7JaRm^L36x)0@vQfwU8+Z$?9S34nd_v}vnIm{Y+Z2tX*$FQOg|w-M15Vf2v+a)+es%YSuHU%ve#8g?H@P@*vwjK7AR`xo|4E5;9wY`{Cl8-_sx{F#s7aKD3N}k>tS7P6{G63Glv`eb66Ojw zl2e)5%ZLyxV8yCc9|VRZm+8*QHJj<5c^xfp@tx^3A!0`44)v>hf=YDfM~l_Ex*+5- zR0ZgNmCn5rW^lvej4zE7US;Q{CMKHbm^H!Eg=sE}q@5RrlYtqk^(m?}Q)B!&DLt-#OmH1qr1kwQ zNJG~9aS`o%c&zPOPq)gaqA?9qW%%G9uXk4vT{l$@P%qD(wbyd-5K{zB;1j@x-2_Hf z1aG|_$uk*1zb?mZNx1?od4tF4@Tj1KzP~q<(MQVeW0wfsa*5*Gm4@TW^r_#Inr zR0X9`*_Y7asVSUJwAkr<&ig0?`;&QAYW)sSiz!kXUO|6Z)&3wSOnkW)X z8osg5uFXA9WRyNMd}6X)dQVPQ5G00DFBW12EaTBB?Wjk(hfjHrhYqMhj9(T~)c-%K z%6l_VH&%0BrpBQRR-eIJj;2w@@4Q~u!EYW@Y>GQHH`890?I(%tv3m2g+il6WP$1&+ zs{Mq!sB^K*XBRAF9CP1$ecO97=U??ZyK*bfSRimbE|MLC&W7}H?SI*^v z*^HyAkJ%yY*J{xxBj%wo8QB)2_7`RFb<1G?&UV>Aa>*Dvg0w0#FWOJgGM!?il8XNj zMMhR01)&U`a2Ac3Q`mu)kmvIjw`E!h{~TkRC(NKC^(j5@CWnxT0C6ptIX$muaF)|% zu~+-Yt8kbchayNkn@o=-s3L8(SdJ~ZIeaCb4ZMPSl~AzpDirEw8- z8zXo{0nL&*4vCL|V=Jh3Q@DLS`9b3MB$={cP~dKyp^!5obt2aPY-7ZJQ(${)egTrh zbR}HT5sD3i&>w7l(Ua4MD11pl94DSz)R!v!UZj)5pXblhAc_r@cqF1F!V2U3-24cX zcjzgdj}%Za%2PpCY(tYiE*t$?RY>xyq5u;?%(G_`M{J~fT<$V;nY5}0CJd6slpFnu zLn?8jqvZqU7SJKW}_Q+u-(q|@}ZpEorHIU3I z(u+&oP7IgH)a%qaA_9|c9vnlgxu5491V9iepBz-iXaR?`$&pn;IZ%XktzgVSwRI(M z@lP;85PNkQcSAu6SMO>6y>PMgnX=TVT48>K*CB!qOVO$1aD^*mI%_TE44}r?Iff`s z80mSa>A{W!f ziWRIV&fqSNTON_^ZEuf$ZB^u6&z9dyF8a3K({)xY>GfHDO<44o=xQc2gee6iM;xL2 z3K`*&XhDxJ25ZvQKo$=GdBev_lKWvgjoAge+K*q;@xs?K{M}UnqYzzC=1`|i1qylh z;^M_2#q1y17t{hh9Y1TbjF4!jdF>OyjoroX_0Z5<=N>d$)rUuh8@bm6{iallV%~~* zg4gC2c?(B!?tt~uKZ;wl_sC!1NqPY1)~THn83qf(H;h}x5NOT{5`iDmINk|{j~!+aAb#?)j^ zN%{9@xp+%WMk;=@xJMfgHW)nNf%^Z!c}VL2148_tQ3bB+eq`#0-Z&{#^v;n4As`EvHCx+^g5#zLdxX?rLRd zu9%`w`TQcxP-~O@gtGtqo@Pzy7_Y;gL+S%pR6%}%BSZs$&5-jz1xcF0>aEHTZpfL0 z$;rKLEP4%I}B^aff|{$2b0H4c^(5&PM-?Uxy5IIh<55OwhVP*h-5W zTfEO8wfob-m7X<}iT>|bbx0K_6h39WDsoR7{4ZucJJ*f!ebHWaV~tVR(mkD-mUP;g=C zQA7qnidGb;RtTOyR;+mM5P}F&?hq!oa@s9m$n@lJ!;GKblmjJh*~l>Zuh$_1PUvIU zWIY}1(Q<=T`T*cafq=?8=824*mf5Sw_!U${1?6efa;t#w+_7TD$?M*YEq@P;P9npU zj6ui4xZuI8w@i+)g%3VR#U>xatLx4xj2pw(E&TR%A*~EKv%JWInEPWjWldT7 z$M2>xYcuc?++Q^sgB3WTM@C^%ATq+`xlOlrk59+ytmGoZ3eb@USF9KJG2;wkba@@} zcsL__gS2)s;{KIxWC4PqI_<~Rk-`q-q15mJV1*iCb{b}B!iUL_Hc$Add0;I$!PmLK zUSuY8rMFi`HT&_tyz{r0Onmg`9=u9RTMfg}!qBiOfK)&HndJB|#yib|hn13IeSlAB zsl@E*xOG$`u}(ENH{Y3@IAh|^XP^-j-?|Uog_)Qm%Q!u#=qsKBzRZnel#4suyj836 zWFm^$;}cqU=ulm@A&^pM`&462DJysB(j}4m38%rH5_dkeeyOka|5!M(b^?S31Ci^8 zo5v|mI+m?W`68#ON=GPl8YqzX*j>*eQ%T|jfjxx?yp#wCFpamIl=Ih0d~>Ga)@95e zvg}n9DdkA9Xb3BHtjB{nf?95pQz9s5R6t5>ge zNNfLD4+C?az^ro|NL3q~lwn0o_2r!T>*nW}WMB_4E_CJ!oLa!% zP%8s)xI*;0lc`qa@)=N~Srbb~F}bBT6VDp{uvA9bT~?NmIb0-?-W_LW1pL~UrVnS- z*6hZJH2OFj{#hxIK8|xDw*{5-o5f$w3smEw}{|fwx|dRUsEu$*z;x{OLFI=$KwXS0oZG+) zUFFG%G#`jjRX!5Agd8Z~=;S2cHzE>}fpi6GcSl@nhVhSgVeJ>6W&Zfne+&JZwn;N; zYAyzK7P^wGLX>M`am>?ux{Zxco90JHssN!~M|Y|myLcFP#WWJa*qKM|t z41u9MHo-ZWD`YgpxZC7t1+efi{W3Fw_f=o~$2FjN>{A|*E#_V#k!76$YYoO0x>Tld zVnRb41K2r?Z$}d;H_^h!r7^E7psCEaLgzwDbP|e40yx?#+VVeuSL#$A*z{AA-_ifq zZ)RnWEoTe_Y&;RBgb~1gY`D*e(B-lT+9(xN3FtC68fMS+2$6c1Tn5$xFxnW(F5PBM`-;oYxG`0LlSBk|M!RvG=e z>xJJs{ks+8-T$vT(M_kfkFB@@CgT;&t!Q1zzT$}klo!n@6wK)MAqa~r7z&Xx12@!( z3H~b~{JucjApv#$1Puh9xB?Okdbbt^9$V5aAn(l~USK;U>5AH16FcZwzndXS=*X&LwL-!GI9OKdd}n>s+w%t9YQjZvpK|6&PVq*cD6 zxE0Yg2H_P+$k&MF8>s7CkADAk*P9m}eOh3Y_o;VrMnSPg$w!OKE#{-NE43Z6anGsF zIsu*MI&65XzCC72sNTHOWv8Z`+1zB4o<{R7p1V6$U0t>I=3ztHu4?WRxb%5$;rCOM zGDGrvU3mYU(r8)Kgy*~Bf^MV~I~v-VMs+l;ZQ`wM{1za?Z?d z0G}DifD1AvmF(63^k<8~8ZL{x7RMW8536qQIocV^W6n1kxa9GawLq;M5+^te2j%u= z%6i)QkKsZ7LF{p5Oi}x@eUE_=?$L9Yz9h+Z7bC;^@-Ndng%OHNL%6CB6+=f==}5;>3wb%QAv@v?TY%B+c;k zA+_jf`8wOn&(apOGtq_0$CsuH!l_v!^go8m8x8{p)&&StU;g1#!(NMKZZRx;7nzeV z6mx+vCnx!sIl*ZcriRekIN>Un{d!}^H=DZL88gMGYUZEd}O>Z#o=(;uHosg=z$^BO+d*ClUk+SjQ3=lfcw1Bl1Q9-Cj`1w5JD z2oqg|x{YG0@xvz#$IM?@N8aXSq3|^A!yh*{$%=O?c35A)jA&20$S;HVzWaQZKG!qh zi*93%h>t_hI0}y$vi`CBaK`G>b$7{cl%R~$aUJ3+X?bt0ZEDb&r>1c_^Fq;) zDOc7s>^9rnHEV}&>JC%G7=F~3;!Mi~p9u4aeGd|r_!NHoN)FBD5&Dzcb-esXwmEM8 zjE2MV*!+2+6UW|9-V)s?l#@7FNh7n5`S*PveD-;)mzDJ%w77;yoQI)GKa*&^JZ-R1 zp&6s%qniZC6}c}S2aU)m<&JOftuDsi8>}(W_5`#_LENO`?KWHVj@ZuyU-j}u5OLvw zO^2qF7@`OJ^{Ll>+=D*$rElOuQ6y`#L-lTcu-UCl*vHWd8YB zt9LHeu&Qhs-=I*=5vWdrLT^G4B`F_pm#2ySrDI{HVEnJ&o`021gXW54z7? zed_eq$KnmD)a~ph{ax0}W_5_lNswUCu*duf#RT!U2Ub>(9bEfpjHA5EnELHL z#;q+08q=<|2{I$!)QiJMb6ViB3_jWS{nCp?6RmL>mHVdm z>CcqYTz)Dpm?7R8ohJ7QPny1O!h%cX3s}E<;SN&uUlfdx^~Rm-eX;0S=Ik7~L(1fN zXP+&3(#qsHiqpvKJNiR!S(%&5M5@Vlm_Lgx8Vj%1G$}5sdV;<=b}!;xTPlB(?SLYW zjOX+7?zxw^#oMQ?ZL;Jhv)Vo?Wh+c$Br%B_ke#AY(vx6#d*}H1d#tRTTID}=Fuh|~ z^uc>kf(bFub^F?;%i7>dakKTlxudT9hy`=bmaX1XbKku1vHZg`>Ml9=EUfqR;P5AJ z=d2i~l+8Ux;SKO~;SGyBKFi(*&H~1`zNN`NjpEO1$5O0qFZpD=7^UM3c4yM80G|)9 zCH$XMU;X-Z)Oo7}Vp&$UZ;3(TBsUivpM7*r{Mr1)_fJdbXuqfsFtSa{0=MzULV}W? zEpweWSb5Zchv~P)^9GO6*_<))Y5+tjXU*7uxwR z*{WS{qvXyzyq`A@Fyh^zjaekBRwgK;b`WCTtg1cYW+vJT#D{u(v89{W$So*+jcKSe zcPqrUC0;T6uTXEkBr!1mN zT3sxB7|Fp$5$}%$WziVS2=G}l1PjU8x<AC-rfDc))!q#b^Q+~ z`G&pQ*4tR{>ix*OXW{C=aX<0t#hW}HnH2Cd%!#&@}p~qx%E|{JZr?LJy}Z&;pPAC^pT5y_`LvDg!*6Pod=R+uitM#M}K6sxM^wWKN^iZ>VQKjVs+H&&}!8>iTlmyKSre>t_y3 zv(*_LnsMU%8#}k6@xvK@|M@y7tcCm^{({um2chN7?s&x6PIqZj)6}%er;z zM!Z_n*gmfP+2cQIn9}#Kd}QC)Pm0DSOCC3qt(<+cg~v2*T&pI|OvM%4wsYMdd!|x?&=Hbo9I+kW6d6VcN6k89b zO7Lf2ofWSskSe$R>>UFFq!V(C(Wn;3;$?T{0|7jpyj;EKOZN|_ZZz#yPyN?*Ode=? ztnchol!|Y!);7IG;vS>do$8lk#f$)#jY(|BdTajr*U`tc1N+XN)jet2j@j#eKdq79 z-#=LL>z^p@dk_8nY0cVNnoi`eAB}J1_v@dz=kBb~_Sb(*j{3&~{vUtDa%^A~ZEbB` z@E-}Jw5=y}^Kk-x%WTjc`X1kjOP_V4hMK>(`@C`UN~`G7j~HT>RehAi1lc46$MzQ2 zSw-C>y57mzq3#wW$7 zCi{j_ac6`aF>^wqb_Sg|cP#nx?=S0lD78W$_5^;Awn@ZBog2ku7)gNz1SM}uejTI2 zdmdRYSJsfagE5DPjT-#=*vYnDPc?I?F8$fz7T4vWwv5>}xanBEN09Av$;^}sPiGI3 zq%SXb+vTsxS;Fi!99j2d&JUw5s zuuJ|k2lAi)@#F0Xn&YYZ`^d51j= zWh)1@Z};*_~los^$-(Vy)Rd5I}nb{_heAV&S!^C;2k19U#=4L zN_*_YR+*Z)KAFy1tf5;=36v9Dav!Fn{gF0-n)}bJ`dS5ciO*eH|(M1(0N;iZ0D9(xL%+m+TL>41bY*&rn}JDq!E^5_})8bE*ZD;?DJ(hCib5% zsdntV^k%cQE!QW02p%|O{?=Yt8s8_W;%cWoW53h%rj%22d6Wg*(6RpWS&Ss5q~)1W zK{`6W?o&&XVtU}q|>&Exae_AUSnUKkLa@>^l5!Y-?4Zd6* z;)AR72@|R4X0DG=)o|`)B5Nb~>FSXwLtL*}BYDW?-oHJvZ~p(o-g`h*b*^2bANv0zEIg(e9Kiin~n$_5ccdbc170*WG{wCGWz9u>C~sWy671Ox<3j8Q6SE`%iK`~LCY``>@uG42?5k8_N|-g~X}uJ?W3`OG<=XBFViU8)H%uf_9G zyqI!DIw1p4#X(A0?()Z?6t^IQo+)PJWh$AR({AS9iNnu@`c$+j#4$ib%(VF3)BVh< zt>67TKCW!elh5tpWS4VoLW7`c1Zf8+6dV)TaPvDQBKN=>ZL#NNBoAra&=s?I#(d3z zoEvCgF{8FbThTI7V{klDzQO}Q)=8>4bH?YsKClSug8Qi-swip^7`8KPFhd3_tt;{9 zeppqcGyk#Q(K%VO-5$FdGmopzOa8h0`qVyOQ4k6kF;vKG-COc!?Whg1tA$Kg%% zL;IY}YRT5jnt%z5c5f5FKtgI!LH?{Wp-3_}mFmpbk(9-zx+Kn<>fN)Qo5UmZ6U@ZE ztc2YQeSZ~r!Xz7BAq6rFj*792e>BYyL;>1Vs{MvBn#ki4!XnlV6x|9psW;@4vwhJMWF0io!-av`{UsZ6-)4k;qf#)i`LlM z(=`cCVv%twS$7s_g>@s)tdhJ+EnbYIamwD8k%30_CS}M@BM7T|+aD?d>FN+V z0D#tvuzt6U*#%s4K;_?NcGiZ>MTe(Xs19&PyRB5b|K4{)8>xz3)w^Gr)wEzH*x^dG zuTE)ByJw{>*^2y9Zczmk&TtV1dxKH^qoY*`?c6TF5E<4o&Wgkk*NugfvA*c`(}&|X z0yw?|osXP1tf#8K*ZMekOpn%(s@u>x&J@ncmPgoL4WT3m7bqHUwR@dT^&r5u8J6Bw zVP1!=itzjtmiwiGM;lJ42VirU$IO8Am?$3>Y0_gej?dlp^cttY1H8utB%WaqgnzRq zyI&l`WJ)Z6L+kM=5ekyFgBCGR%_Sb)`P=v6zK7RtRG!o8sdB4M)FA(2^V2>L1TP|= z0Mic4S8k@*-G2HaPb6=CPES)Z!^-kX(iXRd&NS%{Oh)T0eseEPdR^kYB)7+v$E6Yu zEgWfL^6>Ktt)@&39g};_nGJ=hpT?O@?XyxpQ<<`R6DH@RB#;GmC)C?-Va0?vd%T#$ z43)QqYXJKkmx0SKpRW{&IlI8t*`XX;7s@L3oiG7uF`GdDfkkLpy9KUA{}<3emhkH4 zWc_{~l0lw}2euF5FQ|6@TydZpbtDLmUlK8*r&Qr0WWWyq;J$_2T=PR2o^yGF4jBXDuWn_5pRkyFGKClc&1-c zl@+62(6`yEBbx~fsYVMVVV%e`c3`@0!XB+XZ>2)M(Y<5m6vyW|#Xm|TATcvkg+@m9CMqFLE$rvPnL z2RkI?E8nv?#l-(lXwz)1%YFl3K8Xl!W`XD9NP$4o#=l#z0_w1ayPJO~EKtJk^(hRP zhc}G*PmH^?8U4@%Z-{Z?2iY%J1s_vbtWN#@mpf zTEU|8sqTy#a&=%j{otyPz$A>9{$lVmHiilM`^(IobNsRJRIsn*Nkli+e|uorIj#L= zNU&V1flaYqJcKOt_DQTSb|8M(Hs^E{fq?V1-1T4vG`(U{Uf(??FajqG1=T=X zZ}4C~`oySU@E4*#W>IW(crb#-^zjzkmLkUGeD@f@N7#-SFVp@$lwHp@#9(l{!c*}= zm-IXvjWt%7*ob5fPsX@X@NDqwrHoc`bn(?#7l9zCl2X#KJrZonemHnXq)aTaN23$a zZ15Z=v6WoQ*v=GG1s0lLrQr(TYJXGO?KYggt;iGRWHg~E;%#7?Kw``I$$}2Rx7xV+ zJ(?zvLUZr#^aIp;t~$)`Y>$Ldr;td?Xir?CKB}d5=;Fnt?ZO z`^l`8h`1vW^H(c+cWNWxpKSK1QzOz#J2V(rrA#K^{L7JBa74(jkRa>?B+yfJg9;K& z%I~;(`+d?D<$*(2b@n?#lYHmmf1Fz_XNUFQxRB|!qF0c@pbCWx2#tFq024@_LIJaWZk z2Y4?o_G(Wp*BzoJQa&-dKdmiN(Nktt=(c4wXPg1aFT?FJ92${6ALP!$0z7}D8a+Kp zb;Kzua6a&*@i9K?3M#ht?()%zJ*}V&0vgjF&H=D?IQG$>rXdKZE^DmuRVZe4_-bfn zX{5s+cZrDhdq(D?Y2Hrl#|8vzKw=4}Z*fBdkece6$qM zk1pyrUx=b4A~qgK(8$2TlhsB79*os$=wKrZBmVI5=NUs#AlWLr0ThYmIlWsy44-16 zS^<{@qV9B9Px|Vk6E=~@7zOyg6$qWVtO~jw5MyN`Jq$EkVq4%B>ecm+}Mgk@dnkbDOcb1LY?h^to@k*C3`4@Ew9CT-h{!K=Jh zfN)cP)VgOqdYDMy<(etx^d?StU>~1Yf}hByVaw107Z^Wli!vXhPL^ue=tTlA3`c!0AnKBX`n(U&9ogfFbyGj@@uhr^pWgx(3sUpv9#e%6Q89Q{h z3c2awc{=*?<91Y^T)7D;s}9Ie{1Zn#BkRh&0=nRkPr?>B z50#GzN_qJ+Ly*LeidCXQY6~Q@1st^#8ZXVm=;nJ{Pu&gPIT{-gH$WHU(n%K^28Tn+ z$O;pN8G={Oi>U;$?wvJ(a{&@3yOym{1>%e=8wt`$+gST10Km0H3Bd!;h$Ru{&wqtme;Wi%prXgK5nK?vpsa98kj=z; z7ljhTW^m(^JxxCewCSs_2=%!2LfB@t|1 zy|yl|e|JDz*VW<^iVeyhgU8cod@yZd1c|PK;=(!9?9g(|v0xXv(A6&<!Nt~ z8QM%rFUr0{+@Anxq!E-q3mc(9a`D;RUJLC|4^PHwhVDAz-WoeN3w{ zjglS}KcftnPF?1{L>p(c3n^nGgN`E*;-Ogd*wu}K^|4A%unLxTZ!v+L2^_}_rbYCQ-T3} zN!$WTxdn6zn|{xd|A(8DkE?FJnr51F70R}z1GbD_{LW?ylHHmLJVOYU;RF{AUW$n| zN;Ri3fO3}*(LY0VZ&lbrWljb>%_@ulz_0>k>MDpzg-~WN`llGugRoR+1Hsu>gY+*1 zAGKyuAWuJKH9>KFV!9SO*Bus^&=jkNKW#;1J5V@eqBUM;jTpTHxo3Kdr&*Z$Sy=fDWqbPsPN@;&&-iASPa&p>e>32#5Jtr>rEY44= zAqe+dK$co${+YvbQd8Z>Bt&iuXuT4T>Ns_I3homco_}K0d`v&<(C- z{rJx~quM?)`Wpe`R!?kY^D zdc0mL*$80y5>8!iuQgD+bPEa#TXCDnZrs5h!C-{qKUJ|3M0zS=;t~)GkyAQaUm!lk zJd5Y}o6Pq|k_$5;LuMpmnq8Qy5rt8(*s8<|@nxXzstS2xoX42f@kv7PO(`Pkl~DhK zRY@Q`84bh~E;gYF3x7xqwgjwo=JO+R|NUkaR@a(stZ)b3n7$5w?*_?yxlQ-}JW!?4 zz0rbGrn%qJ5F!#~5*S&09g;FeaN;-C{Xz@==v zqij8tsv?e}DE86G-c|>ELrU3;fWSawFDHb(v>K*iCTFl#+Q7d=BKKjKHa5uy8*b$E zv5=uoiYw0hBpD^W+KHx%7K5J>QKkV1_num%(9q_7W~z0C!#j?t%1%IEPsC~<=QqdK0LCOFJQU1sf^Q}G#)~t_Jdd++Nb@L_U&UgSOOS&u z51$eE_*AW*_&ql6q!2xzHOU!vGhqWC6#MME!t@*C%r`QoNUmbl;4B}#f}=mX;38B` z1u>pt`eS60>7rcFX;BQ-!X9tqsgI-pTu(TRqcv2PT^c~Ag`>ag+`Wxy_hb(-T>01% z>w}ui)K8P2k(k*Sbev5PhH#BKs~2}x zn`O@ARkUBD- zOieXDIX4(aGX?U?+~+6&UU_l65Pp8fU9&5sy?k6(K8Tz13UL|SOc?ERh&n|W_|!MQ zPKpM_@e+(}qW9M*-aUNpJgk=v0Ie8fnwcKkQUAjQkP z=A&2ryQF`T_w3XdB9pTPr_KuHRzzn7AJIErUG5?-x!G>|2+9E^LA$5{(9I zkqI*3)O2Cp#wTzWcA_r`N`QAZ9^ejSlbG>HAW5RlR$nSNGB+&Tj-N`|T!dnDYlmRG zGuoUvQlECSb9yF;Um`6aP^kEpZ+q%(2?`x7IR5VAdd&syf=c`Na;HDk_ZWpjM_)M4 zN+urRo1`K`zTL0_v~X1o7I6_0PjdIR6_qb1m20H{3pcVUOxH@@eDnR+_XSsgk825YFG5!EjdwuE zSv?f4(?lC|q9M{#Nj(@DsQg;$!cpY!!74*(!~uPuaAKK|FX0tZHsdV?%46b*jldqw zM(}dTTo^eAk1@qjN!P=6Ql;8>xwHh6hOhzWvt~fnuo>ottl18}Mfy_g)r6Ssa1p^F zAR17)6A1SzV^oy3Cxv-q#EwsTYyv{Xa$WFZhD$D(;c5QgZ$+C=BCIU z^SNL3cjxsN!uMmnHkt!&Gb|0kX{MNIJ%CgcqCEE~mCDT*A_@Aa2RJBwC0xZ16CHfD zBuGGNA_J}B1_-m1A0{sL*zLarRuFZ}ck4}_0B_H}A+En4t31zwOL zvNxAhO)G~ppN4HpXRVO(GF8@dg6JK^1OZu-WB(Ozij*IPFG|_1_+ccX^8;Y7))U@6 zUlWn}XAR21uyzgjR&0hDM5hbO`&!%;0EZTQt_HZH5SG)E^n-&dX#2tyz92i&PXM$D zrPoc5i%=P7rq-?zttv7K=V|jlJv+=r_MEKFz@ z7Q)lkC)kyD)|bh^rD46Uk(30Sqe(v)@NyCy;ov5sE?5PlM3RL2Q&VAyeTn~~lh$yQaMKH2ew|<^SWih}aB3P` zx*C$$6uH9y)dIG8G066{oSf9OU&2k-_z@$>StEJ3;jxkKNdGPYxX?Xbn25dHie@c3 z{oV@(0g=?|@m($euC{18{Y3CiFkUkIU`CLBerR6C#l+k~N}Bp}7T^c@ zaG=-tG2CONjHqpr{%r-*Y>S{mKQ+&ol0HGj=PM_l1o@bdzG3L=x@b3m%%?(c6B#ze z)F$|2Jb6cvlP4Q#@S9CeQu^s^&&acUDC`v`Bgpj+uu-S^L~9)J4skNqX(yP37KD1D zv6TW6$9aOzC)EE#5tcGJhs3F?A|uzILK;0+2jQp!C$;TxFs=<-YwZ!aq+1!t4&DRe zGxL#1QQm-i#xfv#Z`tzE>VDKNk#Km>8>@W!Gn}nY+EOIOGxF)Jp)Z=ZhNJ|dT4yTW z6-llacgF=D2_rWvUZAE#FxcSctc@;enYx{N3iVeyK$ZPCx;|-Qe5pb@a|3uc#S;3J z85gi8l!A}0FA#Q*!neRGkqG~)zn|S7VD1LLUleA3C#Fj{6!NVjBN-QL94{ibI|F0g z++%Ba{Dd_iGK;iwl#=ykM_@%$#=NH?P-_(`@#TV3f)^^JfOkRyF{ccT=kSvzNfhx| zp@$1DS2jwPB-gb53A4rx_J)nVK(LQDuA! zx|Ivg(r7^Cdq)aV5i&sL(|scX%>6MW!p!$B|dkW8psxy%sH9w>Fe{f z?bkwOBaZ>@x2G-Ho0uW@b{1GqJMh$PM^NXBH+=`)#s&YTU9}hnN}=ip5qmpq!2tlu zpu$wKF3Dw*m!mg6O`4RHkC`6wI+&sFAP$6J%S}v4$Lt6~kEH>f(Pu1e6U^-B;aLqs zBzZMvw%A4aPdfE$Awz4z5$K>M-1f8xkWUC)M{YdVDqoEa8@iXwfGC-eK!Y@Ea+s0+ zd+L(K{2F8isSG!S^U?xXH6!MR&7c+We^~^9WV|MhI|E4B!3yuyUx7 zgjA*@An@DtLJp3eYE@H=%Plynn0w`R*Rpd+RrxEENdvZ4n=leSgP_dDe_{k3t#~#F zS4q$}>y~lhuD^Jttnxi7FcQx5oi*}dV^kmDWh1SJtQlfd>oI4KuVLt~R|+}5xVWN* zzvN59Q3m(VZb`SPm3xgpoyuS?ttjhI_C%5!d)R~TOIb#Jonjzvqat+?Ok)$(4onY_ z_)*vz(gtF3CDU}=eAC=~x~NrCUdeIRi|6$s2&vItNu`haj``M$H@T1?)(;;NSd%wD|L$>26?HNngoV|ejIXt%BM*j_@sRgsLNT2VuzsXqn9jF z%z3N-ehkI}uw|5hIHb68AdO)MSC##d{s$-u-(*tz8+fSH2{k7gtj3{qz1 zI0B8cUcyv-7m5g32a3ZqwvvW3*zaN=Xkfx;JOL&`*}Yrjq-M^YPbaC!bDU6;qP_4Q zSiehP4r*|Pw}Aq}_K#XP8_f-hF^U1m0~IlPD;W#X)NZq(;&3UI1;{EVeq(d(lhNNG zl}Bc8AJjTvyovdkNE4Y?3zDE{>_deKlQ8hGd7Q?+;yf~Hn3Rp@8fcTSE=TbJOt3}_ zAi5JEvpLj2=`RF`52UHRunOI*!y!EW9oP*m5mshh0p%=Wrcx2 zJc-$I03TRWO%2`l^T(`Sl?ePiP2{A}opU=k;kRy8DJWOpNGg_44cyaQTS-~asfD^!Tbg{lGeNP0S4XH!YQnj}L~VT8P+V(C%0{ooT4nH(gh^wB|s z%rux@c#LL)?3oTAphDP}Pmh>$Wg67J!oa+q8fq7?)|rrp1^*PQ^Mus&gn2DDkRnkYo3Kk5|4kY?X74GH$lVtA+HFn@87&e(1V&#wx zAhJn(G3?sG7+5lBiJ&+OD}Y^qdB^R!#e!N4@$2${OX;ciNW=}oCctE0)|v3{ct&n8 z3o_}xqFO;uJ4E%fwX=co1+AH0O9JNH_OxV-B?pd{Elv!mqvnY_bO9EaUXt~K4!3~2 z3DsUq5EaBy>lSKf%;^$$NL@P^g_wxihb7RnCcGMC)wa3>UvP5QC1*RYn->&;Q?jWp^v zwt@sxBhAJc!uWv11^;t&fw7Dp={u;E*ny-xx!qmLW;u5bkV|B=61PptM|iaZNnC&* z8L0~SpZqQ$&RRGf80*P*A~wyp3*Ekgnlm5*jilUL2JY~DkW{P*$FdM(nxTkJ)_dIi znPFIWNK#ZA5>m-1;$w7_8rp+OdVc&R(`%q!MiO!waO5F2#SVK=Q?fI2~wsUT6i->AZl z(u#5^S#b++%7(P0&!tr6&B$Bd>|ot(Mn4>8ykn*-)j*^zEosS~j{euVPhVS4A3E$A zm&=?B=c7|TfQuz4Agd9`W6tiR+YLR#10Mk~82;LiXdMZp(pjGD<^Px^{y$SVJ&*M^=_8iu%L zXOu_WMhUfG^5V#P!8cR~6S)@i6AYm6U067vlc9XoG1SB!e#Co-mxBTBODWPX|D-b@ zaFyzKAo%p;U7Kh%LULqcwm`}z2+Wt3gtelQ?ht=9;!AAT++6$dRq%>}=mC1#$F^Ne?Zs3?IhOY-&>X0maKJDX7~g34By*oh?J3{uI=w@0P!OI@)ZhN*E3 z=`%UwREQ~?aDoYw(YvRZ*%A#xC}F?F8hgfJcn59L1jYzMS=kQdCJOlj`2@O~X4Jyz z=tsOHEQ@2rVDL-;D%I1rbKiM7kNVrsi%cv5i| zj0ieVzIR|DC6;v3##;7`ltn z7?3$6fPo7`5g2Qk9Ku;b{b0!C7o+3dReGickg2x9-1!dNifH(|iW3XB zqee<3UJb4rVr}xTMZ;2(T=G5|CkH!1fu9WOs4VNtI=!{owu3cmV4sU&EIAF%oI8sS z(&V|wp6o{ol$n%Nnk{S@swie+#f)&n(k7Fy0KQ~-HTLv-W}|%4{JWK3yAyVxND@8n znJCMoAN;CVL4aBT@xhFmtMK1EgZ#s(}wAzMT7MK9WF|FGXTe zr3HUzGZ<=7pGxT<1^t7p)Q~Gwqyud$c<=QZXa+X*r06s(YBDiMtQv%vo=FzB-7 ziv0-lFfJ(!5MyD}9k4sfL-FjHua00MuBFvcyQ zVpnLvV@=%a@0@ph@NRO4gN-Rm&wx>c1^}3?1t=OIw_Qv158=o-(n(#=U-6nWeDOTWXT=g7Q_eK%-$MnPrvcX!BV z?nKXe6!uv+`j$5W72d`+*wX0itu|Wy?OlBbZo>aJd@|4E2JWVf4uje~Ws++Tg3)BJip> z`#-}5#r9!r3xX%USGx2sdG!3AGW#yXyUy`xJIOKRrf1oXIa7;Rq6>J~!piB)G=g|Z zHKlvN+PK^P{{E3@wyVwBzkKl+@tsQuQXPrS`Xe8w{InzQV4L+h z6ZvrtYQF?C2S!zabv|=vqNaTBjJw%LJa?d0g>3`;*$71wd($N*Xs8absLx(WI}jL; zabgK&ptowvM;!QBt30v6TcOcOJx}N`kw2?1EB)&P|6kiGB1RbG{b$38=XTN!U4N}@ zN@;psRWT{RwKvbrW_ewB&N|b;<_B`ElQ-<&ed7rlMeSf+ImRKgHrT<)WXWGlIu#yX z9{ zBQ6zQ*JRDXEkOCQQkX?;B5ksrA>1c$`EeVw+n(!F(0aoB^5T^30 ztcE1^0~r}X(ZR-3s6X~wyIL$E>_%USYx)h$m|;{9(T~r+GXnvhwVre3={|C%4&PA8 zxt2wB55fSh&ZK7l_@g%v%FG5-ex*8=YJjcS9Xt9gi>pbbAobbX5>nMOle=IjJS;H@ z%^b1_)*i1J%^5MOJN$j~Um=0q1?E61Wp+OPsnz1ks@AK$HgbHra5lj8v2OPg@HBERAIr5B~&g?{utwu zkNHLo29=J(9;`pAK)P=gwq)*_EM-FcvSqiE#uy$ZVY$%D2h(gd{6wY9@dj#EsOsS% zH$vqsL;v9?tHuJo zpaahYv`j&buy+lJ6Wg={mZfZegs)R3(5DQ{k|5lgLZ&GIbdi{MY0`kt7)F7rph`^D?2?#Yz?KF z%;_aCL@w8Y%aE=h{z!Eo7Q6P)sG~3pAt@gH2^^|D8oGcG+qnVsMaW9vu_B&HPwJ7+>ra`m~-{ss_rK) z4&&j^BUl!f0OHa!P7KdFB9!T zIrw_7O!gj#I+Ft0dc!SSPA&eAh|e#=LximRYF=Jr?&p#t_cR z8DNeuqtIWXK8|G{EU{y9{BswY1{!x`0r=bV1p_Qv%Y%W00j)%3!5sd z?;LrMCjEo*X3IXy4{=TNCh2tRI~)YW)YJh@BwA6>@M$vvfq_|8_~Yka3u--@4v^4f(;sweGw^CnKsy{3)1e7Z9eVF! zAyR$4mi3%r0dZw68qdid(=DWc%k~@?FSFW)*2PP56YjkhrrIYX^|Ak!XTiuw6U!ph zMU(oc;y_iM^z?kxVW9TtTvjS~|7#2D{!i@{4Of87%(k+03;m{yEKTwCQZ_jT^5fV(@g?GP?9s};6Zz?0y+LP6*H*+x+-U7y zHcFC+gu=G_u*^tAs?UG?S|VagEt;NdP9u?FuOdkV$_yM{nej9#zDNCj8@-!!xC=vo zssW+g2S}uYg}*U9wSsFjr3E(olgME=(e8`=4k`925nSK8h--7fZzAH(WQRnBSzM7O z*@-1`?X&_w)V4(jAe^IE0ud~-mKI?efG9zMvAu6UoFDKRDaOvp9)g6`tb(Kuk^Cc; zSpg6f1sOX)3u+wpg1UOmED`D|$x#L>LIG2|Fn?G@WYH9@@C#Q2fex^7EQowBjZYF9 zodGW1!L10O+Z<`KAl)LTAd^nJhT}?NUXunIlIc)jNe*`E_(!>H%b;E6iVC$vv$O(m z-g#Yuo@Ar`KMQgRhoCBg4h`4Bx|a*N5MYOXAgn+&mJviz97yU~Jw;XsrK@3#G>%Kx zo9(v;9PGf$#~!s?W#Ht3DT^IoVIztq2&0Yu;~VTDCKTf)v@i;0x(fFlt`U?qI1I39 z!WSkS|G#c~~JEMu&ILJkt3Uk<`pY=XEFM7%BOIq z3NIU5zr=#`NbweNp%I58SQ{QE)hU*+Qwh0Z@vJZdbzwAx&lvqAtBjr-C|WB4`e#}E zf7oD%N@#Pfzy0z1!$|2}G9>Q>d%*k5EIDb>;P0f$qd?`eb^?V_!M6qwK7Of}$W*jG z=7NAN(00xpA`~W#!&gJ4N7wAhN??O4LVR%n5nK!4w21}xY(pWaf7OCbjK?yu1-$_O zk+#5pf+}ZwH5SCRu%OMlx89f*#yalOAZ+!%6!uc#B(*@08%W3ovxf_lFqToXFuD}m z2&ry*MgedyA&S+}qXkL8s<3IcY8a7EpWXxf42%pb0JjMt&aPK`1l~ z*2BdQ^l#+HH1dtIm&Ubdpau@AMNvIpKcOMqI8?#T?C2>#T%WHwSCbCosU@JZ)YBLgi&4_mOB*et??Qk?g`Ndx2e z$sS?=>*vZS6(irC%Nb0_3m1`yHZe45yIBpE-3i&|+Qh$Bz#8C7S(#K26~ks;Y|zAN zX(HtEDRdJFFB?NxOO+wzUPdDi8p!z`W@)j91-ess6FRGW6pIQL5wsL@$T^^8=hB>Ji!GBQREkIYff1ZBWKCde zoAOZt-sPwC=;$Y93`YS+m4Z24jS8t&1HNS8P3Bk;SvXyX0pBElrjdpvP>;rYF@Nb? zu-b9x`lVuI$XmH%R4AU%IK~!++$daYh9;1SWBld)Wyted0Cnaeh_cgGD{719qXE@= z|6ue+lOalk9X5Ud?FdJY#Tj)$f^%WtHS#gwmdW1=g_c?L-hwk^w((y+gj^!@+4Vzq z?CXIqZ3R2ZRtef=jVLjWMV2uIxa@Ku@atIk9@DugF z;T!n!DMzU@q!!JKCKj=kAlO*O^q*z&u6WQg6Ha*51I-6*J)J*~&rOMYFz_|_(?}a2 ziLH}Zb8@Elm0e+?8Y&G=i7-*C<7WI3eyD4NK{z0TnuNzH2vY~S0od0-A;)FsLhK20 zq+gJhX0ylQUt_g@)>-6K?jV`4VX+jgAQK2*B*vig18nw0XI5L}0L_N8n=Lr?{x=Lc zyuOr@|K51R-$})&V8RS>57)A6{M~%bg4tLJsm5)g(;F-j1U+b@WF8hRe29OTY~ggI zBMm~u9aK}9+V-%CBb^egjL5{cxC69s9V*{%A<_6#A21CWOgF5!Lu|^+E)EGwUQ}Xe z6U3Ia1I|@js?mV7EORh2HyJ!C8U`W=K+z(!fpz8kn)?)V(Ld3e`L9q+4zWSt&awVz z05uynbng-pusYeYM;5I&mB5II1f_PNYv0~micno?5~u2I|KlG!UKTz=+YsznL#6b1 zkD={Vuz<9~8!1j7JiBYMY(YT$fxJlx^?agyTsT2Kjfb6;_i~LQcLCM0nbLpu>0m5G z35O7Z8T*;s&`bi5sLPVx26~R^VZ#F~;w!EV9#Tp!KB4TxCEX8-mGG1imX2{+aT8$f zk1|ZH$6&ZwG4~27W4S-mdf|0egn5i;HivoOSYwNmAeiTpef4^Ged;EZj3nsK_jvc3 zW%{b{k^Sf0(~=~nSISa^2r%D)mQ;!Q)82nE6g&O**je~VnV(@LY+n%T7t#g6ZcWe| zKs(GNd(BP;182reS$Ns_54KhTVq1&`{T`AC6M3T|E)+}X0Fhu z!xPN~2r#*+(UU__gAK#D8zCrh3!t5D5r|1P31Xs;y7h`bIzbKC@gR+V&#$1b36|VC z{g5+kamFU5Libqoc&dgqGASTs@SLS0W7gL1<6cd36F^fy=G-#8B^-&G8h->a=W6Bj zp?%JX7AKkw=x4ZtUWu8mjMWG!bCG_%w^sAZQx;;W*t5E9zWN=Af{;rU+lDS$s+_G4 zLF3E;g?0rZuMt@O;leO_y+vCepawP?Z$v8y>Z!=XDPn$QB@*(6{XaF>!}< zqhRnrZ74?0_%CfN(y!)YS-Qo(0Nx;(LzNChBrX0d#p8-%1sh3zwjf1pv&FP6Myc1D zub&NFbATNfG*&P?`v4PtF0cd$|8oWJxLL6l4}HUpG=!7IA3laK(8MAeNQ|94_%o?; zCwv3e=e^*9p>Ne|9IQ#@+kApAJ(7+H5Ij}Dj)Ck_gVczgb;8MH&qfEHv8&yC(j8%J z#tdblNjNbfoeH)UH0pLjA3DhnLurCeE}TNbaeuEHPvANzeFf5kGXXFbOO07Eoqr^9 zCQkytPyQqn6Gu5nNbLAwn*(V6Sq~Agh%lasC^i?}CLD>aCLjzIGo{A{My6k|dcp+} z1+|earrF&pxM&Evq^v>TB-rQ)FZKV+V#ng-JLXyr2%iH5PVxNIR`0|KwcnvO_t83R}k`F?LE@K7=q5j^HEe z?7)RoktvI06h+?!lq{(w{AKd88w{AaP1wS<*^6W0Ht33_KQtq=q?|5v z-W~A4@P=HgXXoAb?Uq=mLxY}{^joH`oZkR&h}ns|$e;)>AH$32Fpkthk@eF;GY{bF z&OFo}Q90L6?8O~mzsedkYT7f{gor9qy!yPsFG151^Puze`*b3d1%bqy0~?N_%WvP0 zk7Nnm`zj^egjqDGlBs2J7gBp+V?>HnvD0ht%Y1CjVn2$d3xb{cXhIOO5;$(?MZJoG zNjPWlQU5e%Cnvn#UZ7J)M$SC;`p}kG_M?B3jVwA1upbetLWRdBU?$9^UlmW*CPB>h zhUl=kQfTji(n{qZ!OG0@u!;1$dSfkas(>+tM&5tM*s*IL4y{xv`+|>tn}#|*c9@~A zK@;(%mr)^67-&|Un?E_?;l7Z=kfiJIHae>?i-+E zIo^e#OJF#Vh_Hz!DtW;sF;mR4PGt77I}pdTVXz-4-Y>c`bZW|M_hGaVozcV+WY5OO zv0yi%tKEBz@A30DpqFzGemS&gj4Q-gWGWhu&g-08^>slKz1& zwt0MU>3mHcNZSa-9+GN4R7n->B3ZR#1tjEJITyYk6@?t%z@sg$R2-XDb_dhRbn5 z%HRA5wQU$L`Z6f<=c1mFWkfMbRYOE_{kPbs-#ylub?it?xVhx6|_4!L9qZ`kFl&L&m@Z7`_-VBappz zV>^2s+7|a`72gp#ZOYM*g2b2bnI>aC{>^Zgt82XUimD+AvjPlPq%0iyuD(UMzDfE2zj`lAdr`L35LVwn_%--#RbrPh$ch|@$fz)+#{0!w^czhNa zBC=!j)R6L{e=&L6V|FhjQAtLIg=gHdSSm!an|L1$D{gTGEg!AEbEL(IS9Q3u+_15(ZN(ffM=J?6S@L8ornhi= z+<)(h=goV&JG>`*pLsRy@aMlAF%_>#{rs07WzRnUn zm~D1pbNP>z>x(OlzE)BG+Nidz`?}}q?~C%kQq6y}XYmyG%bB?^x;%8B?@h9}SLcyu zy#<@DtZ`B(6toCDB)qe$3(p}lonR$-4}-tKQ#V6@{`u#foh`O_s?aiwqt2S9XV2Dk z_P=}gZcJ6kjT<*i`>IMy=XUQKpbu_cpK|2yvDwaJq*YzM!voIqw>?Bp%gv|@nfUGc ze;Fm1oWqQZ-~Hx!{cjfzQI`BP@wzvaeI^{>qc`!24oI&12h9NUsC zGS{~)_H&=K7|(8W-@Dh&`7S03UhZYfPfi;M*rgMWaqj}6z@_daj(9rpaii}GC0y+94C{bdbH@F@D4hK7($A$YuvmITo4+%LcUGR789 ziw~Y_cU%+;;p4%V0~pR{*tEfx`?ogXai;Z8a z3Ki7$n3+dToCp-%`1RK~Ab~8Gnj?@L&!P6sw&VSnw8)R&JBY^@WFbN|U>6M#WoBi? zp^{gn;Ym(dIJy*~XL%x+{FiX#lmM0)Bw47U`-~qB@u^rH9d+&pRLU|j*P9$*_&}>ykZh4zTXK??rIqf z4Aevp98fJ<_o1tFTKhDlTV33(V ze?ETY38-qi(9Oak6wS6rlqZ@S=jP_h>YepLxA`^nxGls6+xw|dqUWM+G}NhL+Hh=W z`rbWn4t+9a_E~iOY5*3-l)JKhIntpxQ7ocDOKWQ>(&*HkPk#C3JM65hU_~~zu{n#( zYo?oRNky0#vKjtea?Yhs!A$nPfaTgi7p#*aGLGV8sZe$Zv>D(Wg)Ull3cwz1%Q0oL2s6 zz|JBu*@NDTSO+DE4YjpVSS_*N9sZ)j^9dwKEeRl`!mMb&6CWb(%o1mT5%WAFtUDLE z!I>^4wao@51Amc%>80=R1ZtTASkvZ2?AtC9`RY?dzw7bG{f7$Ic-|g$DIW+Lse!yJ=WHUx%$y~s+Sl}ts0vObz zP)uHU(MbQ(Djcq|=j~?H0j)#<+&Ng46qhVnf}kCbOAS_YOG3flc z)(ZUl(`&-@>m$8e>t7zTX8D0n#+Jb32nexrt(lyzMwVaU0X@!w)~q z2lTwTp2Of1TD`uRrz5;}{}ZK78o=DXBF-AghYylW6+-5^J{q;~R^O>B2wAd3?F`47i>U#`euJRZ8E10W%zpo4{DGio@ zF>PSu&Z6SaBsWiOX+eN~q}CR|IbgyQ=$rDD&Y8uQm}RnS&i@OK(o{yPr4K*)=t;4# zl}q+jn{A3;QHd%kuzq@c%U}<=y>h&ix~dNcd@h@ zrI#h`Ougm(N_yY0xf``|r)OMIcgf4KT%CXKmGpbWfOGa)_0>|@(b;et5QqdxE%mgU|Wm9zYozvSwqO0~Zq+9aAKyZBp^`I4+#btN}lbQah*`wEvW z{IT(GAM3jtU_~COo;`m2c)*T=jY!!#6!EWYpZp}_U-*KRI;H_DDWTnR-aFCZYPc$JC-I|Iabli0zsodIe(30XWMg` zy@#Z(ec<1w;Mn|`OXR!4@wm+8b3c`-J-Bnt48PO6Ew@i`t#;p$mh;{q%rEp^y1IRo zl<5wPm)Ondc!6X78iblJPKjIc6;;)=XO36Ak-cQof4wAqeTk?h&an+a-C*Xa>U#O0 zRdbX=TOO+Y!6!=vY3wtYveE^(Q1zF$d5UkI2s@;38PzlXh*-i8wwe5|e=rAU`XKu4 z+jZ@urt2K-Ym?+G*I0NYagIatXC-gle&3GEH<|p2Te{mnw#d%+O3hg=eDlB`*#MUA zR(HzrLwnY->(l>F($v%XzM4SC%q4rU7+oQ~|0l*R4x3{!X4 zI-2gRC)0nqIOt^Sw2mtsZnyID#ZNx;e5>idvb^ts-CKj3?M4mFS=*HEcqLqHSHn}v z6UOE#{8r+gx4F2;GRx)VrO?nYgYyP5<#EQw{q>dNw0-;K+q^rsrytDgSrJ?(vnJ|t zz^nT)5fLKgSPRPx=O<;sdAsDifB2hBz2yQot24~zTwdccKCGE#T zl-mMT<4Vt31t%}XjTm=3H1fud+Ly__!*{lJjdUOT_N4R0^KnM;QNjOGHr{M9Lc2Gy zVSasQboOlZj_~2_FRZGIn$LT0SV}seYn@c{T+*e#p}WYl z%f5bc;^UO%bw$>a_@hUA3g5+kpV~FCEH6aMUp^r2PH;@4{clb_qSy`Dvy^vK#u&Js z*9~2xk+{qLGmGW%!E!RP9szsoH6cb{|arncQ0X>qEPKW>q2 zZPr`AY~T6*h3Ct)1GhMoebo4{E&8Jq{|tZWl^|mpwev``VtBv#{_aojCMlTv=V^RU z?ogr`Ss3WQ?w<>{zdapL)ct*H?wQzi7gqcpIZ(kRL7dz~}(TiO;`J$WJ#+1c5t zPtn(n50o!$9~n}9<@?kzZRrW-iejapRh1LdrW@`nOAN_)Th|`eaywABe1Cm}Y7=fo z&Lee;2IHiFz!N7XDtj-xq3x%)Y}wW0lNUtHXdUPYD9c@2aaUvP^1U5037+OT%C*bg$IG8S zU(|Nr#IxhZ#v4}I0dbz_Ka5@LSKRL!`E!47cz9z+`ndjA z7w)b&5nX>_S!c<@J+J%@jy!XEy?@NVXE#_cYaZn}eXZl*^-D=rdooT*xx_#Hb);e6 zNiC&Mm+y7!dvmS3zu9>0&dtq>%M&I=^mo^%8Xy)LTr1R895pGyYFM{j=OWihhcqSW z3EE*&X*2Bi+n0^h-yfy(w&TH$Ba1ym-*;B@r+wRTGQiS2LoBIWT&im5;wN&iZ8gxl z{!!5JshPTqbw~HwZJT<-B0K5ihNz7(I&Tlp?aI5jwx+)Ry!(=WYx_^0A#QxQ_}qz= zxlx!nBPT901MG;c|NFhzDDO1;^CLt0@Ag<5Kj|2f*%4qD0~Ajid2pR(R_ce3_p3B_ z2T3BvKiO@%di56Fn@&wlri*nWuSt!SF*7*S-c(v%Xe@tIcgoR<2dAP9GS&_IP&q;I zWl@}|+rz6(MIApKn;5k>eBXgj#FFDv)q`tHPP+AEJC8Svi@5u8cI{cw8nIlpSylDZCCH8c-KC~^}fh(+S-3C_V@KYT|-;```7LT zDL1A}{x#y;7i*>EOrtJWx;CsowE5{jR1!_2f4|O@z#z@&*ryq(?b)-%O-)Vptpyt5 zlFU-Oo$X~?GD#Zj-(z~D`DwVPO0i+iYJ~&y9PB$6$E+Eq5Y}-teTAc5YkK00$ENC9 zl6@0Z^lsp~TNKVGHmJlnoUxKb+=nME1Te%y|Y`R2&~F)hnjODoN^Am#F8w1)x3Ch zYG0R5wn(fyFE(V8I<{ryEe0!>+&NbE>m6^C$D5K}A}`h7i{GCcyl^F#sdn#~e7)N@ zbwkU4JdvAjHtw~is_VZpKMW*WyvX2c8p6o$HWy6chB}L&-GPXFcwC>gBm*E*-CL>0jlYK9|#-5a8;4 z&KaE2(-kArc0N(8{37j%WLlc*<2^NsdK!lIMuGVoOJ0nucW@}%GvVO$Q~42&4fADX zWW*;9&AWB?6SE)6<6mA^oK^oKN0NQXrOfhlJKgY0 zommIc?U(K|m{X1zR6b(QsyunK#I}%KG7(lfA+E(?c_Oi_W!&zp-heso(M3T!#&4}~ zx4YFEth)Cimv;Sw#A6;;M?^eqesjvs(dpxN$woSV2D6jLe5}9ipV@OlFEe7@{&?Zc zQ(~*dPvQ)dlPWuQuSisLyJfIMwJrNya#Gm5=Q|q2-*?14jQK&dr6Fr%zy|XRDse_R zSuWc%Ta7*|OUd=>Z<{auC{_1|6Ncph{pBN_CQF{3He1}QsQ%l+Re1B}d$oxRQ}!nB za>5agew}8&J)&~w%ZSglKYw=elz7+oN2^Sy&#EmLd*g9#SdC$XQk;RQx4&#dGM2~AK4g+s$9{_(F<$=g zTFKkpnx5j40M|AB+P+65q7UagosQi>I%;X@O#8c+4~t}QD+|ZZG5SSrf7`=XnS0K= zYGaDuA2xBqt3)e3SJ%xCSG9Uso>OfNjNP%fA^w}rv2y2!9XkH9>}BiAcK7<*VGC#U z9*Ht-uJLxecdyjDV!U;6iBE3W+^LI(=Bv-4zfG97J7&!5c8SO`$K%GPFgdj2m0b0> zx>NWaskV;U#Tue+t%pweSER;IOHZ|U@Yk)~lx&yOBQ$Cd`TPTJY| z^Y_xK+calr1xmaRv~;*yVY^N9=(66kOJhl4=QEQPSJm{AlIHBO`(^x*6S^s8(MxkC zN+SFn(jy)i1G!YJX!I48k4`$d$#V2)x9_jEIfGwq$T?}fak|j&WZ;ZP5BnnGwO*`6 z+7Z#qV#kMx0kdY#w6B`#9ThWTW8f(DY;8&Ig#K7bsY&@N$@3S_o4P$*`!fum1h_UJ z;OAT&(KY+&#Wpq7e?rMeH`l~2o6y!=D*o+e z@$W}vQWGj(Vt$8)g|!u)FRyfa`s1D4O+ByUi$b%{7Wfy$ZP#qvS^W4y!`e7gb&XUn zt0?L0c02d?^dL`q^7xPScgI|ADzhpoDhlrCJ<;W0T|fNg+F!lnC@Lp@40%eclhzBxK>>0o)?MRU3TrN zu-kCRc60n%#FpQh^S-+N&D)}X7@Z{ZG4S#YKAJO3%7wRlPwVx69CLTLvgh)j6Ezu)1B`}czVAwT9fyDVqgA0Lp(i{JR%mg)}@-YKTr8dc1@OFP4~+$^X?`u zDM~lX=x+6VyZPA7CTERX{xgqFa86Cw5+CqMPgR^wc;}+FnuKQME5%X@;cC_nUkAop zomwGlnVazG^}G2SeyhB+D<|pZulDzU|2(#9;T1dUU{s6w-dpQo9bvM@@Y95ZuZNEk zcg-#LRauF5 zjTFkRIyM+M7KbkKuQl28wpq@3>X}b(?h7@(;(OxYRaAw4?_r->l)c>g?y-=Jd2b4X z_FNUaDSMyKjr+TS-m>-QRUA9lnx{8|;KS9(>@IMsTU|WY^ZHo5#5k`| zC7tZ_t)=ymh1K=-aV0f9<#jbb)OS7~A!`tka;Lq}+k9(k!peRxnJ*%4KlC##eH$B) zxp--P^W7K7n{x9Z_IOiW}#qx#YDTV|NN4S=DU?u#z=BRdHRtU z0*ADQCx*%mQLD$PUUoY(s`0U1t3}65O|kyt6(bcs)?YfZAzD_^{NmLk{;x}`?22QT zY1oc!lMAmkX>NaXRPk2cGF497y!OU>zZyD>89zQ6FeP^N{;oZ7FG}sp-)sq56@@_# z#~?RUvuXWT4(oPO%(-jGJ^0O+b_oG$s;XOV|9G#g^W``pIeB3n1p6z>VkY(_zck}HPE3YP089O1yG@iMR9ZK2d_Lz_w_3X4i zF?UkiG-h7^=F`2yfABv2nJ1q4 zt^H|mnF;^ZSp5f8CH%nW{}AIZdt&aV*Pl#BQ=M(!^mWX5vUf;n@VWZ0zsD{rc<{hf z;^*tDAwdHF;Py8W{}SbY^M$Qe?z-k}f1Kd}!2$JZhWPN|!%$#85kni<_~muq^-h{% zY?S|$%eeBmebkWUQ%YmzmHTQ^8;?PC>Feuuo-u1y11e2f?q^|Xi7eukbg)R~rAoYm z)b7{Es}?ws%ZOO8cjeP-eHktW`VzmL-3hD27eU)RezN9&^!DXpHRu1^2Q$+!XqGZc z8%}6Nwo=)pQ>4;f3N5tKrafWAm?Wnk3Q}`gXeX4lp=eZb7V~9kA1>tI^_E8l8!Pm}C!t<+66 z&;k{d=h)%TX0v;R*^e87y?W{ObaD-Vx&%qJD*b3V@GXw0I54yz_i-O*vagRhV+;5B zPI-I_7$^!{(L8?1m>uWlH7SPh6`3fMi=w1%&=H6S6g6>LyLTtU6b~e zD_1ZqqP=aK-$Ehz8}O1-2dfSGBBxh)+2`vLT09o_`A->vGVhYjX;mG5Q5u05TX9P485aago|JN1t^Ul?_zHsSMJrp@i770@m zhoFN=a_X!A$=8@xJRR}uPr2iRUmZ_<9twCleMKzPA5gXAmMx}eJv^ZmOKE_-`zDi2 z>J&>6*Pw;|~#rU1i|pf$zuSU%Jl*%IMUPQ?+lKX-U*n(BYb|hSp;X2b`^KO2(x~ z>8Gl)@suqx19PiP{+z*a*Sx>=Gv|Balfib$1vrNi6l890Q*uXtzbp$}kDmX;`uQEx zbLI$L7!QvB*v#ST|#|LNG!eenv?n?PZZEmx~1`gGH;r8Shu9TJ4V88C}4qv|Z^QL(D>YpdF=Zub^T!xZ|3P<73dN$+C>L$=Gu502t1Rv4u?gUjS|{rob1Jm>!7JL?j4)g2rhF8=Zh9ZHql@bg8gVtIJF zERKC%N$y{t`1iy43PYyF`;f0C<<~NBu*%EbfDb~#A>Ggg+G7r(BD6}^>Z-4(hTz(6)1aV_yRp5 zRbXR5#q7(Mhf+*(wZV*KI)`j)L*L3|?cL1?n!4&dORNiz+WmJtr{^V;vW(ZlRZWZ_ zw>3u}J_!;F-Hx3^qtQf&!&oy3J9vi8ckft`o1fpT_-AD;FaPgt`1$(&{9T9>jEd>s zzx;Z4{6nwJ{Kfg*|7%~!|C*PV+DO<{^$J(a*!eV*V1bRW_YW54=jV&S1c2oxlD%us9$Ro}FVfA=lVAgr3`TuC4523=k4K>|eG601 zD@?GQQft;|A`?N9R(uafK>5Im%@drRVdoNt(`gVV)lmuuU^kR*RD`jpNew$;>>>;? zO0y)5vy(qgO=TUe6@e{<@dS4HJ@0WR4^L04`^8Y%8R0-NHd9no6f`RNbkPy=_wdd8 zu5H$ZJDV@(pS9+YDpLH?Zj_GFhjgC4Xc@(&t`b3 zsj6y1(KiZlRy@e@39zEBuYUUy$Hnf%;jB+!oRUJ0H$VjM-v6NgCaYN7Z`^dL`p6_g-~(Ct$rD!(SK* zN)2tCVet=b;%+sS8!n%jCh_>?{Q2`ZCMyjh3RN~7-mSYXhy>idy9P=ZFLq2>nJV77 zsph*`+ukohHGm@=#86Agm=&48kj&&FZ{hfd4{u-&_B$4n1Zx5cv@EAiHLr8uFn9~W z<#xkH`*%cc$fHwrI#FCs;KDnxcae6ix`p6C5_EVm2{llrSSpxeBhcEIQIt)=WE}_hLpaK@e{IY%hzR-LO-CE?6kCFA8(A}0Ho=*<)C==ZZDb5(V~`ks)s48tj)4@+c=}T0O!6>d z-}4-`sWb^fX2!&BB9sah(OZix)r)XTv7JMu^lP}|Pa|(<+WHyYSS=*`5QMYE^95u5sZJNfZGmE)YG+7?*TV*HZ=ni6dj*9A z41Tl88Q(DQ==&*&<&ElmnLpmg2N9sx>VsMnEs=hUZZ zPe?Q^7W>nj)BNZTlr?Xk#a;gR@goYh1Pt=(z(+~@$dNy${COF?b%;{vgI;%bTDtNt z&xC3YoUt|e!!16j2H|Z8Gbu?`2FDiuC~Klg^VB2TxzI8~YPy#_LXH-a$6M~FL*v}d<>3|4!93M^XYO8pDZRb3X3gT*HA7bdkKKJJ#q}&K zVoXd-bh2!858{9TOokycXJNAc&faSSnAl6yPrDA$M?A)Iq{PIwiHL{@uCof*v2!Q? zk3SyG?1Lxt9m;-?tb*m%g@W*PZ*MZ5rsVqdbwlOBUNEaQxOwv?i3c6p3KF0+N!oUL z_KQ5%B-k}1;sWl(<7izz$WhE_>+RTahD3q5J`(A`cI6pv|AA|0{FPyQ>WMR7v)l$u zL!}K7b3&hD%#rE_h&dl|@87z03mmXOL5qHe27LakioM^W-Rn`V|%;V(xpo!58Mf0Ccw;c-rTubs;U=A!7F{~ zn{EAi*hr9p(ne>GAQBo+x1PuUmX@yE?F0A?&SO2$r{6RA!PCpD-A$-T0i&@!Y4a8= zsK@YzE@Gd6|4jtkbN>EsAreiUjU<3b6fz5luWDkx| z8b`T?+oylArxZ)Yb)XIWGpx37Jj}9JBT(3{b{n zL_E)Z@JQk?PB$?_+96R67A@c6#qk(Z*upz~DTD9=KnRoyi_k+s(p!yKkC$Cd=8DK()G!`>nhD*8JdSHpo{ z%%iAp#PsKW{k0})0GFu=*T|o@LQ6_^<5;FxIh=>zP2a{IrD%ezln@gW!)M8jk#PQA z*Q&22_WP&1eB0PS&1e0v`s&WWGHg>Qs(!L%PbZFPEU5nZC zA$aY;lNAs-ibpk{rGi@@fSY0*cpTH^?}mrh0V_!<$p&_YfC#YPDXwhl2@t~9Q| zOCNRj@DPxdl`X-P;+0#i$b$~5a!N`{0dV9lQQf=uUI+qe>s2L>jC2%jmFbXj86H7s z;tlxer!TNEmakp=M9vE0M{}%kiJD`gIo$fl?qQ3ukDDRMtIT?XVU$Y<==fnu4(|pBS-R7hg~TI)($C6$ zK9&t91G&d2uoz~m{!Bz+L5@NjZh(S9)3LhvD@)7Wagxa;3~L!xpaeXITBIh(f-DLi zd(_~N3~0jf;@AtZ?(K_l_Il@wFk^^Z)(yMnW%Ue#4Ze*GkJm- zY|HVxkG&TT=R!B9h#{}$jQzcoR?tHL)ojwf4!it9n0Ikv7sVzj7#yyV(kHtMT&I>0{1IsAzc$g{>zX!YL%ilv(-0 zPL8r7suz5)nuf}GVm$IgE#5f@U~kZiqTZ)$cse=8mG+t;DNR zbi|w-FNE?tV6j8>B)lQc^F;?4BQ#P+1OY5fJ5bdan~aKJ0%K1)c`nwZlwGM2bvFbC zn)(-i{#h%_rhYwaruYVykRPR^@_fcNBhtrGeTX9y;+P50&SOE#RR-FgtGXBvP=}ct zCW|T%=Oy1VPJDr?gdM{-_VJl^jOurl@8h`H*=TfCFoMN_e%JToAW7F+v>R^i{$_#b zmKLt_myEk%@TybM(9kf#*^UVM>_l%=0PGk@=U=-VOE74*I*t-G&u20lMe*Tbfuo*P z9kM#{LFkI)Y}2%?2)NQ)W%uOPIebRzM)#l`h%=x6?D0oYWVuC#(B*&t?;Bk`bqT@H zVh6qbK+@>R_x8#d`#lHI8A7 zNz&y0UDWPZUl}x{nWe!eEuKa@&@%a(hs2;md~B0D{-Jg2c?gqNZQg8zXy*yHe(fv? z-NcX=ImeG@6=Sf6#jZce*2%QmhX<;GnGx0)?`}p@>6I>Sej$9z^ez`85U}koKQkSi z?JkctySX5{BC+>+8v2j0w+1N4IuEC04N>8E{rYu0u_tg5m-9|V__BtpA;*5Ss=4oy zh_Vqi&svS%GT7!0uGWLVA(HU+fW{jj)0Qa>lQvw|T{C$$E7F2H2ElRv2As`}0~Iup z&~@O1pi8F=M#*bEy~eYSd`gEVgAGrbNX2xbO5&`gmIEb|HhLQf9BCnu^1SrOEQZ(5 zmoA0691W~l<=jdLgJJE*vGXZiJgW=dok8MS=WWU;n=7ssq6vtWUsN=WCOKCrdc99Z zl_HL`LoS6HNaB#GlYAGp&1sOutLJ^{8;v=YotJrX-*N6=z$vHJ{5`A&k71<_y?@^T zYpOmG-l0W=rzj=&n^wPZfM)&*Q&`D;&7PVZuQ3PKQ#3--lf`X4D1F#5s6vmvG2uS1 zHqRy8X!h*c)WRpFBPhgUujw@lerh)V*l&PryplC<-aNH*rw|$By?V6d*Q^4#=hcu7 z$+!lvlD&X_xSS>ZA?sS!G(5#_Vt$B~7aVd(YR)8994EbXDuKmNpeG#=)u>dH-&NG| z?P|aC2T-N0bFmK-z`k#q;B&>`jw{9xTq@z!b3Gw+CHjUrw7V2f_69KtO$pE$?lHH*p!Bk6Scp?g_rb>I%?lY1Fi_w+^qK`hXOAq_>y8~e>hj%=Q$4TU&>mft z(P6YyhL93;Rh^Sa>+wL~Xy6CV8yom#P2^{u;o#5^+okKen@V`ki@Z1oPFuTZTU#3i zk4{rX#_L5mBjR^pT_f;bc!PQ~N9Ys6Z3y&_RBGl%XlXc}JgJ8?u^xF2iFiL2#}gD* zz`tS$#Te`P59s>oRN4pXiG7f70eU^7Ivhhd`6XvbGx$IQ?_@Q4-=oQ3c;#9VKI7;& zYqm0w2z2tKezaWwOGA{aooxVHTl%usacM9$5)Sf)<~7ry)8nsyzI=KAoj)@9OAAZQ z&CP)y__U)D#R896BHC2pH#HNVM@l57){V7R&+Io#?hoYxp{>KADOaoK4t0k{Zk->b zg^g?jRzZAwB49JU;KAySHr&!YpEl+GkkptvV|||$>abUurhnx zjCj~(Y9UcPL&1PWEWygP^-|6RBto+E3AlV?jm#DBcQi}d!xCstm0jN47JC%go3J5t zyUD2jv~cHhY?aC)pC9498;o#epzJ#h)zK&K?;lDRo?j6@{-~t2%2<4XfZQ(s!!hHb zPwZ8yd?NI%GfI>+Z2MqJ6X_uBR=kkQ^8v+D_{Ca1p)rw7pQQWUOxP~S`!1#Yf`Sn@ z9l_NJc{mgqdv=6qU#`DCIQv1N99z@Ee&NY`<2IOho@0Sgt+m??L{>V5v4uX zBL_Hb@&nf=<(B1leEHJA&VEyrKzDU9)vkTN3~*J4b@%QKz(w}Z!froI5=UKTjkiOIo9wY#$!F?|>nWrKqSp}8JsvAOh|0(Q%eLDg7R zTf4eCuX}3hyCq8^`_&#W4y{Ih3md6T)Z%`^>`hxnr?JD~qG<9qBw`XQ!C1W+T6`jP zFrz{SST@&Yrb14fIB{pv?Czn{n7{}{XaK7>1O7#fR(Czmu>7Uo{ypJNm>{xJ5P5A+ z$itE8hc8WW0K++^gtWBu<3VhEwY+uV?Y3BL6BuF5;M;@I+}Hj6spxkjPq|y@VbmN{ z4)BE)_U%Q=y5q=ln0gA=DV0Jej)yL_VucD4P360B1VSA!eb=_0GKG2HLH`#p;L>J~ zP?;#RNpw?VFe1}744LSmMFgC$mbej9h4q*|N`Bv%ZjO}&5NjQuC}Z-(>)dgN?2>WB zJ#{sOG0R;iOTT%L?(XTSmRBsZ=@&kUWk9Y0;2FdFPea$CN2PFiwLVIMIxOir6g@OF zMr#YD@F+|bY+*?$@iroD_sad(lTjpDeGuY~DGyJ+EUX%U$BD&m>uBN(*HthuT~JUU-P~&E<%`3JXy{56B%ckCp*s_S0RcMr1IpES zoUZx?!`FLr=l}!V`zH=#5Q$34go@&E?h=+Zi0KjdFvE|hk_jiXN#5|5`Bl9@{g8o1q>l|FqX{N7MrD+fHTldiL*T6 z$wPyE>P={qHXM2Mb&~Vl+g187IC_foasywq#o2xcR~RPDYnujEEwD{O?Feo|sn8A633W%&t=L!aLAXAR)49~yI9J7$s_E|cO9TZIZ`=^W<4!`G zAfY$kw8%TR?bIiq$Xb5JgEriKI@gr-6gU&^*241I+Wq^L27kJ?Ts;MVixIvVqsiC1 zdDWYTV3G#x*czXX2osso%R&mb(8Q^+F=uzY=~by6mLZLE1IG%p34MU99*eeLcBpgwd zJ9Fkt0Q#R2D-#@0u@>oR0ZRp*pHGu0M-8z z>~EEuJI)2bY^gcOVgxRmy3ME-mY~DUDOiD*+;HaX*-P-}d-VMt#BqN!a|UnCL%B_1 zGR6%-7GRD`P9Wn}g>rLQ>_JRs#@K{i8(fHPF34g&ezz&+o*c8mICOoka^W=NIUVS5 zbUf!E@YLIY%vIo56acJN5Eu|>G$3oJ4_TSf$d!U{(D!F9H!Jd1L6Exy{sR~bU8P(X zoPDKt=l4dL4JbEGaf}sxQRS*f0VQD7hk4lH@vlcZlP-lN@$oNsc#KOx_j=sMQ&GX1 zpO3k|%0bwM$G)LX(mylP`J!zv5Cy<= zQ2-$s=<#N}R3+R9nWjW;zn7QS5>)*gL8MsS&&@ULk1K?GQNjxmct&#zJV7p2wl9Gq zW#%)0#lyRx?-0a>&-UE&>Kbt4w=NoYUf>K{86AA4zhi`MDc zUp}?VM03g4RaWJiIX<12)fLq}4*Xu)d^N~ZEwqmhp_U!`I1H6c!i$!vh*mGNWWY99 z0HA}vo;x>vd;>Zuov9ToRtQod1eCJeLXIxDdp~_=L$1oHW=_Nyy0!6>U*?g->QK;_3!&l+V~T}m}Aj7ab$)&`YR zmi03g;J1_($vyyM%%=F(drh*dYxVsPuToMn(N8sb0t>dI;}(Ck!ie_<_USHg)tvp0 zSZ=AcxTSH(Q7U+CYEAD!V02B%LmQjA9{w}*zYq&zPA0f^ipWh1IYe(Ina(WdqTFK+E?(dGZ9{_!(2rZt*GHW$S5st9_7~ z(Qr0)TzP5f9z_38pt>}DynpMbkH}{SHeULg(Qk$`ZFB(_RF0vp+@G@#VFYRui{T!G z@q`y9#lFhb8{ZrGxB|)`FbJm%q=HkUWq@kE0k%%@C|gz`K7_xmRSE=Tjdr1pOYMSo z7U?!1RvEz+A*>h?R71ruC}1)Wo4`^LExUSU?CSi7CBB-BjC13&ub=XSIe?P1vorK5 zB!NPZ*^53Q(@p+wz0S74Lo*73zrVm(_{$fcHpyil>jOa4~YHs6Epu6Q@s2C zq564Q^pn3ov4KcO5`lYWNLLB0%g1ELk4z~6kiYbZ_qTd!rZSM>fQ9usKbz5HgO3BQ zC=}&BvKIcK#l}@Smf96>R~R?TPkF6bT#5GrB@3Xk1w6adw`)-MYRD~eQ^LH z5I!Yp0n{(Y4E;q$P~;l3IpGhB|I)p0%LWagi#33)}(#XIXU?Pk>%)=im22%$Uwh_k;ysuW-p69HVd$&;LQ& zVa(odc7Jz+nSeezq2emfEU~deMPJF4{)bUUnJ~7jIBRHRkVf;FVUw|Aa6OfHwxB=X8sH4>6db|M!afZF!rK`IE52-_ zQ)nPi|B*0=^b9Ci{rni%;_(*cY%)xl*y48kG2j-B z{#aMr(OrGkIvlnK#MT2nten(@i4Y9Q()W(J=Aeoulfm7^@GEOT?U&q8>=lLc_`-;A zLnbbN_;4paY$I^yYp_kuIA;v>{6NmXG`5Q(}UFUQY%bX zgwP2YDfL|SImgW+?SJ_G`??HEb&}Ok@`4Ow6q&XMT}1;-K*3RbaH-`LJB}_U3e8(0 zx@k#V5pEqtY)U`SM@9BQIg$hgB+k%M1zJqHNNL`8bSwvIOluq+KpHO4x2w>25yOq! zu8BrlyJ|JOr>~-O2CON=0#zAdDKdhcJoFr`1jNLfT>D!LK?^!LI%<*kCloED!$HAC zj`SIIV`MCX@F8PbU;q#;(X=oRXz_X|4$*%ezz=S0?uQ5p8TSbqnNEn)sGmWep><1V zBitMw>wdO}J+muVkI=VYV+RMak3iIVa2PLC9%S)ha&iff z=Y3iy0aBQwec=pz0ykm~BZ?A~8`^b>J`?v~n8LG@z1Hb$@uK|53a`NKL%3-}wrTwzWI{&oqr!CIyOXw!X9aPj7-xIint8yQ*eJ3W3>-O$7& z>PkomMSE$}g6GfgJKn}zc+{;aG~kzjFzjT1$Il;XBg784=Mb5OlNf?5$Or53AfDi2 zLV}qBD16spq1#Cq-#>Bl3Y?m7ogPCLk!d@z7So|uP;Y|6?j>+9=x z03OM%GXKAD(pAH$&;UG%+b`OIHOt8W#vn+9A~Xu9ZgE2OB~${26`LGdj`7JD)dg96 zgw&hGADp7ly2h^dtlQ;I)8oZc*-Ojgjj|HcQ8`r(BJD0;w(Ped5zr`oE@o-@SXgC4 zg5JO$ISQXZH_ecPs&b@-j>54CZ$F@`xio9eSBvp3cSE3%`8CJ_h>0Mu^nFii8oZ34 zhge$v0V1!I5&mpj6hPY-ix3j~eB*vLaE=7Ruh=4QL5f&DWG)J@Rd4JG6iCN{!{(~swgxe-tYEu>v|DRrrif9yhe zwhyo#ffUKvNx0KR(-XFxwbAWPRnMLsKsoaiURYFIqmoY0NzmpD;o7VLYp`SJLjm`0 z+L#J5nLKRBo@kM*32C0y!%>9cw9!=bu{0S0;f%(eiH&K3XdP;kaM&kDp&fx24VgBW zC5rUu&kXHFA>}}j+d*<^Aur@=Dt@tG)NvsdnnGOu*#m7E4R)yIC?TtHq!Vy#jfIsv z!%1RhPaXsSG*jX*{Lac9AAXW;Z>$VOfPhUf9s+K95}+tRI|o3z*WEq5lUNOetn&+n z=}=)2Rd1q}P||A)^Uj;64dxpIU#P5;v-^-CB79K&8B)yjeBYaZ>}6cC7UzJ^oKf1Z z=$nZE2aEt@*&jE!)N+B*N`&C_HKDw`dHk99PD8k7L6bnZY77GpSo3PoZ| z>vdfeS$d);Ro(P7sl}@QX&KZ?{MgS{#gtFb`?$i^F47g%K*+eCy*!OT#2Uj3=- z)@aDB+qda#7}(_8S(T=f0Fn}^CfKFx*Y6pnR#HFLE2SAke)G&GtgV0)SqNn4b~Nz7 zzyKOp?W_qxHLuP_V7`M~iXj4I0+!n7-OVz<$MsO-lO_xb%p_MN5y_dm!E4hG`A{)@ zKlJ}AKaY8%iroPYtXZn!`g}< zy>?~T>LkeA1OW%ap9sjL3}#r#atI}s_aXD3X^&OrflU#p7^F6D-h7~A?aIrB%^Y4o z^Yjg(cQ*W}mfdwZphn$#bV}^1=$+`zu4-s5)iMF{{Qx64Lo|E)bXP)1{B&!_@aSk( z__4_PMp-FmY&#z~gz`}x%He@lwN@W6KFYE*KR>_uV_%B7na?bMe`W&YSHY-^r44~+ zo-00Y5~}J-gavhuwAPhnyS3}ly;RJ(L;|~wl{L{@SF<4PtI&x?JZQjn&`C2@_JKAN zRiZ@2Gb3HeB=F#3-O*Q5%HO&(l!U+&l0#YC=Iiyj%X6yKx$-z@;TA0AS3T^bu?O_p zam@p0;{oH_jo$RqsLB8J*I!XPBodxUz#CLBoW2OMx$W!X339%gcHl^nis=#M-G4c2 z2X%$#Bq14^=egd#eOtV>Y5Jj&QytQn=m)*yq$$K2sn38yS)E;-B5$XF){U8fS508w zW?BRN4-b&@;VwpHV4J11Vns;?Kg+6g&dXPat`FM@*Y>ttX)6dp(rx-mQn5>Xcrpkt zL30O(7bg@|VRb_rwGtYE7cDgv-ZfDXZe&TyqIzHBx>5rA$_m^|B=qVl!*j|6sv0HM znk`@b&#a)34!&QU1p=+^^cE`*;OX(rcve>;|FN=K zi2Z3qf?@J;Ow^SkGH&-c&PpBu<*1%FoG^$rM!?tv6l2m-if9avA&in(5APvz%o}AM zdFzoFr81qWQQ#G9I@pY$g9APrZR%8?UOUg4(UQTQQV)f$jMQF3^DT`~sK~k2VE5Bx z7L5er*kD6+QuW%|2(_2srpLxnn`@GijAlK6J=X|sy#cokA{{y<>ILWl=wa%MPkq`u z+K4lD&~0JDc{|nLD6P#Un!z~XkXT(7ox7Cbs=oYx<`^R0DoZM(bPi@xU;@gV$sB7# zq^GL?{XlyfdcE%UPt#*lyv#XhA?s5QD#9(!AUp0z{)z{KYs}V}Eg$&Dm{I6CthMU^ z7E#RrKpvg2J~AnBx}mZMjlTQtJ9a-)in<4O3k zoGnOnDN0Omw~1(TP~EmxWBCI|)dC`X5A6b#PPhaDcrZ_qf;CF~HAz|Iltmc4=b;Ge z2HvcX7uH2@4jH*MMs;X(9JrdC@}~&+?_u-L*c7_}UZBUucV$ttm(Xsm&P;XpvVb?A$5=UxoG zAjuN?{5qUI0~!3FouHu&*FF-h38@FDi!ZHNAa?Y>9>`U29$*<#fm)Kv1483^NERhf zCJ+oT0aJZxSDyKcZ@_YxMK=(!8G_r@G-3h?+ktGQ5Jxw+MrxIzJ$ZuypKC8(6zIIpVYk=jd_Z#l(~Kpo&vb^od67Iq7B5JFTz_Q z?l>WOgnPwqk-|Iy(5z#4D|(Ws|Kd>ROUTO`pxAqgT`c;c!mk0kc5AR-6QHw*m?8QL zr2-Xo^fVC8Y>+mI<-0Q}f;kSFC_;dcz`P++C{f5%A)^%FK0mq73Jiii$TM*kW;}$F zzM8%8!c684E5Dl^{x1UxJ4;`&%8t#YC!iL^;^_v2O@pRXM8RuJNvx_d@Z9*st4H(x zO8Ch8C!%Qt1Ep6I=nN%09zT-+e9NxCAtQWC%XzY}#B?ls{fU>`6!*+w+@+U!yvXa% zx&f1C!gnnXv~7YwlsaKRl=TSyd_qFW0CJ6dvl&7F=G#iW7yR3^&QDq%NG#Og;F;^P yZ1qvJM4#_t1bwsC>@48(fB%yIUoyL!7JbIqc_CN-4TiHB4AyRqU8y?`o%uh!h!QXW literal 0 HcmV?d00001 diff --git a/runs/fidelity-headroom/run_pilot_simulator.py b/runs/fidelity-headroom/run_pilot_simulator.py index 6db18b4..d1833ec 100644 --- a/runs/fidelity-headroom/run_pilot_simulator.py +++ b/runs/fidelity-headroom/run_pilot_simulator.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Run and score the 12 frozen Frontier P1 primary probes, CPU only.""" +"""Run and score a frozen Frontier probe manifest, CPU only.""" from __future__ import annotations @@ -54,9 +54,16 @@ def git_capture(root: Path, *arguments: str) -> str: def execute(args: argparse.Namespace) -> dict[str, Any]: + args.output = args.output.resolve() prepared = json.loads(args.prepared_manifest.read_text(encoding="utf-8")) if prepared["status"] != "PASS": raise RuntimeError("prepared simulator manifest did not pass") + expected_runs = int(prepared.get("expected_runs", len(prepared["entries"]))) + if expected_runs != len(prepared["entries"]): + raise RuntimeError( + f"prepared manifest expected {expected_runs} runs but contains " + f"{len(prepared['entries'])} entries" + ) driver = load_module( "simfid_execution_driver", args.replayserve_root @@ -71,7 +78,7 @@ def execute(args: argparse.Namespace) -> dict[str, Any]: failures = [] gpu_visibility_disabled = True for sequence, entry in enumerate(prepared["entries"]): - run_root = args.output / f"{sequence:02d}_{entry['fixture_id']}" + run_root = args.output / f"{sequence:03d}_{entry['fixture_id']}" scorer_path = run_root / "scorer_output.json" if scorer_path.is_file() and args.resume: scorer = json.loads(scorer_path.read_text(encoding="utf-8")) @@ -205,8 +212,8 @@ def execute(args: argparse.Namespace) -> dict[str, Any]: red_flags = [] if failures: red_flags.append("frontier_run_failure") - if len(results) != 12: - red_flags.append("runs_not_12") + if len(results) != expected_runs: + red_flags.append("runs_not_expected") if any(not 0.0 <= value <= 1.0 for value in pass_rates): red_flags.append("pass_rate_out_of_range") if any(value <= 0 for value in throughputs): @@ -251,7 +258,8 @@ def execute(args: argparse.Namespace) -> dict[str, Any]: "distinct_n": len(set(runtimes)), }, "invariants": { - "runs_12": len(results) == 12, + "runs_expected": len(results) == expected_runs, + "expected_runs": expected_runs, "zero_failures": not failures, "ratios_bounded": all(0.0 <= value <= 1.0 for value in pass_rates), "nonnegative_metrics": all(value > 0 for value in throughputs), diff --git a/runs/frontier-qwen30-vllm020-profile-v1/analyze_op_trace_delta.py b/runs/frontier-qwen30-vllm020-profile-v1/analyze_op_trace_delta.py new file mode 100644 index 0000000..07883e4 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/analyze_op_trace_delta.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Compare Frontier operator predictions at the identical initial scheduler state.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + + +PATTERN = re.compile( + r"\[OP-TRACE\]\[MONOLITHIC\]\[(?:ATTENTION|MOE)\]\[([^]]+)\] " + r"batch_id=(\d+), layer_id=(\d+), predicted_time_ms=([0-9.eE+-]+)" +) +COMPONENTS = ( + "input_layernorm", + "attn_pre_proj", + "attn_rope", + "attn_kv_cache_save", + "attn_prefill", + "attn_decode", + "attn_post_proj", + "post_attention_layernorm", + "moe_gating_linear", + "moe_gating_routing_topk", + "moe_shuffling", + "moe_grouped_gemm", +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--old", type=Path, required=True) + parser.add_argument("--new", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def first_state(path: Path) -> dict[str, float]: + values: dict[str, set[float]] = {} + with path.open(errors="replace") as handle: + for line in handle: + match = PATTERN.search(line) + if match and int(match[2]) == 0 and int(match[3]) == 0: + values.setdefault(match[1], set()).add(float(match[4])) + ambiguous = {name: sorted(items) for name, items in values.items() if len(items) != 1} + if ambiguous: + raise ValueError(f"ambiguous initial predictions in {path}: {ambiguous}") + result = {name: next(iter(items)) for name, items in values.items()} + result.setdefault("attn_decode", 0.0) + missing = sorted(set(COMPONENTS) - set(result)) + if missing: + raise ValueError(f"missing initial predictions in {path}: {missing}") + return result + + +def main() -> None: + args = parse_args() + old = first_state(args.old) + new = first_state(args.new) + rows = [] + for component in COMPONENTS: + rows.append( + { + "component": component, + "historical_profile_ms": old[component], + "vllm020_profile_ms": new[component], + "new_over_old": new[component] / old[component] + if old[component] != 0 + else None, + } + ) + old_total = sum(old[name] for name in COMPONENTS) + new_total = sum(new[name] for name in COMPONENTS) + output = { + "schema": "frontier-initial-op-trace-delta.v1", + "comparison_contract": { + "fixture": "fidelity_p1_tp1_mns64_low1", + "scheduler_state": "batch_id=0, layer_id=0 before profile-dependent trajectories diverge", + "calibration_a_tp": 1.0, + }, + "layer_component_sum_ms": { + "historical_profile": old_total, + "vllm020_profile": new_total, + "new_over_old": new_total / old_total, + }, + "rows": rows, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n") + print(args.output) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-qwen30-vllm020-profile-v1/analyze_p1_profile_ablation.py b/runs/frontier-qwen30-vllm020-profile-v1/analyze_p1_profile_ablation.py new file mode 100644 index 0000000..67ec218 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/analyze_p1_profile_ablation.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Analyze real, calibrated, old-profile, and new-profile P1 probe outcomes.""" + +from __future__ import annotations + +import argparse +import csv +import json +import statistics +from pathlib import Path +from typing import Any + + +MODES = ("historical-calibrated", "historical-profile-only", "vllm020-profile-only") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--controller-state", type=Path, required=True) + parser.add_argument("--calibrated", type=Path, required=True) + parser.add_argument("--old-profile-only", type=Path, required=True) + parser.add_argument("--new-profile-only", type=Path, required=True) + parser.add_argument("--output-json", type=Path, required=True) + parser.add_argument("--output-csv", type=Path, required=True) + return parser.parse_args() + + +def load_results(path: Path) -> dict[tuple[str, str], dict[str, Any]]: + payload = json.loads(path.read_text()) + if payload["status"] != "PASS" or len(payload["results"]) != 12: + raise ValueError(f"expected 12 passing simulator probes in {path}") + return {(row["cell"], row["role"]): row for row in payload["results"]} + + +def main() -> None: + args = parse_args() + controller = json.loads(args.controller_state.read_text()) + real: dict[tuple[str, str], dict[str, Any]] = {} + for cell, value in controller["cells"].items(): + tp = int(value["tp"]) + for run in value["runs"]: + if run["role"] in ("low1", "high1"): + real[(cell, run["role"])] = { + **run, + "tp": tp, + "offered_req_s_per_gpu": int(run["selected_count"]) / 60 / tp, + } + if len(real) != 12: + raise ValueError(f"expected 12 real P1 probes, found {len(real)}") + + modes = { + "historical-calibrated": load_results(args.calibrated), + "historical-profile-only": load_results(args.old_profile_only), + "vllm020-profile-only": load_results(args.new_profile_only), + } + rows: list[dict[str, Any]] = [] + summaries: dict[str, Any] = {} + for mode in MODES: + predicted = modes[mode] + false_feasible = 0 + false_infeasible = 0 + pass_errors: list[float] = [] + capacity_lower_bounds: dict[str, float] = {} + for key in sorted(real): + real_row = real[key] + sim_row = predicted[key] + scorer = sim_row["scorer"] + sim_feasible = bool(scorer["slo"]["feasible"]) + real_feasible = bool(real_row["feasible"]) + false_feasible += int(sim_feasible and not real_feasible) + false_infeasible += int(real_feasible and not sim_feasible) + pass_error = abs(float(scorer["slo"]["pass_rate"]) - float(real_row["pass_rate"])) + pass_errors.append(pass_error) + rows.append( + { + "mode": mode, + "cell": key[0], + "role": key[1], + "real_feasible": real_feasible, + "sim_feasible": sim_feasible, + "real_pass_rate": float(real_row["pass_rate"]), + "sim_pass_rate": float(scorer["slo"]["pass_rate"]), + "pass_rate_absolute_error": pass_error, + "offered_req_s_per_gpu": float(real_row["offered_req_s_per_gpu"]), + "sim_throughput_req_s_per_gpu": float( + scorer["throughput_requests_per_second_per_gpu"] + ), + } + ) + if sim_feasible: + capacity_lower_bounds[key[0]] = max( + capacity_lower_bounds.get(key[0], 0.0), + float(real_row["offered_req_s_per_gpu"]), + ) + agreement = 12 - false_feasible - false_infeasible + summaries[mode] = { + "probe_classification": { + "agreement": agreement, + "accuracy": agreement / 12, + "false_feasible": false_feasible, + "false_infeasible": false_infeasible, + }, + "pass_rate_mae": statistics.mean(pass_errors), + "feasible_probe_count": sum( + bool(row["scorer"]["slo"]["feasible"]) for row in predicted.values() + ), + "p1_capacity_lower_bounds_req_s_per_gpu": { + cell: capacity_lower_bounds.get(cell, 0.0) + for cell in sorted(controller["cells"]) + }, + "rank_identifiable": bool(capacity_lower_bounds), + } + + output = { + "schema": "frontier-qwen30-p1-profile-ablation.v1", + "scope": { + "cells": 6, + "probes_per_cell": 2, + "roles": ["low1", "high1"], + "reading": "held-out boundary classification, not a complete capacity sweep", + }, + "sources": { + "controller_state": str(args.controller_state.resolve()), + "historical_calibrated": str(args.calibrated.resolve()), + "historical_profile_only": str(args.old_profile_only.resolve()), + "vllm020_profile_only": str(args.new_profile_only.resolve()), + }, + "summaries": summaries, + "rows": rows, + } + args.output_json.parent.mkdir(parents=True, exist_ok=True) + args.output_json.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n") + args.output_csv.parent.mkdir(parents=True, exist_ok=True) + with args.output_csv.open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + print(args.output_json) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-qwen30-vllm020-profile-v1/analyze_routing_mismatch.py b/runs/frontier-qwen30-vllm020-profile-v1/analyze_routing_mismatch.py new file mode 100644 index 0000000..56d8492 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/analyze_routing_mismatch.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Compare captured vLLM expert loads with Frontier's fixed routing prior.""" + +from __future__ import annotations + +import argparse +import json +import math +import statistics +from pathlib import Path +from typing import Any + +import numpy as np + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--routing", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--seed", type=int, default=42) + return parser.parse_args() + + +def gini(values: np.ndarray) -> float: + values = np.asarray(values, dtype=np.float64) + if values.sum() == 0: + return 0.0 + ordered = np.sort(values) + n = len(ordered) + indices = np.arange(1, n + 1, dtype=np.float64) + return float((2 * np.sum(indices * ordered) / np.sum(ordered) - (n + 1)) / n) + + +def stats(values: np.ndarray) -> dict[str, float]: + values = np.asarray(values, dtype=np.float64) + mean = float(np.mean(values)) + return { + "load_cv": float(np.std(values) / mean), + "load_gini": gini(values), + "max_load_ratio": float(np.max(values) / mean), + "expert_utilization": float(np.count_nonzero(values) / len(values)), + } + + +def proportional_counts(total: int, ratios: np.ndarray) -> np.ndarray: + exact = total * ratios / ratios.sum() + counts = np.floor(exact).astype(np.int64) + remainder = total - int(counts.sum()) + order = sorted( + range(len(ratios)), + key=lambda i: (-(exact[i] - counts[i]), -(ratios[i] / ratios.sum()), i), + ) + for index in range(remainder): + counts[order[index % len(order)]] += 1 + assert int(counts.sum()) == total + return counts + + +def correlation(left: np.ndarray, right: np.ndarray) -> float: + value = float(np.corrcoef(left, right)[0, 1]) + return value if math.isfinite(value) else 0.0 + + +def distribution(values: list[float]) -> dict[str, float]: + return { + "min": min(values), + "median": statistics.median(values), + "max": max(values), + } + + +def main() -> None: + args = parse_args() + routing = json.loads(args.routing.read_text()) + phases = routing["phases"] + layer_count = len(phases["prefill"]["per_layer"]) + expert_count = len(phases["prefill"]["per_layer"][0]["counts"]) + + rows: list[dict[str, Any]] = [] + for phase in ("prefill", "decode"): + for layer, actual in enumerate(phases[phase]["per_layer"]): + rng = np.random.RandomState(args.seed + layer) + ratios = rng.uniform(0.1, 1.0, expert_count) + synthetic = proportional_counts(int(actual["total_routed_tokens"]), ratios) + synthetic_stats = stats(synthetic) + for name in synthetic_stats: + if not math.isclose( + stats(np.asarray(actual["counts"]))[name], + float(actual[name]), + rel_tol=0, + abs_tol=1e-12, + ): + raise ValueError(f"captured metric mismatch: {phase} layer {layer} {name}") + rows.append( + { + "phase": phase, + "layer": layer, + "total_routed_tokens": int(actual["total_routed_tokens"]), + "actual": {name: float(actual[name]) for name in synthetic_stats}, + "frontier_simulation": synthetic_stats, + "actual_vs_frontier_pearson": correlation( + np.asarray(actual["counts"], dtype=np.float64), synthetic + ), + } + ) + + phase_summary: dict[str, Any] = {} + for phase in ("prefill", "decode"): + selected = [row for row in rows if row["phase"] == phase] + phase_summary[phase] = { + "token_count": int(phases[phase]["token_count"]), + "actual": { + name: distribution([row["actual"][name] for row in selected]) + for name in selected[0]["actual"] + }, + "frontier_simulation": { + name: distribution([row["frontier_simulation"][name] for row in selected]) + for name in selected[0]["frontier_simulation"] + }, + "actual_vs_frontier_pearson": distribution( + [row["actual_vs_frontier_pearson"] for row in selected] + ), + } + + phase_correlations = [] + for layer in range(layer_count): + prefill = np.asarray(phases["prefill"]["per_layer"][layer]["counts"]) + decode = np.asarray(phases["decode"]["per_layer"][layer]["counts"]) + phase_correlations.append(correlation(prefill, decode)) + + output = { + "schema": "frontier-routing-mismatch.v1", + "source": str(args.routing.resolve()), + "frontier_contract": { + "mode": "simulation", + "seed": args.seed, + "allocation": "per-layer fixed Uniform(0.1, 1.0), normalized once and reused for every batch and phase", + "layer_count": layer_count, + "expert_count": expert_count, + }, + "phase_summary": phase_summary, + "actual_prefill_vs_decode_pearson": distribution(phase_correlations), + "frontier_prefill_vs_decode_pearson": 1.0, + "rows": rows, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n") + print(args.output) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-qwen30-vllm020-profile-v1/analyze_s2_profile_ablation.py b/runs/frontier-qwen30-vllm020-profile-v1/analyze_s2_profile_ablation.py new file mode 100644 index 0000000..ac4dfec --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/analyze_s2_profile_ablation.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +"""Score a replacement-profile S2 sweep against the frozen real oracle.""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import math +import sys +from pathlib import Path +from typing import Any + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--shard-metrics", type=Path, action="append", required=True) + parser.add_argument("--ground-truth", type=Path, required=True) + parser.add_argument("--historical-metrics", type=Path, required=True) + parser.add_argument("--historical-analyzer", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def load_module(path: Path): + spec = importlib.util.spec_from_file_location("simfid_s2_analyzer", path) + if spec is None or spec.loader is None: + raise ImportError(path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def ranking(scores: dict[str, float]) -> list[dict[str, Any]]: + return [ + {"rank": index + 1, "cell": cell, "score": score} + for index, (cell, score) in enumerate( + sorted(scores.items(), key=lambda item: (-item[1], item[0])) + ) + ] + + +def robust_loao( + analyzer: Any, + runs: list[dict[str, Any]], + mode: str, + reading: str, + real_scores: dict[str, float], +) -> dict[str, Any]: + """Historical LOAO with an explicit null range for all-tied concordance.""" + anchors = sorted( + {(row["cell_id"], int(row["probe_index"])) for row in runs}, + key=lambda item: (analyzer.CELL_ORDER.index(item[0]), item[1]), + ) + replicates = [] + undefined = [] + for cell, probe in anchors: + scores, detail = analyzer.cell_scores( + runs, mode, reading, removed=(cell, probe) + ) + if scores is None: + undefined.append( + {"removed_cell_id": cell, "removed_probe_index": probe, **detail} + ) + continue + metric = analyzer.rank_metrics(real_scores, scores) + replicates.append( + { + "removed_cell_id": cell, + "removed_probe_index": probe, + "top1_optimistic_regret": metric["top1"]["optimistic_regret"], + "top1_worst_case_regret": metric["top1"]["worst_case_regret"], + "top5_minimum_exact_five_overlap": metric["top5"]["minimum_exact_five_overlap"], + "top5_maximum_exact_five_overlap": metric["top5"]["maximum_exact_five_overlap"], + "top5_optimistic_regret": metric["top5"]["optimistic_regret"], + "top5_worst_case_regret": metric["top5"]["worst_case_regret"], + "tau_b": metric["kendall_tau_b"]["tau_b"], + "pairwise_exact_sign_accuracy": metric["pairwise_direction"]["exact_sign_accuracy"], + "pairwise_non_tied_concordance": metric["pairwise_direction"]["non_tied_concordance"], + "trap_reproduced": metric["named_interactions"]["trap_reproduced"], + "tp2_mns32_unique_global_best": metric["named_interactions"]["tp2_mns32_unique_global_best"], + } + ) + scalar_keys = ( + "top1_optimistic_regret", + "top1_worst_case_regret", + "top5_minimum_exact_five_overlap", + "top5_maximum_exact_five_overlap", + "top5_optimistic_regret", + "top5_worst_case_regret", + "tau_b", + "pairwise_exact_sign_accuracy", + "pairwise_non_tied_concordance", + ) + ranges = {} + for key in scalar_keys: + values = [row[key] for row in replicates if row[key] is not None] + ranges[key] = { + "min": min(values) if values else None, + "max": max(values) if values else None, + } + return { + "replicate_count": len(anchors), + "defined_replicates": len(replicates), + "undefined_replicates": len(undefined), + "undefined": undefined, + "ranges": ranges, + "trap_reproduced_count": sum(row["trap_reproduced"] for row in replicates), + "tp2_mns32_unique_global_best_count": sum( + row["tp2_mns32_unique_global_best"] for row in replicates + ), + "replicates": replicates, + "range_semantics": "deterministic LOAO sensitivity ranges, not confidence intervals", + } + + +def main() -> None: + args = parse_args() + analyzer = load_module(args.historical_analyzer) + ground = json.loads(args.ground_truth.read_text()) + cells = {str(cell["cell_id"]): cell for cell in ground["cells"]} + if set(cells) != set(analyzer.CELL_ORDER): + raise ValueError("ground-truth cells differ from the frozen 3x4 surface") + + result_rows: list[dict[str, Any]] = [] + sources = [] + for path in args.shard_metrics: + payload = json.loads(path.read_text()) + if payload["status"] != "PASS": + raise ValueError(f"shard did not pass: {path}") + sources.append({"path": str(path.resolve()), "runs": len(payload["results"])}) + result_rows.extend(payload["results"]) + if len(result_rows) != 92: + raise ValueError(f"expected 92 replacement-profile probes, found {len(result_rows)}") + + seen: set[tuple[str, int]] = set() + runs = [] + for row in result_rows: + cell_id = str(row["cell"]) + probe_index = int(row["probe_index"]) + key = (cell_id, probe_index) + if key in seen: + raise ValueError(f"duplicate probe {key}") + seen.add(key) + real_probe = cells[cell_id]["probe_history"][probe_index] + if not math.isclose( + float(real_probe["sampling_u"]), float(row["sampling_u"]), rel_tol=0, abs_tol=1e-15 + ): + raise ValueError(f"sampling_u mismatch for {key}") + if int(real_probe["request_count"]) != int(row["selected_count"]): + raise ValueError(f"request count mismatch for {key}") + runs.append( + { + **row, + "cell_id": cell_id, + "mode": "vllm020-profile-only", + "probe_index": probe_index, + "sampling_u": float(row["sampling_u"]), + "request_count": int(row["selected_count"]), + "tensor_parallel_size": int(row["tensor_parallel_size"]), + "real_anchor": { + "feasible": bool(real_probe["feasible"]), + "pass_rate": float(real_probe["pass_rate"]), + "request_count": int(real_probe["request_count"]), + }, + } + ) + + historical = json.loads(args.historical_metrics.read_text()) + real_scores = {cell: float(value) for cell, value in historical["real_scores"].items()} + analyses = {} + for reading in analyzer.READINGS: + scores, detail = analyzer.cell_scores(runs, "vllm020-profile-only", reading) + if scores is None: + raise ValueError(f"undefined {reading} scores: {detail}") + analysis = { + "reading": reading, + "simulated_scores": scores, + "ranking": ranking(scores), + "cell_score_details": detail["cells"], + "metrics": analyzer.rank_metrics(real_scores, scores), + "loao": robust_loao( + analyzer, runs, "vllm020-profile-only", reading, real_scores + ), + } + if reading == "SLO-gated": + analysis["false_feasibility"] = analyzer.false_feasibility( + runs, "vllm020-profile-only" + ) + analyses[reading] = analysis + + historical_modes = {} + for label, key in ( + ("historical-profile-only", "uncalibrated/SLO-gated"), + ("historical-per-tp-calibration", "frozen-calibrated/SLO-gated"), + ): + value = historical["analyses"][key] + historical_modes[label] = { + "simulated_scores": value["simulated_scores"], + "ranking": ranking(value["simulated_scores"]), + "metrics": value["metrics"], + "false_feasibility": value["false_feasibility"], + } + + output = { + "schema": "frontier-qwen30-s2-profile-ablation.v1", + "scope": { + "cells": len(analyzer.CELL_ORDER), + "probes": len(runs), + "trace_horizon_seconds": 60, + "calibration_a_tp": 1.0, + }, + "sources": { + "replacement_profile_shards": sources, + "ground_truth": str(args.ground_truth.resolve()), + "historical_metrics": str(args.historical_metrics.resolve()), + }, + "real_scores": real_scores, + "historical_modes": historical_modes, + "vllm020_profile_only": analyses, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n") + print(args.output) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-initial-op-trace-delta.json b/runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-initial-op-trace-delta.json new file mode 100644 index 0000000..58346dc --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-initial-op-trace-delta.json @@ -0,0 +1,87 @@ +{ + "comparison_contract": { + "calibration_a_tp": 1.0, + "fixture": "fidelity_p1_tp1_mns64_low1", + "scheduler_state": "batch_id=0, layer_id=0 before profile-dependent trajectories diverge" + }, + "layer_component_sum_ms": { + "historical_profile": 1.148334, + "new_over_old": 2.700194368537377, + "vllm020_profile": 3.1007249999999997 + }, + "rows": [ + { + "component": "input_layernorm", + "historical_profile_ms": 0.010619, + "new_over_old": 1.955833882663151, + "vllm020_profile_ms": 0.020769 + }, + { + "component": "attn_pre_proj", + "historical_profile_ms": 0.204389, + "new_over_old": 1.0199276869107439, + "vllm020_profile_ms": 0.208462 + }, + { + "component": "attn_rope", + "historical_profile_ms": 0.018908, + "new_over_old": 7.394171779141103, + "vllm020_profile_ms": 0.139809 + }, + { + "component": "attn_kv_cache_save", + "historical_profile_ms": 0.014226, + "new_over_old": 1.427386475467454, + "vllm020_profile_ms": 0.020306 + }, + { + "component": "attn_prefill", + "historical_profile_ms": 0.107759, + "new_over_old": 2.670709639102071, + "vllm020_profile_ms": 0.287793 + }, + { + "component": "attn_decode", + "historical_profile_ms": 0.0, + "new_over_old": null, + "vllm020_profile_ms": 0.0 + }, + { + "component": "attn_post_proj", + "historical_profile_ms": 0.134153, + "new_over_old": 0.967559428413826, + "vllm020_profile_ms": 0.129801 + }, + { + "component": "post_attention_layernorm", + "historical_profile_ms": 0.011481, + "new_over_old": 0.9000087100426792, + "vllm020_profile_ms": 0.010333 + }, + { + "component": "moe_gating_linear", + "historical_profile_ms": 0.011802, + "new_over_old": 2.2642772411455687, + "vllm020_profile_ms": 0.026723 + }, + { + "component": "moe_gating_routing_topk", + "historical_profile_ms": 0.009662, + "new_over_old": 2.2922790312564687, + "vllm020_profile_ms": 0.022148 + }, + { + "component": "moe_shuffling", + "historical_profile_ms": 0.030431, + "new_over_old": 0.0, + "vllm020_profile_ms": 0.0 + }, + { + "component": "moe_grouped_gemm", + "historical_profile_ms": 0.594904, + "new_over_old": 3.756204362384519, + "vllm020_profile_ms": 2.234581 + } + ], + "schema": "frontier-initial-op-trace-delta.v1" +} diff --git a/runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-p1-profile-ablation.csv b/runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-p1-profile-ablation.csv new file mode 100644 index 0000000..2750398 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-p1-profile-ablation.csv @@ -0,0 +1,37 @@ +mode,cell,role,real_feasible,sim_feasible,real_pass_rate,sim_pass_rate,pass_rate_absolute_error,offered_req_s_per_gpu,sim_throughput_req_s_per_gpu +historical-calibrated,tp1_mns64,high1,False,True,0.24581005586592178,0.9888268156424581,0.7430167597765364,2.9833333333333334,2.8823345959582403 +historical-calibrated,tp1_mns64,low1,True,True,1.0,1.0,0.0,2.033333333333333,1.9806287931485522 +historical-calibrated,tp1_mns8,high1,False,False,0.20670391061452514,0.16759776536312848,0.03910614525139666,2.9833333333333334,2.162661799726284 +historical-calibrated,tp1_mns8,low1,True,True,1.0,0.9754098360655737,0.024590163934426257,2.033333333333333,1.9806313066623409 +historical-calibrated,tp2_mns64,high1,True,True,1.0,1.0,0.0,2.875,2.809583898291986 +historical-calibrated,tp2_mns64,low1,True,True,1.0,1.0,0.0,1.9583333333333333,1.911740196570019 +historical-calibrated,tp2_mns8,high1,False,False,0.20238095238095238,0.15476190476190477,0.047619047619047616,2.8,2.083217332903509 +historical-calibrated,tp2_mns8,low1,True,True,1.0,0.9868995633187773,0.013100436681222738,1.9083333333333334,1.8637394408880168 +historical-calibrated,tp4_mns16,high1,False,False,0.10666666666666667,0.2693333333333333,0.16266666666666663,3.125,2.6829294600763345 +historical-calibrated,tp4_mns16,low1,False,True,0.6196078431372549,1.0,0.3803921568627451,2.125,2.0865667413849516 +historical-calibrated,tp4_mns64,high1,True,True,1.0,1.0,0.0,3.125,3.0718260341342285 +historical-calibrated,tp4_mns64,low1,True,True,1.0,1.0,0.0,2.125,2.0967062495193307 +historical-profile-only,tp1_mns64,high1,False,False,0.24581005586592178,0.24581005586592178,0.0,2.9833333333333334,2.82546810606982 +historical-profile-only,tp1_mns64,low1,True,False,1.0,0.7377049180327869,0.2622950819672131,2.033333333333333,1.9427723213024963 +historical-profile-only,tp1_mns8,high1,False,False,0.20670391061452514,0.09497206703910614,0.111731843575419,2.9833333333333334,1.5728783582441022 +historical-profile-only,tp1_mns8,low1,True,False,1.0,0.1721311475409836,0.8278688524590164,2.033333333333333,1.5629164685645112 +historical-profile-only,tp2_mns64,high1,True,False,1.0,0.2608695652173913,0.7391304347826086,2.875,2.7121507155597584 +historical-profile-only,tp2_mns64,low1,True,False,1.0,0.8382978723404255,0.16170212765957448,1.9583333333333333,1.849101894293262 +historical-profile-only,tp2_mns8,high1,False,False,0.20238095238095238,0.041666666666666664,0.16071428571428573,2.8,0.9859710483293953 +historical-profile-only,tp2_mns8,low1,True,False,1.0,0.0611353711790393,0.9388646288209607,1.9083333333333334,0.9771135206177947 +historical-profile-only,tp4_mns16,high1,False,False,0.10666666666666667,0.028,0.07866666666666668,3.125,0.9604809617478992 +historical-profile-only,tp4_mns16,low1,False,False,0.6196078431372549,0.058823529411764705,0.5607843137254902,2.125,0.9495221159963836 +historical-profile-only,tp4_mns64,high1,True,False,1.0,0.0026666666666666666,0.9973333333333333,3.125,2.281094127221267 +historical-profile-only,tp4_mns64,low1,True,False,1.0,0.4627450980392157,0.5372549019607843,2.125,1.9793960278798877 +vllm020-profile-only,tp1_mns64,high1,False,False,0.24581005586592178,0.0,0.24581005586592178,2.9833333333333334,2.074240230678028 +vllm020-profile-only,tp1_mns64,low1,True,False,1.0,0.0,1.0,2.033333333333333,1.6189525827713862 +vllm020-profile-only,tp1_mns8,high1,False,False,0.20670391061452514,0.0,0.20670391061452514,2.9833333333333334,0.5271192714936628 +vllm020-profile-only,tp1_mns8,low1,True,False,1.0,0.0,1.0,2.033333333333333,0.5225514701978402 +vllm020-profile-only,tp2_mns64,high1,True,False,1.0,0.0,1.0,2.875,1.7237810220598582 +vllm020-profile-only,tp2_mns64,low1,True,False,1.0,0.0,1.0,1.9583333333333333,1.608919184557297 +vllm020-profile-only,tp2_mns8,high1,False,False,0.20238095238095238,0.0,0.20238095238095238,2.8,0.39315152956277477 +vllm020-profile-only,tp2_mns8,low1,True,False,1.0,0.0,1.0,1.9083333333333334,0.3894201655965407 +vllm020-profile-only,tp4_mns16,high1,False,False,0.10666666666666667,0.0,0.10666666666666667,3.125,0.43700447014112276 +vllm020-profile-only,tp4_mns16,low1,False,False,0.6196078431372549,0.0,0.6196078431372549,2.125,0.43475717185302865 +vllm020-profile-only,tp4_mns64,high1,True,False,1.0,0.0,1.0,3.125,1.2598449015844073 +vllm020-profile-only,tp4_mns64,low1,True,False,1.0,0.0,1.0,2.125,1.2025831680037373 diff --git a/runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-p1-profile-ablation.json b/runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-p1-profile-ablation.json new file mode 100644 index 0000000..0bf3077 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-p1-profile-ablation.json @@ -0,0 +1,511 @@ +{ + "rows": [ + { + "cell": "tp1_mns64", + "mode": "historical-calibrated", + "offered_req_s_per_gpu": 2.9833333333333334, + "pass_rate_absolute_error": 0.7430167597765364, + "real_feasible": false, + "real_pass_rate": 0.24581005586592178, + "role": "high1", + "sim_feasible": true, + "sim_pass_rate": 0.9888268156424581, + "sim_throughput_req_s_per_gpu": 2.8823345959582403 + }, + { + "cell": "tp1_mns64", + "mode": "historical-calibrated", + "offered_req_s_per_gpu": 2.033333333333333, + "pass_rate_absolute_error": 0.0, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "low1", + "sim_feasible": true, + "sim_pass_rate": 1.0, + "sim_throughput_req_s_per_gpu": 1.9806287931485522 + }, + { + "cell": "tp1_mns8", + "mode": "historical-calibrated", + "offered_req_s_per_gpu": 2.9833333333333334, + "pass_rate_absolute_error": 0.03910614525139666, + "real_feasible": false, + "real_pass_rate": 0.20670391061452514, + "role": "high1", + "sim_feasible": false, + "sim_pass_rate": 0.16759776536312848, + "sim_throughput_req_s_per_gpu": 2.162661799726284 + }, + { + "cell": "tp1_mns8", + "mode": "historical-calibrated", + "offered_req_s_per_gpu": 2.033333333333333, + "pass_rate_absolute_error": 0.024590163934426257, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "low1", + "sim_feasible": true, + "sim_pass_rate": 0.9754098360655737, + "sim_throughput_req_s_per_gpu": 1.9806313066623409 + }, + { + "cell": "tp2_mns64", + "mode": "historical-calibrated", + "offered_req_s_per_gpu": 2.875, + "pass_rate_absolute_error": 0.0, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "high1", + "sim_feasible": true, + "sim_pass_rate": 1.0, + "sim_throughput_req_s_per_gpu": 2.809583898291986 + }, + { + "cell": "tp2_mns64", + "mode": "historical-calibrated", + "offered_req_s_per_gpu": 1.9583333333333333, + "pass_rate_absolute_error": 0.0, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "low1", + "sim_feasible": true, + "sim_pass_rate": 1.0, + "sim_throughput_req_s_per_gpu": 1.911740196570019 + }, + { + "cell": "tp2_mns8", + "mode": "historical-calibrated", + "offered_req_s_per_gpu": 2.8, + "pass_rate_absolute_error": 0.047619047619047616, + "real_feasible": false, + "real_pass_rate": 0.20238095238095238, + "role": "high1", + "sim_feasible": false, + "sim_pass_rate": 0.15476190476190477, + "sim_throughput_req_s_per_gpu": 2.083217332903509 + }, + { + "cell": "tp2_mns8", + "mode": "historical-calibrated", + "offered_req_s_per_gpu": 1.9083333333333334, + "pass_rate_absolute_error": 0.013100436681222738, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "low1", + "sim_feasible": true, + "sim_pass_rate": 0.9868995633187773, + "sim_throughput_req_s_per_gpu": 1.8637394408880168 + }, + { + "cell": "tp4_mns16", + "mode": "historical-calibrated", + "offered_req_s_per_gpu": 3.125, + "pass_rate_absolute_error": 0.16266666666666663, + "real_feasible": false, + "real_pass_rate": 0.10666666666666667, + "role": "high1", + "sim_feasible": false, + "sim_pass_rate": 0.2693333333333333, + "sim_throughput_req_s_per_gpu": 2.6829294600763345 + }, + { + "cell": "tp4_mns16", + "mode": "historical-calibrated", + "offered_req_s_per_gpu": 2.125, + "pass_rate_absolute_error": 0.3803921568627451, + "real_feasible": false, + "real_pass_rate": 0.6196078431372549, + "role": "low1", + "sim_feasible": true, + "sim_pass_rate": 1.0, + "sim_throughput_req_s_per_gpu": 2.0865667413849516 + }, + { + "cell": "tp4_mns64", + "mode": "historical-calibrated", + "offered_req_s_per_gpu": 3.125, + "pass_rate_absolute_error": 0.0, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "high1", + "sim_feasible": true, + "sim_pass_rate": 1.0, + "sim_throughput_req_s_per_gpu": 3.0718260341342285 + }, + { + "cell": "tp4_mns64", + "mode": "historical-calibrated", + "offered_req_s_per_gpu": 2.125, + "pass_rate_absolute_error": 0.0, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "low1", + "sim_feasible": true, + "sim_pass_rate": 1.0, + "sim_throughput_req_s_per_gpu": 2.0967062495193307 + }, + { + "cell": "tp1_mns64", + "mode": "historical-profile-only", + "offered_req_s_per_gpu": 2.9833333333333334, + "pass_rate_absolute_error": 0.0, + "real_feasible": false, + "real_pass_rate": 0.24581005586592178, + "role": "high1", + "sim_feasible": false, + "sim_pass_rate": 0.24581005586592178, + "sim_throughput_req_s_per_gpu": 2.82546810606982 + }, + { + "cell": "tp1_mns64", + "mode": "historical-profile-only", + "offered_req_s_per_gpu": 2.033333333333333, + "pass_rate_absolute_error": 0.2622950819672131, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "low1", + "sim_feasible": false, + "sim_pass_rate": 0.7377049180327869, + "sim_throughput_req_s_per_gpu": 1.9427723213024963 + }, + { + "cell": "tp1_mns8", + "mode": "historical-profile-only", + "offered_req_s_per_gpu": 2.9833333333333334, + "pass_rate_absolute_error": 0.111731843575419, + "real_feasible": false, + "real_pass_rate": 0.20670391061452514, + "role": "high1", + "sim_feasible": false, + "sim_pass_rate": 0.09497206703910614, + "sim_throughput_req_s_per_gpu": 1.5728783582441022 + }, + { + "cell": "tp1_mns8", + "mode": "historical-profile-only", + "offered_req_s_per_gpu": 2.033333333333333, + "pass_rate_absolute_error": 0.8278688524590164, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "low1", + "sim_feasible": false, + "sim_pass_rate": 0.1721311475409836, + "sim_throughput_req_s_per_gpu": 1.5629164685645112 + }, + { + "cell": "tp2_mns64", + "mode": "historical-profile-only", + "offered_req_s_per_gpu": 2.875, + "pass_rate_absolute_error": 0.7391304347826086, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "high1", + "sim_feasible": false, + "sim_pass_rate": 0.2608695652173913, + "sim_throughput_req_s_per_gpu": 2.7121507155597584 + }, + { + "cell": "tp2_mns64", + "mode": "historical-profile-only", + "offered_req_s_per_gpu": 1.9583333333333333, + "pass_rate_absolute_error": 0.16170212765957448, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "low1", + "sim_feasible": false, + "sim_pass_rate": 0.8382978723404255, + "sim_throughput_req_s_per_gpu": 1.849101894293262 + }, + { + "cell": "tp2_mns8", + "mode": "historical-profile-only", + "offered_req_s_per_gpu": 2.8, + "pass_rate_absolute_error": 0.16071428571428573, + "real_feasible": false, + "real_pass_rate": 0.20238095238095238, + "role": "high1", + "sim_feasible": false, + "sim_pass_rate": 0.041666666666666664, + "sim_throughput_req_s_per_gpu": 0.9859710483293953 + }, + { + "cell": "tp2_mns8", + "mode": "historical-profile-only", + "offered_req_s_per_gpu": 1.9083333333333334, + "pass_rate_absolute_error": 0.9388646288209607, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "low1", + "sim_feasible": false, + "sim_pass_rate": 0.0611353711790393, + "sim_throughput_req_s_per_gpu": 0.9771135206177947 + }, + { + "cell": "tp4_mns16", + "mode": "historical-profile-only", + "offered_req_s_per_gpu": 3.125, + "pass_rate_absolute_error": 0.07866666666666668, + "real_feasible": false, + "real_pass_rate": 0.10666666666666667, + "role": "high1", + "sim_feasible": false, + "sim_pass_rate": 0.028, + "sim_throughput_req_s_per_gpu": 0.9604809617478992 + }, + { + "cell": "tp4_mns16", + "mode": "historical-profile-only", + "offered_req_s_per_gpu": 2.125, + "pass_rate_absolute_error": 0.5607843137254902, + "real_feasible": false, + "real_pass_rate": 0.6196078431372549, + "role": "low1", + "sim_feasible": false, + "sim_pass_rate": 0.058823529411764705, + "sim_throughput_req_s_per_gpu": 0.9495221159963836 + }, + { + "cell": "tp4_mns64", + "mode": "historical-profile-only", + "offered_req_s_per_gpu": 3.125, + "pass_rate_absolute_error": 0.9973333333333333, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "high1", + "sim_feasible": false, + "sim_pass_rate": 0.0026666666666666666, + "sim_throughput_req_s_per_gpu": 2.281094127221267 + }, + { + "cell": "tp4_mns64", + "mode": "historical-profile-only", + "offered_req_s_per_gpu": 2.125, + "pass_rate_absolute_error": 0.5372549019607843, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "low1", + "sim_feasible": false, + "sim_pass_rate": 0.4627450980392157, + "sim_throughput_req_s_per_gpu": 1.9793960278798877 + }, + { + "cell": "tp1_mns64", + "mode": "vllm020-profile-only", + "offered_req_s_per_gpu": 2.9833333333333334, + "pass_rate_absolute_error": 0.24581005586592178, + "real_feasible": false, + "real_pass_rate": 0.24581005586592178, + "role": "high1", + "sim_feasible": false, + "sim_pass_rate": 0.0, + "sim_throughput_req_s_per_gpu": 2.074240230678028 + }, + { + "cell": "tp1_mns64", + "mode": "vllm020-profile-only", + "offered_req_s_per_gpu": 2.033333333333333, + "pass_rate_absolute_error": 1.0, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "low1", + "sim_feasible": false, + "sim_pass_rate": 0.0, + "sim_throughput_req_s_per_gpu": 1.6189525827713862 + }, + { + "cell": "tp1_mns8", + "mode": "vllm020-profile-only", + "offered_req_s_per_gpu": 2.9833333333333334, + "pass_rate_absolute_error": 0.20670391061452514, + "real_feasible": false, + "real_pass_rate": 0.20670391061452514, + "role": "high1", + "sim_feasible": false, + "sim_pass_rate": 0.0, + "sim_throughput_req_s_per_gpu": 0.5271192714936628 + }, + { + "cell": "tp1_mns8", + "mode": "vllm020-profile-only", + "offered_req_s_per_gpu": 2.033333333333333, + "pass_rate_absolute_error": 1.0, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "low1", + "sim_feasible": false, + "sim_pass_rate": 0.0, + "sim_throughput_req_s_per_gpu": 0.5225514701978402 + }, + { + "cell": "tp2_mns64", + "mode": "vllm020-profile-only", + "offered_req_s_per_gpu": 2.875, + "pass_rate_absolute_error": 1.0, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "high1", + "sim_feasible": false, + "sim_pass_rate": 0.0, + "sim_throughput_req_s_per_gpu": 1.7237810220598582 + }, + { + "cell": "tp2_mns64", + "mode": "vllm020-profile-only", + "offered_req_s_per_gpu": 1.9583333333333333, + "pass_rate_absolute_error": 1.0, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "low1", + "sim_feasible": false, + "sim_pass_rate": 0.0, + "sim_throughput_req_s_per_gpu": 1.608919184557297 + }, + { + "cell": "tp2_mns8", + "mode": "vllm020-profile-only", + "offered_req_s_per_gpu": 2.8, + "pass_rate_absolute_error": 0.20238095238095238, + "real_feasible": false, + "real_pass_rate": 0.20238095238095238, + "role": "high1", + "sim_feasible": false, + "sim_pass_rate": 0.0, + "sim_throughput_req_s_per_gpu": 0.39315152956277477 + }, + { + "cell": "tp2_mns8", + "mode": "vllm020-profile-only", + "offered_req_s_per_gpu": 1.9083333333333334, + "pass_rate_absolute_error": 1.0, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "low1", + "sim_feasible": false, + "sim_pass_rate": 0.0, + "sim_throughput_req_s_per_gpu": 0.3894201655965407 + }, + { + "cell": "tp4_mns16", + "mode": "vllm020-profile-only", + "offered_req_s_per_gpu": 3.125, + "pass_rate_absolute_error": 0.10666666666666667, + "real_feasible": false, + "real_pass_rate": 0.10666666666666667, + "role": "high1", + "sim_feasible": false, + "sim_pass_rate": 0.0, + "sim_throughput_req_s_per_gpu": 0.43700447014112276 + }, + { + "cell": "tp4_mns16", + "mode": "vllm020-profile-only", + "offered_req_s_per_gpu": 2.125, + "pass_rate_absolute_error": 0.6196078431372549, + "real_feasible": false, + "real_pass_rate": 0.6196078431372549, + "role": "low1", + "sim_feasible": false, + "sim_pass_rate": 0.0, + "sim_throughput_req_s_per_gpu": 0.43475717185302865 + }, + { + "cell": "tp4_mns64", + "mode": "vllm020-profile-only", + "offered_req_s_per_gpu": 3.125, + "pass_rate_absolute_error": 1.0, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "high1", + "sim_feasible": false, + "sim_pass_rate": 0.0, + "sim_throughput_req_s_per_gpu": 1.2598449015844073 + }, + { + "cell": "tp4_mns64", + "mode": "vllm020-profile-only", + "offered_req_s_per_gpu": 2.125, + "pass_rate_absolute_error": 1.0, + "real_feasible": true, + "real_pass_rate": 1.0, + "role": "low1", + "sim_feasible": false, + "sim_pass_rate": 0.0, + "sim_throughput_req_s_per_gpu": 1.2025831680037373 + } + ], + "schema": "frontier-qwen30-p1-profile-ablation.v1", + "scope": { + "cells": 6, + "probes_per_cell": 2, + "reading": "held-out boundary classification, not a complete capacity sweep", + "roles": [ + "low1", + "high1" + ] + }, + "sources": { + "controller_state": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/real/p1b/controller-state.json", + "historical_calibrated": "/home/gahow/phd/replayserve/runs/fidelity_p1_frontier_committed_20260714/results/metrics.json", + "historical_profile_only": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/comparison/old-profile-only-v2/results/metrics.json", + "vllm020_profile_only": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/comparison/new-profile-only-v2/results/metrics.json" + }, + "summaries": { + "historical-calibrated": { + "feasible_probe_count": 9, + "p1_capacity_lower_bounds_req_s_per_gpu": { + "tp1_mns64": 2.9833333333333334, + "tp1_mns8": 2.033333333333333, + "tp2_mns64": 2.875, + "tp2_mns8": 1.9083333333333334, + "tp4_mns16": 2.125, + "tp4_mns64": 3.125 + }, + "pass_rate_mae": 0.11754094806600345, + "probe_classification": { + "accuracy": 0.8333333333333334, + "agreement": 10, + "false_feasible": 2, + "false_infeasible": 0 + }, + "rank_identifiable": true + }, + "historical-profile-only": { + "feasible_probe_count": 0, + "p1_capacity_lower_bounds_req_s_per_gpu": { + "tp1_mns64": 0.0, + "tp1_mns8": 0.0, + "tp2_mns64": 0.0, + "tp2_mns8": 0.0, + "tp4_mns16": 0.0, + "tp4_mns64": 0.0 + }, + "pass_rate_mae": 0.44802887255544604, + "probe_classification": { + "accuracy": 0.4166666666666667, + "agreement": 5, + "false_feasible": 0, + "false_infeasible": 7 + }, + "rank_identifiable": false + }, + "vllm020-profile-only": { + "feasible_probe_count": 0, + "p1_capacity_lower_bounds_req_s_per_gpu": { + "tp1_mns64": 0.0, + "tp1_mns8": 0.0, + "tp2_mns64": 0.0, + "tp2_mns8": 0.0, + "tp4_mns16": 0.0, + "tp4_mns64": 0.0 + }, + "pass_rate_mae": 0.69843078572211, + "probe_classification": { + "accuracy": 0.4166666666666667, + "agreement": 5, + "false_feasible": 0, + "false_infeasible": 7 + }, + "rank_identifiable": false + } + } +} diff --git a/runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-p1b-opprof-summary.json b/runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-p1b-opprof-summary.json new file mode 100644 index 0000000..34efb09 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-p1b-opprof-summary.json @@ -0,0 +1,2171 @@ +{ + "cell_root": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/runs/p1b/cells", + "cells": [ + { + "cell": "tp1_mns64", + "cudagraph_runtime_mode_counts": { + "FULL": 26305, + "NONE": 698, + "PIECEWISE": 141 + }, + "dropped_records_before_max": 0, + "groups": [ + { + "cudagraph_runtime_mode": "FULL", + "decode_tokens": { + "max": 33, + "mean": 4.150661496350365, + "min": 1, + "p50": 3, + "p95": 13, + "p99": 21 + }, + "kv_usage": { + "max": 0.35797781996203415, + "mean": 0.03727774678883895, + "min": 0.00029973024278151517, + "p50": 0.023728644220201822, + "p95": 0.13018283544809672, + "p99": 0.21860325706863826 + }, + "phase": "pure_decode", + "prefill_tokens": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 33, + "mean": 4.150661496350365, + "min": 1, + "p50": 3, + "p95": 13, + "p99": 21 + }, + "steps": 26304, + "submit_to_complete_ms": { + "max": 771.351779, + "mean": 20.43450573646594, + "min": 5.345856, + "p50": 15.161645, + "p95": 35.769253, + "p99": 179.487345 + }, + "total_tokens": { + "max": 33, + "mean": 4.150661496350365, + "min": 1, + "p50": 3, + "p95": 13, + "p99": 21 + } + }, + { + "cudagraph_runtime_mode": "FULL", + "decode_tokens": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "kv_usage": { + "max": 0.02532720551503642, + "mean": 0.02532720551503642, + "min": 0.02532720551503642, + "p50": 0.02532720551503642, + "p95": 0.02532720551503642, + "p99": 0.02532720551503642 + }, + "phase": "pure_prefill", + "prefill_tokens": { + "max": 1, + "mean": 1.0, + "min": 1, + "p50": 1, + "p95": 1, + "p99": 1 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 1, + "mean": 1.0, + "min": 1, + "p50": 1, + "p95": 1, + "p99": 1 + }, + "steps": 1, + "submit_to_complete_ms": { + "max": 7.226884, + "mean": 7.226884, + "min": 7.226884, + "p50": 7.226884, + "p95": 7.226884, + "p99": 7.226884 + }, + "total_tokens": { + "max": 1, + "mean": 1.0, + "min": 1, + "p50": 1, + "p95": 1, + "p99": 1 + } + }, + { + "cudagraph_runtime_mode": "NONE", + "decode_tokens": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "kv_usage": { + "max": 0.022329903087221492, + "mean": 0.00819839067915799, + "min": 0.000549505445099463, + "p50": 0.007043660705365218, + "p95": 0.020731341792386893, + "p99": 0.022329903087221492 + }, + "phase": "pure_prefill", + "prefill_tokens": { + "max": 7089, + "mean": 2337.153846153846, + "min": 163, + "p50": 1828, + "p95": 6569, + "p99": 7089 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 1, + "mean": 1.0, + "min": 1, + "p50": 1, + "p95": 1, + "p99": 1 + }, + "steps": 52, + "submit_to_complete_ms": { + "max": 462.299574, + "mean": 154.1463595769231, + "min": 41.988631, + "p50": 121.390432, + "p95": 426.234755, + "p99": 462.299574 + }, + "total_tokens": { + "max": 7089, + "mean": 2337.153846153846, + "min": 163, + "p50": 1828, + "p95": 6569, + "p99": 7089 + } + }, + { + "cudagraph_runtime_mode": "PIECEWISE", + "decode_tokens": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "kv_usage": { + "max": 0.011539614347087612, + "mean": 0.002140381349093522, + "min": 0.00029973024278151517, + "p50": 0.0007992806474172998, + "p95": 0.011539614347087612, + "p99": 0.011539614347087612 + }, + "phase": "pure_prefill", + "prefill_tokens": { + "max": 118, + "mean": 43.07692307692308, + "min": 6, + "p50": 27, + "p95": 118, + "p99": 118 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 1, + "mean": 1.0, + "min": 1, + "p50": 1, + "p95": 1, + "p99": 1 + }, + "steps": 13, + "submit_to_complete_ms": { + "max": 20.969793, + "mean": 15.511612692307692, + "min": 10.185672, + "p50": 15.302969, + "p95": 20.969793, + "p99": 20.969793 + }, + "total_tokens": { + "max": 118, + "mean": 43.07692307692308, + "min": 6, + "p50": 27, + "p95": 118, + "p99": 118 + } + }, + { + "cudagraph_runtime_mode": "NONE", + "decode_tokens": { + "max": 32, + "mean": 7.1671826625387, + "min": 1, + "p50": 5, + "p95": 20, + "p99": 27 + }, + "kv_usage": { + "max": 0.35622939354580874, + "mean": 0.08236302498030411, + "min": 0.003446897791987258, + "p50": 0.06049555400139872, + "p95": 0.2183534818663203, + "p99": 0.3386452193026276 + }, + "phase": "true_mixed", + "prefill_tokens": { + "max": 8189, + "mean": 3407.3668730650156, + "min": 119, + "p50": 2823, + "p95": 7968, + "p99": 8188 + }, + "queue_waiting": { + "max": 1, + "mean": 0.009287925696594427, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 33, + "mean": 8.292569659442725, + "min": 2, + "p50": 6, + "p95": 21, + "p99": 28 + }, + "steps": 646, + "submit_to_complete_ms": { + "max": 1031.099195, + "mean": 262.67329438235294, + "min": 41.593929, + "p50": 208.770469, + "p95": 560.110521, + "p99": 836.430251 + }, + "total_tokens": { + "max": 8192, + "mean": 3414.534055727554, + "min": 131, + "p50": 2838, + "p95": 7973, + "p99": 8192 + } + }, + { + "cudagraph_runtime_mode": "PIECEWISE", + "decode_tokens": { + "max": 26, + "mean": 5.8203125, + "min": 1, + "p50": 4, + "p95": 15, + "p99": 25 + }, + "kv_usage": { + "max": 0.31431711459686285, + "mean": 0.05598086721950245, + "min": 0.0004995504046357846, + "p50": 0.03536816864821657, + "p95": 0.13977420321710465, + "p99": 0.29343590768308525 + }, + "phase": "true_mixed", + "prefill_tokens": { + "max": 122, + "mean": 38.9140625, + "min": 2, + "p50": 27, + "p95": 94, + "p99": 119 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 27, + "mean": 6.8203125, + "min": 2, + "p50": 5, + "p95": 16, + "p99": 26 + }, + "steps": 128, + "submit_to_complete_ms": { + "max": 549.993756, + "mean": 38.9609057265625, + "min": 13.155768, + "p50": 24.716054, + "p95": 86.104443, + "p99": 407.701862 + }, + "total_tokens": { + "max": 123, + "mean": 44.734375, + "min": 3, + "p50": 35, + "p95": 105, + "p99": 120 + } + } + ], + "max_num_seqs": 64, + "phase_counts": { + "pure_decode": 26304, + "pure_prefill": 66, + "true_mixed": 774 + }, + "prefix": { + "hit_rate": 0.1199154023961761, + "hits": 317232, + "queries": 2645465 + }, + "records": 27144, + "stream": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/runs/p1b/cells/tp1_mns64/opprof/opprof-v1-dp0-pid234511-1784005631788937266.jsonl", + "tensor_parallel_size": 1 + }, + { + "cell": "tp1_mns8", + "cudagraph_runtime_mode_counts": { + "FULL": 27496, + "NONE": 800, + "PIECEWISE": 13 + }, + "dropped_records_before_max": 0, + "groups": [ + { + "cudagraph_runtime_mode": "FULL", + "decode_tokens": { + "max": 8, + "mean": 3.827968721585743, + "min": 1, + "p50": 3, + "p95": 8, + "p99": 8 + }, + "kv_usage": { + "max": 0.13572814567888947, + "mean": 0.03436450743578334, + "min": 0.0002985223145429705, + "p50": 0.02666799343250903, + "p95": 0.09512911090103982, + "p99": 0.11562764316632668 + }, + "phase": "pure_decode", + "prefill_tokens": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "queue_waiting": { + "max": 13, + "mean": 0.7083106019276232, + "min": 0, + "p50": 0, + "p95": 5, + "p99": 9 + }, + "scheduled_requests": { + "max": 8, + "mean": 3.827968721585743, + "min": 1, + "p50": 3, + "p95": 8, + "p99": 8 + }, + "steps": 27495, + "submit_to_complete_ms": { + "max": 561.758474, + "mean": 19.407119342862337, + "min": 5.267886, + "p50": 15.55276, + "p95": 24.123839, + "p99": 151.668642 + }, + "total_tokens": { + "max": 8, + "mean": 3.827968721585743, + "min": 1, + "p50": 3, + "p95": 8, + "p99": 8 + } + }, + { + "cudagraph_runtime_mode": "FULL", + "decode_tokens": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "kv_usage": { + "max": 0.025225135578884506, + "mean": 0.025225135578884506, + "min": 0.025225135578884506, + "p50": 0.025225135578884506, + "p95": 0.025225135578884506, + "p99": 0.025225135578884506 + }, + "phase": "pure_prefill", + "prefill_tokens": { + "max": 1, + "mean": 1.0, + "min": 1, + "p50": 1, + "p95": 1, + "p99": 1 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 1, + "mean": 1.0, + "min": 1, + "p50": 1, + "p95": 1, + "p99": 1 + }, + "steps": 1, + "submit_to_complete_ms": { + "max": 7.195164, + "mean": 7.195164, + "min": 7.195164, + "p50": 7.195164, + "p95": 7.195164, + "p99": 7.195164 + }, + "total_tokens": { + "max": 1, + "mean": 1.0, + "min": 1, + "p50": 1, + "p95": 1, + "p99": 1 + } + }, + { + "cudagraph_runtime_mode": "NONE", + "decode_tokens": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "kv_usage": { + "max": 0.02318523309617393, + "mean": 0.007211759417999546, + "min": 0.0002985223145429705, + "p50": 0.005373401661774246, + "p95": 0.02064779342255829, + "p99": 0.02318523309617393 + }, + "phase": "pure_prefill", + "prefill_tokens": { + "max": 7089, + "mean": 2017.593220338983, + "min": 18, + "p50": 1501, + "p95": 6569, + "p99": 7089 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 1, + "mean": 1.0, + "min": 1, + "p50": 1, + "p95": 1, + "p99": 1 + }, + "steps": 59, + "submit_to_complete_ms": { + "max": 463.900987, + "mean": 140.730381559322, + "min": 41.733004, + "p50": 101.529539, + "p95": 425.237592, + "p99": 463.900987 + }, + "total_tokens": { + "max": 7089, + "mean": 2017.593220338983, + "min": 18, + "p50": 1501, + "p95": 6569, + "p99": 7089 + } + }, + { + "cudagraph_runtime_mode": "PIECEWISE", + "decode_tokens": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "kv_usage": { + "max": 0.011493109109905975, + "mean": 0.004925618189959698, + "min": 0.0005472909099955015, + "p50": 0.0027364545499776183, + "p95": 0.011493109109905975, + "p99": 0.011493109109905975 + }, + "phase": "pure_prefill", + "prefill_tokens": { + "max": 10, + "mean": 8.0, + "min": 6, + "p50": 8, + "p95": 10, + "p99": 10 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 1, + "mean": 1.0, + "min": 1, + "p50": 1, + "p95": 1, + "p99": 1 + }, + "steps": 3, + "submit_to_complete_ms": { + "max": 11.597111, + "mean": 10.493325333333333, + "min": 9.464417, + "p50": 10.418448, + "p95": 11.597111, + "p99": 11.597111 + }, + "total_tokens": { + "max": 10, + "mean": 8.0, + "min": 6, + "p50": 8, + "p95": 10, + "p99": 10 + } + }, + { + "cudagraph_runtime_mode": "NONE", + "decode_tokens": { + "max": 7, + "mean": 4.6396761133603235, + "min": 1, + "p50": 5, + "p95": 7, + "p99": 7 + }, + "kv_usage": { + "max": 0.13562863824070848, + "mean": 0.05433522417609084, + "min": 0.0004477834718145113, + "p50": 0.05353500174138015, + "p95": 0.10617443653913128, + "p99": 0.12702124483805166 + }, + "phase": "true_mixed", + "prefill_tokens": { + "max": 8189, + "mean": 2800.905533063428, + "min": 13, + "p50": 1881, + "p95": 7847, + "p99": 8186 + }, + "queue_waiting": { + "max": 12, + "mean": 1.2199730094466936, + "min": 0, + "p50": 0, + "p95": 7, + "p99": 10 + }, + "scheduled_requests": { + "max": 8, + "mean": 5.711201079622132, + "min": 2, + "p50": 6, + "p95": 8, + "p99": 8 + }, + "steps": 741, + "submit_to_complete_ms": { + "max": 1024.27287, + "mean": 224.85357696356274, + "min": 40.32457, + "p50": 144.763835, + "p95": 549.585886, + "p99": 785.810047 + }, + "total_tokens": { + "max": 8192, + "mean": 2805.545209176788, + "min": 17, + "p50": 1884, + "p95": 7854, + "p99": 8192 + } + }, + { + "cudagraph_runtime_mode": "PIECEWISE", + "decode_tokens": { + "max": 5, + "mean": 2.4, + "min": 1, + "p50": 2, + "p95": 5, + "p99": 5 + }, + "kv_usage": { + "max": 0.06567490919946262, + "mean": 0.033126026170456235, + "min": 0.008557639683566376, + "p50": 0.02617045624160408, + "p95": 0.06567490919946262, + "p99": 0.06567490919946262 + }, + "phase": "true_mixed", + "prefill_tokens": { + "max": 13, + "mean": 7.5, + "min": 2, + "p50": 7, + "p95": 13, + "p99": 13 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 6, + "mean": 3.4, + "min": 2, + "p50": 3, + "p95": 6, + "p99": 6 + }, + "steps": 10, + "submit_to_complete_ms": { + "max": 22.147232, + "mean": 17.1428369, + "min": 13.210445, + "p50": 15.229545, + "p95": 22.147232, + "p99": 22.147232 + }, + "total_tokens": { + "max": 15, + "mean": 9.9, + "min": 3, + "p50": 9, + "p95": 15, + "p99": 15 + } + } + ], + "max_num_seqs": 8, + "phase_counts": { + "pure_decode": 27495, + "pure_prefill": 63, + "true_mixed": 751 + }, + "prefix": { + "hit_rate": 0.12292016112369349, + "hits": 307568, + "queries": 2502177 + }, + "records": 28309, + "stream": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/runs/p1b/cells/tp1_mns8/opprof/opprof-v1-dp0-pid227734-1784005177936445360.jsonl", + "tensor_parallel_size": 1 + }, + { + "cell": "tp2_mns64", + "cudagraph_runtime_mode_counts": { + "FULL": 30746, + "NONE": 1555, + "PIECEWISE": 298 + }, + "dropped_records_before_max": 0, + "groups": [ + { + "cudagraph_runtime_mode": "FULL", + "decode_tokens": { + "max": 39, + "mean": 7.531678917582775, + "min": 1, + "p50": 6, + "p95": 21, + "p99": 30 + }, + "kv_usage": { + "max": 0.10309634347542496, + "mean": 0.016984430152386328, + "min": 7.673713693745476e-05, + "p50": 0.011983782885060523, + "p95": 0.05189988361534226, + "p99": 0.07517681515302665 + }, + "phase": "pure_decode", + "prefill_tokens": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 39, + "mean": 7.531678917582775, + "min": 1, + "p50": 6, + "p95": 21, + "p99": 30 + }, + "steps": 30746, + "submit_to_complete_ms": { + "max": 327.486803, + "mean": 19.866698748357507, + "min": 6.465612, + "p50": 13.818435, + "p95": 29.678643, + "p99": 212.284891 + }, + "total_tokens": { + "max": 39, + "mean": 7.531678917582775, + "min": 1, + "p50": 6, + "p95": 21, + "p99": 30 + } + }, + { + "cudagraph_runtime_mode": "NONE", + "decode_tokens": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "kv_usage": { + "max": 0.0061261814321708075, + "mean": 0.0026679719731172026, + "min": 0.00014068475105188938, + "p50": 0.0020719026973103905, + "p95": 0.005857601452889827, + "p99": 0.0061261814321708075 + }, + "phase": "pure_prefill", + "prefill_tokens": { + "max": 7612, + "mean": 2967.242424242424, + "min": 129, + "p50": 2331, + "p95": 7273, + "p99": 7612 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 1, + "mean": 1.0, + "min": 1, + "p50": 1, + "p95": 1, + "p99": 1 + }, + "steps": 33, + "submit_to_complete_ms": { + "max": 285.377954, + "mean": 125.60165333333333, + "min": 58.085137, + "p50": 87.591746, + "p95": 269.63345, + "p99": 285.377954 + }, + "total_tokens": { + "max": 7612, + "mean": 2967.242424242424, + "min": 129, + "p50": 2331, + "p95": 7273, + "p99": 7612 + } + }, + { + "cudagraph_runtime_mode": "PIECEWISE", + "decode_tokens": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "kv_usage": { + "max": 0.0007545818465513054, + "mean": 0.00022381664940082646, + "min": 7.673713693745476e-05, + "p50": 8.952665976036389e-05, + "p95": 0.0007545818465513054, + "p99": 0.0007545818465513054 + }, + "phase": "pure_prefill", + "prefill_tokens": { + "max": 93, + "mean": 46.25, + "min": 10, + "p50": 32, + "p95": 93, + "p99": 93 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 1, + "mean": 1.0, + "min": 1, + "p50": 1, + "p95": 1, + "p99": 1 + }, + "steps": 8, + "submit_to_complete_ms": { + "max": 17.211201, + "mean": 12.989633, + "min": 10.521338, + "p50": 13.034104, + "p95": 17.211201, + "p99": 17.211201 + }, + "total_tokens": { + "max": 93, + "mean": 46.25, + "min": 10, + "p50": 32, + "p95": 93, + "p99": 93 + } + }, + { + "cudagraph_runtime_mode": "NONE", + "decode_tokens": { + "max": 37, + "mean": 11.961892247043364, + "min": 1, + "p50": 11, + "p95": 28, + "p99": 32 + }, + "kv_usage": { + "max": 0.10278939492767525, + "mean": 0.03137185077117546, + "min": 0.0009336351660719222, + "p50": 0.02681962935962856, + "p95": 0.07204338206141525, + "p99": 0.08896392075611659 + }, + "phase": "true_mixed", + "prefill_tokens": { + "max": 8191, + "mean": 3282.0696452036796, + "min": 114, + "p50": 2770, + "p95": 7855, + "p99": 8176 + }, + "queue_waiting": { + "max": 1, + "mean": 0.0032851511169513796, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 39, + "mean": 13.06898817345598, + "min": 2, + "p50": 12, + "p95": 29, + "p99": 34 + }, + "steps": 1522, + "submit_to_complete_ms": { + "max": 560.812857, + "mean": 146.33081431603154, + "min": 56.009573, + "p50": 112.614117, + "p95": 307.930472, + "p99": 403.669994 + }, + "total_tokens": { + "max": 8192, + "mean": 3294.0315374507227, + "min": 130, + "p50": 2779, + "p95": 7865, + "p99": 8192 + } + }, + { + "cudagraph_runtime_mode": "PIECEWISE", + "decode_tokens": { + "max": 34, + "mean": 10.627586206896552, + "min": 1, + "p50": 9, + "p95": 25, + "p99": 33 + }, + "kv_usage": { + "max": 0.10290450063308143, + "mean": 0.025070992877117823, + "min": 0.0008313189834887602, + "p50": 0.021217818363196916, + "p95": 0.06711941577459746, + "p99": 0.08507590581795388 + }, + "phase": "true_mixed", + "prefill_tokens": { + "max": 122, + "mean": 38.53793103448276, + "min": 2, + "p50": 27, + "p95": 96, + "p99": 119 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 35, + "mean": 11.63448275862069, + "min": 2, + "p50": 10, + "p95": 26, + "p99": 34 + }, + "steps": 290, + "submit_to_complete_ms": { + "max": 310.419123, + "mean": 25.218022886206896, + "min": 10.997474, + "p50": 20.06169, + "p95": 35.895051, + "p99": 196.114372 + }, + "total_tokens": { + "max": 127, + "mean": 49.16551724137931, + "min": 5, + "p50": 41, + "p95": 104, + "p99": 124 + } + } + ], + "max_num_seqs": 64, + "phase_counts": { + "pure_decode": 30746, + "pure_prefill": 41, + "true_mixed": 1812 + }, + "prefix": { + "hit_rate": 0.1232028946400688, + "hits": 717296, + "queries": 5822071 + }, + "records": 32599, + "stream": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/runs/p1b/cells/tp2_mns64/opprof/opprof-v1-dp0-pid247791-1784006495467408663.jsonl", + "tensor_parallel_size": 2 + }, + { + "cell": "tp2_mns8", + "cudagraph_runtime_mode_counts": { + "FULL": 27697, + "NONE": 1201, + "PIECEWISE": 13 + }, + "dropped_records_before_max": 0, + "groups": [ + { + "cudagraph_runtime_mode": "FULL", + "decode_tokens": { + "max": 8, + "mean": 5.417373722785861, + "min": 1, + "p50": 6, + "p95": 8, + "p99": 8 + }, + "kv_usage": { + "max": 0.03395735232188657, + "mean": 0.012474261139679475, + "min": 7.656674706180144e-05, + "p50": 0.011918890292612616, + "p95": 0.026530377856896714, + "p99": 0.03005244822173725 + }, + "phase": "pure_decode", + "prefill_tokens": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "queue_waiting": { + "max": 24, + "mean": 2.201646387695418, + "min": 0, + "p50": 0, + "p95": 13, + "p99": 20 + }, + "scheduled_requests": { + "max": 8, + "mean": 5.417373722785861, + "min": 1, + "p50": 6, + "p95": 8, + "p99": 8 + }, + "steps": 27697, + "submit_to_complete_ms": { + "max": 314.563834, + "mean": 16.962280882044986, + "min": 6.575488, + "p50": 13.532942, + "p95": 15.503119, + "p99": 174.775316 + }, + "total_tokens": { + "max": 8, + "mean": 5.417373722785861, + "min": 1, + "p50": 6, + "p95": 8, + "p99": 8 + } + }, + { + "cudagraph_runtime_mode": "NONE", + "decode_tokens": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "kv_usage": { + "max": 0.005844595025713661, + "mean": 0.0019588326123298008, + "min": 7.656674706180144e-05, + "p50": 0.001365440322601219, + "p95": 0.005704222656100488, + "p99": 0.005844595025713661 + }, + "phase": "pure_prefill", + "prefill_tokens": { + "max": 7273, + "mean": 2182.5666666666666, + "min": 24, + "p50": 1060, + "p95": 7073, + "p99": 7273 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 1, + "mean": 1.0, + "min": 1, + "p50": 1, + "p95": 1, + "p99": 1 + }, + "steps": 30, + "submit_to_complete_ms": { + "max": 269.442406, + "mean": 103.43858230000001, + "min": 55.279405, + "p50": 66.596689, + "p95": 260.579, + "p99": 269.442406 + }, + "total_tokens": { + "max": 7273, + "mean": 2182.5666666666666, + "min": 24, + "p50": 1060, + "p95": 7073, + "p99": 7273 + } + }, + { + "cudagraph_runtime_mode": "PIECEWISE", + "decode_tokens": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "kv_usage": { + "max": 0.00014037236961317312, + "mean": 0.00014037236961317312, + "min": 0.00014037236961317312, + "p50": 0.00014037236961317312, + "p95": 0.00014037236961317312, + "p99": 0.00014037236961317312 + }, + "phase": "pure_prefill", + "prefill_tokens": { + "max": 10, + "mean": 10.0, + "min": 10, + "p50": 10, + "p95": 10, + "p99": 10 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 1, + "mean": 1.0, + "min": 1, + "p50": 1, + "p95": 1, + "p99": 1 + }, + "steps": 1, + "submit_to_complete_ms": { + "max": 10.336689, + "mean": 10.336689, + "min": 10.336689, + "p50": 10.336689, + "p95": 10.336689, + "p99": 10.336689 + }, + "total_tokens": { + "max": 10, + "mean": 10.0, + "min": 10, + "p50": 10, + "p95": 10, + "p99": 10 + } + }, + { + "cudagraph_runtime_mode": "NONE", + "decode_tokens": { + "max": 7, + "mean": 5.772843723313407, + "min": 1, + "p50": 7, + "p95": 7, + "p99": 7 + }, + "kv_usage": { + "max": 0.03394459119737636, + "mean": 0.0159228647453862, + "min": 0.0008167119686586233, + "p50": 0.016079016882967756, + "p95": 0.02733432870104513, + "p99": 0.0316731110345444 + }, + "phase": "true_mixed", + "prefill_tokens": { + "max": 8191, + "mean": 2652.5926558497013, + "min": 13, + "p50": 1837, + "p95": 7634, + "p99": 8066 + }, + "queue_waiting": { + "max": 23, + "mean": 2.9812126387702818, + "min": 0, + "p50": 1, + "p95": 13, + "p99": 21 + }, + "scheduled_requests": { + "max": 8, + "mean": 6.7933390264731, + "min": 2, + "p50": 8, + "p95": 8, + "p99": 8 + }, + "steps": 1171, + "submit_to_complete_ms": { + "max": 506.291757, + "mean": 121.45648444577284, + "min": 53.060612, + "p50": 81.564408, + "p95": 292.93805, + "p99": 310.013083 + }, + "total_tokens": { + "max": 8192, + "mean": 2658.3654995730144, + "min": 17, + "p50": 1840, + "p95": 7641, + "p99": 8073 + } + }, + { + "cudagraph_runtime_mode": "PIECEWISE", + "decode_tokens": { + "max": 7, + "mean": 3.3333333333333335, + "min": 1, + "p50": 3, + "p95": 7, + "p99": 7 + }, + "kv_usage": { + "max": 0.01494327680155172, + "mean": 0.009233737010238682, + "min": 0.0034837869913096897, + "p50": 0.009315620892513032, + "p95": 0.01494327680155172, + "p99": 0.01494327680155172 + }, + "phase": "true_mixed", + "prefill_tokens": { + "max": 13, + "mean": 6.916666666666667, + "min": 2, + "p50": 6, + "p95": 13, + "p99": 13 + }, + "queue_waiting": { + "max": 11, + "mean": 0.9166666666666666, + "min": 0, + "p50": 0, + "p95": 11, + "p99": 11 + }, + "scheduled_requests": { + "max": 8, + "mean": 4.333333333333333, + "min": 2, + "p50": 4, + "p95": 8, + "p99": 8 + }, + "steps": 12, + "submit_to_complete_ms": { + "max": 15.924106, + "mean": 13.055620500000002, + "min": 10.64629, + "p50": 12.616157, + "p95": 15.924106, + "p99": 15.924106 + }, + "total_tokens": { + "max": 16, + "mean": 10.25, + "min": 5, + "p50": 9, + "p95": 16, + "p99": 16 + } + } + ], + "max_num_seqs": 8, + "phase_counts": { + "pure_decode": 27697, + "pure_prefill": 31, + "true_mixed": 1183 + }, + "prefix": { + "hit_rate": 0.12439998012356517, + "hits": 450624, + "queries": 3622380 + }, + "records": 28911, + "stream": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/runs/p1b/cells/tp2_mns8/opprof/opprof-v1-dp0-pid241594-1784006096655191299.jsonl", + "tensor_parallel_size": 2 + }, + { + "cell": "tp4_mns16", + "cudagraph_runtime_mode_counts": { + "FULL": 17418, + "NONE": 1752, + "PIECEWISE": 53 + }, + "dropped_records_before_max": 0, + "groups": [ + { + "cudagraph_runtime_mode": "FULL", + "decode_tokens": { + "max": 16, + "mean": 13.292530286501695, + "min": 1, + "p50": 16, + "p95": 16, + "p99": 16 + }, + "kv_usage": { + "max": 0.02384243790712981, + "mean": 0.011910732119220589, + "min": 0.00021424527002555926, + "p50": 0.012732290332947316, + "p95": 0.01879237082795593, + "p99": 0.02112866448680606 + }, + "phase": "pure_decode", + "prefill_tokens": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "queue_waiting": { + "max": 48, + "mean": 11.14313601653557, + "min": 0, + "p50": 5, + "p95": 48, + "p99": 48 + }, + "scheduled_requests": { + "max": 16, + "mean": 13.292530286501695, + "min": 1, + "p50": 16, + "p95": 16, + "p99": 16 + }, + "steps": 17417, + "submit_to_complete_ms": { + "max": 185.583096, + "mean": 18.308918529310446, + "min": 6.71116, + "p50": 12.899641, + "p95": 64.79132, + "p99": 141.517546 + }, + "total_tokens": { + "max": 16, + "mean": 13.292530286501695, + "min": 1, + "p50": 16, + "p95": 16, + "p99": 16 + } + }, + { + "cudagraph_runtime_mode": "NONE", + "decode_tokens": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "kv_usage": { + "max": 0.0014436050337436335, + "mean": 0.0005439024265827402, + "min": 0.0003213679050383389, + "p50": 0.0003570754500425988, + "p95": 0.0014436050337436335, + "p99": 0.0014436050337436335 + }, + "phase": "pure_prefill", + "prefill_tokens": { + "max": 4459, + "mean": 1152.125, + "min": 201, + "p50": 699, + "p95": 4459, + "p99": 4459 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 1, + "mean": 1.0, + "min": 1, + "p50": 1, + "p95": 1, + "p99": 1 + }, + "steps": 8, + "submit_to_complete_ms": { + "max": 169.859824, + "mean": 79.969349875, + "min": 59.109261, + "p50": 63.446202, + "p95": 169.859824, + "p99": 169.859824 + }, + "total_tokens": { + "max": 4459, + "mean": 1152.125, + "min": 201, + "p50": 699, + "p95": 4459, + "p99": 4459 + } + }, + { + "cudagraph_runtime_mode": "PIECEWISE", + "decode_tokens": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "kv_usage": { + "max": 0.0014334028780281782, + "mean": 0.0006869451515105393, + "min": 0.0002805592821762959, + "p50": 0.00034687329432714353, + "p95": 0.0014334028780281782, + "p99": 0.0014334028780281782 + }, + "phase": "pure_prefill", + "prefill_tokens": { + "max": 23, + "mean": 13.0, + "min": 3, + "p50": 13, + "p95": 23, + "p99": 23 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 1, + "mean": 1.0, + "min": 1, + "p50": 1, + "p95": 1, + "p99": 1 + }, + "steps": 3, + "submit_to_complete_ms": { + "max": 13.064208, + "mean": 12.249279000000001, + "min": 10.993105, + "p50": 12.690524, + "p95": 13.064208, + "p99": 13.064208 + }, + "total_tokens": { + "max": 23, + "mean": 13.0, + "min": 3, + "p50": 13, + "p95": 23, + "p99": 23 + } + }, + { + "cudagraph_runtime_mode": "FULL", + "decode_tokens": { + "max": 15, + "mean": 15.0, + "min": 15, + "p50": 15, + "p95": 15, + "p99": 15 + }, + "kv_usage": { + "max": 0.014119783510255668, + "mean": 0.014119783510255668, + "min": 0.014119783510255668, + "p50": 0.014119783510255668, + "p95": 0.014119783510255668, + "p99": 0.014119783510255668 + }, + "phase": "true_mixed", + "prefill_tokens": { + "max": 1, + "mean": 1.0, + "min": 1, + "p50": 1, + "p95": 1, + "p99": 1 + }, + "queue_waiting": { + "max": 3, + "mean": 3.0, + "min": 3, + "p50": 3, + "p95": 3, + "p99": 3 + }, + "scheduled_requests": { + "max": 16, + "mean": 16.0, + "min": 16, + "p50": 16, + "p95": 16, + "p99": 16 + }, + "steps": 1, + "submit_to_complete_ms": { + "max": 31.93902, + "mean": 31.93902, + "min": 31.93902, + "p50": 31.93902, + "p95": 31.93902, + "p99": 31.93902 + }, + "total_tokens": { + "max": 16, + "mean": 16.0, + "min": 16, + "p50": 16, + "p95": 16, + "p99": 16 + } + }, + { + "cudagraph_runtime_mode": "NONE", + "decode_tokens": { + "max": 15, + "mean": 13.41112385321101, + "min": 1, + "p50": 15, + "p95": 15, + "p99": 15 + }, + "kv_usage": { + "max": 0.02384243790712981, + "mean": 0.013269559018371043, + "min": 0.0002397506593143639, + "p50": 0.013645383269484834, + "p95": 0.019067829032274553, + "p99": 0.021705086284732 + }, + "phase": "true_mixed", + "prefill_tokens": { + "max": 8191, + "mean": 2915.523509174312, + "min": 18, + "p50": 2078, + "p95": 7887, + "p99": 8180 + }, + "queue_waiting": { + "max": 48, + "mean": 13.461582568807339, + "min": 0, + "p50": 10, + "p95": 47, + "p99": 47 + }, + "scheduled_requests": { + "max": 16, + "mean": 14.555045871559633, + "min": 2, + "p50": 16, + "p95": 16, + "p99": 16 + }, + "steps": 1744, + "submit_to_complete_ms": { + "max": 379.907708, + "mean": 108.41418502752293, + "min": 56.308147, + "p50": 83.525673, + "p95": 217.228507, + "p99": 297.197569 + }, + "total_tokens": { + "max": 8192, + "mean": 2928.934633027523, + "min": 33, + "p50": 2093, + "p95": 7900, + "p99": 8192 + } + }, + { + "cudagraph_runtime_mode": "PIECEWISE", + "decode_tokens": { + "max": 15, + "mean": 9.54, + "min": 1, + "p50": 9, + "p95": 15, + "p99": 15 + }, + "kv_usage": { + "max": 0.022806919102006273, + "mean": 0.009802027168340671, + "min": 0.0015405255130409579, + "p50": 0.008386171998143177, + "p95": 0.018516912623637416, + "p99": 0.022806919102006273 + }, + "phase": "true_mixed", + "prefill_tokens": { + "max": 25, + "mean": 13.94, + "min": 2, + "p50": 13, + "p95": 23, + "p99": 25 + }, + "queue_waiting": { + "max": 47, + "mean": 5.82, + "min": 0, + "p50": 0, + "p95": 47, + "p99": 47 + }, + "scheduled_requests": { + "max": 16, + "mean": 10.54, + "min": 2, + "p50": 10, + "p95": 16, + "p99": 16 + }, + "steps": 50, + "submit_to_complete_ms": { + "max": 91.408766, + "mean": 24.5079404, + "min": 11.206266, + "p50": 14.216713, + "p95": 83.023614, + "p99": 91.408766 + }, + "total_tokens": { + "max": 32, + "mean": 23.48, + "min": 4, + "p50": 25, + "p95": 32, + "p99": 32 + } + } + ], + "max_num_seqs": 16, + "phase_counts": { + "pure_decode": 17417, + "pure_prefill": 11, + "true_mixed": 1795 + }, + "prefix": { + "hit_rate": 0.13543755577854244, + "hits": 798096, + "queries": 5892723 + }, + "records": 19223, + "stream": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/runs/p1b/cells/tp4_mns16/opprof/opprof-v1-dp0-pid256090-1784007028237607411.jsonl", + "tensor_parallel_size": 4 + }, + { + "cell": "tp4_mns64", + "cudagraph_runtime_mode_counts": { + "FULL": 21812, + "NONE": 2809, + "PIECEWISE": 354 + }, + "dropped_records_before_max": 0, + "groups": [ + { + "cudagraph_runtime_mode": "FULL", + "decode_tokens": { + "max": 64, + "mean": 20.639510361269025, + "min": 1, + "p50": 15, + "p95": 62, + "p99": 64 + }, + "kv_usage": { + "max": 0.07507819047813735, + "mean": 0.01804654466537005, + "min": 4.088391013712389e-05, + "p50": 0.013655225985813257, + "p95": 0.05230074204296897, + "p99": 0.062281526605204474 + }, + "phase": "pure_decode", + "prefill_tokens": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 64, + "mean": 20.639510361269025, + "min": 1, + "p50": 15, + "p95": 62, + "p99": 64 + }, + "steps": 21812, + "submit_to_complete_ms": { + "max": 198.715632, + "mean": 21.35240870649184, + "min": 6.440609, + "p50": 13.906458, + "p95": 68.920454, + "p99": 155.977421 + }, + "total_tokens": { + "max": 64, + "mean": 20.639510361269025, + "min": 1, + "p50": 15, + "p95": 62, + "p99": 64 + } + }, + { + "cudagraph_runtime_mode": "NONE", + "decode_tokens": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "kv_usage": { + "max": 0.0025552443835727967, + "mean": 0.0008350538645516159, + "min": 0.00032196079233015595, + "p50": 0.00038839714630312105, + "p95": 0.0025552443835727967, + "p99": 0.0025552443835727967 + }, + "phase": "pure_prefill", + "prefill_tokens": { + "max": 7950, + "mean": 2158.0, + "min": 201, + "p50": 959, + "p95": 7950, + "p99": 7950 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 1, + "mean": 1.0, + "min": 1, + "p50": 1, + "p95": 1, + "p99": 1 + }, + "steps": 10, + "submit_to_complete_ms": { + "max": 173.889993, + "mean": 88.2164954, + "min": 60.32347, + "p50": 64.173068, + "p95": 173.889993, + "p99": 173.889993 + }, + "total_tokens": { + "max": 7950, + "mean": 2158.0, + "min": 201, + "p50": 959, + "p95": 7950, + "p99": 7950 + } + }, + { + "cudagraph_runtime_mode": "PIECEWISE", + "decode_tokens": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "kv_usage": { + "max": 0.00034751323616588614, + "mean": 0.0003142950591794591, + "min": 0.00028107688219303206, + "p50": 0.00028107688219303206, + "p95": 0.00034751323616588614, + "p99": 0.00034751323616588614 + }, + "phase": "pure_prefill", + "prefill_tokens": { + "max": 23, + "mean": 13.0, + "min": 3, + "p50": 3, + "p95": 23, + "p99": 23 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 1, + "mean": 1.0, + "min": 1, + "p50": 1, + "p95": 1, + "p99": 1 + }, + "steps": 2, + "submit_to_complete_ms": { + "max": 14.426163, + "mean": 12.918343, + "min": 11.410523, + "p50": 11.410523, + "p95": 14.426163, + "p99": 14.426163 + }, + "total_tokens": { + "max": 23, + "mean": 13.0, + "min": 3, + "p50": 3, + "p95": 23, + "p99": 23 + } + }, + { + "cudagraph_runtime_mode": "NONE", + "decode_tokens": { + "max": 63, + "mean": 31.312254376563057, + "min": 1, + "p50": 27, + "p95": 62, + "p99": 63 + }, + "kv_usage": { + "max": 0.07563012326498908, + "mean": 0.02885461746554652, + "min": 0.00024019297205579715, + "p50": 0.025511559925591287, + "p95": 0.05791716919806211, + "p99": 0.06878206831701383 + }, + "phase": "true_mixed", + "prefill_tokens": { + "max": 8191, + "mean": 3841.2243658449447, + "min": 73, + "p50": 3432, + "p95": 8154, + "p99": 8181 + }, + "queue_waiting": { + "max": 11, + "mean": 0.1907824222936763, + "min": 0, + "p50": 0, + "p95": 1, + "p99": 5 + }, + "scheduled_requests": { + "max": 64, + "mean": 32.843515541264736, + "min": 2, + "p50": 28, + "p95": 63, + "p99": 64 + }, + "steps": 2799, + "submit_to_complete_ms": { + "max": 373.577501, + "mean": 134.91752637549124, + "min": 55.209363, + "p50": 114.740321, + "p95": 314.91308, + "p99": 343.860853 + }, + "total_tokens": { + "max": 8192, + "mean": 3872.5366202215077, + "min": 129, + "p50": 3456, + "p95": 8192, + "p99": 8192 + } + }, + { + "cudagraph_runtime_mode": "PIECEWISE", + "decode_tokens": { + "max": 63, + "mean": 25.65340909090909, + "min": 1, + "p50": 20, + "p95": 60, + "p99": 63 + }, + "kv_usage": { + "max": 0.07241562583045447, + "mean": 0.022061428307276035, + "min": 0.00047016496657736884, + "p50": 0.017590302336515484, + "p95": 0.05158016312680147, + "p99": 0.05824424047915944 + }, + "phase": "true_mixed", + "prefill_tokens": { + "max": 114, + "mean": 35.32670454545455, + "min": 2, + "p50": 25, + "p95": 87, + "p99": 106 + }, + "queue_waiting": { + "max": 0, + "mean": 0.0, + "min": 0, + "p50": 0, + "p95": 0, + "p99": 0 + }, + "scheduled_requests": { + "max": 64, + "mean": 26.678977272727273, + "min": 2, + "p50": 21, + "p95": 61, + "p99": 64 + }, + "steps": 352, + "submit_to_complete_ms": { + "max": 196.960993, + "mean": 27.942972144886365, + "min": 11.037722, + "p50": 17.402246, + "p95": 97.041082, + "p99": 187.504426 + }, + "total_tokens": { + "max": 128, + "mean": 60.98011363636363, + "min": 4, + "p50": 60, + "p95": 114, + "p99": 126 + } + } + ], + "max_num_seqs": 64, + "phase_counts": { + "pure_decode": 21812, + "pure_prefill": 12, + "true_mixed": 3151 + }, + "prefix": { + "hit_rate": 0.13386788146702433, + "hits": 1667008, + "queries": 12452636 + }, + "records": 24975, + "stream": "/home/admin/cpfs/wjh/fidelity-prefix-pilot-20260714/runs/p1b/cells/tp4_mns64/opprof/opprof-v1-dp0-pid262293-1784007376383569515.jsonl", + "tensor_parallel_size": 4 + } + ], + "contains_prompt_text": false, + "measurement_semantics": "complete_mono_ns - submit_mono_ns from the existing vLLM opprof record; this includes the instrumented submit-to-completion interval and is not claimed to be a single-kernel CUDA-event duration", + "schema_version": "qwen30_vllm020_opprof_summary.v1" +} diff --git a/runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-routing-mismatch.json b/runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-routing-mismatch.json new file mode 100644 index 0000000..e2a4fb0 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-routing-mismatch.json @@ -0,0 +1,1853 @@ +{ + "actual_prefill_vs_decode_pearson": { + "max": 0.9436904928897797, + "median": 0.6667880423793499, + "min": 0.48720283009127985 + }, + "frontier_contract": { + "allocation": "per-layer fixed Uniform(0.1, 1.0), normalized once and reused for every batch and phase", + "expert_count": 128, + "layer_count": 48, + "mode": "simulation", + "seed": 42 + }, + "frontier_prefill_vs_decode_pearson": 1.0, + "phase_summary": { + "decode": { + "actual": { + "expert_utilization": { + "max": 1.0, + "median": 0.96875, + "min": 0.921875 + }, + "load_cv": { + "max": 1.5122257286912568, + "median": 1.106820300518243, + "min": 0.7532423759874015 + }, + "load_gini": { + "max": 0.647222736853249, + "median": 0.5427848362430705, + "min": 0.3623759672389899 + }, + "max_load_ratio": { + "max": 8.630736064059132, + "median": 5.642131198028951, + "min": 3.392670157068063 + } + }, + "actual_vs_frontier_pearson": { + "max": 0.17269736915133127, + "median": -0.024141635934415796, + "min": -0.20201754939516248 + }, + "frontier_simulation": { + "expert_utilization": { + "max": 1.0, + "median": 1.0, + "min": 1.0 + }, + "load_cv": { + "max": 0.5319387026702922, + "median": 0.4852775824549782, + "min": 0.4143052194562559 + }, + "load_gini": { + "max": 0.3057696286957192, + "median": 0.2796991393497844, + "min": 0.2373612300777641 + }, + "max_load_ratio": { + "max": 2.0104712041884816, + "median": 1.8392362180474282, + "min": 1.6482907299045273 + } + }, + "token_count": 6494 + }, + "prefill": { + "actual": { + "expert_utilization": { + "max": 1.0, + "median": 0.984375, + "min": 0.9453125 + }, + "load_cv": { + "max": 1.636444303545179, + "median": 1.0764214888570005, + "min": 0.802893981590087 + }, + "load_gini": { + "max": 0.6967902002938451, + "median": 0.5262265677660404, + "min": 0.42613218222079463 + }, + "max_load_ratio": { + "max": 10.381092808695286, + "median": 6.121413826515567, + "min": 3.8360365673842525 + } + }, + "actual_vs_frontier_pearson": { + "max": 0.1797880356509229, + "median": -0.01729201163458405, + "min": -0.20008636491506096 + }, + "frontier_simulation": { + "expert_utilization": { + "max": 1.0, + "median": 1.0, + "min": 1.0 + }, + "load_cv": { + "max": 0.5319950183538887, + "median": 0.48529878894960354, + "min": 0.41421985083210333 + }, + "load_gini": { + "max": 0.3057985522338542, + "median": 0.2797113699735645, + "min": 0.23730966555377675 + }, + "max_load_ratio": { + "max": 2.0093524876774658, + "median": 1.838817036693769, + "min": 1.649408097063656 + } + }, + "token_count": 23737 + } + }, + "rows": [ + { + "actual": { + "expert_utilization": 1.0, + "load_cv": 1.0209765758908849, + "load_gini": 0.46969517196254795, + "max_load_ratio": 5.4827484517841345 + }, + "actual_vs_frontier_pearson": -0.09214641606187762, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5013611454809463, + "load_gini": 0.28879565986118716, + "max_load_ratio": 1.8509499936807516 + }, + "layer": 0, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 1.0, + "load_cv": 0.9656310061810012, + "load_gini": 0.4405106117295783, + "max_load_ratio": 6.154105405063825 + }, + "actual_vs_frontier_pearson": -0.14455565868271067, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4633214169333404, + "load_gini": 0.2672016031143363, + "max_load_ratio": 1.7579306567805535 + }, + "layer": 1, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9921875, + "load_cv": 1.4100541859296152, + "load_gini": 0.5967419337426803, + "max_load_ratio": 8.47284829590934 + }, + "actual_vs_frontier_pearson": 0.10746117235272135, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4915202242607809, + "load_gini": 0.28343845117853994, + "max_load_ratio": 1.8556683658423558 + }, + "layer": 2, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 1.0, + "load_cv": 1.1750115312430194, + "load_gini": 0.49178514160382525, + "max_load_ratio": 7.965286261953912 + }, + "actual_vs_frontier_pearson": 0.07854011611932744, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5153981688841472, + "load_gini": 0.29577933447781946, + "max_load_ratio": 1.899481821628681 + }, + "layer": 3, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 1.0, + "load_cv": 1.2753111721870192, + "load_gini": 0.5858468253675697, + "max_load_ratio": 7.488056620465939 + }, + "actual_vs_frontier_pearson": 0.047570378008613835, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5319950183538887, + "load_gini": 0.3057985522338542, + "max_load_ratio": 2.0093524876774658 + }, + "layer": 4, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 1.2689973799039043, + "load_gini": 0.5545393103330243, + "max_load_ratio": 9.569532796899356 + }, + "actual_vs_frontier_pearson": -0.06358262657671977, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.46397269621367815, + "load_gini": 0.2675475161404137, + "max_load_ratio": 1.7828706239204617 + }, + "layer": 5, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9765625, + "load_cv": 1.3617880106985483, + "load_gini": 0.5929317066973501, + "max_load_ratio": 10.381092808695286 + }, + "actual_vs_frontier_pearson": -0.01786861982721008, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5065069021962776, + "load_gini": 0.29121334835910173, + "max_load_ratio": 1.9075704596200025 + }, + "layer": 6, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9765625, + "load_cv": 1.4508622399317908, + "load_gini": 0.633230738799132, + "max_load_ratio": 8.563845473311707 + }, + "actual_vs_frontier_pearson": -0.018533706086765106, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4174667228740326, + "load_gini": 0.23938283981758435, + "max_load_ratio": 1.649408097063656 + }, + "layer": 7, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 1.0194348783142033, + "load_gini": 0.5101870727924758, + "max_load_ratio": 4.746008341407928 + }, + "actual_vs_frontier_pearson": -0.07019410060413582, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.45521406973427264, + "load_gini": 0.2624579704153853, + "max_load_ratio": 1.7660192947718751 + }, + "layer": 8, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 1.0499203578258598, + "load_gini": 0.5348460473101067, + "max_load_ratio": 4.87475249610313 + }, + "actual_vs_frontier_pearson": 0.055984358629187964, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4981686541927147, + "load_gini": 0.28707712050280154, + "max_load_ratio": 1.935880692589628 + }, + "layer": 9, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9921875, + "load_cv": 0.9182859260805778, + "load_gini": 0.4667757285566836, + "max_load_ratio": 6.065130387159287 + }, + "actual_vs_frontier_pearson": 0.04566572558971987, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4410838523022549, + "load_gini": 0.2539255039600623, + "max_load_ratio": 1.7262501579812108 + }, + "layer": 10, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9921875, + "load_cv": 0.8938335625889317, + "load_gini": 0.46192282552028474, + "max_load_ratio": 4.6934321944643385 + }, + "actual_vs_frontier_pearson": 0.05102058572481217, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4344763177468921, + "load_gini": 0.250220186312508, + "max_load_ratio": 1.7592787631124405 + }, + "layer": 11, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9921875, + "load_cv": 0.9842127914363633, + "load_gini": 0.48561951344420096, + "max_load_ratio": 6.4446223195854575 + }, + "actual_vs_frontier_pearson": -0.117474412760527, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4881943608268652, + "load_gini": 0.28113027986371497, + "max_load_ratio": 1.86038673800396 + }, + "layer": 12, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9921875, + "load_cv": 1.073206640427139, + "load_gini": 0.5187799341481654, + "max_load_ratio": 6.354973248514977 + }, + "actual_vs_frontier_pearson": -0.020599525349797518, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4551370538191022, + "load_gini": 0.2622560506803724, + "max_load_ratio": 1.7660192947718751 + }, + "layer": 13, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 1.0501051337772571, + "load_gini": 0.5237924640329443, + "max_load_ratio": 4.951594557020685 + }, + "actual_vs_frontier_pearson": -0.1404264118915223, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4843196647309149, + "load_gini": 0.2793572699003666, + "max_load_ratio": 1.8071365378944264 + }, + "layer": 14, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9921875, + "load_cv": 1.1589582379096603, + "load_gini": 0.5428402257024898, + "max_load_ratio": 6.021990984538905 + }, + "actual_vs_frontier_pearson": 0.010817413166755824, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4523916462775862, + "load_gini": 0.2604694642067238, + "max_load_ratio": 1.8084846442263134 + }, + "layer": 15, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9609375, + "load_cv": 1.1496287137851553, + "load_gini": 0.5617946797194253, + "max_load_ratio": 6.50933142351603 + }, + "actual_vs_frontier_pearson": 0.1797880356509229, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.48641030098466176, + "load_gini": 0.2805311032091249, + "max_load_ratio": 1.8334246113662216 + }, + "layer": 16, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9765625, + "load_cv": 1.2394125145202786, + "load_gini": 0.5729841103814719, + "max_load_ratio": 7.101150103214391 + }, + "actual_vs_frontier_pearson": -0.20008636491506096, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.443419246367453, + "load_gini": 0.25561211794877203, + "max_load_ratio": 1.7754560390950835 + }, + "layer": 17, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9609375, + "load_cv": 1.2168439400279965, + "load_gini": 0.575970613520032, + "max_load_ratio": 8.145258457260816 + }, + "actual_vs_frontier_pearson": -0.06530349438725558, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4516257974939683, + "load_gini": 0.25957785774845177, + "max_load_ratio": 1.8010700594009352 + }, + "layer": 18, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9765625, + "load_cv": 1.2427942152510565, + "load_gini": 0.5980218283955427, + "max_load_ratio": 6.7688418924042635 + }, + "actual_vs_frontier_pearson": -0.04180797903095555, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4681222179773095, + "load_gini": 0.26999935161878086, + "max_load_ratio": 1.8111808568900871 + }, + "layer": 19, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.96875, + "load_cv": 0.9706255724368628, + "load_gini": 0.5072024621371698, + "max_load_ratio": 4.138012385726924 + }, + "actual_vs_frontier_pearson": 0.007542212699855623, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4876159778421919, + "load_gini": 0.2809273727461348, + "max_load_ratio": 1.7734338795972533 + }, + "layer": 20, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9765625, + "load_cv": 0.9454143633213303, + "load_gini": 0.5064155182310317, + "max_load_ratio": 3.8980494586510512 + }, + "actual_vs_frontier_pearson": -0.005313631721923769, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4407612899555989, + "load_gini": 0.25342531240784427, + "max_load_ratio": 1.7006361376753591 + }, + "layer": 21, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 0.8037705358927761, + "load_gini": 0.42613218222079463, + "max_load_ratio": 5.447697687155074 + }, + "actual_vs_frontier_pearson": 0.006893096788933473, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5096984088007344, + "load_gini": 0.292796698587648, + "max_load_ratio": 1.9850865737035008 + }, + "layer": 22, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9765625, + "load_cv": 0.802893981590087, + "load_gini": 0.43984914769136796, + "max_load_ratio": 3.953321818258415 + }, + "actual_vs_frontier_pearson": 0.055518113087740455, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5020829253028674, + "load_gini": 0.28819319193137294, + "max_load_ratio": 1.9695833508868013 + }, + "layer": 23, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9765625, + "load_cv": 0.8487498379405487, + "load_gini": 0.44496173234296665, + "max_load_ratio": 4.682647343809243 + }, + "actual_vs_frontier_pearson": -0.11069406175185324, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4937827979946325, + "load_gini": 0.28478219657075443, + "max_load_ratio": 1.8812823861482075 + }, + "layer": 24, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9921875, + "load_cv": 0.8842495554569121, + "load_gini": 0.4609973189272023, + "max_load_ratio": 5.068205754728904 + }, + "actual_vs_frontier_pearson": 0.04041373985743766, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5094957270929035, + "load_gini": 0.29290695630766317, + "max_load_ratio": 1.9264439482664195 + }, + "layer": 25, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 0.910991140491345, + "load_gini": 0.47597056415090355, + "max_load_ratio": 4.189240426338627 + }, + "actual_vs_frontier_pearson": -0.001853343235349625, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5141430671669248, + "load_gini": 0.2956995210536293, + "max_load_ratio": 1.870497535493112 + }, + "layer": 26, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.96875, + "load_cv": 1.0416423824930179, + "load_gini": 0.5048694416944011, + "max_load_ratio": 5.7442810801701985 + }, + "actual_vs_frontier_pearson": 0.021178350576558223, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4563109042451374, + "load_gini": 0.2625302961884399, + "max_load_ratio": 1.8846526519779248 + }, + "layer": 27, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.953125, + "load_cv": 1.0513699281689028, + "load_gini": 0.5286606714991364, + "max_load_ratio": 5.484096558116021 + }, + "actual_vs_frontier_pearson": -0.195005081355267, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4931389294946141, + "load_gini": 0.28378707950667725, + "max_load_ratio": 1.7983738467371615 + }, + "layer": 28, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.953125, + "load_cv": 1.079636337286862, + "load_gini": 0.5304158262943928, + "max_load_ratio": 6.738509499936807 + }, + "actual_vs_frontier_pearson": -0.016715403441958022, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4500354635096657, + "load_gini": 0.2587639253854741, + "max_load_ratio": 1.7316425833087585 + }, + "layer": 29, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9453125, + "load_cv": 1.1836591453389935, + "load_gini": 0.5595659927012682, + "max_load_ratio": 7.912710115010322 + }, + "actual_vs_frontier_pearson": 0.009080640196552386, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4501844468319078, + "load_gini": 0.25952453908981754, + "max_load_ratio": 1.845557568353204 + }, + "layer": 30, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9609375, + "load_cv": 1.1837797460931923, + "load_gini": 0.5779810067089355, + "max_load_ratio": 6.173652946876185 + }, + "actual_vs_frontier_pearson": -0.11789359921128652, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5029280507742879, + "load_gini": 0.2893184789305725, + "max_load_ratio": 1.92846610776425 + }, + "layer": 31, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.96875, + "load_cv": 0.9485322853469729, + "load_gini": 0.4766967017472723, + "max_load_ratio": 5.981547794582298 + }, + "actual_vs_frontier_pearson": -0.11341887598082027, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4862779131682921, + "load_gini": 0.2800654700467624, + "max_load_ratio": 1.8307283987024476 + }, + "layer": 32, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 0.9147261414047101, + "load_gini": 0.48626378056936437, + "max_load_ratio": 4.4292033534145006 + }, + "actual_vs_frontier_pearson": -0.0734554681462204, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4826868638135814, + "load_gini": 0.278262015787589, + "max_load_ratio": 1.8347727176981083 + }, + "layer": 33, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9921875, + "load_cv": 0.822096011945377, + "load_gini": 0.4335047210051819, + "max_load_ratio": 5.387032902220162 + }, + "actual_vs_frontier_pearson": 0.041109767501958036, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5205338188216663, + "load_gini": 0.2990128478219658, + "max_load_ratio": 1.84286135568943 + }, + "layer": 34, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9765625, + "load_cv": 0.8350377321986099, + "load_gini": 0.4458607441704934, + "max_load_ratio": 3.8360365673842525 + }, + "actual_vs_frontier_pearson": -0.002498639457316309, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4931172415832444, + "load_gini": 0.28332819345852456, + "max_load_ratio": 1.9008299279605678 + }, + "layer": 35, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 0.936559272961729, + "load_gini": 0.4615142959830645, + "max_load_ratio": 5.575767788684333 + }, + "actual_vs_frontier_pearson": -0.04463576992397696, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.47788893428675927, + "load_gini": 0.27538099801996885, + "max_load_ratio": 1.7997219530690483 + }, + "layer": 36, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 1.0424990294361898, + "load_gini": 0.5039767655716814, + "max_load_ratio": 6.859839069806631 + }, + "actual_vs_frontier_pearson": -0.05047148070882558, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.41421985083210333, + "load_gini": 0.23730966555377675, + "max_load_ratio": 1.6965918186796984 + }, + "layer": 37, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9921875, + "load_cv": 1.081067450691553, + "load_gini": 0.515087617037747, + "max_load_ratio": 6.495850360197161 + }, + "actual_vs_frontier_pearson": -0.03189531872213542, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.48198382691720465, + "load_gini": 0.27773105081202343, + "max_load_ratio": 1.8037662720647092 + }, + "layer": 38, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 1.140261877881891, + "load_gini": 0.5301265232021739, + "max_load_ratio": 6.088722247967309 + }, + "actual_vs_frontier_pearson": -0.1047856089108959, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.49823287732736005, + "load_gini": 0.28601938692758133, + "max_load_ratio": 1.9102666722837764 + }, + "layer": 39, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.96875, + "load_cv": 1.16210598515629, + "load_gini": 0.5750575314909214, + "max_load_ratio": 5.264355226018452 + }, + "actual_vs_frontier_pearson": 0.07101910025349224, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5231673221525147, + "load_gini": 0.30064416838690655, + "max_load_ratio": 1.9163331507772676 + }, + "layer": 40, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9765625, + "load_cv": 1.335821482050506, + "load_gini": 0.6172422174506045, + "max_load_ratio": 8.129755234444117 + }, + "actual_vs_frontier_pearson": 0.04271119117719951, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4592680576699305, + "load_gini": 0.26492856616253113, + "max_load_ratio": 1.8145511227198046 + }, + "layer": 41, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 1.5340944192835082, + "load_gini": 0.6802103190693853, + "max_load_ratio": 9.211610565783376 + }, + "actual_vs_frontier_pearson": 0.012121305370270835, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.493325400417425, + "load_gini": 0.28234122230062764, + "max_load_ratio": 1.9055483001221722 + }, + "layer": 42, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 1.5076855954950434, + "load_gini": 0.6597892728124868, + "max_load_ratio": 7.8324977882630495 + }, + "actual_vs_frontier_pearson": 0.1787454639272607, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4651216460072469, + "load_gini": 0.267678755739984, + "max_load_ratio": 1.870497535493112 + }, + "layer": 43, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9921875, + "load_cv": 1.4676145461991041, + "load_gini": 0.666736825683532, + "max_load_ratio": 6.086700088469478 + }, + "actual_vs_frontier_pearson": -0.1318454052741815, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.47511235321400247, + "load_gini": 0.2737368414816532, + "max_load_ratio": 1.84286135568943 + }, + "layer": 44, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.9765625, + "load_cv": 1.5817687300685335, + "load_gini": 0.6967902002938451, + "max_load_ratio": 6.617853983232927 + }, + "actual_vs_frontier_pearson": -0.042931525341602716, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.47921880192242294, + "load_gini": 0.2763617157549396, + "max_load_ratio": 1.7754560390950835 + }, + "layer": 45, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 1.5934457162349587, + "load_gini": 0.6958883908823776, + "max_load_ratio": 7.270337447866201 + }, + "actual_vs_frontier_pearson": 0.02211603536084719, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5297987184237168, + "load_gini": 0.30463451046888834, + "max_load_ratio": 1.8799342798163206 + }, + "layer": 46, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 1.636444303545179, + "load_gini": 0.6918075387317268, + "max_load_ratio": 8.385895437502633 + }, + "actual_vs_frontier_pearson": -0.01870726023327083, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4952385440074709, + "load_gini": 0.28391033776382857, + "max_load_ratio": 1.9170072039432111 + }, + "layer": 47, + "phase": "prefill", + "total_routed_tokens": 189896 + }, + { + "actual": { + "expert_utilization": 1.0, + "load_cv": 0.7564860612847418, + "load_gini": 0.3623759672389899, + "max_load_ratio": 4.043116723129042 + }, + "actual_vs_frontier_pearson": -0.11076278775511662, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5013193007157182, + "load_gini": 0.288770162842624, + "max_load_ratio": 1.8503233754234678 + }, + "layer": 0, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 1.0, + "load_cv": 0.7532423759874015, + "load_gini": 0.36718870303356943, + "max_load_ratio": 4.479211579919926 + }, + "actual_vs_frontier_pearson": -0.20201754939516248, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4633429809491391, + "load_gini": 0.2672133050700647, + "max_load_ratio": 1.7566984909146905 + }, + "layer": 1, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9765625, + "load_cv": 0.9982932903554329, + "load_gini": 0.48608902207807203, + "max_load_ratio": 5.430243301509085 + }, + "actual_vs_frontier_pearson": 0.12507453392144424, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4914341503043837, + "load_gini": 0.2833871891361257, + "max_load_ratio": 1.8552510009239298 + }, + "layer": 2, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9921875, + "load_cv": 0.887995113903465, + "load_gini": 0.4407307466507546, + "max_load_ratio": 4.843855866954112 + }, + "actual_vs_frontier_pearson": 0.09916207755901395, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5153762623410879, + "load_gini": 0.2957582840891593, + "max_load_ratio": 1.8995996304280875 + }, + "layer": 3, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 1.0374295217881528, + "load_gini": 0.5126796730635972, + "max_load_ratio": 5.198644902987373 + }, + "actual_vs_frontier_pearson": 0.1087630450097359, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5319387026702922, + "load_gini": 0.3057696286957192, + "max_load_ratio": 2.0104712041884816 + }, + "layer": 4, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 0.9864025871023987, + "load_gini": 0.5000595501616878, + "max_load_ratio": 6.716353557129658 + }, + "actual_vs_frontier_pearson": -0.060788024591760324, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.46389740539367647, + "load_gini": 0.26750173236833996, + "max_load_ratio": 1.7838004311672313 + }, + "layer": 5, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9375, + "load_cv": 1.2953358972433757, + "load_gini": 0.629114675565907, + "max_load_ratio": 5.74314752078842 + }, + "actual_vs_frontier_pearson": -0.07165433962410654, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5065225414304516, + "load_gini": 0.2912228475323375, + "max_load_ratio": 1.9069910686787803 + }, + "layer": 6, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9375, + "load_cv": 1.2699902455381382, + "load_gini": 0.6262021312942716, + "max_load_ratio": 6.66954111487527 + }, + "actual_vs_frontier_pearson": 0.01205569571775735, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.41747315017256753, + "load_gini": 0.2393889431590699, + "max_load_ratio": 1.6482907299045273 + }, + "layer": 7, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9765625, + "load_cv": 1.0541527297772746, + "load_gini": 0.5379860857137357, + "max_load_ratio": 5.171542962734832 + }, + "actual_vs_frontier_pearson": -0.04958458083380716, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4553076424255581, + "load_gini": 0.2625118498806591, + "max_load_ratio": 1.7665537419156143 + }, + "layer": 8, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9765625, + "load_cv": 1.192957622384299, + "load_gini": 0.5506200435016939, + "max_load_ratio": 8.320295657530028 + }, + "actual_vs_frontier_pearson": -0.05509740986705497, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4981311718832551, + "load_gini": 0.28705553924776717, + "max_load_ratio": 1.9365568216815523 + }, + "layer": 9, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 0.8076732412246237, + "load_gini": 0.4410110534724361, + "max_load_ratio": 3.392670157068063 + }, + "actual_vs_frontier_pearson": -0.09000721184282558, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4410207617483597, + "load_gini": 0.2538882044964583, + "max_load_ratio": 1.7271327379119188 + }, + "layer": 10, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 0.946297012928892, + "load_gini": 0.4790344332268248, + "max_load_ratio": 4.728056667693255 + }, + "actual_vs_frontier_pearson": 0.034097483022671266, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4344893792424141, + "load_gini": 0.25022677182784103, + "max_load_ratio": 1.7591623036649215 + }, + "layer": 11, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9765625, + "load_cv": 0.9479122820728094, + "load_gini": 0.48800966517169697, + "max_load_ratio": 5.097628580227902 + }, + "actual_vs_frontier_pearson": -0.02153501737741947, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.48821731504828314, + "load_gini": 0.28114623835463504, + "max_load_ratio": 1.8601786264243918 + }, + "layer": 12, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9921875, + "load_cv": 0.9047451307311554, + "load_gini": 0.4740120688327687, + "max_load_ratio": 4.555589775177086 + }, + "actual_vs_frontier_pearson": -0.05894159287248561, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4550660315489719, + "load_gini": 0.26222101651524476, + "max_load_ratio": 1.7665537419156143 + }, + "layer": 13, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.96875, + "load_cv": 1.1102662993215426, + "load_gini": 0.550685007314444, + "max_load_ratio": 5.5214043732676314 + }, + "actual_vs_frontier_pearson": -0.07473125180903177, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4843214689622731, + "load_gini": 0.2793576281952572, + "max_load_ratio": 1.80597474591931 + }, + "layer": 14, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9453125, + "load_cv": 1.0424321211996845, + "load_gini": 0.5389009927433015, + "max_load_ratio": 4.885740683708038 + }, + "actual_vs_frontier_pearson": -0.122334501258848, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.45239738453493195, + "load_gini": 0.26047060267169697, + "max_load_ratio": 1.808438558669541 + }, + "layer": 15, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9453125, + "load_cv": 1.1000158272634826, + "load_gini": 0.5458891139898367, + "max_load_ratio": 5.321835540498922 + }, + "actual_vs_frontier_pearson": 0.06285259077675373, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.48638660674407463, + "load_gini": 0.2805170517978133, + "max_load_ratio": 1.833076686171851 + }, + "layer": 16, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9609375, + "load_cv": 1.1000359607690584, + "load_gini": 0.5519229288574068, + "max_load_ratio": 5.201108715737604 + }, + "actual_vs_frontier_pearson": -0.14790969026732273, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.44351655698688397, + "load_gini": 0.25566538583692644, + "max_load_ratio": 1.7764089929165383 + }, + "layer": 17, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.921875, + "load_cv": 1.275279013013624, + "load_gini": 0.6363662611641516, + "max_load_ratio": 5.637203572528488 + }, + "actual_vs_frontier_pearson": -0.017099613299867974, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4514828672543027, + "load_gini": 0.259494942446874, + "max_load_ratio": 1.801047120418848 + }, + "layer": 18, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.921875, + "load_cv": 1.2773056983164457, + "load_gini": 0.6308308029527256, + "max_load_ratio": 5.979673544810594 + }, + "actual_vs_frontier_pearson": -0.08479638082455722, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4680898752103892, + "load_gini": 0.2699814853133662, + "max_load_ratio": 1.810902371419772 + }, + "layer": 19, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9453125, + "load_cv": 1.142557882037356, + "load_gini": 0.573428958461657, + "max_load_ratio": 5.856482907299045 + }, + "actual_vs_frontier_pearson": -0.16520423265299047, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.48760009420143696, + "load_gini": 0.2809182634932246, + "max_load_ratio": 1.7739451801663073 + }, + "layer": 20, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9375, + "load_cv": 1.2250892374067721, + "load_gini": 0.5839970068524791, + "max_load_ratio": 6.6547582383738835 + }, + "actual_vs_frontier_pearson": 0.030713168827124045, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.44079057860501963, + "load_gini": 0.25344789421004, + "max_load_ratio": 1.7000307976593778 + }, + "layer": 21, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 0.9505437008130998, + "load_gini": 0.49268164603865117, + "max_load_ratio": 5.647058823529412 + }, + "actual_vs_frontier_pearson": -0.06756357177648568, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5096150033498628, + "load_gini": 0.2927473918232215, + "max_load_ratio": 1.9858330766861718 + }, + "layer": 22, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9609375, + "load_cv": 1.0080029689110093, + "load_gini": 0.521078351170311, + "max_load_ratio": 5.109947643979058 + }, + "actual_vs_frontier_pearson": 0.051969177479756205, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.502148823760943, + "load_gini": 0.288229700011549, + "max_load_ratio": 1.968586387434555 + }, + "layer": 23, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.96875, + "load_cv": 1.0336730047933789, + "load_gini": 0.5219367156221126, + "max_load_ratio": 6.2309824453341545 + }, + "actual_vs_frontier_pearson": 0.046624159557719196, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4937452666192137, + "load_gini": 0.2847580458885124, + "max_load_ratio": 1.8798891284262396 + }, + "layer": 24, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 0.9370799623414752, + "load_gini": 0.49417972359100704, + "max_load_ratio": 4.348629504157684 + }, + "actual_vs_frontier_pearson": 0.003630591230348502, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5094100439322153, + "load_gini": 0.29286198077071135, + "max_load_ratio": 1.9267015706806283 + }, + "layer": 25, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9765625, + "load_cv": 1.1307693127116234, + "load_gini": 0.562033222975054, + "max_load_ratio": 5.900831536803203 + }, + "actual_vs_frontier_pearson": 0.12889829096423783, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5140701353957097, + "load_gini": 0.2956587330612874, + "max_load_ratio": 1.8700338774253156 + }, + "layer": 26, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9453125, + "load_cv": 1.1066234149756957, + "load_gini": 0.5586361569333231, + "max_load_ratio": 5.615029257776409 + }, + "actual_vs_frontier_pearson": 0.027435906393056993, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4562891247455973, + "load_gini": 0.26251997035725294, + "max_load_ratio": 1.8848167539267016 + }, + "layer": 27, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9375, + "load_cv": 1.15194374689878, + "load_gini": 0.5561218167731752, + "max_load_ratio": 6.691715429627348 + }, + "actual_vs_frontier_pearson": -0.0851905104644269, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4931885285350055, + "load_gini": 0.28381426605327986, + "max_load_ratio": 1.7985833076686173 + }, + "layer": 28, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9296875, + "load_cv": 1.1356431750011955, + "load_gini": 0.5496513007198953, + "max_load_ratio": 5.863874345549738 + }, + "actual_vs_frontier_pearson": -0.016979392910092525, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.45012219999295283, + "load_gini": 0.25880680724514926, + "max_load_ratio": 1.7320603634123806 + }, + "layer": 29, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9296875, + "load_cv": 1.3264510778008716, + "load_gini": 0.647222736853249, + "max_load_ratio": 5.84416384354789 + }, + "actual_vs_frontier_pearson": -0.08251025635608537, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.45023597421285283, + "load_gini": 0.25955389109177696, + "max_load_ratio": 1.8453957499230058 + }, + "layer": 30, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.921875, + "load_cv": 1.2804663472936682, + "load_gini": 0.6305014725130891, + "max_load_ratio": 5.723437018786572 + }, + "actual_vs_frontier_pearson": -0.07964706640411469, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5029240802777508, + "load_gini": 0.2893154378079765, + "max_load_ratio": 1.9291653834308593 + }, + "layer": 31, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.96875, + "load_cv": 1.1507881607782275, + "load_gini": 0.5729281957383738, + "max_load_ratio": 6.021558361564521 + }, + "actual_vs_frontier_pearson": -0.10452258144829583, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.48623369594768334, + "load_gini": 0.2800406505043116, + "max_load_ratio": 1.83061287342162 + }, + "layer": 32, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9375, + "load_cv": 1.2686811219716572, + "load_gini": 0.583570832210502, + "max_load_ratio": 7.844779796735448 + }, + "actual_vs_frontier_pearson": -0.01888683352026748, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.48261072157815704, + "load_gini": 0.2782189569217739, + "max_load_ratio": 1.8355404989220818 + }, + "layer": 33, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9921875, + "load_cv": 0.8839459404636472, + "load_gini": 0.4794891799160763, + "max_load_ratio": 3.9618109023714196 + }, + "actual_vs_frontier_pearson": -0.028493286082986517, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5205697308164966, + "load_gini": 0.2990305353980598, + "max_load_ratio": 1.8429319371727748 + }, + "layer": 34, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.96875, + "load_cv": 0.9362312679691519, + "load_gini": 0.4916611728133662, + "max_load_ratio": 4.747767169695103 + }, + "actual_vs_frontier_pearson": -0.13290690148552678, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.49297481569205803, + "load_gini": 0.2832476372420696, + "max_load_ratio": 1.8995996304280875 + }, + "layer": 35, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9765625, + "load_cv": 0.9574253286736439, + "load_gini": 0.49605465140899296, + "max_load_ratio": 4.767477671696951 + }, + "actual_vs_frontier_pearson": -0.02212359104446765, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.47796935463002527, + "load_gini": 0.27542581373190633, + "max_load_ratio": 1.7985833076686173 + }, + "layer": 36, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 0.897443530050085, + "load_gini": 0.46608167154296276, + "max_load_ratio": 4.484139205420388 + }, + "actual_vs_frontier_pearson": -0.019587227325278997, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4143052194562559, + "load_gini": 0.2373612300777641, + "max_load_ratio": 1.6975669849091468 + }, + "layer": 37, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 1.0030386322057563, + "load_gini": 0.5019200415768401, + "max_load_ratio": 5.198644902987373 + }, + "actual_vs_frontier_pearson": 0.028921565758140203, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.48203207236683104, + "load_gini": 0.27775819506467503, + "max_load_ratio": 1.803510933169079 + }, + "layer": 38, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9765625, + "load_cv": 1.1070171860607907, + "load_gini": 0.5396805584963043, + "max_load_ratio": 5.164151524484139 + }, + "actual_vs_frontier_pearson": -0.042234189776402496, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.49820199968589846, + "load_gini": 0.2860022833577147, + "max_load_ratio": 1.9094548814290113 + }, + "layer": 39, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.96875, + "load_cv": 1.1143666092181908, + "load_gini": 0.5215875351285801, + "max_load_ratio": 6.679396365876193 + }, + "actual_vs_frontier_pearson": -0.0013066971352570613, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5231859746946843, + "load_gini": 0.3006501193409301, + "max_load_ratio": 1.9168463196797043 + }, + "layer": 40, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9375, + "load_cv": 1.3163699264356221, + "load_gini": 0.576891890591315, + "max_load_ratio": 6.450261780104712 + }, + "actual_vs_frontier_pearson": 0.038349159249901725, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.45930246027155186, + "load_gini": 0.2649497974091468, + "max_load_ratio": 1.815829996920234 + }, + "layer": 41, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9609375, + "load_cv": 1.4367616763803446, + "load_gini": 0.6064525307014168, + "max_load_ratio": 8.630736064059132 + }, + "actual_vs_frontier_pearson": -0.005385768830336229, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.49323689442136087, + "load_gini": 0.2822840073529411, + "max_load_ratio": 1.9069910686787803 + }, + "layer": 42, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.9453125, + "load_cv": 1.5122257286912568, + "load_gini": 0.6255329438712658, + "max_load_ratio": 8.19956883276871 + }, + "actual_vs_frontier_pearson": 0.17269736915133127, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4650946236339636, + "load_gini": 0.2676644426586079, + "max_load_ratio": 1.8700338774253156 + }, + "layer": 43, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 1.4453751038205662, + "load_gini": 0.6368104813096704, + "max_load_ratio": 6.94548814290114 + }, + "actual_vs_frontier_pearson": -0.18470982231222025, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4751830338518044, + "load_gini": 0.27377735698336925, + "max_load_ratio": 1.8429319371727748 + }, + "layer": 44, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.96875, + "load_cv": 1.464195386895173, + "load_gini": 0.6367434121881737, + "max_load_ratio": 7.145056975669849 + }, + "actual_vs_frontier_pearson": -0.026159680824363943, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4792291952239595, + "load_gini": 0.2763707966007083, + "max_load_ratio": 1.7764089929165383 + }, + "layer": 45, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 1.3411915327005308, + "load_gini": 0.5801415850207885, + "max_load_ratio": 6.0437326763165995 + }, + "actual_vs_frontier_pearson": 0.02225870145809893, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.5297396161604814, + "load_gini": 0.3045975732406838, + "max_load_ratio": 1.8798891284262396 + }, + "layer": 46, + "phase": "decode", + "total_routed_tokens": 51952 + }, + { + "actual": { + "expert_utilization": 0.984375, + "load_cv": 1.351786104177472, + "load_gini": 0.536063938828149, + "max_load_ratio": 6.908530951647675 + }, + "actual_vs_frontier_pearson": -0.04368832283746608, + "frontier_simulation": { + "expert_utilization": 1.0, + "load_cv": 0.4952080715601842, + "load_gini": 0.2838945685440406, + "max_load_ratio": 1.9168463196797043 + }, + "layer": 47, + "phase": "decode", + "total_routed_tokens": 51952 + } + ], + "schema": "frontier-routing-mismatch.v1", + "source": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-trace-routing-20260716-v1-20260716T151552356215Z/artifacts/artifacts/trace-routing-v1/raw/routing.json" +} diff --git a/runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-s2-profile-ablation.json b/runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-s2-profile-ablation.json new file mode 100644 index 0000000..32d8bdf --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/artifacts/qwen30-s2-profile-ablation.json @@ -0,0 +1,5222 @@ +{ + "historical_modes": { + "historical-per-tp-calibration": { + "false_feasibility": { + "overall": { + "agreement": 64, + "false_feasible": 21, + "false_infeasible": 7 + }, + "per_cell": { + "tp1_mns16": { + "agreement": 7, + "false_feasible": 1, + "false_infeasible": 0 + }, + "tp1_mns32": { + "agreement": 6, + "false_feasible": 2, + "false_infeasible": 0 + }, + "tp1_mns64": { + "agreement": 6, + "false_feasible": 2, + "false_infeasible": 0 + }, + "tp1_mns8": { + "agreement": 6, + "false_feasible": 0, + "false_infeasible": 2 + }, + "tp2_mns16": { + "agreement": 7, + "false_feasible": 1, + "false_infeasible": 0 + }, + "tp2_mns32": { + "agreement": 3, + "false_feasible": 5, + "false_infeasible": 0 + }, + "tp2_mns64": { + "agreement": 2, + "false_feasible": 6, + "false_infeasible": 0 + }, + "tp2_mns8": { + "agreement": 3, + "false_feasible": 0, + "false_infeasible": 5 + }, + "tp4_mns16": { + "agreement": 6, + "false_feasible": 1, + "false_infeasible": 0 + }, + "tp4_mns32": { + "agreement": 6, + "false_feasible": 1, + "false_infeasible": 0 + }, + "tp4_mns64": { + "agreement": 6, + "false_feasible": 1, + "false_infeasible": 0 + }, + "tp4_mns8": { + "agreement": 6, + "false_feasible": 1, + "false_infeasible": 0 + } + } + }, + "metrics": { + "kendall_tau_b": { + "both_tied": 4, + "concordant": 58, + "discordant": 0, + "pair_count": 66, + "real_only_tied": 1, + "sim_only_tied": 3, + "tau_b": 0.9668009539030813 + }, + "named_interactions": { + "relations": { + "tp2_mns32_gt_tp4_mns16": true, + "tp4_mns16_eq_tp4_mns32": true, + "tp4_mns16_eq_tp4_mns64": true, + "tp4_mns16_gt_tp1_mns16": true, + "tp4_mns16_gt_tp2_mns16": true, + "tp4_mns16_gt_tp4_mns8": true + }, + "simulator_top_bucket_cells": [ + "tp2_mns32", + "tp2_mns64" + ], + "tp2_mns32_unique_global_best": false, + "trap_reproduced": true + }, + "pairwise_direction": { + "exact_sign_accuracy": 0.9393939393939394, + "exact_sign_agreement": 62, + "non_tied_concordance": 1.0, + "non_tied_concordant": 58, + "non_tied_pair_count": 58, + "pair_count": 66 + }, + "per_mns_tp_direction": { + "accuracy": 1.0, + "comparisons": 8, + "exact_sign_agreement": 8, + "lines": [ + { + "comparisons": 2, + "exact_sign_agreement": 2, + "line_class_agrees": true, + "mns": 8, + "real_adjacent_signs": [ + 1, + -1 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + 1, + -1 + ], + "sim_line_class": "non-monotonic" + }, + { + "comparisons": 2, + "exact_sign_agreement": 2, + "line_class_agrees": true, + "mns": 16, + "real_adjacent_signs": [ + -1, + 1 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + -1, + 1 + ], + "sim_line_class": "non-monotonic" + }, + { + "comparisons": 2, + "exact_sign_agreement": 2, + "line_class_agrees": true, + "mns": 32, + "real_adjacent_signs": [ + 1, + -1 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + 1, + -1 + ], + "sim_line_class": "non-monotonic" + }, + { + "comparisons": 2, + "exact_sign_agreement": 2, + "line_class_agrees": true, + "mns": 64, + "real_adjacent_signs": [ + 1, + -1 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + 1, + -1 + ], + "sim_line_class": "non-monotonic" + } + ] + }, + "per_tp_mns_direction": { + "accuracy": 0.6666666666666666, + "comparisons": 9, + "exact_sign_agreement": 6, + "lines": [ + { + "comparisons": 3, + "exact_sign_agreement": 2, + "line_class_agrees": false, + "real_adjacent_signs": [ + 1, + -1, + 0 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + 1, + 0, + 0 + ], + "sim_line_class": "increasing", + "tp": 1 + }, + { + "comparisons": 3, + "exact_sign_agreement": 1, + "line_class_agrees": false, + "real_adjacent_signs": [ + 0, + 1, + -1 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + 1, + 1, + 0 + ], + "sim_line_class": "increasing", + "tp": 2 + }, + { + "comparisons": 3, + "exact_sign_agreement": 3, + "line_class_agrees": true, + "real_adjacent_signs": [ + 1, + 0, + 0 + ], + "real_line_class": "increasing", + "sim_adjacent_signs": [ + 1, + 0, + 0 + ], + "sim_line_class": "increasing", + "tp": 4 + } + ] + }, + "tie_buckets": { + "real": { + "buckets": { + "tp1_mns16": 715736, + "tp1_mns32": 695431, + "tp1_mns64": 695431, + "tp1_mns8": 639593, + "tp2_mns16": 692893, + "tp2_mns32": 1000000, + "tp2_mns64": 992385, + "tp2_mns8": 692893, + "tp4_mns16": 743654, + "tp4_mns32": 743654, + "tp4_mns64": 743654, + "tp4_mns8": 390862 + }, + "occupied_intervals": [ + { + "bucket_id": 1000000, + "cells": [ + "tp2_mns32" + ], + "interval": [ + 3.2833333333333328, + 3.2833366166666664 + ] + }, + { + "bucket_id": 992385, + "cells": [ + "tp2_mns64" + ], + "interval": [ + 3.25833075, + 3.258334033333333 + ] + }, + { + "bucket_id": 743654, + "cells": [ + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "interval": [ + 2.4416639666666664, + 2.4416672499999996 + ] + }, + { + "bucket_id": 715736, + "cells": [ + "tp1_mns16" + ], + "interval": [ + 2.3499998666666664, + 2.3500031499999996 + ] + }, + { + "bucket_id": 695431, + "cells": [ + "tp1_mns32", + "tp1_mns64" + ], + "interval": [ + 2.283331783333333, + 2.2833350666666665 + ] + }, + { + "bucket_id": 692893, + "cells": [ + "tp2_mns8", + "tp2_mns16" + ], + "interval": [ + 2.274998683333333, + 2.2750019666666663 + ] + }, + { + "bucket_id": 639593, + "cells": [ + "tp1_mns8" + ], + "interval": [ + 2.0999970166666664, + 2.1000002999999996 + ] + }, + { + "bucket_id": 390862, + "cells": [ + "tp4_mns8" + ], + "interval": [ + 1.2833302333333332, + 1.2833335166666666 + ] + } + ], + "tolerance": 3.283333333333333e-06 + }, + "simulator": { + "buckets": { + "tp1_mns16": 635555, + "tp1_mns32": 635555, + "tp1_mns64": 635555, + "tp1_mns8": 457777, + "tp2_mns16": 613333, + "tp2_mns32": 1000000, + "tp2_mns64": 1000000, + "tp2_mns8": 464444, + "tp4_mns16": 666666, + "tp4_mns32": 666666, + "tp4_mns64": 666666, + "tp4_mns8": 352222 + }, + "occupied_intervals": [ + { + "bucket_id": 1000000, + "cells": [ + "tp2_mns32", + "tp2_mns64" + ], + "interval": [ + 3.7499999999999996, + 3.75000375 + ] + }, + { + "bucket_id": 666666, + "cells": [ + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "interval": [ + 2.4999974999999997, + 2.50000125 + ] + }, + { + "bucket_id": 635555, + "cells": [ + "tp1_mns16", + "tp1_mns32", + "tp1_mns64" + ], + "interval": [ + 2.38333125, + 2.3833349999999998 + ] + }, + { + "bucket_id": 613333, + "cells": [ + "tp2_mns16" + ], + "interval": [ + 2.29999875, + 2.3000024999999997 + ] + }, + { + "bucket_id": 464444, + "cells": [ + "tp2_mns8" + ], + "interval": [ + 1.7416649999999998, + 1.7416687499999999 + ] + }, + { + "bucket_id": 457777, + "cells": [ + "tp1_mns8" + ], + "interval": [ + 1.71666375, + 1.7166674999999998 + ] + }, + { + "bucket_id": 352222, + "cells": [ + "tp4_mns8" + ], + "interval": [ + 1.3208324999999999, + 1.32083625 + ] + } + ], + "tolerance": 3.7499999999999997e-06 + } + }, + "top1": { + "candidate_cells": [ + "tp2_mns32", + "tp2_mns64" + ], + "effective_k": 2, + "optimistic_regret": 0.0, + "worst_case_regret": 0.0076142131979695165 + }, + "top5": { + "maximum_exact_five_overlap": 5, + "minimum_exact_five_overlap": 5, + "optimistic_regret": 0.0, + "real_cutoff": { + "boundary": [ + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "choose_from_boundary": 3, + "expanded_effective_k": 5, + "higher": [ + "tp2_mns32", + "tp2_mns64" + ] + }, + "simulator_cutoff": { + "boundary": [ + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "choose_from_boundary": 3, + "expanded_effective_k": 5, + "higher": [ + "tp2_mns32", + "tp2_mns64" + ] + }, + "worst_case_regret": 0.0 + } + }, + "ranking": [ + { + "cell": "tp2_mns32", + "rank": 1, + "score": 3.75 + }, + { + "cell": "tp2_mns64", + "rank": 2, + "score": 3.75 + }, + { + "cell": "tp4_mns16", + "rank": 3, + "score": 2.5 + }, + { + "cell": "tp4_mns32", + "rank": 4, + "score": 2.5 + }, + { + "cell": "tp4_mns64", + "rank": 5, + "score": 2.5 + }, + { + "cell": "tp1_mns16", + "rank": 6, + "score": 2.3833333333333333 + }, + { + "cell": "tp1_mns32", + "rank": 7, + "score": 2.3833333333333333 + }, + { + "cell": "tp1_mns64", + "rank": 8, + "score": 2.3833333333333333 + }, + { + "cell": "tp2_mns16", + "rank": 9, + "score": 2.3 + }, + { + "cell": "tp2_mns8", + "rank": 10, + "score": 1.7416666666666667 + }, + { + "cell": "tp1_mns8", + "rank": 11, + "score": 1.7166666666666666 + }, + { + "cell": "tp4_mns8", + "rank": 12, + "score": 1.3208333333333333 + } + ], + "simulated_scores": { + "tp1_mns16": 2.3833333333333333, + "tp1_mns32": 2.3833333333333333, + "tp1_mns64": 2.3833333333333333, + "tp1_mns8": 1.7166666666666666, + "tp2_mns16": 2.3, + "tp2_mns32": 3.75, + "tp2_mns64": 3.75, + "tp2_mns8": 1.7416666666666667, + "tp4_mns16": 2.5, + "tp4_mns32": 2.5, + "tp4_mns64": 2.5, + "tp4_mns8": 1.3208333333333333 + } + }, + "historical-profile-only": { + "false_feasibility": { + "overall": { + "agreement": 37, + "false_feasible": 0, + "false_infeasible": 55 + }, + "per_cell": { + "tp1_mns16": { + "agreement": 3, + "false_feasible": 0, + "false_infeasible": 5 + }, + "tp1_mns32": { + "agreement": 4, + "false_feasible": 0, + "false_infeasible": 4 + }, + "tp1_mns64": { + "agreement": 4, + "false_feasible": 0, + "false_infeasible": 4 + }, + "tp1_mns8": { + "agreement": 5, + "false_feasible": 0, + "false_infeasible": 3 + }, + "tp2_mns16": { + "agreement": 2, + "false_feasible": 0, + "false_infeasible": 6 + }, + "tp2_mns32": { + "agreement": 5, + "false_feasible": 0, + "false_infeasible": 3 + }, + "tp2_mns64": { + "agreement": 6, + "false_feasible": 0, + "false_infeasible": 2 + }, + "tp2_mns8": { + "agreement": 1, + "false_feasible": 0, + "false_infeasible": 7 + }, + "tp4_mns16": { + "agreement": 1, + "false_feasible": 0, + "false_infeasible": 6 + }, + "tp4_mns32": { + "agreement": 2, + "false_feasible": 0, + "false_infeasible": 5 + }, + "tp4_mns64": { + "agreement": 2, + "false_feasible": 0, + "false_infeasible": 5 + }, + "tp4_mns8": { + "agreement": 2, + "false_feasible": 0, + "false_infeasible": 5 + } + } + }, + "metrics": { + "kendall_tau_b": { + "both_tied": 2, + "concordant": 23, + "discordant": 23, + "pair_count": 66, + "real_only_tied": 3, + "sim_only_tied": 15, + "tau_b": 0.0 + }, + "named_interactions": { + "relations": { + "tp2_mns32_gt_tp4_mns16": false, + "tp4_mns16_eq_tp4_mns32": false, + "tp4_mns16_eq_tp4_mns64": false, + "tp4_mns16_gt_tp1_mns16": false, + "tp4_mns16_gt_tp2_mns16": false, + "tp4_mns16_gt_tp4_mns8": false + }, + "simulator_top_bucket_cells": [ + "tp4_mns32", + "tp4_mns64" + ], + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + "pairwise_direction": { + "exact_sign_accuracy": 0.3787878787878788, + "exact_sign_agreement": 25, + "non_tied_concordance": 0.5, + "non_tied_concordant": 23, + "non_tied_pair_count": 46, + "pair_count": 66 + }, + "per_mns_tp_direction": { + "accuracy": 0.0, + "comparisons": 8, + "exact_sign_agreement": 0, + "lines": [ + { + "comparisons": 2, + "exact_sign_agreement": 0, + "line_class_agrees": false, + "mns": 8, + "real_adjacent_signs": [ + 1, + -1 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + -1, + 0 + ], + "sim_line_class": "decreasing" + }, + { + "comparisons": 2, + "exact_sign_agreement": 0, + "line_class_agrees": true, + "mns": 16, + "real_adjacent_signs": [ + -1, + 1 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + 1, + -1 + ], + "sim_line_class": "non-monotonic" + }, + { + "comparisons": 2, + "exact_sign_agreement": 0, + "line_class_agrees": true, + "mns": 32, + "real_adjacent_signs": [ + 1, + -1 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + -1, + 1 + ], + "sim_line_class": "non-monotonic" + }, + { + "comparisons": 2, + "exact_sign_agreement": 0, + "line_class_agrees": true, + "mns": 64, + "real_adjacent_signs": [ + 1, + -1 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + -1, + 1 + ], + "sim_line_class": "non-monotonic" + } + ] + }, + "per_tp_mns_direction": { + "accuracy": 0.2222222222222222, + "comparisons": 9, + "exact_sign_agreement": 2, + "lines": [ + { + "comparisons": 3, + "exact_sign_agreement": 1, + "line_class_agrees": false, + "real_adjacent_signs": [ + 1, + -1, + 0 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + 0, + 0, + 0 + ], + "sim_line_class": "flat", + "tp": 1 + }, + { + "comparisons": 3, + "exact_sign_agreement": 0, + "line_class_agrees": true, + "real_adjacent_signs": [ + 0, + 1, + -1 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + 1, + -1, + 0 + ], + "sim_line_class": "non-monotonic", + "tp": 2 + }, + { + "comparisons": 3, + "exact_sign_agreement": 1, + "line_class_agrees": true, + "real_adjacent_signs": [ + 1, + 0, + 0 + ], + "real_line_class": "increasing", + "sim_adjacent_signs": [ + 0, + 1, + 0 + ], + "sim_line_class": "increasing", + "tp": 4 + } + ] + }, + "tie_buckets": { + "real": { + "buckets": { + "tp1_mns16": 715736, + "tp1_mns32": 695431, + "tp1_mns64": 695431, + "tp1_mns8": 639593, + "tp2_mns16": 692893, + "tp2_mns32": 1000000, + "tp2_mns64": 992385, + "tp2_mns8": 692893, + "tp4_mns16": 743654, + "tp4_mns32": 743654, + "tp4_mns64": 743654, + "tp4_mns8": 390862 + }, + "occupied_intervals": [ + { + "bucket_id": 1000000, + "cells": [ + "tp2_mns32" + ], + "interval": [ + 3.2833333333333328, + 3.2833366166666664 + ] + }, + { + "bucket_id": 992385, + "cells": [ + "tp2_mns64" + ], + "interval": [ + 3.25833075, + 3.258334033333333 + ] + }, + { + "bucket_id": 743654, + "cells": [ + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "interval": [ + 2.4416639666666664, + 2.4416672499999996 + ] + }, + { + "bucket_id": 715736, + "cells": [ + "tp1_mns16" + ], + "interval": [ + 2.3499998666666664, + 2.3500031499999996 + ] + }, + { + "bucket_id": 695431, + "cells": [ + "tp1_mns32", + "tp1_mns64" + ], + "interval": [ + 2.283331783333333, + 2.2833350666666665 + ] + }, + { + "bucket_id": 692893, + "cells": [ + "tp2_mns8", + "tp2_mns16" + ], + "interval": [ + 2.274998683333333, + 2.2750019666666663 + ] + }, + { + "bucket_id": 639593, + "cells": [ + "tp1_mns8" + ], + "interval": [ + 2.0999970166666664, + 2.1000002999999996 + ] + }, + { + "bucket_id": 390862, + "cells": [ + "tp4_mns8" + ], + "interval": [ + 1.2833302333333332, + 1.2833335166666666 + ] + } + ], + "tolerance": 3.283333333333333e-06 + }, + "simulator": { + "buckets": { + "tp1_mns16": 832807, + "tp1_mns32": 832807, + "tp1_mns64": 832807, + "tp1_mns8": 832807, + "tp2_mns16": 902208, + "tp2_mns32": 0, + "tp2_mns64": 0, + "tp2_mns8": 0, + "tp4_mns16": 0, + "tp4_mns32": 1000000, + "tp4_mns64": 1000000, + "tp4_mns8": 0 + }, + "occupied_intervals": [ + { + "bucket_id": 1000000, + "cells": [ + "tp4_mns32", + "tp4_mns64" + ], + "interval": [ + 1.3208333333333333, + 1.3208346541666667 + ] + }, + { + "bucket_id": 902208, + "cells": [ + "tp2_mns16" + ], + "interval": [ + 1.1916664, + 1.1916677208333333 + ] + }, + { + "bucket_id": 832807, + "cells": [ + "tp1_mns8", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64" + ], + "interval": [ + 1.0999992458333334, + 1.1000005666666666 + ] + }, + { + "bucket_id": 0, + "cells": [ + "tp2_mns8", + "tp2_mns32", + "tp2_mns64", + "tp4_mns8", + "tp4_mns16" + ], + "interval": [ + 0.0, + 1.3208333333333333e-06 + ] + } + ], + "tolerance": 1.3208333333333333e-06 + } + }, + "top1": { + "candidate_cells": [ + "tp4_mns32", + "tp4_mns64" + ], + "effective_k": 2, + "optimistic_regret": 0.25634517766497456, + "worst_case_regret": 0.25634517766497456 + }, + "top5": { + "maximum_exact_five_overlap": 2, + "minimum_exact_five_overlap": 2, + "optimistic_regret": 0.25634517766497456, + "real_cutoff": { + "boundary": [ + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "choose_from_boundary": 3, + "expanded_effective_k": 5, + "higher": [ + "tp2_mns32", + "tp2_mns64" + ] + }, + "simulator_cutoff": { + "boundary": [ + "tp1_mns8", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64" + ], + "choose_from_boundary": 2, + "expanded_effective_k": 7, + "higher": [ + "tp4_mns32", + "tp4_mns64", + "tp2_mns16" + ] + }, + "worst_case_regret": 0.25634517766497456 + } + }, + "ranking": [ + { + "cell": "tp4_mns32", + "rank": 1, + "score": 1.3208333333333333 + }, + { + "cell": "tp4_mns64", + "rank": 2, + "score": 1.3208333333333333 + }, + { + "cell": "tp2_mns16", + "rank": 3, + "score": 1.1916666666666667 + }, + { + "cell": "tp1_mns16", + "rank": 4, + "score": 1.1 + }, + { + "cell": "tp1_mns32", + "rank": 5, + "score": 1.1 + }, + { + "cell": "tp1_mns64", + "rank": 6, + "score": 1.1 + }, + { + "cell": "tp1_mns8", + "rank": 7, + "score": 1.1 + }, + { + "cell": "tp2_mns32", + "rank": 8, + "score": 0.0 + }, + { + "cell": "tp2_mns64", + "rank": 9, + "score": 0.0 + }, + { + "cell": "tp2_mns8", + "rank": 10, + "score": 0.0 + }, + { + "cell": "tp4_mns16", + "rank": 11, + "score": 0.0 + }, + { + "cell": "tp4_mns8", + "rank": 12, + "score": 0.0 + } + ], + "simulated_scores": { + "tp1_mns16": 1.1, + "tp1_mns32": 1.1, + "tp1_mns64": 1.1, + "tp1_mns8": 1.1, + "tp2_mns16": 1.1916666666666667, + "tp2_mns32": 0.0, + "tp2_mns64": 0.0, + "tp2_mns8": 0.0, + "tp4_mns16": 0.0, + "tp4_mns32": 1.3208333333333333, + "tp4_mns64": 1.3208333333333333, + "tp4_mns8": 0.0 + } + } + }, + "real_scores": { + "tp1_mns16": 2.35, + "tp1_mns32": 2.283333333333333, + "tp1_mns64": 2.283333333333333, + "tp1_mns8": 2.1, + "tp2_mns16": 2.275, + "tp2_mns32": 3.283333333333333, + "tp2_mns64": 3.2583333333333333, + "tp2_mns8": 2.275, + "tp4_mns16": 2.441666666666667, + "tp4_mns32": 2.441666666666667, + "tp4_mns64": 2.441666666666667, + "tp4_mns8": 1.2833333333333334 + }, + "schema": "frontier-qwen30-s2-profile-ablation.v1", + "scope": { + "calibration_a_tp": 1.0, + "cells": 12, + "probes": 92, + "trace_horizon_seconds": 60 + }, + "sources": { + "ground_truth": "/home/gahow/phd/replayserve/docs/assets/simfid_s2r/ground_truth.json", + "historical_metrics": "/home/gahow/phd/replayserve/runs/simfid_s2rb/results/metrics.json", + "replacement_profile_shards": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/comparison/new-profile-only-v2-s2/results-shard-00/metrics.json", + "runs": 46 + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/comparison/new-profile-only-v2-s2/results-shard-01/metrics.json", + "runs": 46 + } + ] + }, + "vllm020_profile_only": { + "SLO-gated": { + "cell_score_details": { + "tp1_mns16": { + "feasible": false, + "feasible_anchor_count": 0, + "recorded_anchor_count_after_removal": 8, + "score": 0.0, + "selected_probe_index": null, + "selected_sampling_u": null + }, + "tp1_mns32": { + "feasible": false, + "feasible_anchor_count": 0, + "recorded_anchor_count_after_removal": 8, + "score": 0.0, + "selected_probe_index": null, + "selected_sampling_u": null + }, + "tp1_mns64": { + "feasible": false, + "feasible_anchor_count": 0, + "recorded_anchor_count_after_removal": 8, + "score": 0.0, + "selected_probe_index": null, + "selected_sampling_u": null + }, + "tp1_mns8": { + "feasible": false, + "feasible_anchor_count": 0, + "recorded_anchor_count_after_removal": 8, + "score": 0.0, + "selected_probe_index": null, + "selected_sampling_u": null + }, + "tp2_mns16": { + "feasible": false, + "feasible_anchor_count": 0, + "recorded_anchor_count_after_removal": 8, + "score": 0.0, + "selected_probe_index": null, + "selected_sampling_u": null + }, + "tp2_mns32": { + "feasible": false, + "feasible_anchor_count": 0, + "recorded_anchor_count_after_removal": 8, + "score": 0.0, + "selected_probe_index": null, + "selected_sampling_u": null + }, + "tp2_mns64": { + "feasible": false, + "feasible_anchor_count": 0, + "recorded_anchor_count_after_removal": 8, + "score": 0.0, + "selected_probe_index": null, + "selected_sampling_u": null + }, + "tp2_mns8": { + "feasible": false, + "feasible_anchor_count": 0, + "recorded_anchor_count_after_removal": 8, + "score": 0.0, + "selected_probe_index": null, + "selected_sampling_u": null + }, + "tp4_mns16": { + "feasible": false, + "feasible_anchor_count": 0, + "recorded_anchor_count_after_removal": 7, + "score": 0.0, + "selected_probe_index": null, + "selected_sampling_u": null + }, + "tp4_mns32": { + "feasible": false, + "feasible_anchor_count": 0, + "recorded_anchor_count_after_removal": 7, + "score": 0.0, + "selected_probe_index": null, + "selected_sampling_u": null + }, + "tp4_mns64": { + "feasible": false, + "feasible_anchor_count": 0, + "recorded_anchor_count_after_removal": 7, + "score": 0.0, + "selected_probe_index": null, + "selected_sampling_u": null + }, + "tp4_mns8": { + "feasible": false, + "feasible_anchor_count": 0, + "recorded_anchor_count_after_removal": 7, + "score": 0.0, + "selected_probe_index": null, + "selected_sampling_u": null + } + }, + "false_feasibility": { + "overall": { + "agreement": 30, + "false_feasible": 0, + "false_infeasible": 62 + }, + "per_cell": { + "tp1_mns16": { + "agreement": 2, + "false_feasible": 0, + "false_infeasible": 6 + }, + "tp1_mns32": { + "agreement": 3, + "false_feasible": 0, + "false_infeasible": 5 + }, + "tp1_mns64": { + "agreement": 3, + "false_feasible": 0, + "false_infeasible": 5 + }, + "tp1_mns8": { + "agreement": 4, + "false_feasible": 0, + "false_infeasible": 4 + }, + "tp2_mns16": { + "agreement": 1, + "false_feasible": 0, + "false_infeasible": 7 + }, + "tp2_mns32": { + "agreement": 5, + "false_feasible": 0, + "false_infeasible": 3 + }, + "tp2_mns64": { + "agreement": 6, + "false_feasible": 0, + "false_infeasible": 2 + }, + "tp2_mns8": { + "agreement": 1, + "false_feasible": 0, + "false_infeasible": 7 + }, + "tp4_mns16": { + "agreement": 1, + "false_feasible": 0, + "false_infeasible": 6 + }, + "tp4_mns32": { + "agreement": 1, + "false_feasible": 0, + "false_infeasible": 6 + }, + "tp4_mns64": { + "agreement": 1, + "false_feasible": 0, + "false_infeasible": 6 + }, + "tp4_mns8": { + "agreement": 2, + "false_feasible": 0, + "false_infeasible": 5 + } + } + }, + "loao": { + "defined_replicates": 92, + "range_semantics": "deterministic LOAO sensitivity ranges, not confidence intervals", + "ranges": { + "pairwise_exact_sign_accuracy": { + "max": 0.07575757575757576, + "min": 0.07575757575757576 + }, + "pairwise_non_tied_concordance": { + "max": null, + "min": null + }, + "tau_b": { + "max": 0.0, + "min": 0.0 + }, + "top1_optimistic_regret": { + "max": 0.0, + "min": 0.0 + }, + "top1_worst_case_regret": { + "max": 0.6091370558375634, + "min": 0.6091370558375634 + }, + "top5_maximum_exact_five_overlap": { + "max": 5, + "min": 5 + }, + "top5_minimum_exact_five_overlap": { + "max": 0, + "min": 0 + }, + "top5_optimistic_regret": { + "max": 0.0, + "min": 0.0 + }, + "top5_worst_case_regret": { + "max": 0.30456852791878175, + "min": 0.30456852791878175 + } + }, + "replicate_count": 92, + "replicates": [ + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns8", + "removed_probe_index": 0, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns8", + "removed_probe_index": 1, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns8", + "removed_probe_index": 2, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns8", + "removed_probe_index": 3, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns8", + "removed_probe_index": 4, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns8", + "removed_probe_index": 5, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns8", + "removed_probe_index": 6, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns8", + "removed_probe_index": 7, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns16", + "removed_probe_index": 0, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns16", + "removed_probe_index": 1, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns16", + "removed_probe_index": 2, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns16", + "removed_probe_index": 3, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns16", + "removed_probe_index": 4, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns16", + "removed_probe_index": 5, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns16", + "removed_probe_index": 6, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns16", + "removed_probe_index": 7, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns32", + "removed_probe_index": 0, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns32", + "removed_probe_index": 1, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns32", + "removed_probe_index": 2, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns32", + "removed_probe_index": 3, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns32", + "removed_probe_index": 4, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns32", + "removed_probe_index": 5, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns32", + "removed_probe_index": 6, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns32", + "removed_probe_index": 7, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns64", + "removed_probe_index": 0, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns64", + "removed_probe_index": 1, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns64", + "removed_probe_index": 2, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns64", + "removed_probe_index": 3, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns64", + "removed_probe_index": 4, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns64", + "removed_probe_index": 5, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns64", + "removed_probe_index": 6, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp1_mns64", + "removed_probe_index": 7, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns8", + "removed_probe_index": 0, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns8", + "removed_probe_index": 1, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns8", + "removed_probe_index": 2, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns8", + "removed_probe_index": 3, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns8", + "removed_probe_index": 4, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns8", + "removed_probe_index": 5, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns8", + "removed_probe_index": 6, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns8", + "removed_probe_index": 7, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns16", + "removed_probe_index": 0, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns16", + "removed_probe_index": 1, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns16", + "removed_probe_index": 2, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns16", + "removed_probe_index": 3, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns16", + "removed_probe_index": 4, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns16", + "removed_probe_index": 5, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns16", + "removed_probe_index": 6, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns16", + "removed_probe_index": 7, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns32", + "removed_probe_index": 0, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns32", + "removed_probe_index": 1, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns32", + "removed_probe_index": 2, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns32", + "removed_probe_index": 3, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns32", + "removed_probe_index": 4, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns32", + "removed_probe_index": 5, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns32", + "removed_probe_index": 6, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns32", + "removed_probe_index": 7, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns64", + "removed_probe_index": 0, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns64", + "removed_probe_index": 1, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns64", + "removed_probe_index": 2, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns64", + "removed_probe_index": 3, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns64", + "removed_probe_index": 4, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns64", + "removed_probe_index": 5, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns64", + "removed_probe_index": 6, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp2_mns64", + "removed_probe_index": 7, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns8", + "removed_probe_index": 0, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns8", + "removed_probe_index": 1, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns8", + "removed_probe_index": 2, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns8", + "removed_probe_index": 3, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns8", + "removed_probe_index": 4, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns8", + "removed_probe_index": 5, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns8", + "removed_probe_index": 6, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns16", + "removed_probe_index": 0, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns16", + "removed_probe_index": 1, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns16", + "removed_probe_index": 2, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns16", + "removed_probe_index": 3, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns16", + "removed_probe_index": 4, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns16", + "removed_probe_index": 5, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns16", + "removed_probe_index": 6, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns32", + "removed_probe_index": 0, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns32", + "removed_probe_index": 1, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns32", + "removed_probe_index": 2, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns32", + "removed_probe_index": 3, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns32", + "removed_probe_index": 4, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns32", + "removed_probe_index": 5, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns32", + "removed_probe_index": 6, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns64", + "removed_probe_index": 0, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns64", + "removed_probe_index": 1, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns64", + "removed_probe_index": 2, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns64", + "removed_probe_index": 3, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns64", + "removed_probe_index": 4, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns64", + "removed_probe_index": 5, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.07575757575757576, + "pairwise_non_tied_concordance": null, + "removed_cell_id": "tp4_mns64", + "removed_probe_index": 6, + "tau_b": 0.0, + "top1_optimistic_regret": 0.0, + "top1_worst_case_regret": 0.6091370558375634, + "top5_maximum_exact_five_overlap": 5, + "top5_minimum_exact_five_overlap": 0, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.30456852791878175, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + } + ], + "tp2_mns32_unique_global_best_count": 0, + "trap_reproduced_count": 0, + "undefined": [], + "undefined_replicates": 0 + }, + "metrics": { + "kendall_tau_b": { + "both_tied": 5, + "concordant": 0, + "discordant": 0, + "pair_count": 66, + "real_only_tied": 0, + "sim_only_tied": 61, + "tau_b": 0.0 + }, + "named_interactions": { + "relations": { + "tp2_mns32_gt_tp4_mns16": false, + "tp4_mns16_eq_tp4_mns32": true, + "tp4_mns16_eq_tp4_mns64": true, + "tp4_mns16_gt_tp1_mns16": false, + "tp4_mns16_gt_tp2_mns16": false, + "tp4_mns16_gt_tp4_mns8": false + }, + "simulator_top_bucket_cells": [ + "tp1_mns8", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp2_mns8", + "tp2_mns16", + "tp2_mns32", + "tp2_mns64", + "tp4_mns8", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + "pairwise_direction": { + "exact_sign_accuracy": 0.07575757575757576, + "exact_sign_agreement": 5, + "non_tied_concordance": null, + "non_tied_concordant": 0, + "non_tied_pair_count": 0, + "pair_count": 66 + }, + "per_mns_tp_direction": { + "accuracy": 0.0, + "comparisons": 8, + "exact_sign_agreement": 0, + "lines": [ + { + "comparisons": 2, + "exact_sign_agreement": 0, + "line_class_agrees": false, + "mns": 8, + "real_adjacent_signs": [ + 1, + -1 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + 0, + 0 + ], + "sim_line_class": "flat" + }, + { + "comparisons": 2, + "exact_sign_agreement": 0, + "line_class_agrees": false, + "mns": 16, + "real_adjacent_signs": [ + -1, + 1 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + 0, + 0 + ], + "sim_line_class": "flat" + }, + { + "comparisons": 2, + "exact_sign_agreement": 0, + "line_class_agrees": false, + "mns": 32, + "real_adjacent_signs": [ + 1, + -1 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + 0, + 0 + ], + "sim_line_class": "flat" + }, + { + "comparisons": 2, + "exact_sign_agreement": 0, + "line_class_agrees": false, + "mns": 64, + "real_adjacent_signs": [ + 1, + -1 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + 0, + 0 + ], + "sim_line_class": "flat" + } + ] + }, + "per_tp_mns_direction": { + "accuracy": 0.4444444444444444, + "comparisons": 9, + "exact_sign_agreement": 4, + "lines": [ + { + "comparisons": 3, + "exact_sign_agreement": 1, + "line_class_agrees": false, + "real_adjacent_signs": [ + 1, + -1, + 0 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + 0, + 0, + 0 + ], + "sim_line_class": "flat", + "tp": 1 + }, + { + "comparisons": 3, + "exact_sign_agreement": 1, + "line_class_agrees": false, + "real_adjacent_signs": [ + 0, + 1, + -1 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + 0, + 0, + 0 + ], + "sim_line_class": "flat", + "tp": 2 + }, + { + "comparisons": 3, + "exact_sign_agreement": 2, + "line_class_agrees": false, + "real_adjacent_signs": [ + 1, + 0, + 0 + ], + "real_line_class": "increasing", + "sim_adjacent_signs": [ + 0, + 0, + 0 + ], + "sim_line_class": "flat", + "tp": 4 + } + ] + }, + "tie_buckets": { + "real": { + "buckets": { + "tp1_mns16": 715736, + "tp1_mns32": 695431, + "tp1_mns64": 695431, + "tp1_mns8": 639593, + "tp2_mns16": 692893, + "tp2_mns32": 1000000, + "tp2_mns64": 992385, + "tp2_mns8": 692893, + "tp4_mns16": 743654, + "tp4_mns32": 743654, + "tp4_mns64": 743654, + "tp4_mns8": 390862 + }, + "occupied_intervals": [ + { + "bucket_id": 1000000, + "cells": [ + "tp2_mns32" + ], + "interval": [ + 3.2833333333333328, + 3.2833366166666664 + ] + }, + { + "bucket_id": 992385, + "cells": [ + "tp2_mns64" + ], + "interval": [ + 3.25833075, + 3.258334033333333 + ] + }, + { + "bucket_id": 743654, + "cells": [ + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "interval": [ + 2.4416639666666664, + 2.4416672499999996 + ] + }, + { + "bucket_id": 715736, + "cells": [ + "tp1_mns16" + ], + "interval": [ + 2.3499998666666664, + 2.3500031499999996 + ] + }, + { + "bucket_id": 695431, + "cells": [ + "tp1_mns32", + "tp1_mns64" + ], + "interval": [ + 2.283331783333333, + 2.2833350666666665 + ] + }, + { + "bucket_id": 692893, + "cells": [ + "tp2_mns8", + "tp2_mns16" + ], + "interval": [ + 2.274998683333333, + 2.2750019666666663 + ] + }, + { + "bucket_id": 639593, + "cells": [ + "tp1_mns8" + ], + "interval": [ + 2.0999970166666664, + 2.1000002999999996 + ] + }, + { + "bucket_id": 390862, + "cells": [ + "tp4_mns8" + ], + "interval": [ + 1.2833302333333332, + 1.2833335166666666 + ] + } + ], + "tolerance": 3.283333333333333e-06 + }, + "simulator": { + "buckets": { + "tp1_mns16": 0, + "tp1_mns32": 0, + "tp1_mns64": 0, + "tp1_mns8": 0, + "tp2_mns16": 0, + "tp2_mns32": 0, + "tp2_mns64": 0, + "tp2_mns8": 0, + "tp4_mns16": 0, + "tp4_mns32": 0, + "tp4_mns64": 0, + "tp4_mns8": 0 + }, + "occupied_intervals": [ + { + "bucket_id": 0, + "cells": [ + "tp1_mns8", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp2_mns8", + "tp2_mns16", + "tp2_mns32", + "tp2_mns64", + "tp4_mns8", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "interval": [ + 0.0, + 1e-09 + ] + } + ], + "tolerance": 1e-09 + } + }, + "top1": { + "candidate_cells": [ + "tp1_mns8", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp2_mns8", + "tp2_mns16", + "tp2_mns32", + "tp2_mns64", + "tp4_mns8", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "effective_k": 12, + "optimistic_regret": 0.0, + "worst_case_regret": 0.6091370558375634 + }, + "top5": { + "maximum_exact_five_overlap": 5, + "minimum_exact_five_overlap": 0, + "optimistic_regret": 0.0, + "real_cutoff": { + "boundary": [ + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "choose_from_boundary": 3, + "expanded_effective_k": 5, + "higher": [ + "tp2_mns32", + "tp2_mns64" + ] + }, + "simulator_cutoff": { + "boundary": [ + "tp1_mns8", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp2_mns8", + "tp2_mns16", + "tp2_mns32", + "tp2_mns64", + "tp4_mns8", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "choose_from_boundary": 5, + "expanded_effective_k": 12, + "higher": [] + }, + "worst_case_regret": 0.30456852791878175 + } + }, + "ranking": [ + { + "cell": "tp1_mns16", + "rank": 1, + "score": 0.0 + }, + { + "cell": "tp1_mns32", + "rank": 2, + "score": 0.0 + }, + { + "cell": "tp1_mns64", + "rank": 3, + "score": 0.0 + }, + { + "cell": "tp1_mns8", + "rank": 4, + "score": 0.0 + }, + { + "cell": "tp2_mns16", + "rank": 5, + "score": 0.0 + }, + { + "cell": "tp2_mns32", + "rank": 6, + "score": 0.0 + }, + { + "cell": "tp2_mns64", + "rank": 7, + "score": 0.0 + }, + { + "cell": "tp2_mns8", + "rank": 8, + "score": 0.0 + }, + { + "cell": "tp4_mns16", + "rank": 9, + "score": 0.0 + }, + { + "cell": "tp4_mns32", + "rank": 10, + "score": 0.0 + }, + { + "cell": "tp4_mns64", + "rank": 11, + "score": 0.0 + }, + { + "cell": "tp4_mns8", + "rank": 12, + "score": 0.0 + } + ], + "reading": "SLO-gated", + "simulated_scores": { + "tp1_mns16": 0.0, + "tp1_mns32": 0.0, + "tp1_mns64": 0.0, + "tp1_mns8": 0.0, + "tp2_mns16": 0.0, + "tp2_mns32": 0.0, + "tp2_mns64": 0.0, + "tp2_mns8": 0.0, + "tp4_mns16": 0.0, + "tp4_mns32": 0.0, + "tp4_mns64": 0.0, + "tp4_mns8": 0.0 + } + }, + "throughput-proxy": { + "cell_score_details": { + "tp1_mns16": { + "recorded_anchor_count_after_removal": 8, + "score": 0.8453202347375465, + "selected_probe_index": 1, + "selected_sampling_u": 0.25, + "valid_anchor_count": 8 + }, + "tp1_mns32": { + "recorded_anchor_count_after_removal": 8, + "score": 1.4213857973581445, + "selected_probe_index": 0, + "selected_sampling_u": 0.5, + "valid_anchor_count": 8 + }, + "tp1_mns64": { + "recorded_anchor_count_after_removal": 8, + "score": 2.1437448608949605, + "selected_probe_index": 0, + "selected_sampling_u": 0.5, + "valid_anchor_count": 8 + }, + "tp1_mns8": { + "recorded_anchor_count_after_removal": 8, + "score": 0.5319246206192596, + "selected_probe_index": 1, + "selected_sampling_u": 0.25, + "valid_anchor_count": 8 + }, + "tp2_mns16": { + "recorded_anchor_count_after_removal": 8, + "score": 0.6441840113363596, + "selected_probe_index": 4, + "selected_sampling_u": 0.46875, + "valid_anchor_count": 8 + }, + "tp2_mns32": { + "recorded_anchor_count_after_removal": 8, + "score": 1.093439153724593, + "selected_probe_index": 4, + "selected_sampling_u": 0.78125, + "valid_anchor_count": 8 + }, + "tp2_mns64": { + "recorded_anchor_count_after_removal": 8, + "score": 1.7227463064785613, + "selected_probe_index": 2, + "selected_sampling_u": 0.875, + "valid_anchor_count": 8 + }, + "tp2_mns8": { + "recorded_anchor_count_after_removal": 8, + "score": 0.3900292387118045, + "selected_probe_index": 4, + "selected_sampling_u": 0.46875, + "valid_anchor_count": 8 + }, + "tp4_mns16": { + "recorded_anchor_count_after_removal": 7, + "score": 0.4355571852372527, + "selected_probe_index": 5, + "selected_sampling_u": 0.033182214016, + "valid_anchor_count": 7 + }, + "tp4_mns32": { + "recorded_anchor_count_after_removal": 7, + "score": 0.760215065879823, + "selected_probe_index": 5, + "selected_sampling_u": 0.033182214016, + "valid_anchor_count": 7 + }, + "tp4_mns64": { + "recorded_anchor_count_after_removal": 7, + "score": 1.2313111686958826, + "selected_probe_index": 5, + "selected_sampling_u": 0.033182214016, + "valid_anchor_count": 7 + }, + "tp4_mns8": { + "recorded_anchor_count_after_removal": 7, + "score": 0.25160540206585935, + "selected_probe_index": 0, + "selected_sampling_u": 0.034252608017, + "valid_anchor_count": 7 + } + }, + "loao": { + "defined_replicates": 92, + "range_semantics": "deterministic LOAO sensitivity ranges, not confidence intervals", + "ranges": { + "pairwise_exact_sign_accuracy": { + "max": 0.6515151515151515, + "min": 0.6515151515151515 + }, + "pairwise_non_tied_concordance": { + "max": 0.7049180327868853, + "min": 0.7049180327868853 + }, + "tau_b": { + "max": 0.39400626128204935, + "min": 0.39400626128204935 + }, + "top1_optimistic_regret": { + "max": 0.30456852791878175, + "min": 0.30456852791878175 + }, + "top1_worst_case_regret": { + "max": 0.30456852791878175, + "min": 0.30456852791878175 + }, + "top5_maximum_exact_five_overlap": { + "max": 3, + "min": 3 + }, + "top5_minimum_exact_five_overlap": { + "max": 3, + "min": 3 + }, + "top5_optimistic_regret": { + "max": 0.0, + "min": 0.0 + }, + "top5_worst_case_regret": { + "max": 0.0, + "min": 0.0 + } + }, + "replicate_count": 92, + "replicates": [ + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns8", + "removed_probe_index": 0, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns8", + "removed_probe_index": 1, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns8", + "removed_probe_index": 2, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns8", + "removed_probe_index": 3, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns8", + "removed_probe_index": 4, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns8", + "removed_probe_index": 5, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns8", + "removed_probe_index": 6, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns8", + "removed_probe_index": 7, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns16", + "removed_probe_index": 0, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns16", + "removed_probe_index": 1, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns16", + "removed_probe_index": 2, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns16", + "removed_probe_index": 3, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns16", + "removed_probe_index": 4, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns16", + "removed_probe_index": 5, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns16", + "removed_probe_index": 6, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns16", + "removed_probe_index": 7, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns32", + "removed_probe_index": 0, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns32", + "removed_probe_index": 1, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns32", + "removed_probe_index": 2, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns32", + "removed_probe_index": 3, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns32", + "removed_probe_index": 4, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns32", + "removed_probe_index": 5, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns32", + "removed_probe_index": 6, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns32", + "removed_probe_index": 7, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns64", + "removed_probe_index": 0, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns64", + "removed_probe_index": 1, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns64", + "removed_probe_index": 2, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns64", + "removed_probe_index": 3, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns64", + "removed_probe_index": 4, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns64", + "removed_probe_index": 5, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns64", + "removed_probe_index": 6, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp1_mns64", + "removed_probe_index": 7, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns8", + "removed_probe_index": 0, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns8", + "removed_probe_index": 1, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns8", + "removed_probe_index": 2, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns8", + "removed_probe_index": 3, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns8", + "removed_probe_index": 4, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns8", + "removed_probe_index": 5, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns8", + "removed_probe_index": 6, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns8", + "removed_probe_index": 7, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns16", + "removed_probe_index": 0, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns16", + "removed_probe_index": 1, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns16", + "removed_probe_index": 2, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns16", + "removed_probe_index": 3, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns16", + "removed_probe_index": 4, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns16", + "removed_probe_index": 5, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns16", + "removed_probe_index": 6, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns16", + "removed_probe_index": 7, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns32", + "removed_probe_index": 0, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns32", + "removed_probe_index": 1, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns32", + "removed_probe_index": 2, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns32", + "removed_probe_index": 3, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns32", + "removed_probe_index": 4, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns32", + "removed_probe_index": 5, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns32", + "removed_probe_index": 6, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns32", + "removed_probe_index": 7, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns64", + "removed_probe_index": 0, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns64", + "removed_probe_index": 1, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns64", + "removed_probe_index": 2, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns64", + "removed_probe_index": 3, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns64", + "removed_probe_index": 4, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns64", + "removed_probe_index": 5, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns64", + "removed_probe_index": 6, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp2_mns64", + "removed_probe_index": 7, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns8", + "removed_probe_index": 0, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns8", + "removed_probe_index": 1, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns8", + "removed_probe_index": 2, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns8", + "removed_probe_index": 3, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns8", + "removed_probe_index": 4, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns8", + "removed_probe_index": 5, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns8", + "removed_probe_index": 6, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns16", + "removed_probe_index": 0, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns16", + "removed_probe_index": 1, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns16", + "removed_probe_index": 2, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns16", + "removed_probe_index": 3, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns16", + "removed_probe_index": 4, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns16", + "removed_probe_index": 5, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns16", + "removed_probe_index": 6, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns32", + "removed_probe_index": 0, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns32", + "removed_probe_index": 1, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns32", + "removed_probe_index": 2, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns32", + "removed_probe_index": 3, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns32", + "removed_probe_index": 4, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns32", + "removed_probe_index": 5, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns32", + "removed_probe_index": 6, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns64", + "removed_probe_index": 0, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns64", + "removed_probe_index": 1, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns64", + "removed_probe_index": 2, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns64", + "removed_probe_index": 3, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns64", + "removed_probe_index": 4, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns64", + "removed_probe_index": 5, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + { + "pairwise_exact_sign_accuracy": 0.6515151515151515, + "pairwise_non_tied_concordance": 0.7049180327868853, + "removed_cell_id": "tp4_mns64", + "removed_probe_index": 6, + "tau_b": 0.39400626128204935, + "top1_optimistic_regret": 0.30456852791878175, + "top1_worst_case_regret": 0.30456852791878175, + "top5_maximum_exact_five_overlap": 3, + "top5_minimum_exact_five_overlap": 3, + "top5_optimistic_regret": 0.0, + "top5_worst_case_regret": 0.0, + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + } + ], + "tp2_mns32_unique_global_best_count": 0, + "trap_reproduced_count": 0, + "undefined": [], + "undefined_replicates": 0 + }, + "metrics": { + "kendall_tau_b": { + "both_tied": 0, + "concordant": 43, + "discordant": 18, + "pair_count": 66, + "real_only_tied": 5, + "sim_only_tied": 0, + "tau_b": 0.39400626128204935 + }, + "named_interactions": { + "relations": { + "tp2_mns32_gt_tp4_mns16": true, + "tp4_mns16_eq_tp4_mns32": false, + "tp4_mns16_eq_tp4_mns64": false, + "tp4_mns16_gt_tp1_mns16": false, + "tp4_mns16_gt_tp2_mns16": false, + "tp4_mns16_gt_tp4_mns8": true + }, + "simulator_top_bucket_cells": [ + "tp1_mns64" + ], + "tp2_mns32_unique_global_best": false, + "trap_reproduced": false + }, + "pairwise_direction": { + "exact_sign_accuracy": 0.6515151515151515, + "exact_sign_agreement": 43, + "non_tied_concordance": 0.7049180327868853, + "non_tied_concordant": 43, + "non_tied_pair_count": 61, + "pair_count": 66 + }, + "per_mns_tp_direction": { + "accuracy": 0.5, + "comparisons": 8, + "exact_sign_agreement": 4, + "lines": [ + { + "comparisons": 2, + "exact_sign_agreement": 1, + "line_class_agrees": false, + "mns": 8, + "real_adjacent_signs": [ + 1, + -1 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + -1, + -1 + ], + "sim_line_class": "decreasing" + }, + { + "comparisons": 2, + "exact_sign_agreement": 1, + "line_class_agrees": false, + "mns": 16, + "real_adjacent_signs": [ + -1, + 1 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + -1, + -1 + ], + "sim_line_class": "decreasing" + }, + { + "comparisons": 2, + "exact_sign_agreement": 1, + "line_class_agrees": false, + "mns": 32, + "real_adjacent_signs": [ + 1, + -1 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + -1, + -1 + ], + "sim_line_class": "decreasing" + }, + { + "comparisons": 2, + "exact_sign_agreement": 1, + "line_class_agrees": false, + "mns": 64, + "real_adjacent_signs": [ + 1, + -1 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + -1, + -1 + ], + "sim_line_class": "decreasing" + } + ] + }, + "per_tp_mns_direction": { + "accuracy": 0.3333333333333333, + "comparisons": 9, + "exact_sign_agreement": 3, + "lines": [ + { + "comparisons": 3, + "exact_sign_agreement": 1, + "line_class_agrees": false, + "real_adjacent_signs": [ + 1, + -1, + 0 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + 1, + 1, + 1 + ], + "sim_line_class": "increasing", + "tp": 1 + }, + { + "comparisons": 3, + "exact_sign_agreement": 1, + "line_class_agrees": false, + "real_adjacent_signs": [ + 0, + 1, + -1 + ], + "real_line_class": "non-monotonic", + "sim_adjacent_signs": [ + 1, + 1, + 1 + ], + "sim_line_class": "increasing", + "tp": 2 + }, + { + "comparisons": 3, + "exact_sign_agreement": 1, + "line_class_agrees": true, + "real_adjacent_signs": [ + 1, + 0, + 0 + ], + "real_line_class": "increasing", + "sim_adjacent_signs": [ + 1, + 1, + 1 + ], + "sim_line_class": "increasing", + "tp": 4 + } + ] + }, + "tie_buckets": { + "real": { + "buckets": { + "tp1_mns16": 715736, + "tp1_mns32": 695431, + "tp1_mns64": 695431, + "tp1_mns8": 639593, + "tp2_mns16": 692893, + "tp2_mns32": 1000000, + "tp2_mns64": 992385, + "tp2_mns8": 692893, + "tp4_mns16": 743654, + "tp4_mns32": 743654, + "tp4_mns64": 743654, + "tp4_mns8": 390862 + }, + "occupied_intervals": [ + { + "bucket_id": 1000000, + "cells": [ + "tp2_mns32" + ], + "interval": [ + 3.2833333333333328, + 3.2833366166666664 + ] + }, + { + "bucket_id": 992385, + "cells": [ + "tp2_mns64" + ], + "interval": [ + 3.25833075, + 3.258334033333333 + ] + }, + { + "bucket_id": 743654, + "cells": [ + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "interval": [ + 2.4416639666666664, + 2.4416672499999996 + ] + }, + { + "bucket_id": 715736, + "cells": [ + "tp1_mns16" + ], + "interval": [ + 2.3499998666666664, + 2.3500031499999996 + ] + }, + { + "bucket_id": 695431, + "cells": [ + "tp1_mns32", + "tp1_mns64" + ], + "interval": [ + 2.283331783333333, + 2.2833350666666665 + ] + }, + { + "bucket_id": 692893, + "cells": [ + "tp2_mns8", + "tp2_mns16" + ], + "interval": [ + 2.274998683333333, + 2.2750019666666663 + ] + }, + { + "bucket_id": 639593, + "cells": [ + "tp1_mns8" + ], + "interval": [ + 2.0999970166666664, + 2.1000002999999996 + ] + }, + { + "bucket_id": 390862, + "cells": [ + "tp4_mns8" + ], + "interval": [ + 1.2833302333333332, + 1.2833335166666666 + ] + } + ], + "tolerance": 3.283333333333333e-06 + }, + "simulator": { + "buckets": { + "tp1_mns16": 394319, + "tp1_mns32": 663038, + "tp1_mns64": 1000000, + "tp1_mns8": 248128, + "tp2_mns16": 300494, + "tp2_mns32": 510060, + "tp2_mns64": 803615, + "tp2_mns8": 181938, + "tp4_mns16": 203175, + "tp4_mns32": 354620, + "tp4_mns64": 574373, + "tp4_mns8": 117367 + }, + "occupied_intervals": [ + { + "bucket_id": 1000000, + "cells": [ + "tp1_mns64" + ], + "interval": [ + 2.1437448608949605, + 2.1437470046398217 + ] + }, + { + "bucket_id": 803615, + "cells": [ + "tp2_mns64" + ], + "interval": [ + 1.7227455263881037, + 1.7227476701329647 + ] + }, + { + "bucket_id": 663038, + "cells": [ + "tp1_mns32" + ], + "interval": [ + 1.421384305078073, + 1.4213864488229337 + ] + }, + { + "bucket_id": 574373, + "cells": [ + "tp4_mns64" + ], + "interval": [ + 1.2313091669868212, + 1.2313113107316822 + ] + }, + { + "bucket_id": 510060, + "cells": [ + "tp2_mns32" + ], + "interval": [ + 1.0934385037480836, + 1.0934406474929446 + ] + }, + { + "bucket_id": 394319, + "cells": [ + "tp1_mns16" + ], + "interval": [ + 0.84531932980324, + 0.8453214735481008 + ] + }, + { + "bucket_id": 354620, + "cells": [ + "tp4_mns32" + ], + "interval": [ + 0.760214802570571, + 0.7602169463154318 + ] + }, + { + "bucket_id": 300494, + "cells": [ + "tp2_mns16" + ], + "interval": [ + 0.6441824682297703, + 0.6441846119746312 + ] + }, + { + "bucket_id": 248128, + "cells": [ + "tp1_mns8" + ], + "interval": [ + 0.5319231248441448, + 0.5319252685890057 + ] + }, + { + "bucket_id": 203175, + "cells": [ + "tp4_mns16" + ], + "interval": [ + 0.43555536211233364, + 0.4355575058571945 + ] + }, + { + "bucket_id": 181938, + "cells": [ + "tp2_mns8" + ], + "interval": [ + 0.39002865250150737, + 0.3900307962463682 + ] + }, + { + "bucket_id": 117367, + "cells": [ + "tp4_mns8" + ], + "interval": [ + 0.25160490308865885, + 0.25160704683351975 + ] + } + ], + "tolerance": 2.1437448608949606e-06 + } + }, + "top1": { + "candidate_cells": [ + "tp1_mns64" + ], + "effective_k": 1, + "optimistic_regret": 0.30456852791878175, + "worst_case_regret": 0.30456852791878175 + }, + "top5": { + "maximum_exact_five_overlap": 3, + "minimum_exact_five_overlap": 3, + "optimistic_regret": 0.0, + "real_cutoff": { + "boundary": [ + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "choose_from_boundary": 3, + "expanded_effective_k": 5, + "higher": [ + "tp2_mns32", + "tp2_mns64" + ] + }, + "simulator_cutoff": { + "boundary": [ + "tp2_mns32" + ], + "choose_from_boundary": 1, + "expanded_effective_k": 5, + "higher": [ + "tp1_mns64", + "tp2_mns64", + "tp1_mns32", + "tp4_mns64" + ] + }, + "worst_case_regret": 0.0 + } + }, + "ranking": [ + { + "cell": "tp1_mns64", + "rank": 1, + "score": 2.1437448608949605 + }, + { + "cell": "tp2_mns64", + "rank": 2, + "score": 1.7227463064785613 + }, + { + "cell": "tp1_mns32", + "rank": 3, + "score": 1.4213857973581445 + }, + { + "cell": "tp4_mns64", + "rank": 4, + "score": 1.2313111686958826 + }, + { + "cell": "tp2_mns32", + "rank": 5, + "score": 1.093439153724593 + }, + { + "cell": "tp1_mns16", + "rank": 6, + "score": 0.8453202347375465 + }, + { + "cell": "tp4_mns32", + "rank": 7, + "score": 0.760215065879823 + }, + { + "cell": "tp2_mns16", + "rank": 8, + "score": 0.6441840113363596 + }, + { + "cell": "tp1_mns8", + "rank": 9, + "score": 0.5319246206192596 + }, + { + "cell": "tp4_mns16", + "rank": 10, + "score": 0.4355571852372527 + }, + { + "cell": "tp2_mns8", + "rank": 11, + "score": 0.3900292387118045 + }, + { + "cell": "tp4_mns8", + "rank": 12, + "score": 0.25160540206585935 + } + ], + "reading": "throughput-proxy", + "simulated_scores": { + "tp1_mns16": 0.8453202347375465, + "tp1_mns32": 1.4213857973581445, + "tp1_mns64": 2.1437448608949605, + "tp1_mns8": 0.5319246206192596, + "tp2_mns16": 0.6441840113363596, + "tp2_mns32": 1.093439153724593, + "tp2_mns64": 1.7227463064785613, + "tp2_mns8": 0.3900292387118045, + "tp4_mns16": 0.4355571852372527, + "tp4_mns32": 0.760215065879823, + "tp4_mns64": 1.2313111686958826, + "tp4_mns8": 0.25160540206585935 + } + } + } +} diff --git a/runs/frontier-qwen30-vllm020-profile-v1/artifacts/trace-support/support.json b/runs/frontier-qwen30-vllm020-profile-v1/artifacts/trace-support/support.json new file mode 100644 index 0000000..5bd1e82 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/artifacts/trace-support/support.json @@ -0,0 +1,469 @@ +{ + "block_size": 64, + "cohort_output": "/home/admin/cpfs/wjh/frontier_qwen30_vllm020_profiles/trace-support/cohort.jsonl", + "cohort_sha256": "9ef15a4c8843127a5c52ff8455bdeb9945ac6921f2977dd11070ed6944eed7c8", + "full_downsampled_cohort": { + "hash_blocks_per_request": { + "max": 128.0, + "min": 2.0, + "p10": 4.0, + "p25": 15.0, + "p50": 38.0, + "p75": 82.0, + "p90": 111.0, + "p95": 122.0, + "p99": 127.0 + }, + "input_tokens": { + "max": 8186.0, + "min": 71.0, + "p10": 208.0, + "p25": 933.0, + "p50": 2421.0, + "p75": 5244.0, + "p90": 7103.0, + "p95": 7775.0, + "p99": 8074.0 + }, + "interarrival_s": { + "max": 0.20259999999998257, + "min": 0.06279999999997088, + "p10": 0.0917999999999708, + "p25": 0.10240000000003846, + "p50": 0.1163999999999703, + "p75": 0.13009999999999167, + "p90": 0.14210000000002765, + "p95": 0.15119999999997447, + "p99": 0.17959999999997933 + }, + "multi_turn_fraction": 0.140625, + "output_tokens": { + "max": 128.0, + "min": 128.0, + "p10": 128.0, + "p25": 128.0, + "p50": 128.0, + "p75": 128.0, + "p90": 128.0, + "p95": 128.0, + "p99": 128.0 + }, + "prefix_reuse_upper_bound": { + "any_position_reusable_block_ratio": 0.09274807669777795, + "leading_reusable_blocks_per_request": { + "max": 55.0, + "min": 0.0, + "p10": 0.0, + "p25": 0.0, + "p50": 1.0, + "p75": 13.0, + "p90": 13.0, + "p95": 13.0, + "p99": 49.0 + }, + "leading_reusable_tokens_per_request": { + "max": 3520.0, + "min": 0.0, + "p10": 0.0, + "p25": 0.0, + "p50": 64.0, + "p75": 832.0, + "p90": 832.0, + "p95": 832.0, + "p99": 3136.0 + }, + "rows_with_hash_ids": 512, + "semantics": "arrival-ordered, infinite-capacity/no-eviction upper bound; not an observed KV-cache hit rate", + "total_blocks": 25607, + "unique_blocks": 23232 + }, + "request_count": 512, + "sampling_u": { + "max": 0.9979913161468553, + "min": 0.0017297900650431776, + "p10": 0.10365756051317851, + "p25": 0.22877823992809823, + "p50": 0.4667483893139273, + "p75": 0.7338360613813898, + "p90": 0.8969698510725516, + "p95": 0.9558957048665733, + "p99": 0.991130680217395 + } + }, + "limits": [ + "Trace length/hash support does not determine dynamic decode or mixed batch shapes.", + "Those shapes depend jointly on arrival history, SLO pressure, TP execution time, MNS, chunking, and KV eviction.", + "MoE expert routing is not present in the trace and must be measured from model execution." + ], + "loader_contract": { + "completion_tokens_override": 128, + "downsampling": "AITuner _downsample_requests before sampling_u threshold", + "input_length_filter": { + "max": 8192, + "min": 0 + }, + "max_requests_per_probe": 512, + "ordering": "arrival_s", + "replay_time_scale": 0.1 + }, + "schema_version": "qwen30_trace_profile_support.v1", + "study": "/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/configs/examples/dash0_qwen30b_a3b_community_vllm020_noharness.json", + "study_sha256": "cd0b9e2385751478f459faeae2cf3d12c58229866176911051421bc043bf99d4", + "threshold_cohorts": { + "0.125": { + "hash_blocks_per_request": { + "max": 127.0, + "min": 2.0, + "p10": 7.0, + "p25": 15.0, + "p50": 30.0, + "p75": 76.0, + "p90": 105.0, + "p95": 111.0, + "p99": 127.0 + }, + "input_tokens": { + "max": 8110.0, + "min": 74.0, + "p10": 392.0, + "p25": 922.0, + "p50": 1864.0, + "p75": 4844.0, + "p90": 6686.0, + "p95": 7054.0, + "p99": 8110.0 + }, + "interarrival_s": { + "max": 6.217200000000002, + "min": 0.10639999999998517, + "p10": 0.16680000000001272, + "p25": 0.30810000000001736, + "p50": 0.6001999999999938, + "p75": 1.1201000000000016, + "p90": 1.8216999999999643, + "p95": 2.419699999999967, + "p99": 6.217200000000002 + }, + "multi_turn_fraction": 0.15151515151515152, + "output_tokens": { + "max": 128.0, + "min": 128.0, + "p10": 128.0, + "p25": 128.0, + "p50": 128.0, + "p75": 128.0, + "p90": 128.0, + "p95": 128.0, + "p99": 128.0 + }, + "prefix_reuse_upper_bound": { + "any_position_reusable_block_ratio": 0.10466666666666667, + "leading_reusable_blocks_per_request": { + "max": 13.0, + "min": 0.0, + "p10": 0.0, + "p25": 0.0, + "p50": 1.0, + "p75": 13.0, + "p90": 13.0, + "p95": 13.0, + "p99": 13.0 + }, + "leading_reusable_tokens_per_request": { + "max": 832.0, + "min": 0.0, + "p10": 0.0, + "p25": 0.0, + "p50": 64.0, + "p75": 832.0, + "p90": 832.0, + "p95": 832.0, + "p99": 832.0 + }, + "rows_with_hash_ids": 66, + "semantics": "arrival-ordered, infinite-capacity/no-eviction upper bound; not an observed KV-cache hit rate", + "total_blocks": 3000, + "unique_blocks": 2686 + }, + "request_count": 66, + "sampling_u": { + "max": 0.12321850600323363, + "min": 0.0017297900650431776, + "p10": 0.01599560440427848, + "p25": 0.03636425746001375, + "p50": 0.06451196858246099, + "p75": 0.10162658487005978, + "p90": 0.1138971596689021, + "p95": 0.12001874701158237, + "p99": 0.12321850600323363 + } + }, + "0.25": { + "hash_blocks_per_request": { + "max": 128.0, + "min": 2.0, + "p10": 4.0, + "p25": 14.0, + "p50": 34.0, + "p75": 87.0, + "p90": 107.0, + "p95": 122.0, + "p99": 127.0 + }, + "input_tokens": { + "max": 8149.0, + "min": 72.0, + "p10": 197.0, + "p25": 867.0, + "p50": 2167.0, + "p75": 5568.0, + "p90": 6841.0, + "p95": 7745.0, + "p99": 8110.0 + }, + "interarrival_s": { + "max": 2.3110999999999855, + "min": 0.07340000000003499, + "p10": 0.11439999999997497, + "p25": 0.14210000000002765, + "p50": 0.3411000000000044, + "p75": 0.5650000000000093, + "p90": 0.8215000000000146, + "p95": 1.088799999999992, + "p99": 1.7577999999999996 + }, + "multi_turn_fraction": 0.11888111888111888, + "output_tokens": { + "max": 128.0, + "min": 128.0, + "p10": 128.0, + "p25": 128.0, + "p50": 128.0, + "p75": 128.0, + "p90": 128.0, + "p95": 128.0, + "p99": 128.0 + }, + "prefix_reuse_upper_bound": { + "any_position_reusable_block_ratio": 0.07953890489913544, + "leading_reusable_blocks_per_request": { + "max": 13.0, + "min": 0.0, + "p10": 0.0, + "p25": 0.0, + "p50": 1.0, + "p75": 13.0, + "p90": 13.0, + "p95": 13.0, + "p99": 13.0 + }, + "leading_reusable_tokens_per_request": { + "max": 832.0, + "min": 0.0, + "p10": 0.0, + "p25": 0.0, + "p50": 64.0, + "p75": 832.0, + "p90": 832.0, + "p95": 832.0, + "p99": 832.0 + }, + "rows_with_hash_ids": 143, + "semantics": "arrival-ordered, infinite-capacity/no-eviction upper bound; not an observed KV-cache hit rate", + "total_blocks": 6940, + "unique_blocks": 6388 + }, + "request_count": 143, + "sampling_u": { + "max": 0.24941730181025415, + "min": 0.0017297900650431776, + "p10": 0.03095529459918134, + "p25": 0.07431293234041883, + "p50": 0.13301201741611407, + "p75": 0.1994148366998705, + "p90": 0.2292686392189136, + "p95": 0.24107176388894497, + "p99": 0.24671643798343437 + } + }, + "0.5": { + "hash_blocks_per_request": { + "max": 128.0, + "min": 2.0, + "p10": 2.0, + "p25": 15.0, + "p50": 42.0, + "p75": 86.0, + "p90": 109.0, + "p95": 117.0, + "p99": 127.0 + }, + "input_tokens": { + "max": 8149.0, + "min": 71.0, + "p10": 113.0, + "p25": 905.0, + "p50": 2648.0, + "p75": 5490.0, + "p90": 6926.0, + "p95": 7476.0, + "p99": 8074.0 + }, + "interarrival_s": { + "max": 1.2561000000000142, + "min": 0.06749999999997236, + "p10": 0.10019999999999563, + "p25": 0.11840000000002249, + "p50": 0.15819999999994394, + "p75": 0.2599999999999483, + "p90": 0.3867999999999938, + "p95": 0.5083999999999804, + "p99": 0.696700000000007 + }, + "multi_turn_fraction": 0.14130434782608695, + "output_tokens": { + "max": 128.0, + "min": 128.0, + "p10": 128.0, + "p25": 128.0, + "p50": 128.0, + "p75": 128.0, + "p90": 128.0, + "p95": 128.0, + "p99": 128.0 + }, + "prefix_reuse_upper_bound": { + "any_position_reusable_block_ratio": 0.08318264014466546, + "leading_reusable_blocks_per_request": { + "max": 52.0, + "min": 0.0, + "p10": 0.0, + "p25": 0.0, + "p50": 1.0, + "p75": 13.0, + "p90": 13.0, + "p95": 13.0, + "p99": 24.0 + }, + "leading_reusable_tokens_per_request": { + "max": 3328.0, + "min": 0.0, + "p10": 0.0, + "p25": 0.0, + "p50": 64.0, + "p75": 832.0, + "p90": 832.0, + "p95": 832.0, + "p99": 1536.0 + }, + "rows_with_hash_ids": 276, + "semantics": "arrival-ordered, infinite-capacity/no-eviction upper bound; not an observed KV-cache hit rate", + "total_blocks": 13825, + "unique_blocks": 12675 + }, + "request_count": 276, + "sampling_u": { + "max": 0.49832600818075473, + "min": 0.0017297900650431776, + "p10": 0.056494245471656455, + "p25": 0.1313132001019981, + "p50": 0.24403265916374703, + "p75": 0.3706937275751198, + "p90": 0.4530058956953501, + "p95": 0.48171254599865826, + "p99": 0.49639910750753047 + } + }, + "1.0": { + "hash_blocks_per_request": { + "max": 128.0, + "min": 2.0, + "p10": 4.0, + "p25": 15.0, + "p50": 38.0, + "p75": 82.0, + "p90": 111.0, + "p95": 122.0, + "p99": 127.0 + }, + "input_tokens": { + "max": 8186.0, + "min": 71.0, + "p10": 208.0, + "p25": 933.0, + "p50": 2421.0, + "p75": 5244.0, + "p90": 7103.0, + "p95": 7775.0, + "p99": 8074.0 + }, + "interarrival_s": { + "max": 0.20259999999998257, + "min": 0.06279999999997088, + "p10": 0.0917999999999708, + "p25": 0.10240000000003846, + "p50": 0.1163999999999703, + "p75": 0.13009999999999167, + "p90": 0.14210000000002765, + "p95": 0.15119999999997447, + "p99": 0.17959999999997933 + }, + "multi_turn_fraction": 0.140625, + "output_tokens": { + "max": 128.0, + "min": 128.0, + "p10": 128.0, + "p25": 128.0, + "p50": 128.0, + "p75": 128.0, + "p90": 128.0, + "p95": 128.0, + "p99": 128.0 + }, + "prefix_reuse_upper_bound": { + "any_position_reusable_block_ratio": 0.09274807669777795, + "leading_reusable_blocks_per_request": { + "max": 55.0, + "min": 0.0, + "p10": 0.0, + "p25": 0.0, + "p50": 1.0, + "p75": 13.0, + "p90": 13.0, + "p95": 13.0, + "p99": 49.0 + }, + "leading_reusable_tokens_per_request": { + "max": 3520.0, + "min": 0.0, + "p10": 0.0, + "p25": 0.0, + "p50": 64.0, + "p75": 832.0, + "p90": 832.0, + "p95": 832.0, + "p99": 3136.0 + }, + "rows_with_hash_ids": 512, + "semantics": "arrival-ordered, infinite-capacity/no-eviction upper bound; not an observed KV-cache hit rate", + "total_blocks": 25607, + "unique_blocks": 23232 + }, + "request_count": 512, + "sampling_u": { + "max": 0.9979913161468553, + "min": 0.0017297900650431776, + "p10": 0.10365756051317851, + "p25": 0.22877823992809823, + "p50": 0.4667483893139273, + "p75": 0.7338360613813898, + "p90": 0.8969698510725516, + "p95": 0.9558957048665733, + "p99": 0.991130680217395 + } + } + }, + "trace": "/home/admin/cpfs/wjh/aituner/aituner/trace_windows/traces/chat_w20260311_1000.jsonl", + "trace_sha256": "f539f38eb0ee0f750e3c23ff47df6eed3faf723a25f1444d55665a85871750b9", + "window_id": "chat_w20260311_1000" +} diff --git a/runs/frontier-qwen30-vllm020-profile-v1/experiment-card.md b/runs/frontier-qwen30-vllm020-profile-v1/experiment-card.md index a51365c..4f38358 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/experiment-card.md +++ b/runs/frontier-qwen30-vllm020-profile-v1/experiment-card.md @@ -1,6 +1,6 @@ # 实验 EXP-SIMFID-Q30-P020：同栈 per-TP operator profile 能否恢复 Frontier ranking？ -> **状态：** 运行中（用户已批准 5-step campaign） +> **状态：** 完成；H1 rejected，结果支持 H2（2026-07-16） ## Claim 与决策 @@ -22,8 +22,8 @@ ## 预期产物与 review -- **预期数据：** TP1/2/4 attention、KV-cache update、linear、MoE、collective raw profiles；冻结 manifest/sha256；12-cell simulation outputs；三 baseline comparison table。 -- **Figure prototype：** `mock-profile-ablation.png`；x 为 12 configs，y 为 normalized capacity；series 为 real、old profile-only、new profile-only、calibrated。图中数值明确标为 mock，不进入结论。 +- **实际数据：** TP1/2/4 attention、KV-cache update、linear、MoE、collective raw profiles；冻结 manifest/sha256；92/92 new profile-only runs；old/new/calibrated comparison；scheduler graph-mode 与 native MoE routing diagnostics。 +- **Figure：** `../../docs/assets/simulator-fidelity/qwen30-vllm020-profile-ablation.png`；四个 panels 分别展示完整 ranking、routing skew、phase-dependent graph mode 和 held-out P1 boundary accuracy。图中只使用实测或冻结分析结果。 - **人工 review：** 已批准。用户明确要求推进 5 个步骤，并要求 smoke 通过后直接完整运行。 - **Review 意见：** 不能把 TP–MNS 耦合拆成互不相关问题；profile 只冻结 execution counterfactual 的证据，最终仍以完整 config ranking 判断。 @@ -32,12 +32,27 @@ - **Code：** AITuner 当前 branch `codex/fidelity-prefix-pilot-20260714`；Frontier canonical commit `d9cfeb6d8791fbf2f295dd9744c56a666171776e`；vLLM commit/tag `88d34c6409e9fb3c7b8ca0c04756f061d2099eb1` / `0.20.0`。 - **Environment：** `/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1`；hardware/driver/package freeze 写入每个 profile artifact。 - **产物路径：** remote `/home/admin/cpfs/wjh/frontier_qwen30_vllm020_profiles/`; local harvest `runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/`。 -- **已知 deviation：** 历史 real ground truth 来自 dash1；本次 profile 按用户要求在 dash0 生成。driver 也已从历史 570.133.20 漂移到 dash0 当前 580.95.05。结果只能归因于“profile stack alignment 在当前 dash0 上的效果”，不能声称严格复现旧 dash1 execution latency。 +- **已知 deviation：** 历史 real ground truth 与本次 profile 都来自 dash0；早期文档中的主机 provenance 标注错误。driver 从历史 570.133.20 漂移到当前 580.95.05，因此结果仍不能解释为严格的逐 kernel latency 复现，只能作为同一 H20 主机上 profile-stack alignment 的对照。 ## 结果 -- **观察事实：** 待完整 profile 与 simulation。 -- **异常：** Frontier 原生 attention profiler 只实现 `FLASHINFER`/`NO_OP`，不能调用真实 serving 的 `FLASH_ATTN`；其 MoE wrapper声明并硬编码 vLLM 0.10.x API。vLLM 0.20.0 自带的 attention benchmark可调用实际 FlashAttention backend，但 mixed prefill+decode 在 FA3 中是一个 fused varlen call，而 Frontier schema将其拆成 `attn_prefill` 与 `attn_decode` 两项。 -- **Interpretation 与剩余 alternatives：** 这既是 profiler compatibility gap，也是潜在 representational gap。先用真实 kernel smoke 判断能否无损物化 Frontier profile；不能无损时同时保留 fused total 与 schema projection，避免误把 projection error 当 kernel error。 -- **Claim update：** unchanged。 -- **下一步：** 通过 vLLM 0.20 exact-kernel smoke；冻结 trace-derived shape support；完整 TP1/2/4 profiling；profile-only rerun。 +- **Step 1—同栈固定：** dash0 8×H20；Qwen3-30B-A3B BF16；community vLLM `0.20.0+cu129` / source commit `88d34c6409e9fb3c7b8ca0c04756f061d2099eb1`。真实 serving 与 profiles 使用同一主机和 model family；历史 driver 570.133.20 到当前 580.95.05 的漂移保留为限制。 +- **Step 2—per-TP profiles：** profile-v2 冻结 attention 132 rows、直接测量 fused-mixed diagnostic 30 rows、linear 36 rows、MoE 72 rows、all-reduce 24 rows。attention/linear/MoE 通过单 H20 local-shard shape 表达 TP；all-reduce 使用真实 TP2/4 ranks。base comparison 保持 historical analytical CC backend，measured all-reduce 仅作独立诊断。 +- **Step 3—trace execution coverage：** 6 个 P1 cells 共 161,161 scheduler/model steps：pure decode 151,471（100% FULL graph），pure prefill 224（192 NONE、30 PIECEWISE、2 FULL），true mixed 9,466（8,623 NONE、842 PIECEWISE、1 FULL）。exact-trace 8-request native routing probe 表明实际 median max/mean expert load 为 prefill 6.12、decode 5.64，而 Frontier fixed prior 为 1.84；actual-vs-prior Pearson median 接近 0。该小 cohort 用于机制否证，不作为 routing population estimate。 +- **Step 4—冻结 profile-only：** profile-v1 因 true-mixed schema 缺少 `attn_decode_in_mixed` 在第一个 mixed batch fail。profile-v2 使用 same-TP pure prefill/decode ratio 做 total-conserving compatibility split，同时保留直接实测 fused total。两个 46-run CPU shards 均 PASS，92/92 runs 无 crash，总 simulator runtime 2,394.9 s（0.665 CPU-hour）。 +- **Step 5—ranking comparison：** real optimum 为 TP2/MNS32。old profile-only top set 为 TP4/MNS32/64，τ-b=0、worst regret=25.63%；new vLLM 0.20 profile-only 在所有 92 anchors 上都 SLO-infeasible，12 configs 全并列，τ-b=0、exact pair sign=7.58%、worst tie-break regret=60.91%；historical frozen per-TP calibration top set 为 TP2/MNS32/64，τ-b=0.9668、worst regret=0.76%。在 held-out P1 12 boundary labels 上，new profile-only accuracy 41.67%、pass-rate MAE 69.84%，也没有优于 old profile-only；calibration 分别为 83.33% 和 11.75%。 +- **Same-state delta：** 在 `tp1_mns64` 的 batch 0/layer 0、profile-dependent trajectories 尚未分叉时，new/old layer component sum 已为 2.70×；MoE grouped GEMM 3.76×、RoPE 7.39×、attention prefill 2.67×。误差不是可由一个 version correction 解释的一致 scale。 +- **异常与 schema gap：** Frontier 原生 attention profiler 只实现 `FLASHINFER`/`NO_OP`，不能调用真实 serving 的 `FLASH_ATTN`；其 MoE wrapper声明并硬编码 vLLM 0.10.x API。vLLM 0.20.0 的 FA3 mixed prefill+decode 是一个 fused varlen call，而 Frontier schema要求 `attn_prefill` 和 `attn_decode_in_mixed` 两项。兼容 split 是归因，不是观测。 +- **Interpretation：** H1 被否证。同栈 isolated operator profile 不等价于真实 execution counterfactual。TP 与 MNS 共同改变 local shape、collective、queue/batch composition、phase mixture、routing 和 CUDA graph regime；static operator sum 在 scheduler 推进后产生 feedback amplification。历史 per-TP calibration 是 composition 之后的外部 E2E scale，能吸收这些残差，但不是 Frontier 原生 per-TP profiling 或机制解释。 +- **Claim update：** 支持 H2。新的 research target 是验证 execution-context-conditioned composition，而不是继续把更多 isolated kernel rows 当作 fidelity 的充分条件。 +- **下一步：** 分别做 graph-conditioned step profile、fused mixed total、trace-conditioned routing 和 measured collective 四个单变量消融，再回到完整 TP×MNS joint surface 评测；frozen calibration 仅作上界，不参与 mechanism fitting。 + +## 产物索引 + +- Frozen profile：`frozen/profile-v2/manifest.json` +- Full-surface comparison：`artifacts/qwen30-s2-profile-ablation.json` +- P1 boundary comparison：`artifacts/qwen30-p1-profile-ablation.json` +- Scheduler/graph-mode summary：`artifacts/qwen30-p1b-opprof-summary.json` +- Native routing mismatch：`artifacts/qwen30-routing-mismatch.json` +- Same-state operator delta：`artifacts/qwen30-initial-op-trace-delta.json` +- Figure script：`plot_profile_ablation.py` diff --git a/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/allreduce.json b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/allreduce.json new file mode 100644 index 0000000..aae1f0a --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/allreduce.json @@ -0,0 +1,824 @@ +{ + "environment": [ + { + "backend_env": { + "VLLM_ALLREDUCE_USE_FLASHINFER": "1", + "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm" + }, + "gpu": "NVIDIA H20", + "model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B", + "torch_cuda": "12.9", + "torch_version": "2.11.0+cu129", + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + }, + { + "backend_env": { + "VLLM_ALLREDUCE_USE_FLASHINFER": "1", + "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm" + }, + "gpu": "NVIDIA H20", + "model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B", + "torch_cuda": "12.9", + "torch_version": "2.11.0+cu129", + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + } + ], + "frontier_consumption": "diagnostic_only_in_base_profile_only_run; measured lookup requires a separate CC-backend injection ablation", + "rows": [ + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08288000151515007, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 1, + "payload_bytes": 4096, + "per_rank_time_ms": [ + { + "max": 0.4872959852218628, + "mean": 0.1310015980154276, + "median": 0.07679999992251396, + "min": 0.06217600032687187, + "std": 0.12790721677293843 + }, + { + "max": 0.4402880072593689, + "mean": 0.12842560112476348, + "median": 0.08288000151515007, + "min": 0.0655680000782013, + "std": 0.11220176524616535 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.0793600007891655, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 8, + "payload_bytes": 32768, + "per_rank_time_ms": [ + { + "max": 0.12716799974441528, + "mean": 0.07871360033750534, + "median": 0.0759200006723404, + "min": 0.06032000109553337, + "std": 0.019331314939874535 + }, + { + "max": 0.12380799651145935, + "mean": 0.08059840016067028, + "median": 0.0793600007891655, + "min": 0.06217600032687187, + "std": 0.01687088356254092 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.0713919997215271, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 16, + "payload_bytes": 65536, + "per_rank_time_ms": [ + { + "max": 0.12697599828243256, + "mean": 0.0767391998320818, + "median": 0.07078400254249573, + "min": 0.05910399928689003, + "std": 0.018775178979463278 + }, + { + "max": 0.11430399864912033, + "mean": 0.07594559974968433, + "median": 0.0713919997215271, + "min": 0.06124800071120262, + "std": 0.015719922150631036 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08056000247597694, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 32, + "payload_bytes": 131072, + "per_rank_time_ms": [ + { + "max": 0.1037760004401207, + "mean": 0.07954559996724128, + "median": 0.08056000247597694, + "min": 0.05955199897289276, + "std": 0.0135697420393132 + }, + { + "max": 0.10608000308275223, + "mean": 0.07971520014107228, + "median": 0.07593599706888199, + "min": 0.06028800085186958, + "std": 0.015319849772775456 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.0865279994904995, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 64, + "payload_bytes": 262144, + "per_rank_time_ms": [ + { + "max": 0.14470399916172028, + "mean": 0.09121599942445754, + "median": 0.0865279994904995, + "min": 0.06441599875688553, + "std": 0.024893837894277456 + }, + { + "max": 0.12438400089740753, + "mean": 0.08531199917197227, + "median": 0.08031999692320824, + "min": 0.06364800035953522, + "std": 0.01878029830059533 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.07135999947786331, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 128, + "payload_bytes": 524288, + "per_rank_time_ms": [ + { + "max": 0.11753600090742111, + "mean": 0.07606079950928687, + "median": 0.07135999947786331, + "min": 0.05843200162053108, + "std": 0.01755519771639284 + }, + { + "max": 0.1103999987244606, + "mean": 0.07607359997928143, + "median": 0.07073600217700005, + "min": 0.05721599981188774, + "std": 0.016904445220949783 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.07321599870920181, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 256, + "payload_bytes": 1048576, + "per_rank_time_ms": [ + { + "max": 0.11740799993276596, + "mean": 0.07749119997024537, + "median": 0.07203200086951256, + "min": 0.05862399935722351, + "std": 0.017701381594822835 + }, + { + "max": 0.11382400244474411, + "mean": 0.07733759954571724, + "median": 0.07321599870920181, + "min": 0.059039998799562454, + "std": 0.017188890882557036 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.09025600180029869, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 512, + "payload_bytes": 2097152, + "per_rank_time_ms": [ + { + "max": 0.13980799913406372, + "mean": 0.0950367994606495, + "median": 0.09025600180029869, + "min": 0.06815999746322632, + "std": 0.022829835127539378 + }, + { + "max": 0.14764800667762756, + "mean": 0.09710080176591873, + "median": 0.08720000088214874, + "min": 0.07152000069618225, + "std": 0.02500574954063789 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08083200082182884, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 1024, + "payload_bytes": 4194304, + "per_rank_time_ms": [ + { + "max": 0.15574400126934052, + "mean": 0.08760640025138855, + "median": 0.07846399769186974, + "min": 0.07097599655389786, + "std": 0.024487311423551025 + }, + { + "max": 0.16284799575805664, + "mean": 0.08963519930839539, + "median": 0.08083200082182884, + "min": 0.07100799679756165, + "std": 0.026047320562445356 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.10891199856996536, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 2048, + "payload_bytes": 8388608, + "per_rank_time_ms": [ + { + "max": 0.1582079976797104, + "mean": 0.11541439890861512, + "median": 0.10836799815297127, + "min": 0.09062399715185165, + "std": 0.018285136316576037 + }, + { + "max": 0.1578879952430725, + "mean": 0.11537599861621857, + "median": 0.10891199856996536, + "min": 0.0960640013217926, + "std": 0.018246538626977286 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.1703840047121048, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 4096, + "payload_bytes": 16777216, + "per_rank_time_ms": [ + { + "max": 0.19327999651432037, + "mean": 0.1707327976822853, + "median": 0.1703840047121048, + "min": 0.14815999567508698, + "std": 0.014211056022719618 + }, + { + "max": 0.19276799261569977, + "mean": 0.1658592015504837, + "median": 0.16379200667142868, + "min": 0.14560000598430634, + "std": 0.013840249648693638 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.25539200007915497, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 8192, + "payload_bytes": 33554432, + "per_rank_time_ms": [ + { + "max": 0.2807359993457794, + "mean": 0.25750079900026324, + "median": 0.25539200007915497, + "min": 0.24624000489711761, + "std": 0.008925204570802302 + }, + { + "max": 0.2863999903202057, + "mean": 0.2585055992007256, + "median": 0.255280002951622, + "min": 0.24371199309825897, + "std": 0.012059738582656496 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.1021759994328022, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 1, + "payload_bytes": 4096, + "per_rank_time_ms": [ + { + "max": 0.9770879745483398, + "mean": 0.1974783968180418, + "median": 0.10099200159311295, + "min": 0.05913599953055382, + "std": 0.2660581738745569 + }, + { + "max": 0.892799973487854, + "mean": 0.18164799660444259, + "median": 0.1021759994328022, + "min": 0.06435199826955795, + "std": 0.23947520188197013 + }, + { + "max": 0.6467199921607971, + "mean": 0.15839359983801843, + "median": 0.10100800171494484, + "min": 0.06800000369548798, + "std": 0.16617013141866102 + }, + { + "max": 0.6725760102272034, + "mean": 0.15686400160193442, + "median": 0.10044800117611885, + "min": 0.06063999980688095, + "std": 0.17523222161300497 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.12694399803876877, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 8, + "payload_bytes": 32768, + "per_rank_time_ms": [ + { + "max": 0.6659520268440247, + "mean": 0.1927648030221462, + "median": 0.12694399803876877, + "min": 0.07609599828720093, + "std": 0.16697784531907736 + }, + { + "max": 0.695360004901886, + "mean": 0.19356480240821838, + "median": 0.11726400256156921, + "min": 0.0796160027384758, + "std": 0.17588096678867862 + }, + { + "max": 0.5939840078353882, + "mean": 0.1868800014257431, + "median": 0.12379200011491776, + "min": 0.07427199929952621, + "std": 0.1463231714943902 + }, + { + "max": 0.6635839939117432, + "mean": 0.1874335989356041, + "median": 0.12014400213956833, + "min": 0.07526399940252304, + "std": 0.16680959677760304 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.09161599725484848, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 16, + "payload_bytes": 65536, + "per_rank_time_ms": [ + { + "max": 0.12665599584579468, + "mean": 0.09443839862942696, + "median": 0.09124799817800522, + "min": 0.06431999802589417, + "std": 0.0205690155775906 + }, + { + "max": 0.1303039938211441, + "mean": 0.09712959825992584, + "median": 0.09161599725484848, + "min": 0.07648000121116638, + "std": 0.019072048129173236 + }, + { + "max": 0.13836799561977386, + "mean": 0.09821119979023933, + "median": 0.09148800000548363, + "min": 0.0727040022611618, + "std": 0.021314066545189116 + }, + { + "max": 0.12992000579833984, + "mean": 0.09272959977388381, + "median": 0.08931199833750725, + "min": 0.06406400352716446, + "std": 0.021360803830354765 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08580800145864487, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 32, + "payload_bytes": 131072, + "per_rank_time_ms": [ + { + "max": 0.13449600338935852, + "mean": 0.08960640132427215, + "median": 0.08299200236797333, + "min": 0.06796800345182419, + "std": 0.020633597898445998 + }, + { + "max": 0.14735999703407288, + "mean": 0.09248319901525974, + "median": 0.08580800145864487, + "min": 0.05913599953055382, + "std": 0.025960234928829564 + }, + { + "max": 0.13705599308013916, + "mean": 0.08947199806571007, + "median": 0.08460799977183342, + "min": 0.0634239986538887, + "std": 0.02122838946992339 + }, + { + "max": 0.13846400380134583, + "mean": 0.0859104000031948, + "median": 0.08308799937367439, + "min": 0.05974400043487549, + "std": 0.022373631424433445 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.09867199882864952, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 64, + "payload_bytes": 262144, + "per_rank_time_ms": [ + { + "max": 0.1279039978981018, + "mean": 0.09748799875378608, + "median": 0.09678399935364723, + "min": 0.0655359998345375, + "std": 0.02315719144614622 + }, + { + "max": 0.1356479972600937, + "mean": 0.10018239840865135, + "median": 0.09532799944281578, + "min": 0.06185600161552429, + "std": 0.02330700253650042 + }, + { + "max": 0.13142399489879608, + "mean": 0.09778879955410957, + "median": 0.09492799639701843, + "min": 0.06560000032186508, + "std": 0.02307579212430045 + }, + { + "max": 0.1276479959487915, + "mean": 0.09611519873142242, + "median": 0.09867199882864952, + "min": 0.0642239972949028, + "std": 0.02273612181973431 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.09646400064229965, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 128, + "payload_bytes": 524288, + "per_rank_time_ms": [ + { + "max": 0.14060799777507782, + "mean": 0.09770880043506622, + "median": 0.09115200117230415, + "min": 0.06950400024652481, + "std": 0.024752645089849798 + }, + { + "max": 0.14377599954605103, + "mean": 0.09824960008263588, + "median": 0.08999999985098839, + "min": 0.07103999704122543, + "std": 0.025403407389046027 + }, + { + "max": 0.13680000603199005, + "mean": 0.09993600100278854, + "median": 0.09646400064229965, + "min": 0.06790400296449661, + "std": 0.022297424273985882 + }, + { + "max": 0.1391039937734604, + "mean": 0.09769919961690902, + "median": 0.09601600095629692, + "min": 0.06835199892520905, + "std": 0.023899922100804445 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08377600088715553, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 256, + "payload_bytes": 1048576, + "per_rank_time_ms": [ + { + "max": 0.1464959979057312, + "mean": 0.09276160076260567, + "median": 0.08377600088715553, + "min": 0.06777600198984146, + "std": 0.024751086465295658 + }, + { + "max": 0.14319999516010284, + "mean": 0.09080640003085136, + "median": 0.080400001257658, + "min": 0.06796800345182419, + "std": 0.023365718881708488 + }, + { + "max": 0.1382399946451187, + "mean": 0.09063360020518303, + "median": 0.08193599805235863, + "min": 0.06627199798822403, + "std": 0.02372618650854302 + }, + { + "max": 0.14313599467277527, + "mean": 0.09044799953699112, + "median": 0.08128000050783157, + "min": 0.06652799993753433, + "std": 0.02462486862033686 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.1128000020980835, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 512, + "payload_bytes": 2097152, + "per_rank_time_ms": [ + { + "max": 0.1600320041179657, + "mean": 0.11426239982247352, + "median": 0.1128000020980835, + "min": 0.07657600194215775, + "std": 0.030240087702350687 + }, + { + "max": 0.15881599485874176, + "mean": 0.11206399947404862, + "median": 0.10628800094127655, + "min": 0.0772159993648529, + "std": 0.029933135345483627 + }, + { + "max": 0.15612800419330597, + "mean": 0.10761600062251091, + "median": 0.09860799834132195, + "min": 0.07689599692821503, + "std": 0.02634237020678647 + }, + { + "max": 0.15865600109100342, + "mean": 0.11094079986214637, + "median": 0.10979199782013893, + "min": 0.07583999633789062, + "std": 0.028619217028542445 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08755199983716011, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 1024, + "payload_bytes": 4194304, + "per_rank_time_ms": [ + { + "max": 0.11999999731779099, + "mean": 0.0843871995806694, + "median": 0.08755199983716011, + "min": 0.06332799792289734, + "std": 0.016638543890716024 + }, + { + "max": 0.1218239963054657, + "mean": 0.08518079966306687, + "median": 0.08032000064849854, + "min": 0.06393600255250931, + "std": 0.019900899429956945 + }, + { + "max": 0.11849600076675415, + "mean": 0.0843968003988266, + "median": 0.08702399954199791, + "min": 0.06297600269317627, + "std": 0.017031908773433545 + }, + { + "max": 0.12300799787044525, + "mean": 0.0846304003149271, + "median": 0.08139199763536453, + "min": 0.06195199862122536, + "std": 0.020106523698622265 + } + ], + "selected_backend": "nccl_fallback", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.12361599877476692, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 2048, + "payload_bytes": 8388608, + "per_rank_time_ms": [ + { + "max": 0.17155200242996216, + "mean": 0.12168959975242614, + "median": 0.12055999785661697, + "min": 0.09609600156545639, + "std": 0.022565485532483928 + }, + { + "max": 0.9246399998664856, + "mean": 0.19978560134768486, + "median": 0.12361599877476692, + "min": 0.09715200215578079, + "std": 0.24230694305662265 + }, + { + "max": 0.9317439794540405, + "mean": 0.20037759989500045, + "median": 0.12327999994158745, + "min": 0.09603200107812881, + "std": 0.24450903278000383 + }, + { + "max": 0.9321280121803284, + "mean": 0.19875840097665787, + "median": 0.12230399996042252, + "min": 0.0950080007314682, + "std": 0.24519252220785093 + } + ], + "selected_backend": "nccl_fallback", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.20030399411916733, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 4096, + "payload_bytes": 16777216, + "per_rank_time_ms": [ + { + "max": 0.3261440098285675, + "mean": 0.20826880186796187, + "median": 0.19974400103092194, + "min": 0.1409280002117157, + "std": 0.051397264855718945 + }, + { + "max": 0.3248000144958496, + "mean": 0.20548800230026246, + "median": 0.1979840025305748, + "min": 0.141184002161026, + "std": 0.04980002399656717 + }, + { + "max": 0.32547199726104736, + "mean": 0.21280319690704347, + "median": 0.20030399411916733, + "min": 0.14127999544143677, + "std": 0.05304243085685509 + }, + { + "max": 0.26047998666763306, + "mean": 0.1969312012195587, + "median": 0.18079999834299088, + "min": 0.14057600498199463, + "std": 0.042174123638424224 + } + ], + "selected_backend": "nccl_fallback", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.2924960106611252, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 8192, + "payload_bytes": 33554432, + "per_rank_time_ms": [ + { + "max": 0.45052799582481384, + "mean": 0.3013375997543335, + "median": 0.28273600339889526, + "min": 0.21334399282932281, + "std": 0.07214223764527236 + }, + { + "max": 0.4466240108013153, + "mean": 0.2968191936612129, + "median": 0.27796798944473267, + "min": 0.21241599321365356, + "std": 0.07400678240849741 + }, + { + "max": 0.3830080032348633, + "mean": 0.2946112036705017, + "median": 0.2924960106611252, + "min": 0.21084800362586975, + "std": 0.05323627615746332 + }, + { + "max": 0.4609600007534027, + "mean": 0.3054272010922432, + "median": 0.289792001247406, + "min": 0.21062399446964264, + "std": 0.07608482904865672 + } + ], + "selected_backend": "nccl_fallback", + "tensor_parallel_size": 4 + } + ], + "schema_version": "qwen30_vllm020_allreduce_frozen.v1" +} diff --git a/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/attention.csv b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/attention.csv new file mode 100644 index 0000000..e53b40b --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/attention.csv @@ -0,0 +1,103 @@ +time_stats.attn_input_reshape.min,time_stats.attn_input_reshape.max,time_stats.attn_input_reshape.mean,time_stats.attn_input_reshape.median,time_stats.attn_input_reshape.std,time_stats.attn_kv_cache_save.min,time_stats.attn_kv_cache_save.max,time_stats.attn_kv_cache_save.mean,time_stats.attn_kv_cache_save.median,time_stats.attn_kv_cache_save.std,time_stats.attn_prefill.min,time_stats.attn_prefill.max,time_stats.attn_prefill.mean,time_stats.attn_prefill.median,time_stats.attn_prefill.std,time_stats.attn_decode.min,time_stats.attn_decode.max,time_stats.attn_decode.mean,time_stats.attn_decode.median,time_stats.attn_decode.std,time_stats.attn_output_reshape.min,time_stats.attn_output_reshape.max,time_stats.attn_output_reshape.mean,time_stats.attn_output_reshape.median,time_stats.attn_output_reshape.std,n_embd,n_q_head,n_kv_head,block_size,num_tensor_parallel_workers,max_model_len,batch_size,prefill_chunk_size,kv_cache_size,is_prefill,attention_backend,is_mixed_batch,mode,seq_lens,total_tokens,max_seq_len,min_seq_len,avg_seq_len,equal_seq_len,seq_len_variance,seq_len_std,seq_len_cv,is_chunked_prefill_sample,chunk_start_token,chunk_end_token,total_prefill_tokens,profiling_precision,model_arch,quant_signature,measurement_type,is_true_mixed_batch,prefill_seq_lens,prefill_kv_cache_sizes,decode_kv_cache_sizes,num_prefill_seqs,num_decode_seqs,decode_batch_size,total_batch_size,total_decode_tokens,decode_avg_kv_cache_size,batch_composition_ratio,batch_spec,projection_policy +0.0,0.0,0.0,0.0,0.0,0.01414399966597557,0.028863999992609024,0.019705599918961526,0.01771199982613325,0.005157200849836681,0.047968000173568726,0.07046400010585785,0.05810240097343922,0.05810240097343922,0.007477463486041561,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.04947200044989586,0.020412799902260303,0.01635199971497059,0.010107497379722417,0.046560000628232956,0.08323200047016144,0.05587520003318787,0.05587520003318787,0.011126758739503428,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.022207999601960182,0.017033600155264138,0.015312000177800655,0.002819235991970241,0.05104000121355057,0.07692799717187881,0.056396800279617305,0.056396800279617305,0.007481982178637539,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015072000212967396,0.022272000089287758,0.01706880023702979,0.01616000011563301,0.002460889579319197,0.06931199878454208,0.0838719978928566,0.07432000041007995,0.07432000041007995,0.004777766433175866,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.018592000007629395,0.028543999418616295,0.02095999978482723,0.019183999858796597,0.003198175496053494,0.12179200351238251,0.15408000349998474,0.1307712011039257,0.1307712011039257,0.00858807797538298,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.027775999158620834,0.03385600075125694,0.030131200328469276,0.029680000618100166,0.0021152558103575215,0.32678401470184326,0.3450239896774292,0.33396480381488797,0.33396480381488797,0.0045872424917606375,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.04438399896025658,0.05084799975156784,0.046540799736976626,0.04531199857592583,0.002277905811237223,1.0959680080413818,1.1151360273361206,1.0999775886535645,1.0999775886535645,0.005694403246120485,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.078015998005867,0.08691199868917465,0.08114239946007729,0.08019199967384338,0.00292795706334475,4.070400238037109,4.113152027130127,4.087088012695312,4.087088012695312,0.013660567012509554,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.05167999863624573,0.022115200012922286,0.017583999782800674,0.010340094822340818,0.05196800082921982,0.09011200070381165,0.06328320093452933,0.06328320093452933,0.012557341255467452,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01583999954164028,0.026623999699950218,0.018927999772131443,0.017376000061631203,0.003514650316260619,0.06681600213050842,0.07993599772453308,0.0725280001759529,0.0725280001759529,0.004343502558613716,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01648000068962574,0.030880000442266464,0.01945280022919178,0.017967999912798405,0.004096211183007485,0.1311360001564026,0.1546880006790161,0.13908160030841826,0.13908160030841826,0.007511906874366178,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.03558399900794029,0.020851199887692927,0.018559999763965607,0.005235911594130716,0.32950401306152344,0.350271999835968,0.33912960588932034,0.33912960588932034,0.006027400986663648,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.019328000023961067,0.040608000010252,0.022790400311350822,0.020704000256955624,0.006113051965778337,1.1415679454803467,1.1518720388412476,1.144483208656311,1.144483208656311,0.0032332311374389127,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015807999297976494,0.030688000842928886,0.019596799835562707,0.01774400006979704,0.004343384771033462,0.0,0.0,0.0,0.0,0.0,0.049056001007556915,0.07580800354480743,0.05948160067200661,0.05948160067200661,0.009031541471446955,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.02486399933695793,0.01923839971423149,0.018079999834299088,0.0032282528537266424,0.0,0.0,0.0,0.0,0.0,0.05142400041222572,0.07353600114583969,0.059328000620007516,0.059328000620007516,0.0073307807735143084,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016543999314308167,0.03977600112557411,0.021379199624061585,0.018511999398469925,0.006593576176246171,0.0,0.0,0.0,0.0,0.0,0.0488319993019104,0.06435199826955795,0.05479039996862411,0.05479039996862411,0.005672522998491864,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01635199971497059,0.02844800055027008,0.019267200119793416,0.017952000722289085,0.0035068687666949577,0.0,0.0,0.0,0.0,0.0,0.049855999648571014,0.07798399776220322,0.05986879989504815,0.05986879989504815,0.01043914754878828,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016383999958634377,0.026079999282956123,0.01923519968986511,0.017791999503970146,0.0032161974331284568,0.0,0.0,0.0,0.0,0.0,0.058111999183893204,0.1045759990811348,0.06708480007946492,0.06708480007946492,0.013479022462646494,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.04057599976658821,0.019100800156593323,0.015455999877303839,0.007512281243011577,0.0,0.0,0.0,0.0,0.0,0.05363199859857559,0.07782399654388428,0.06090559959411622,0.06090559959411622,0.007544620176348091,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.02191999927163124,0.016684799920767546,0.01563199982047081,0.0024293118621811216,0.0,0.0,0.0,0.0,0.0,0.0629120022058487,0.07891199737787247,0.06891520097851753,0.06891520097851753,0.005472695665695425,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.038943998515605927,0.018313600029796363,0.01561600062996149,0.007127270260115769,0.0,0.0,0.0,0.0,0.0,0.08675199747085571,0.10662399977445602,0.09391999915242194,0.09391999915242194,0.006988099589086635,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.054687999188899994,0.021439999900758268,0.01539199985563755,0.012052764849597775,0.0,0.0,0.0,0.0,0.0,0.13488000631332397,0.1528639942407608,0.1431359991431236,0.1431359991431236,0.005436271464033599,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.041919998824596405,0.01899839974939823,0.015343999955803156,0.007989843526623287,0.0,0.0,0.0,0.0,0.0,0.06176000088453293,0.08374399691820145,0.06747519969940186,0.06747519969940186,0.0066067747128778,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016256000846624374,0.11353600025177002,0.042761600017547606,0.028960000723600388,0.029104301538020762,0.0,0.0,0.0,0.0,0.0,0.09734400361776352,0.14422400295734406,0.11392960175871848,0.11392960175871848,0.013198594600417867,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.034143999218940735,0.018918400071561335,0.01643200032413006,0.005500943993080684,0.0,0.0,0.0,0.0,0.0,0.12918399274349213,0.15087999403476715,0.13807999789714814,0.13807999789714814,0.007658330538677587,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.03641600161790848,0.0198208000510931,0.01780799962580204,0.0057264128169845765,0.0,0.0,0.0,0.0,0.0,0.22099199891090393,0.23904000222682953,0.2293503984808922,0.2293503984808922,0.004861342907006028,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.035840000957250595,0.018908800091594458,0.015792000107467175,0.0064374817924757475,0.0,0.0,0.0,0.0,0.0,0.10134399682283401,0.12201599776744843,0.10896319895982742,0.10896319895982742,0.006336330809165179,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.03417599946260452,0.017846399918198586,0.014800000004470348,0.006495007539635255,0.0,0.0,0.0,0.0,0.0,0.13126400113105774,0.15561600029468536,0.1389280006289482,0.1389280006289482,0.008381472811075022,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03519999980926514,0.019168000388890504,0.01600000075995922,0.005995522477654695,0.0,0.0,0.0,0.0,0.0,0.21728000044822693,0.2343679964542389,0.2231455981731415,0.2231455981731415,0.004720730646739123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.03670400008559227,0.018441599886864425,0.015023999847471714,0.006596535162793127,0.0,0.0,0.0,0.0,0.0,0.39321601390838623,0.4524799883365631,0.4058080047369003,0.4058080047369003,0.01578349755088941,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.022143999114632607,0.016912000067532063,0.015168000012636185,0.0028156089295136347,0.0,0.0,0.0,0.0,0.0,0.15587200224399567,0.3079040050506592,0.17838079929351805,0.17838079929351805,0.04355865575265927,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.02112000063061714,0.015516800060868263,0.01488000014796853,0.001940870731593904,0.0,0.0,0.0,0.0,0.0,0.21587200462818146,0.23561599850654602,0.22250880002975468,0.22250880002975468,0.006181951170646666,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.039264000952243805,0.023609600123018028,0.02131200022995472,0.007236979625711548,0.0,0.0,0.0,0.0,0.0,0.408735990524292,0.470335990190506,0.4336863994598388,0.4336863994598388,0.01844662383160074,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.025407999753952026,0.016227199975401164,0.014960000291466713,0.0031617375441736185,0.0,0.0,0.0,0.0,0.0,0.7412800192832947,0.7627840042114258,0.7464000046253203,0.7464000046253203,0.006112167448837547,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.02364799939095974,0.01809599995613098,0.016352000646293163,0.0035481127058959038,0.04822399839758873,0.08566399663686752,0.05961279980838299,0.05961279980838299,0.011445665413968877,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.020864000543951988,0.01668160008266568,0.015664000064134598,0.0025769063833097584,0.049695998430252075,0.08057600259780884,0.05882879942655563,0.05882879942655563,0.009515126108519331,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.028704000636935234,0.017430400010198355,0.015056000091135502,0.004349294613335555,0.049855999648571014,0.07366400212049484,0.05459520071744919,0.05459520071744919,0.0069610920757925574,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.03161599859595299,0.017788799852132796,0.015711999963968992,0.005005385723318659,0.06102399900555611,0.08179199695587158,0.06650560013949873,0.06650560013949873,0.006947995595801105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.04416000097990036,0.019200000166893005,0.015488000120967627,0.008561241323364038,0.08054400235414505,0.09196799993515015,0.08607039973139763,0.08607039973139763,0.004145329035483754,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.023391999304294586,0.019667199812829494,0.018463999964296818,0.0022577669687832585,0.18729600310325623,0.20585599541664124,0.19546559900045393,0.19546559900045393,0.006339068824303663,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.026240000501275063,0.03446400165557861,0.028297600522637367,0.027088000439107418,0.0025148441522922374,0.5754240155220032,0.5889919996261597,0.5800191938877105,0.5800191938877105,0.003858869273829596,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.04182400181889534,0.047200001776218414,0.043036799877882004,0.0423360001295805,0.0017073405772076728,2.063199996948242,2.0787200927734375,2.067151999473572,2.067151999473572,0.004959651271127835,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.05648000165820122,0.02270399993285537,0.017935999669134617,0.012377915150727689,0.049536000937223434,0.07196799665689468,0.056015999615192415,0.056015999615192415,0.0070476637552742884,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01408000010997057,0.021344000473618507,0.01611520005390048,0.014928000047802925,0.0025499884993961702,0.07072000205516815,0.2642880082130432,0.1588256008923054,0.1588256008923054,0.053220347086102376,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01360000018030405,0.03014400042593479,0.016975999902933837,0.015056000091135502,0.004715159697364111,0.08393599838018417,0.11036799848079681,0.09160000011324881,0.09160000011324881,0.007911669434472792,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.020767999812960625,0.016233599931001663,0.015392000321298838,0.002202615907995427,0.1998399943113327,0.22070400416851044,0.20855360180139543,0.20855360180139543,0.00689307200230667,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.033440001308918,0.018684800155460833,0.016048000194132328,0.00553991005639174,0.6043199896812439,0.635807991027832,0.6126143991947173,0.6126143991947173,0.008745933953408096,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013824000023305416,0.03359999880194664,0.017811199743300678,0.014944000169634819,0.005926140483565472,0.0,0.0,0.0,0.0,0.0,0.045471999794244766,0.07539200037717819,0.054758400097489356,0.054758400097489356,0.010253548506101549,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02409599907696247,0.01641279999166727,0.01473599998280406,0.003347158638713987,0.0,0.0,0.0,0.0,0.0,0.0461760014295578,0.07529599964618683,0.05460800044238568,0.05460800044238568,0.009748937798340135,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.028416000306606293,0.018611199874430894,0.01598400017246604,0.005209875093863647,0.0,0.0,0.0,0.0,0.0,0.048128001391887665,0.07897599786520004,0.061353600397706036,0.061353600397706036,0.010488153655157845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.025728000327944756,0.016092800162732603,0.01512000011280179,0.003300888123052605,0.0,0.0,0.0,0.0,0.0,0.04864000156521797,0.07648000121116638,0.05810560062527656,0.05810560062527656,0.009473544218404408,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.03551999852061272,0.018588799890130757,0.016064000315964222,0.006071057437992447,0.0,0.0,0.0,0.0,0.0,0.04822399839758873,0.07862400263547897,0.05660480037331582,0.05660480037331582,0.009565394213730401,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.028831999748945236,0.01699519995599985,0.015024000313133001,0.004285000744868188,0.0,0.0,0.0,0.0,0.0,0.04854400083422661,0.06719999760389328,0.05778240002691746,0.05778240002691746,0.006852125554679805,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.0144640002399683,0.024927999824285507,0.0161183999851346,0.01521599991247058,0.0029774808524673907,0.0,0.0,0.0,0.0,0.0,0.05379199981689453,0.08966399729251862,0.06270079985260964,0.06270079985260964,0.009983591277092696,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03001599945127964,0.017648000083863736,0.01547200046479702,0.004585126309484848,0.0,0.0,0.0,0.0,0.0,0.061664000153541565,0.07692799717187881,0.06704320013523103,0.06704320013523103,0.005500191798490629,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.026367999613285065,0.016947199776768684,0.015039999969303608,0.0037951566103550205,0.0,0.0,0.0,0.0,0.0,0.08799999952316284,0.111455999314785,0.0964031994342804,0.0964031994342804,0.007397541615558088,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01369599997997284,0.021247999742627144,0.015359999984502793,0.014640000183135271,0.0020942770950814317,0.0,0.0,0.0,0.0,0.0,0.051711998879909515,0.07065600156784058,0.058054400235414506,0.058054400235414506,0.006633034815910025,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.0307839997112751,0.017148799914866685,0.015039999969303608,0.004882374686312284,0.0,0.0,0.0,0.0,0.0,0.061919998377561576,0.07843200117349625,0.06628479920327664,0.06628479920327664,0.004962852689801192,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013887999579310417,0.02969600073993206,0.017500799987465142,0.015232000034302473,0.00467273319705314,0.0,0.0,0.0,0.0,0.0,0.08819200098514557,0.11097600311040878,0.09493440166115762,0.09493440166115762,0.007509235042985577,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.022112000733613968,0.01751680001616478,0.016047999262809753,0.0030306621792915785,0.0,0.0,0.0,0.0,0.0,0.13065600395202637,0.15110400319099426,0.13857279866933822,0.13857279866933822,0.00750137841249771,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.04012800008058548,0.01892479993402958,0.01550400024279952,0.007459545267816961,0.0,0.0,0.0,0.0,0.0,0.06278400123119354,0.08259200304746628,0.0704512007534504,0.0704512007534504,0.005979055984382744,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.02236800082027912,0.016332800220698118,0.014928000047802925,0.002784044363186731,0.0,0.0,0.0,0.0,0.0,0.1003199964761734,0.1279360055923462,0.10921279862523078,0.10921279862523078,0.00862769617046716,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0225600004196167,0.016128000058233737,0.014800000004470348,0.002958953332547708,0.0,0.0,0.0,0.0,0.0,0.13116799294948578,0.14812800288200378,0.13783999979496003,0.13783999979496003,0.005696384361148053,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.029983999207615852,0.017468800116330386,0.01508800033479929,0.0047379550962483065,0.0,0.0,0.0,0.0,0.0,0.217631995677948,0.2447360008955002,0.22715839892625808,0.22715839892625808,0.008831828221847138,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02304000034928322,0.016512000095099212,0.014512000139802694,0.0035026774828624254,0.0,0.0,0.0,0.0,0.0,0.11020799726247787,0.12307199835777283,0.11600959971547126,0.11600959971547126,0.004667637266950902,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.042080000042915344,0.017510399967432023,0.014607999939471483,0.00823117883530167,0.0,0.0,0.0,0.0,0.0,0.15702399611473083,0.17587199807167053,0.16399359852075576,0.16399359852075576,0.006588299074676393,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0208320003002882,0.015299199987202883,0.01462399959564209,0.001916768086505386,0.0,0.0,0.0,0.0,0.0,0.21779200434684753,0.2415360063314438,0.2260768011212349,0.2260768011212349,0.007251352080685236,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014015999622642994,0.021088000386953354,0.015619200188666582,0.01473599998280406,0.002013227452302141,0.0,0.0,0.0,0.0,0.0,0.3959999978542328,0.4152640104293823,0.40332479774951924,0.40332479774951924,0.006942401431914052,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.022784000262618065,0.016435200069099664,0.015519999898970127,0.0023688301421469523,0.04879999905824661,0.09139200299978256,0.06054079942405224,0.06054079942405224,0.012152608702448775,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014816000126302242,0.02844800055027008,0.017008000146597625,0.015696000307798386,0.003941466294662679,0.047807998955249786,0.07356800138950348,0.05626560002565384,0.05626560002565384,0.00842179125412236,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.053408000618219376,0.019353600218892097,0.01532800029963255,0.01138302400841626,0.048448000103235245,0.0785600021481514,0.0556256003677845,0.0556256003677845,0.009348274502616908,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.03139200061559677,0.01823679991066456,0.016159999649971724,0.004860071238302512,0.055424001067876816,0.08505599945783615,0.0640383992344141,0.0640383992344141,0.00921448636178623,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.026528000831604004,0.017324799951165915,0.01536000007763505,0.0036004822686428305,0.07660800218582153,0.0942080020904541,0.08209280073642732,0.08209280073642732,0.00515895587669752,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.021247999742627144,0.016403199825435876,0.01532800029963255,0.001995897022647934,0.11395200341939926,0.15113599598407745,0.12431039959192276,0.12431039959192276,0.011164431123683732,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.017184000462293625,0.028672000393271446,0.019865600019693376,0.017823999747633934,0.0035798491315929977,0.3171840012073517,0.3341119885444641,0.3261695951223373,0.3261695951223373,0.005111046452878918,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.024639999493956566,0.030400000512599945,0.02656640000641346,0.02556800004094839,0.0020189846603237303,1.0648640394210815,1.0828479528427124,1.0715327858924866,1.0715327858924866,0.005558639263049851,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014976000413298607,0.021695999428629875,0.017305599898099898,0.01593599934130907,0.0025718046517268054,0.04956800118088722,0.07932800054550171,0.06228480041027069,0.06228480041027069,0.01027160349757827,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01539199985563755,0.03580800071358681,0.019686400331556796,0.017136000096797943,0.005918387811304003,0.05158400163054466,0.10713600367307663,0.06364160068333148,0.06364160068333148,0.015832957809696766,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02223999984562397,0.016672000009566545,0.015887999907135963,0.0020934945946034563,0.06521599739789963,0.08902399986982346,0.07520959973335266,0.07520959973335266,0.007913840684102929,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015231999568641186,0.04028800129890442,0.019971200078725816,0.01646399963647127,0.007351305886577286,0.14467200636863708,0.16844800114631653,0.15363519936800005,0.15363519936800005,0.008174435899956223,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015519999898970127,0.02304000034928322,0.01775679988786578,0.016784000210464,0.0025077243712082584,0.33740800619125366,0.35343998670578003,0.3445120006799698,0.3445120006799698,0.00445648463590358,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.02393599972128868,0.01809599995613098,0.01654400024563074,0.002998393102466254,0.0,0.0,0.0,0.0,0.0,0.0504320003092289,0.0843840017914772,0.060083200410008426,0.060083200410008426,0.00986959318572296,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.030047999694943428,0.019510399922728537,0.017680000513792038,0.004065185265963332,0.0,0.0,0.0,0.0,0.0,0.05004800111055374,0.07036799937486649,0.059315200522542,0.059315200522542,0.006537768821329647,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.02393599972128868,0.018035199772566558,0.016512000001966953,0.0030667689116777724,0.0,0.0,0.0,0.0,0.0,0.05100800096988678,0.06735999882221222,0.058387200161814694,0.058387200161814694,0.005832787739241381,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.026208000257611275,0.017500799987465142,0.01646399963647127,0.0030666103306165714,0.0,0.0,0.0,0.0,0.0,0.05023999884724617,0.07254400104284286,0.057254400476813315,0.057254400476813315,0.006890853653068151,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.027327999472618103,0.0179776000790298,0.01648000068962574,0.003783417447531392,0.0,0.0,0.0,0.0,0.0,0.04819199815392494,0.07100799679756165,0.05621119923889638,0.05621119923889638,0.007824391484124725,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,127.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.036448001861572266,0.019136000238358975,0.016704000532627106,0.006076530095624803,0.0,0.0,0.0,0.0,0.0,0.047488000243902206,0.08441600203514099,0.0588383998721838,0.0588383998721838,0.011275940726332522,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015135999768972397,0.033952001482248306,0.02119360016658902,0.018000000156462193,0.007136646614305304,0.0,0.0,0.0,0.0,0.0,0.04931199923157692,0.06992000341415405,0.05755840018391609,0.05755840018391609,0.007297587694315226,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.034272000193595886,0.0204415999352932,0.017311999574303627,0.00695404701803013,0.0,0.0,0.0,0.0,0.0,0.05215999856591225,0.0735040009021759,0.06117440015077591,0.06117440015077591,0.007381118436894922,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03743999823927879,0.02012479966506362,0.01775999926030636,0.006181045509079057,0.0,0.0,0.0,0.0,0.0,0.06355199962854385,0.08508799970149994,0.07019200026988984,0.07019200026988984,0.0062246580442117845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,1023.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.032607998698949814,0.01959999995306134,0.017487999983131886,0.004882475068460749,0.0,0.0,0.0,0.0,0.0,0.04918399825692177,0.0865280032157898,0.05973760038614274,0.05973760038614274,0.011546847942113974,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02675200067460537,0.017430400010198355,0.016560000367462635,0.003243135094239819,0.0,0.0,0.0,0.0,0.0,0.052671998739242554,0.08367999643087387,0.06238719932734965,0.06238719932734965,0.011207824780630104,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.03612799942493439,0.019327999837696553,0.01688000001013279,0.006058251425153085,0.0,0.0,0.0,0.0,0.0,0.06364800035953522,0.07878399640321732,0.06935679838061332,0.06935679838061332,0.004515588328677643,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.03711999952793121,0.019670399930328132,0.017152000218629837,0.006165994646522264,0.0,0.0,0.0,0.0,0.0,0.09071999788284302,0.11507199704647064,0.10252480059862136,0.10252480059862136,0.008782544051535657,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,2047.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.026528000831604004,0.018396800104528665,0.01601599995046854,0.004279788048986283,0.0,0.0,0.0,0.0,0.0,0.05331199988722801,0.07977599650621414,0.06076480001211167,0.06076480001211167,0.008761579122069606,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.02643200010061264,0.01726400004699826,0.015855999663472176,0.003210008417242029,0.0,0.0,0.0,0.0,0.0,0.062431998550891876,0.08246400207281113,0.06970879957079888,0.06970879957079888,0.007016341676068233,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.02223999984562397,0.0166015999391675,0.015887999907135963,0.00204913969129354,0.0,0.0,0.0,0.0,0.0,0.10127999633550644,0.1141119971871376,0.10621120035648347,0.10621120035648347,0.004029318311754605,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03363199904561043,0.019305599946528675,0.016671999357640743,0.0055445582307981234,0.0,0.0,0.0,0.0,0.0,0.1319040060043335,0.15839999914169312,0.14040640145540234,0.14040640145540234,0.007998041073596942,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,4095.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014368000440299511,0.04447999969124794,0.018908800091594458,0.0157279996201396,0.0086814937461721,0.0,0.0,0.0,0.0,0.0,0.06428799778223038,0.08982399851083755,0.07349760085344315,0.07349760085344315,0.008605284512197258,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.028672000393271446,0.01890560006722808,0.016080000437796116,0.004797584627815021,0.0,0.0,0.0,0.0,0.0,0.11123199760913849,0.14115199446678162,0.11942399889230729,0.11942399889230729,0.008513394706791027,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.024927999824285507,0.016915200091898442,0.015216000378131866,0.003374147499442635,0.0,0.0,0.0,0.0,0.0,0.15887999534606934,0.18111999332904816,0.16934399753808976,0.16934399753808976,0.007415181260761123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.026944000273942947,0.017737600207328796,0.015199999790638685,0.00415598714375255,0.0,0.0,0.0,0.0,0.0,0.2192319929599762,0.23472000658512115,0.22809920012950893,0.22809920012950893,0.004730841376327335,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,8191.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median diff --git a/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/attention_true_mixed_fused.csv b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/attention_true_mixed_fused.csv new file mode 100644 index 0000000..1b7a751 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/attention_true_mixed_fused.csv @@ -0,0 +1,31 @@ +num_tensor_parallel_workers,batch_spec,num_prefill_seqs,num_decode_seqs,total_prefill_tokens,total_decode_tokens,decode_avg_kv_cache_size,attention_core_mean_ms,attention_core_mean_as_median_ms,kv_cache_update_median_ms,representation +1,q64_8q1s512,1,8,64,8,511.0,0.06604800000786781,0.06604800000786781,0.021359999664127827,one_fused_FA3_call_not_split_into_prefill_decode +1,q128_8q1s1k,1,8,128,8,1023.0,0.07483199909329416,0.07483199909329416,0.024255999363958836,one_fused_FA3_call_not_split_into_prefill_decode +1,q128_16q1s1k,1,16,128,16,1023.0,0.10597119852900506,0.10597119852900506,0.018112000077962875,one_fused_FA3_call_not_split_into_prefill_decode +1,q256_16q1s2k,1,16,256,16,2047.0,0.13697599917650224,0.13697599917650224,0.01673599984496832,one_fused_FA3_call_not_split_into_prefill_decode +1,q256_32q1s2k,1,32,256,32,2047.0,0.2163648009300232,0.2163648009300232,0.01515199989080429,one_fused_FA3_call_not_split_into_prefill_decode +1,q512_32q1s4k,1,32,512,32,4095.0,0.3746495962142944,0.3746495962142944,0.016191999427974224,one_fused_FA3_call_not_split_into_prefill_decode +1,q512_64q1s4k,1,64,512,64,4095.0,0.6718560099601746,0.6718560099601746,0.024848000146448612,one_fused_FA3_call_not_split_into_prefill_decode +1,q1k_64q1s8k,1,64,1024,64,8191.0,1.3472000002861022,1.3472000002861022,0.019600000232458115,one_fused_FA3_call_not_split_into_prefill_decode +1,q2k_32q1s4k,1,32,2048,32,4095.0,0.6351647913455962,0.6351647913455962,0.028256000019609928,one_fused_FA3_call_not_split_into_prefill_decode +1,q4k_16q1s4k,1,16,4096,16,4095.0,1.2515872120857237,1.2515872120857237,0.04395199939608574,one_fused_FA3_call_not_split_into_prefill_decode +2,q64_8q1s512,1,8,64,8,511.0,0.05652800053358078,0.05652800053358078,0.019600000232458115,one_fused_FA3_call_not_split_into_prefill_decode +2,q128_8q1s1k,1,8,128,8,1023.0,0.07362559959292413,0.07362559959292413,0.014431999996304512,one_fused_FA3_call_not_split_into_prefill_decode +2,q128_16q1s1k,1,16,128,16,1023.0,0.0736224003136158,0.0736224003136158,0.014479999896138906,one_fused_FA3_call_not_split_into_prefill_decode +2,q256_16q1s2k,1,16,256,16,2047.0,0.10047360062599181,0.10047360062599181,0.01425600005313754,one_fused_FA3_call_not_split_into_prefill_decode +2,q256_32q1s2k,1,32,256,32,2047.0,0.14032640159130094,0.14032640159130094,0.01566399959847331,one_fused_FA3_call_not_split_into_prefill_decode +2,q512_32q1s4k,1,32,512,32,4095.0,0.22105600088834762,0.22105600088834762,0.016032000072300434,one_fused_FA3_call_not_split_into_prefill_decode +2,q512_64q1s4k,1,64,512,64,4095.0,0.36867519915103913,0.36867519915103913,0.015263999812304974,one_fused_FA3_call_not_split_into_prefill_decode +2,q1k_64q1s8k,1,64,1024,64,8191.0,0.6853824079036711,0.6853824079036711,0.015856000129133463,one_fused_FA3_call_not_split_into_prefill_decode +2,q2k_32q1s4k,1,32,2048,32,4095.0,0.3503839999437332,0.3503839999437332,0.018432000651955605,one_fused_FA3_call_not_split_into_prefill_decode +2,q4k_16q1s4k,1,16,4096,16,4095.0,0.7185311973094941,0.7185311973094941,0.026575999334454536,one_fused_FA3_call_not_split_into_prefill_decode +4,q64_8q1s512,1,8,64,8,511.0,0.0562208004295826,0.0562208004295826,0.015520000364631414,one_fused_FA3_call_not_split_into_prefill_decode +4,q128_8q1s1k,1,8,128,8,1023.0,0.061199999228119854,0.061199999228119854,0.018240000121295452,one_fused_FA3_call_not_split_into_prefill_decode +4,q128_16q1s1k,1,16,128,16,1023.0,0.06104319989681244,0.06104319989681244,0.016207999549806118,one_fused_FA3_call_not_split_into_prefill_decode +4,q256_16q1s2k,1,16,256,16,2047.0,0.07356479987502099,0.07356479987502099,0.015584000386297703,one_fused_FA3_call_not_split_into_prefill_decode +4,q256_32q1s2k,1,32,256,32,2047.0,0.09806400015950203,0.09806400015950203,0.015008000191301107,one_fused_FA3_call_not_split_into_prefill_decode +4,q512_32q1s4k,1,32,512,32,4095.0,0.1509471982717514,0.1509471982717514,0.014864000026136637,one_fused_FA3_call_not_split_into_prefill_decode +4,q512_64q1s4k,1,64,512,64,4095.0,0.21687040030956264,0.21687040030956264,0.015343999955803156,one_fused_FA3_call_not_split_into_prefill_decode +4,q1k_64q1s8k,1,64,1024,64,8191.0,0.3848575979471207,0.3848575979471207,0.015647999942302704,one_fused_FA3_call_not_split_into_prefill_decode +4,q2k_32q1s4k,1,32,2048,32,4095.0,0.2785151988267898,0.2785151988267898,0.01726400014013052,one_fused_FA3_call_not_split_into_prefill_decode +4,q4k_16q1s4k,1,16,4096,16,4095.0,0.456512001156807,0.456512001156807,0.017680000513792038,one_fused_FA3_call_not_split_into_prefill_decode diff --git a/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/linear_op.csv b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/linear_op.csv new file mode 100644 index 0000000..2803f02 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/linear_op.csv @@ -0,0 +1,37 @@ +time_stats.emb.min,time_stats.emb.max,time_stats.emb.mean,time_stats.emb.median,time_stats.emb.std,time_stats.input_layernorm.min,time_stats.input_layernorm.max,time_stats.input_layernorm.mean,time_stats.input_layernorm.median,time_stats.input_layernorm.std,time_stats.attn_pre_proj.min,time_stats.attn_pre_proj.max,time_stats.attn_pre_proj.mean,time_stats.attn_pre_proj.median,time_stats.attn_pre_proj.std,time_stats.attn_rope.min,time_stats.attn_rope.max,time_stats.attn_rope.mean,time_stats.attn_rope.median,time_stats.attn_rope.std,time_stats.attn_post_proj.min,time_stats.attn_post_proj.max,time_stats.attn_post_proj.mean,time_stats.attn_post_proj.median,time_stats.attn_post_proj.std,time_stats.post_attention_layernorm.min,time_stats.post_attention_layernorm.max,time_stats.post_attention_layernorm.mean,time_stats.post_attention_layernorm.median,time_stats.post_attention_layernorm.std,n_head,n_kv_head,n_embd,n_expanded_embd,vocab_size,use_gated_mlp,use_qk_norm,attn_output_gate,num_tokens,num_tensor_parallel_workers,padded_n_embd,padded_n_expanded_embd,model_arch,is_step2_mini,share_expert_dim,share_q_dim,measurement_type,profiling_precision,quant_signature +0.029184000566601753,0.06780800223350525,0.03157280012965202,0.030736000277101994,0.005874173435341216,0.033215999603271484,0.04825599864125252,0.03443359974771738,0.0337119996547699,0.0031825678429048183,1.438431978225708,1.505568027496338,1.446228802204132,1.4429279565811157,0.014128607642643025,0.538752019405365,0.5440319776535034,0.5416463971138,0.5420799851417542,0.0016099306164432847,1.0073280334472656,1.0163840055465698,1.0098415970802308,1.0081279873847961,0.0032980457197329728,0.04022400081157684,0.041728001087903976,0.04091359991580248,0.04081599973142147,0.0004728505416767937,32,4,2048,768,151936,True,True,False,8192,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.01360000018030405,0.06784000247716904,0.017755200061947106,0.0179840000346303,0.00837681421401443,0.01836800016462803,0.03651199862360954,0.019606399815529585,0.018719999119639397,0.0038821958848767424,0.7512000203132629,0.8208960294723511,0.7566704005002975,0.75382399559021,0.014793092586577971,0.28995200991630554,0.29337599873542786,0.29135999977588656,0.29150401055812836,0.000998381071258815,0.5149760246276855,0.5169600248336792,0.5160208016633987,0.5158880054950714,0.0005038652783291977,0.02051199972629547,0.021503999829292297,0.021067200042307378,0.021104000508785248,0.00023542855306902367,32,4,2048,768,151936,True,True,False,4096,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.0080960001796484,0.04281599819660187,0.011092799971811474,0.011039999779313803,0.005489135752949302,0.012223999947309494,0.026335999369621277,0.013187199970707298,0.01247999956831336,0.0030236510562153375,0.3928639888763428,0.45372799038887024,0.3976895987987518,0.39528000354766846,0.012905176716136006,0.1547199934720993,0.15884800255298615,0.15712319910526276,0.1573439985513687,0.001165121945135037,0.26633599400520325,0.268095999956131,0.26719200164079665,0.2671840041875839,0.0004242740199415328,0.013024000450968742,0.013887999579310417,0.013489600038155913,0.013520000036805868,0.0002382817554057738,32,4,2048,768,151936,True,True,False,2048,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.00825599953532219,0.04755200073122978,0.02398160002194345,0.01961600035429001,0.00874683840888523,0.018880000337958336,0.03868800029158592,0.021590400114655496,0.0208320003002882,0.004057015751746236,0.24316799640655518,0.2710399925708771,0.2521967992186546,0.25065599381923676,0.007998418528894075,0.09644799679517746,0.19120000302791595,0.10407840013504029,0.09963199868798256,0.020043722414992166,0.13600000739097595,0.18892799317836761,0.15760480016469955,0.15760000050067902,0.0112223677907762,0.009664000011980534,0.010015999898314476,0.009836799977347255,0.009824000298976898,9.016971952948177e-05,32,4,2048,768,151936,True,True,False,1024,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.017376000061631203,0.044704001396894455,0.026318399980664254,0.027312000282108784,0.007565344034680866,0.01836800016462803,0.03014400042593479,0.021276800055056812,0.020655999891459942,0.002632977855097951,0.1438719928264618,0.17132799327373505,0.152497598528862,0.15012799948453903,0.007947150319625347,0.1430719941854477,0.19305600225925446,0.16630879789590836,0.1685439944267273,0.014628024163894684,0.08899199962615967,0.10467199981212616,0.0943599995225668,0.09374399855732918,0.003472669494074113,0.007615999784320593,0.007935999892652035,0.007769599952735007,0.0077760000713169575,7.680004540222077e-05,32,4,2048,768,151936,True,True,False,512,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.016992000862956047,0.0544000007212162,0.02573199989274144,0.026016000658273697,0.00803323401720434,0.018432000651955605,0.02348800003528595,0.020648000109940768,0.02062400057911873,0.0013939985047930988,0.10220800340175629,0.1361600011587143,0.11850560046732425,0.11684799939393997,0.010637754898360304,0.16710400581359863,0.21110400557518005,0.19078560024499894,0.19409599900245667,0.013714352646558832,0.06265600025653839,0.0740479975938797,0.06842879951000214,0.0690080001950264,0.0031292240446560557,0.00979200005531311,0.033440001308918,0.01864320016466081,0.017280000261962414,0.006957998188876383,32,4,2048,768,151936,True,True,False,256,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.017152000218629837,0.04396799951791763,0.025907999789342284,0.02598400041460991,0.007714554737915267,0.018400000408291817,0.03667199984192848,0.024132800102233887,0.02112000063061714,0.006120348733354326,0.10678400099277496,0.1363839954137802,0.1193264003843069,0.11583999916911125,0.009838647443214228,0.17017599940299988,0.22748799622058868,0.18853759989142418,0.18433599919080734,0.015728078443174653,0.04569600149989128,0.06652799993753433,0.05192639995366335,0.05151999928057194,0.004516605694976449,0.021856000646948814,0.026335999369621277,0.02384479995816946,0.023599999956786633,0.0011301153013314744,32,4,2048,768,151936,True,True,False,128,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.017343999817967415,1.0683200359344482,0.05748240072280168,0.029504000209271908,0.16366824814254827,0.018688000738620758,0.3317759931087494,0.03685439983382821,0.021151999942958355,0.06767422466731164,0.10255999863147736,0.9434880018234253,0.16412640027701855,0.11956800147891045,0.17964606281728834,0.1714559942483902,2.1306240558624268,0.3011296011507511,0.1926399990916252,0.42310127734378766,0.03574400022625923,0.6859520077705383,0.08389280084520578,0.04279999993741512,0.14321647071615612,0.020479999482631683,0.1831360012292862,0.033024000097066165,0.023856000043451786,0.03470987082429836,32,4,2048,768,151936,True,True,False,64,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.016095999628305435,0.05142400041222572,0.026363200135529043,0.02676799986511469,0.008637725852473854,0.01849599927663803,0.03577600046992302,0.022193600237369538,0.020848000422120094,0.004615266181181815,0.10540799796581268,0.15014399588108063,0.12211520001292228,0.11896000057458878,0.012772013396624768,0.17315199971199036,0.21478399634361267,0.1881632000207901,0.1873439997434616,0.011761164657572015,0.03481600061058998,0.058079998940229416,0.042200000025331974,0.03969600051641464,0.006461281521769989,0.02143999934196472,0.038816001266241074,0.02466559996828437,0.023856000043451786,0.0035418033748569927,32,4,2048,768,151936,True,True,False,32,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.015904000028967857,0.03753599897027016,0.023127999808639287,0.02527999971061945,0.0054230027890305385,0.018400000408291817,0.03001599945127964,0.020488000102341176,0.020096000283956528,0.002585688394137697,0.10355199873447418,0.1438400000333786,0.11536479964852334,0.11124800145626068,0.011136028294919255,0.16502399742603302,0.2072959989309311,0.18646399974822997,0.19075199961662292,0.013296437761247597,0.03142400085926056,0.05215999856591225,0.03888959977775812,0.03750399872660637,0.0057399302067536314,0.020128000527620316,0.04064000025391579,0.023937600292265417,0.023648000322282314,0.004144197400898248,32,4,2048,768,151936,True,True,False,16,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.015200000256299973,0.037151999771595,0.02144480012357235,0.02195199951529503,0.005576364091933697,0.017952000722289085,0.0226879995316267,0.019934400077909233,0.019952000118792057,0.0011812499470458758,0.10063999891281128,0.13468800485134125,0.11595199964940547,0.1207519993185997,0.011974301621092394,0.16332800686359406,0.20748800039291382,0.18162400051951408,0.1767839938402176,0.01474120743720334,0.03222399950027466,0.043455999344587326,0.03829439990222454,0.03859200142323971,0.0031378131240041122,0.020959999412298203,0.03587200120091438,0.023795200139284135,0.023423999547958374,0.0031339489695198443,32,4,2048,768,151936,True,True,False,8,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.014944000169634819,0.04022400081157684,0.02162720002233982,0.022463999688625336,0.006017219317026815,0.018464000895619392,0.026528000831604004,0.021264000236988066,0.020896000787615776,0.001934095017586082,0.10156799852848053,0.1703999936580658,0.12565439902245998,0.1244799979031086,0.01641776722785409,0.1653759926557541,0.23865599930286407,0.1969360001385212,0.19223999977111816,0.02058903050274278,0.03254399821162224,0.06752000004053116,0.0443536002188921,0.041519999504089355,0.00984829071098989,0.020096000283956528,0.040031999349594116,0.026934400014579297,0.02478400059044361,0.00562071702648593,32,4,2048,768,151936,True,True,False,1,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.7531200051307678,0.8461440205574036,0.7618160009384155,0.7576479911804199,0.019464233617982506,0.2922559976577759,0.2985599935054779,0.2953856036067009,0.29576000571250916,0.001575901077689139,0.510047972202301,0.5140479803085327,0.5121696025133133,0.5123839974403381,0.0012263137313476844,,,,,,32,4,2048,768,151936,True,True,False,8192,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.39529600739479065,0.47279998660087585,0.40216960161924364,0.39825600385665894,0.0162749796240819,0.15625600516796112,0.16211199760437012,0.15959519892930984,0.15988799929618835,0.0011493418942396922,0.2635200023651123,0.2642880082130432,0.2639120012521744,0.26392000913619995,0.0001903593401384135,,,,,,32,4,2048,768,151936,True,True,False,4096,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.2531839907169342,0.32419198751449585,0.26924319565296173,0.26049599051475525,0.01657194262368116,0.10051199793815613,0.2375359982252121,0.12411200068891048,0.10311999917030334,0.039466165428540506,0.15881599485874176,0.19289599359035492,0.16896959990262986,0.1640480011701584,0.009162416086418127,,,,,,32,4,2048,768,151936,True,True,False,2048,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.15302400290966034,0.1780800074338913,0.16284480094909667,0.16113600134849548,0.00740355661356144,0.14521600306034088,0.20233599841594696,0.17807039842009545,0.1796799972653389,0.013908448560094403,0.0907519981265068,0.09750399738550186,0.09460479989647866,0.09478399902582169,0.0017965487667361475,,,,,,32,4,2048,768,151936,True,True,False,1024,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11740799993276596,0.17958399653434753,0.13262080028653145,0.12878400087356567,0.014146060532423056,0.17468799650669098,0.21161599457263947,0.1910431995987892,0.18966399878263474,0.01270903647700971,0.05926400050520897,0.07577600330114365,0.06530559975653887,0.06404799968004227,0.00479458462514625,,,,,,32,4,2048,768,151936,True,True,False,512,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11260800063610077,0.17209599912166595,0.13610880002379416,0.13809599727392197,0.017675922458993677,0.18729600310325623,0.26822400093078613,0.20815680101513861,0.2078079953789711,0.01700718593658771,0.04499199986457825,0.06537599861621857,0.0507551996037364,0.04787199944257736,0.005727123232692158,,,,,,32,4,2048,768,151936,True,True,False,256,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11257600039243698,0.23452800512313843,0.13866880126297473,0.13964799791574478,0.026590047697062115,0.1828799992799759,0.24751999974250793,0.20735519900918006,0.20670399814844131,0.01500438131586336,0.03654399886727333,0.05593600124120712,0.0408239996060729,0.03892800025641918,0.004864409450710354,,,,,,32,4,2048,768,151936,True,True,False,128,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11734399944543839,0.18726399540901184,0.13890240006148816,0.13308800011873245,0.019086167340006257,0.16991999745368958,0.2443840056657791,0.19185120090842248,0.19257599860429764,0.018017536159219673,0.03033600002527237,0.04956800118088722,0.038387199863791466,0.03750400058925152,0.00500897018702887,,,,,,32,4,2048,768,151936,True,True,False,64,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11420799791812897,0.18115200102329254,0.13938880078494548,0.1393439993262291,0.018141313962922536,0.1693439930677414,0.221343994140625,0.19187839925289155,0.19438399374485016,0.01430752121272052,0.03017600066959858,0.06019200012087822,0.03866560012102127,0.0364960003644228,0.0074199790031205266,,,,,,32,4,2048,768,151936,True,True,False,32,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11276800185441971,0.16223999857902527,0.1331360016018152,0.13305599987506866,0.01432493740801146,0.17187200486660004,0.23625600337982178,0.19287680014967917,0.1913280040025711,0.01754628831589704,0.029823999851942062,0.07574400305747986,0.03900959976017475,0.036927999928593636,0.009277465083962879,,,,,,32,4,2048,768,151936,True,True,False,16,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.1130559965968132,0.19814400374889374,0.13379519879817964,0.12531199678778648,0.020074427302248836,0.16841599345207214,0.21139200031757355,0.1893615983426571,0.19092799723148346,0.012418720676762711,0.029503999277949333,0.07558400183916092,0.040144000016152856,0.03728000074625015,0.010223239196498205,,,,,,32,4,2048,768,151936,True,True,False,8,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.115167997777462,0.2640640139579773,0.13805920109152794,0.13118399679660797,0.03132849683515479,0.17103999853134155,0.2977280020713806,0.19899839907884598,0.1976960003376007,0.028928046763067948,0.0297279991209507,0.05990400165319443,0.03834720011800528,0.0363520011305809,0.0070885330713495905,,,,,,32,4,2048,768,151936,True,True,False,1,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.3940800130367279,0.46483200788497925,0.39923040121793746,0.3957759886980057,0.015108903906233569,0.16022400557994843,0.1634880006313324,0.16203359961509706,0.16228799521923065,0.0010220720912414007,0.26073598861694336,0.2627840042114258,0.26138080209493636,0.2613760083913803,0.00041640363494172775,,,,,,32,4,2048,768,151936,True,True,False,8192,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.2536959946155548,0.27161601185798645,0.25960480123758317,0.2577280104160309,0.005142017404245015,0.09849599748849869,0.10281600058078766,0.10057279989123344,0.10063999891281128,0.0008836232237268523,0.13913600146770477,0.17606399953365326,0.15875840038061143,0.15988799929618835,0.010058952112389172,,,,,,32,4,2048,768,151936,True,True,False,4096,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.15142400562763214,0.1780479997396469,0.16139679849147798,0.1602879986166954,0.0070656055731385115,0.14601600170135498,0.23343999683856964,0.17913119941949845,0.179967999458313,0.023065274309176566,0.09388799965381622,0.12108799815177917,0.10317599996924401,0.10073599964380264,0.007364345618470178,,,,,,32,4,2048,768,151936,True,True,False,2048,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.12198399752378464,0.18406400084495544,0.13770400024950505,0.13232000172138214,0.01623164885687898,0.1773120015859604,0.20688000321388245,0.1870912007987499,0.18535999953746796,0.00825628058448234,0.05766399949789047,0.06739199906587601,0.06187200043350458,0.061216000467538834,0.003004312467037863,,,,,,32,4,2048,768,151936,True,True,False,1024,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11737599968910217,0.18614399433135986,0.13109439946711063,0.12771200388669968,0.01372168725934724,0.17587199807167053,0.2699519991874695,0.19926720038056372,0.19075199961662292,0.02327478982490538,0.041728001087903976,0.06224000081419945,0.05000480003654957,0.049375999718904495,0.006185850944273961,,,,,,32,4,2048,768,151936,True,True,False,512,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.1159679964184761,0.1610880047082901,0.12988320142030715,0.12494400516152382,0.011865375783184065,0.1767680048942566,0.27379199862480164,0.21035519987344742,0.21275199949741364,0.023206964017550125,0.037087999284267426,0.062144000083208084,0.04439679980278015,0.04283200018107891,0.006155226392511492,,,,,,32,4,2048,768,151936,True,True,False,256,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.12015999853610992,0.158720001578331,0.13573280088603495,0.1343199983239174,0.010666830836014414,0.1737920045852661,0.22127999365329742,0.20261440128087999,0.20321600139141083,0.011153965849067301,0.03587200120091438,0.04944000020623207,0.038265600241720675,0.037328001111745834,0.0030431580113575636,,,,,,32,4,2048,768,151936,True,True,False,128,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.1212799996137619,0.15139199793338776,0.13183839991688728,0.13014400005340576,0.00851207547022104,0.17257599532604218,0.2250880002975464,0.1872655987739563,0.18193599581718445,0.013638284975248818,0.02969600073993206,0.0525440014898777,0.03840640028938651,0.03444799967110157,0.007598511754254174,,,,,,32,4,2048,768,151936,True,True,False,64,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11270400136709213,0.1624639928340912,0.13508000001311302,0.13180799782276154,0.014243564451606775,0.17052799463272095,0.23715199530124664,0.1953311987221241,0.1966560035943985,0.01500670718978398,0.030239999294281006,0.05270399898290634,0.03807039987295866,0.03742399998009205,0.004906165602021684,,,,,,32,4,2048,768,151936,True,True,False,32,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.12319999933242798,0.17919999361038208,0.14059039913117885,0.14156799763441086,0.016217662982107223,0.1701119989156723,0.21987199783325195,0.18903039917349815,0.1870879977941513,0.014117854507841239,0.030368000268936157,0.06406400352716446,0.04045119984075427,0.03710399940609932,0.00878942205983628,,,,,,32,4,2048,768,151936,True,True,False,16,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11296000331640244,0.16761599481105804,0.13843199908733367,0.13814399391412735,0.014925286935582404,0.1711679995059967,0.21561600267887115,0.1906527981162071,0.1876479983329773,0.011803992622137974,0.0306560005992651,0.09216000139713287,0.041129599791020155,0.036847999319434166,0.013739721468154687,,,,,,32,4,2048,768,151936,True,True,False,8,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11929599940776825,0.2008959949016571,0.14535359852015972,0.13971199840307236,0.024769473061576282,0.16803200542926788,0.25123199820518494,0.19573760256171227,0.19366399943828583,0.023196011081705884,0.03049599938094616,0.08089599758386612,0.04739360017701984,0.0453919991850853,0.01286389846889463,,,,,,32,4,2048,768,151936,True,True,False,1,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none diff --git a/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/manifest.json b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/manifest.json new file mode 100644 index 0000000..6873602 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/manifest.json @@ -0,0 +1,53 @@ +{ + "attention_tp_coverage": [ + 1, + 2, + 4 + ], + "environment_contract": { + "dtype": "bfloat16", + "frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "hardware": "NVIDIA H20", + "model": "Qwen3-30B-A3B", + "tensor_parallel_sizes": [ + 1, + 2, + 4 + ], + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + }, + "inputs": { + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931" + }, + "outputs": { + "allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3", + "attention.csv": "674e010598eea4c06bdee04ccba413f4c34194fbe2d70b143c6ec2169b638cc5", + "attention_true_mixed_fused.csv": "92cd22f8b669abc7af488cf3d067ed880a6cced2dba1a3213a93a4f02a0001a8", + "linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "moe.csv": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0" + }, + "profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4", + "projection_contract": { + "allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity", + "attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target", + "attention_true_mixed": "Preserved as fused-total diagnostics and excluded from attention.csv because Frontier requires an unobservable prefill/decode split", + "linear": "Frontier profiler using vLLM 0.20 CUDA operators", + "moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero" + }, + "row_counts": { + "allreduce": 24, + "attention_frontier_compatible": 102, + "attention_true_mixed_fused_diagnostic": 30, + "linear": 36, + "moe": 72 + }, + "schema_version": "frontier_qwen30_vllm020_frozen_profile.v1" +} diff --git a/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/moe.csv b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/moe.csv new file mode 100644 index 0000000..e5bf507 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v1/moe.csv @@ -0,0 +1,73 @@ +time_stats.moe_gating_linear.min,time_stats.moe_gating_linear.max,time_stats.moe_gating_linear.mean,time_stats.moe_gating_linear.median,time_stats.moe_gating_linear.std,time_stats.moe_gating_routing_topk.min,time_stats.moe_gating_routing_topk.max,time_stats.moe_gating_routing_topk.mean,time_stats.moe_gating_routing_topk.median,time_stats.moe_gating_routing_topk.std,time_stats.moe_shuffling.min,time_stats.moe_shuffling.max,time_stats.moe_shuffling.mean,time_stats.moe_shuffling.median,time_stats.moe_shuffling.std,time_stats.moe_grouped_gemm.min,time_stats.moe_grouped_gemm.max,time_stats.moe_grouped_gemm.mean,time_stats.moe_grouped_gemm.median,time_stats.moe_grouped_gemm.std,num_tokens,num_experts,num_experts_per_device,expert_parallel_size,routing_runtime_path,routing_assignment_policy,routing_weight_policy,routing_uses_router_logits,gating_runtime_context,gating_runtime_context_impl,router_topk,hidden_dim,expert_hidden_dim,use_gated,num_tensor_parallel_workers,total_routed_tokens,model_expansion_ratio,tokens_per_expert_avg,tokens_to_experts_ratio,expert_utilization,min_load_ratio,load_imbalance_cv,max_load_ratio,load_entropy,load_gini_coefficient,load_distribution,seed,moe_grouped_gemm_backend,measurement_type,profiling_precision,model_arch,quant_signature,router_median_nonadditivity_ratio,projection_policy +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.33926400542259216,0.405023992061615,0.36780479848384856,0.36507199704647064,0.01690507644474779,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,1.1943039894104004,1.286784052848816,1.228384006023407,1.2273280024528503,0.02832547242381263,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.3984375,0.0,1.346291201783626,4.0,5.59375,0.661865234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,1.679744005203247,1.766144037246704,1.7095808148384095,1.7015680074691772,0.02438921262998535,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.625,0.0,1.015504800579495,5.0,6.15516433212955,0.529052734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,2.1576640605926514,2.2921600341796875,2.2097824096679686,2.188944101333618,0.045572321842012986,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.875,0.0,0.6343057228182637,2.5,6.64370748444639,0.35369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,2.422368049621582,2.5130879878997803,2.4516672134399413,2.434159994125366,0.03278900287381846,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.984375,0.0,0.4921254921257382,2.25,6.817190042344769,0.272369384765625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,2.2217600345611572,2.289599895477295,2.2571327924728393,2.263375997543335,0.021660416089449488,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,1.0,0.125,0.3486861500690843,1.875,6.908192310183997,0.197662353515625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,2.18668794631958,2.318079948425293,2.2218016147613526,2.211087942123413,0.035380897213135316,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,1.0,0.4375,0.2525504668006971,1.875,6.953347743053017,0.1410369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,2.2291839122772217,2.3929600715637207,2.2908096313476562,2.2804640531539917,0.04479348924786221,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,1.0,0.65625,0.15765965680164504,1.5625,6.98229848728205,0.08779525756835938,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,2.0678720474243164,2.1297600269317627,2.0837119817733765,2.0779199600219727,0.017880044357986735,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,1.0,0.625,0.12169081635504074,1.3125,6.9892029662356325,0.06879425048828125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,2.916032075881958,3.0819520950317383,2.9805248022079467,2.9656319618225098,0.05482195799019572,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,1.0,0.796875,0.07935434147688751,1.1796875,6.9954297964750305,0.044734954833984375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,4.421599864959717,4.535359859466553,4.486294317245483,4.497056007385254,0.036990243787549344,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,1.0,0.8203125,0.060849326483103046,1.17578125,6.9973188375859685,0.033740997314453125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,7.302591800689697,7.402751922607422,7.354758310317993,7.3464319705963135,0.032142662400335566,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,1.0,0.890625,0.0412323087266341,1.08984375,6.998772433185578,0.02334284782409668,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.35280001163482666,0.39692801237106323,0.37662720382213594,0.37196800112724304,0.013401318050665304,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.4692479968070984,0.5523840188980103,0.5134752035140991,0.5100640058517456,0.02291433464135784,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.34652799367904663,0.4119040071964264,0.3789471983909607,0.38550400733947754,0.02073945105803335,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.31462401151657104,0.7456960082054138,0.38617280423641204,0.34545600414276123,0.12230201266253077,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.32521599531173706,0.419871985912323,0.36325119733810424,0.34968000650405884,0.03161798848223672,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.289792001247406,0.4663360118865967,0.4091839998960495,0.41655999422073364,0.0446001986506615,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.3761279881000519,0.4416320025920868,0.40686399936676027,0.40540799498558044,0.02257778769899645,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.7172480225563049,0.8663039803504944,0.7723807990550995,0.7591840028762817,0.04164772379451242,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.0195519924163818,1.2216639518737793,1.1253888130187988,1.1453600525856018,0.06548005322243594,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7490559816360474,1.9644800424575806,1.8529024004936219,1.814303994178772,0.08042565617327288,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,3.2479360103607178,3.385279893875122,3.296070408821106,3.2800960540771484,0.04525529026442046,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,6.344799995422363,6.517856121063232,6.464438438415527,6.478623867034912,0.05116674443145098,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.2648000121116638,0.325439989566803,0.28852800130844114,0.28390398621559143,0.01933778377635077,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.7347840070724487,0.862496018409729,0.7769344031810761,0.769216001033783,0.03290485328796285,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.421875,0.0,1.346291201783626,6.0,5.652114648336087,0.636962890625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.9198399782180786,0.9646080136299133,0.9412063956260681,0.9411839842796326,0.014939365085478117,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.5703125,0.0,1.118033988749895,5.0,6.008641773518898,0.580810546875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,1.2796800136566162,1.3484159708023071,1.3006976008415223,1.2929120063781738,0.020807998177176254,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.8828125,0.0,0.6959705453537527,3.0,6.60872850615583,0.38055419921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,1.3630399703979492,1.4430400133132935,1.3909215927124023,1.3892319798469543,0.022335744492366926,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.984375,0.0,0.5201036555341637,3.0,6.798826509158851,0.28302001953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,1.27948796749115,1.3904000520706177,1.3176063895225525,1.309440016746521,0.038060887827312775,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,1.0,0.25,0.3511282039725661,1.875,6.91002266305238,0.1970977783203125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,1.264415979385376,1.3145920038223267,1.2791999936103822,1.2753440141677856,0.014130605249568332,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,1.0,0.375,0.24692938483248605,1.6875,6.955481130775285,0.13909912109375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,1.3081920146942139,1.347648024559021,1.3292255997657776,1.329967975616455,0.014558863679016933,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,1.0,0.625,0.17143053326165383,1.5625,6.9786675275754035,0.09520339965820312,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.242751955986023,1.3112000226974487,1.2747935891151427,1.266207993030548,0.021093073517695057,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,1.0,0.78125,0.11000099875256815,1.296875,6.991308871213679,0.062183380126953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7388160228729248,1.8077759742736816,1.772764801979065,1.772704005241394,0.021056644077284283,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,1.0,0.78125,0.0864630150197678,1.1796875,6.994552526394139,0.048796653747558594,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,2.6563520431518555,2.7063679695129395,2.6785055875778196,2.6791679859161377,0.01639963463052944,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,1.0,0.8671875,0.06127686514721937,1.16015625,6.997291583027146,0.03497934341430664,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,4.386879920959473,4.452256202697754,4.4108480453491214,4.406303882598877,0.019768161791937636,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,1.0,0.884765625,0.041723768525324195,1.1171875,6.998746434318934,0.02298593521118164,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.24208000302314758,0.4028480052947998,0.3041536003351212,0.277103990316391,0.05660721484881584,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.2447039932012558,0.30502399802207947,0.26446720361709597,0.26265600323677063,0.016744548364435372,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.2337920069694519,0.2881599962711334,0.26074880361557007,0.264384001493454,0.016469850143940968,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.23369599878787994,0.28591999411582947,0.25465920120477675,0.25385600328445435,0.01593204652619194,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.2295999974012375,0.26556798815727234,0.24674240052700042,0.2497600018978119,0.010345732066199003,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.21721599996089935,0.29020801186561584,0.2394208014011383,0.2346400022506714,0.018747330357768585,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2717759907245636,0.305184006690979,0.28813759982585907,0.28809599578380585,0.01183422600545854,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.3917759954929352,0.43772798776626587,0.41130879521369934,0.4131519943475723,0.012992473640805227,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.6176639795303345,0.7009919881820679,0.642767995595932,0.6330719888210297,0.024074084919938756,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.0820800065994263,1.1674879789352417,1.1034304022789,1.0977439880371094,0.0233067292981566,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.9809919595718384,2.0415360927581787,2.003715181350708,1.992751955986023,0.022066076434645737,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,3.790112018585205,3.8651199340820312,3.829139161109924,3.8230879306793213,0.025177464160110564,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.212351992726326,0.24383999407291412,0.22760000079870224,0.22723200172185898,0.01050568575837594,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.47494399547576904,0.5184000134468079,0.4920704007148743,0.49169600009918213,0.011991064701471855,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.3984375,0.0,1.3919410907075054,6.0,5.570159765557392,0.667236328125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.6360960006713867,0.7004479765892029,0.6608384013175964,0.6572319865226746,0.020416877242438597,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.625,0.0,1.0307764064044151,4.0,6.138251855282827,0.5382080078125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.780896008014679,0.8301439881324768,0.80346559882164,0.8030399978160858,0.016801230312128875,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.859375,0.0,0.6903350635742038,3.0,6.5943747091218174,0.38067626953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.8607040047645569,0.9195200204849243,0.8783008038997651,0.8751039803028107,0.01719059253115595,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.9765625,0.0,0.49410588440130926,2.75,6.814452474347134,0.271270751953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.833952009677887,0.894752025604248,0.8619967997074127,0.863215982913971,0.018716378851797198,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,1.0,0.25,0.33693529145074724,2.125,6.9186075263155535,0.1867218017578125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.834879994392395,0.8871039748191833,0.8651552021503448,0.8638879954814911,0.015262430894400969,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,1.0,0.375,0.25567294018677456,1.8125,6.952441049154937,0.14349365234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.8518080115318298,0.9097599983215332,0.8810272097587586,0.8751040101051331,0.017005819633271906,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,1.0,0.625,0.1747801353218523,1.53125,6.978069554482723,0.09820938110351562,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.8470079898834229,0.9010239839553833,0.8694015920162201,0.8716959953308105,0.016138881171190216,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,1.0,0.65625,0.1158122428154187,1.3125,6.9901908183358845,0.06445503234863281,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.1698240041732788,1.2311359643936157,1.1888479948043824,1.1890720129013062,0.017978797794492758,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,1.0,0.78125,0.08347181893108634,1.1796875,6.994921772573154,0.046871185302734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.7702720165252686,1.8097599744796753,1.7919103980064393,1.7956640124320984,0.012641295676021557,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,1.0,0.78125,0.06866734477822484,1.20703125,6.996602562938728,0.03801727294921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.968672037124634,3.0278079509735107,2.9899007797241213,2.9824799299240112,0.01816414122631667,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,1.0,0.8984375,0.04399546833977376,1.126953125,6.998607314922362,0.024699926376342773,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.19574399292469025,0.2512960135936737,0.21939200013875962,0.2199999988079071,0.017532156418212565,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.20483200252056122,0.24006399512290955,0.22215040028095245,0.22433599829673767,0.00969639786132892,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.20559999346733093,0.24726399779319763,0.22126719802618028,0.22207999974489212,0.01328093478485499,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.20003199577331543,0.2301120012998581,0.21453119963407516,0.21598400175571442,0.010402239855151332,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.19551999866962433,0.22972799837589264,0.21238719969987868,0.21488000452518463,0.010706095835489097,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.19420799612998962,0.2903999984264374,0.2211231991648674,0.21476799994707108,0.025955008886196004,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2290560007095337,0.289247989654541,0.2543327987194061,0.24939200282096863,0.01663236753503115,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.30831998586654663,0.36953601241111755,0.3324000000953674,0.3288639932870865,0.018617999572156707,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.462911993265152,0.5497919917106628,0.4893856018781662,0.4816960096359253,0.023348152887178286,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,0.7662079930305481,0.8717759847640991,0.788454395532608,0.7744799852371216,0.03174746482604812,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.363935947418213,1.4143040180206299,1.3812703967094422,1.3798720240592957,0.014770450075530007,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.5507519245147705,2.680704116821289,2.579859209060669,2.566223978996277,0.03673283558365955,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize diff --git a/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/allreduce.json b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/allreduce.json new file mode 100644 index 0000000..aae1f0a --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/allreduce.json @@ -0,0 +1,824 @@ +{ + "environment": [ + { + "backend_env": { + "VLLM_ALLREDUCE_USE_FLASHINFER": "1", + "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm" + }, + "gpu": "NVIDIA H20", + "model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B", + "torch_cuda": "12.9", + "torch_version": "2.11.0+cu129", + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + }, + { + "backend_env": { + "VLLM_ALLREDUCE_USE_FLASHINFER": "1", + "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm" + }, + "gpu": "NVIDIA H20", + "model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B", + "torch_cuda": "12.9", + "torch_version": "2.11.0+cu129", + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + } + ], + "frontier_consumption": "diagnostic_only_in_base_profile_only_run; measured lookup requires a separate CC-backend injection ablation", + "rows": [ + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08288000151515007, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 1, + "payload_bytes": 4096, + "per_rank_time_ms": [ + { + "max": 0.4872959852218628, + "mean": 0.1310015980154276, + "median": 0.07679999992251396, + "min": 0.06217600032687187, + "std": 0.12790721677293843 + }, + { + "max": 0.4402880072593689, + "mean": 0.12842560112476348, + "median": 0.08288000151515007, + "min": 0.0655680000782013, + "std": 0.11220176524616535 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.0793600007891655, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 8, + "payload_bytes": 32768, + "per_rank_time_ms": [ + { + "max": 0.12716799974441528, + "mean": 0.07871360033750534, + "median": 0.0759200006723404, + "min": 0.06032000109553337, + "std": 0.019331314939874535 + }, + { + "max": 0.12380799651145935, + "mean": 0.08059840016067028, + "median": 0.0793600007891655, + "min": 0.06217600032687187, + "std": 0.01687088356254092 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.0713919997215271, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 16, + "payload_bytes": 65536, + "per_rank_time_ms": [ + { + "max": 0.12697599828243256, + "mean": 0.0767391998320818, + "median": 0.07078400254249573, + "min": 0.05910399928689003, + "std": 0.018775178979463278 + }, + { + "max": 0.11430399864912033, + "mean": 0.07594559974968433, + "median": 0.0713919997215271, + "min": 0.06124800071120262, + "std": 0.015719922150631036 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08056000247597694, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 32, + "payload_bytes": 131072, + "per_rank_time_ms": [ + { + "max": 0.1037760004401207, + "mean": 0.07954559996724128, + "median": 0.08056000247597694, + "min": 0.05955199897289276, + "std": 0.0135697420393132 + }, + { + "max": 0.10608000308275223, + "mean": 0.07971520014107228, + "median": 0.07593599706888199, + "min": 0.06028800085186958, + "std": 0.015319849772775456 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.0865279994904995, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 64, + "payload_bytes": 262144, + "per_rank_time_ms": [ + { + "max": 0.14470399916172028, + "mean": 0.09121599942445754, + "median": 0.0865279994904995, + "min": 0.06441599875688553, + "std": 0.024893837894277456 + }, + { + "max": 0.12438400089740753, + "mean": 0.08531199917197227, + "median": 0.08031999692320824, + "min": 0.06364800035953522, + "std": 0.01878029830059533 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.07135999947786331, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 128, + "payload_bytes": 524288, + "per_rank_time_ms": [ + { + "max": 0.11753600090742111, + "mean": 0.07606079950928687, + "median": 0.07135999947786331, + "min": 0.05843200162053108, + "std": 0.01755519771639284 + }, + { + "max": 0.1103999987244606, + "mean": 0.07607359997928143, + "median": 0.07073600217700005, + "min": 0.05721599981188774, + "std": 0.016904445220949783 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.07321599870920181, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 256, + "payload_bytes": 1048576, + "per_rank_time_ms": [ + { + "max": 0.11740799993276596, + "mean": 0.07749119997024537, + "median": 0.07203200086951256, + "min": 0.05862399935722351, + "std": 0.017701381594822835 + }, + { + "max": 0.11382400244474411, + "mean": 0.07733759954571724, + "median": 0.07321599870920181, + "min": 0.059039998799562454, + "std": 0.017188890882557036 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.09025600180029869, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 512, + "payload_bytes": 2097152, + "per_rank_time_ms": [ + { + "max": 0.13980799913406372, + "mean": 0.0950367994606495, + "median": 0.09025600180029869, + "min": 0.06815999746322632, + "std": 0.022829835127539378 + }, + { + "max": 0.14764800667762756, + "mean": 0.09710080176591873, + "median": 0.08720000088214874, + "min": 0.07152000069618225, + "std": 0.02500574954063789 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08083200082182884, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 1024, + "payload_bytes": 4194304, + "per_rank_time_ms": [ + { + "max": 0.15574400126934052, + "mean": 0.08760640025138855, + "median": 0.07846399769186974, + "min": 0.07097599655389786, + "std": 0.024487311423551025 + }, + { + "max": 0.16284799575805664, + "mean": 0.08963519930839539, + "median": 0.08083200082182884, + "min": 0.07100799679756165, + "std": 0.026047320562445356 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.10891199856996536, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 2048, + "payload_bytes": 8388608, + "per_rank_time_ms": [ + { + "max": 0.1582079976797104, + "mean": 0.11541439890861512, + "median": 0.10836799815297127, + "min": 0.09062399715185165, + "std": 0.018285136316576037 + }, + { + "max": 0.1578879952430725, + "mean": 0.11537599861621857, + "median": 0.10891199856996536, + "min": 0.0960640013217926, + "std": 0.018246538626977286 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.1703840047121048, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 4096, + "payload_bytes": 16777216, + "per_rank_time_ms": [ + { + "max": 0.19327999651432037, + "mean": 0.1707327976822853, + "median": 0.1703840047121048, + "min": 0.14815999567508698, + "std": 0.014211056022719618 + }, + { + "max": 0.19276799261569977, + "mean": 0.1658592015504837, + "median": 0.16379200667142868, + "min": 0.14560000598430634, + "std": 0.013840249648693638 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.25539200007915497, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 8192, + "payload_bytes": 33554432, + "per_rank_time_ms": [ + { + "max": 0.2807359993457794, + "mean": 0.25750079900026324, + "median": 0.25539200007915497, + "min": 0.24624000489711761, + "std": 0.008925204570802302 + }, + { + "max": 0.2863999903202057, + "mean": 0.2585055992007256, + "median": 0.255280002951622, + "min": 0.24371199309825897, + "std": 0.012059738582656496 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.1021759994328022, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 1, + "payload_bytes": 4096, + "per_rank_time_ms": [ + { + "max": 0.9770879745483398, + "mean": 0.1974783968180418, + "median": 0.10099200159311295, + "min": 0.05913599953055382, + "std": 0.2660581738745569 + }, + { + "max": 0.892799973487854, + "mean": 0.18164799660444259, + "median": 0.1021759994328022, + "min": 0.06435199826955795, + "std": 0.23947520188197013 + }, + { + "max": 0.6467199921607971, + "mean": 0.15839359983801843, + "median": 0.10100800171494484, + "min": 0.06800000369548798, + "std": 0.16617013141866102 + }, + { + "max": 0.6725760102272034, + "mean": 0.15686400160193442, + "median": 0.10044800117611885, + "min": 0.06063999980688095, + "std": 0.17523222161300497 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.12694399803876877, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 8, + "payload_bytes": 32768, + "per_rank_time_ms": [ + { + "max": 0.6659520268440247, + "mean": 0.1927648030221462, + "median": 0.12694399803876877, + "min": 0.07609599828720093, + "std": 0.16697784531907736 + }, + { + "max": 0.695360004901886, + "mean": 0.19356480240821838, + "median": 0.11726400256156921, + "min": 0.0796160027384758, + "std": 0.17588096678867862 + }, + { + "max": 0.5939840078353882, + "mean": 0.1868800014257431, + "median": 0.12379200011491776, + "min": 0.07427199929952621, + "std": 0.1463231714943902 + }, + { + "max": 0.6635839939117432, + "mean": 0.1874335989356041, + "median": 0.12014400213956833, + "min": 0.07526399940252304, + "std": 0.16680959677760304 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.09161599725484848, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 16, + "payload_bytes": 65536, + "per_rank_time_ms": [ + { + "max": 0.12665599584579468, + "mean": 0.09443839862942696, + "median": 0.09124799817800522, + "min": 0.06431999802589417, + "std": 0.0205690155775906 + }, + { + "max": 0.1303039938211441, + "mean": 0.09712959825992584, + "median": 0.09161599725484848, + "min": 0.07648000121116638, + "std": 0.019072048129173236 + }, + { + "max": 0.13836799561977386, + "mean": 0.09821119979023933, + "median": 0.09148800000548363, + "min": 0.0727040022611618, + "std": 0.021314066545189116 + }, + { + "max": 0.12992000579833984, + "mean": 0.09272959977388381, + "median": 0.08931199833750725, + "min": 0.06406400352716446, + "std": 0.021360803830354765 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08580800145864487, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 32, + "payload_bytes": 131072, + "per_rank_time_ms": [ + { + "max": 0.13449600338935852, + "mean": 0.08960640132427215, + "median": 0.08299200236797333, + "min": 0.06796800345182419, + "std": 0.020633597898445998 + }, + { + "max": 0.14735999703407288, + "mean": 0.09248319901525974, + "median": 0.08580800145864487, + "min": 0.05913599953055382, + "std": 0.025960234928829564 + }, + { + "max": 0.13705599308013916, + "mean": 0.08947199806571007, + "median": 0.08460799977183342, + "min": 0.0634239986538887, + "std": 0.02122838946992339 + }, + { + "max": 0.13846400380134583, + "mean": 0.0859104000031948, + "median": 0.08308799937367439, + "min": 0.05974400043487549, + "std": 0.022373631424433445 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.09867199882864952, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 64, + "payload_bytes": 262144, + "per_rank_time_ms": [ + { + "max": 0.1279039978981018, + "mean": 0.09748799875378608, + "median": 0.09678399935364723, + "min": 0.0655359998345375, + "std": 0.02315719144614622 + }, + { + "max": 0.1356479972600937, + "mean": 0.10018239840865135, + "median": 0.09532799944281578, + "min": 0.06185600161552429, + "std": 0.02330700253650042 + }, + { + "max": 0.13142399489879608, + "mean": 0.09778879955410957, + "median": 0.09492799639701843, + "min": 0.06560000032186508, + "std": 0.02307579212430045 + }, + { + "max": 0.1276479959487915, + "mean": 0.09611519873142242, + "median": 0.09867199882864952, + "min": 0.0642239972949028, + "std": 0.02273612181973431 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.09646400064229965, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 128, + "payload_bytes": 524288, + "per_rank_time_ms": [ + { + "max": 0.14060799777507782, + "mean": 0.09770880043506622, + "median": 0.09115200117230415, + "min": 0.06950400024652481, + "std": 0.024752645089849798 + }, + { + "max": 0.14377599954605103, + "mean": 0.09824960008263588, + "median": 0.08999999985098839, + "min": 0.07103999704122543, + "std": 0.025403407389046027 + }, + { + "max": 0.13680000603199005, + "mean": 0.09993600100278854, + "median": 0.09646400064229965, + "min": 0.06790400296449661, + "std": 0.022297424273985882 + }, + { + "max": 0.1391039937734604, + "mean": 0.09769919961690902, + "median": 0.09601600095629692, + "min": 0.06835199892520905, + "std": 0.023899922100804445 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08377600088715553, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 256, + "payload_bytes": 1048576, + "per_rank_time_ms": [ + { + "max": 0.1464959979057312, + "mean": 0.09276160076260567, + "median": 0.08377600088715553, + "min": 0.06777600198984146, + "std": 0.024751086465295658 + }, + { + "max": 0.14319999516010284, + "mean": 0.09080640003085136, + "median": 0.080400001257658, + "min": 0.06796800345182419, + "std": 0.023365718881708488 + }, + { + "max": 0.1382399946451187, + "mean": 0.09063360020518303, + "median": 0.08193599805235863, + "min": 0.06627199798822403, + "std": 0.02372618650854302 + }, + { + "max": 0.14313599467277527, + "mean": 0.09044799953699112, + "median": 0.08128000050783157, + "min": 0.06652799993753433, + "std": 0.02462486862033686 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.1128000020980835, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 512, + "payload_bytes": 2097152, + "per_rank_time_ms": [ + { + "max": 0.1600320041179657, + "mean": 0.11426239982247352, + "median": 0.1128000020980835, + "min": 0.07657600194215775, + "std": 0.030240087702350687 + }, + { + "max": 0.15881599485874176, + "mean": 0.11206399947404862, + "median": 0.10628800094127655, + "min": 0.0772159993648529, + "std": 0.029933135345483627 + }, + { + "max": 0.15612800419330597, + "mean": 0.10761600062251091, + "median": 0.09860799834132195, + "min": 0.07689599692821503, + "std": 0.02634237020678647 + }, + { + "max": 0.15865600109100342, + "mean": 0.11094079986214637, + "median": 0.10979199782013893, + "min": 0.07583999633789062, + "std": 0.028619217028542445 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08755199983716011, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 1024, + "payload_bytes": 4194304, + "per_rank_time_ms": [ + { + "max": 0.11999999731779099, + "mean": 0.0843871995806694, + "median": 0.08755199983716011, + "min": 0.06332799792289734, + "std": 0.016638543890716024 + }, + { + "max": 0.1218239963054657, + "mean": 0.08518079966306687, + "median": 0.08032000064849854, + "min": 0.06393600255250931, + "std": 0.019900899429956945 + }, + { + "max": 0.11849600076675415, + "mean": 0.0843968003988266, + "median": 0.08702399954199791, + "min": 0.06297600269317627, + "std": 0.017031908773433545 + }, + { + "max": 0.12300799787044525, + "mean": 0.0846304003149271, + "median": 0.08139199763536453, + "min": 0.06195199862122536, + "std": 0.020106523698622265 + } + ], + "selected_backend": "nccl_fallback", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.12361599877476692, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 2048, + "payload_bytes": 8388608, + "per_rank_time_ms": [ + { + "max": 0.17155200242996216, + "mean": 0.12168959975242614, + "median": 0.12055999785661697, + "min": 0.09609600156545639, + "std": 0.022565485532483928 + }, + { + "max": 0.9246399998664856, + "mean": 0.19978560134768486, + "median": 0.12361599877476692, + "min": 0.09715200215578079, + "std": 0.24230694305662265 + }, + { + "max": 0.9317439794540405, + "mean": 0.20037759989500045, + "median": 0.12327999994158745, + "min": 0.09603200107812881, + "std": 0.24450903278000383 + }, + { + "max": 0.9321280121803284, + "mean": 0.19875840097665787, + "median": 0.12230399996042252, + "min": 0.0950080007314682, + "std": 0.24519252220785093 + } + ], + "selected_backend": "nccl_fallback", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.20030399411916733, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 4096, + "payload_bytes": 16777216, + "per_rank_time_ms": [ + { + "max": 0.3261440098285675, + "mean": 0.20826880186796187, + "median": 0.19974400103092194, + "min": 0.1409280002117157, + "std": 0.051397264855718945 + }, + { + "max": 0.3248000144958496, + "mean": 0.20548800230026246, + "median": 0.1979840025305748, + "min": 0.141184002161026, + "std": 0.04980002399656717 + }, + { + "max": 0.32547199726104736, + "mean": 0.21280319690704347, + "median": 0.20030399411916733, + "min": 0.14127999544143677, + "std": 0.05304243085685509 + }, + { + "max": 0.26047998666763306, + "mean": 0.1969312012195587, + "median": 0.18079999834299088, + "min": 0.14057600498199463, + "std": 0.042174123638424224 + } + ], + "selected_backend": "nccl_fallback", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.2924960106611252, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 8192, + "payload_bytes": 33554432, + "per_rank_time_ms": [ + { + "max": 0.45052799582481384, + "mean": 0.3013375997543335, + "median": 0.28273600339889526, + "min": 0.21334399282932281, + "std": 0.07214223764527236 + }, + { + "max": 0.4466240108013153, + "mean": 0.2968191936612129, + "median": 0.27796798944473267, + "min": 0.21241599321365356, + "std": 0.07400678240849741 + }, + { + "max": 0.3830080032348633, + "mean": 0.2946112036705017, + "median": 0.2924960106611252, + "min": 0.21084800362586975, + "std": 0.05323627615746332 + }, + { + "max": 0.4609600007534027, + "mean": 0.3054272010922432, + "median": 0.289792001247406, + "min": 0.21062399446964264, + "std": 0.07608482904865672 + } + ], + "selected_backend": "nccl_fallback", + "tensor_parallel_size": 4 + } + ], + "schema_version": "qwen30_vllm020_allreduce_frozen.v1" +} diff --git a/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/attention.csv b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/attention.csv new file mode 100644 index 0000000..3220b8b --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/attention.csv @@ -0,0 +1,133 @@ +time_stats.attn_input_reshape.min,time_stats.attn_input_reshape.max,time_stats.attn_input_reshape.mean,time_stats.attn_input_reshape.median,time_stats.attn_input_reshape.std,time_stats.attn_kv_cache_save.min,time_stats.attn_kv_cache_save.max,time_stats.attn_kv_cache_save.mean,time_stats.attn_kv_cache_save.median,time_stats.attn_kv_cache_save.std,time_stats.attn_prefill.min,time_stats.attn_prefill.max,time_stats.attn_prefill.mean,time_stats.attn_prefill.median,time_stats.attn_prefill.std,time_stats.attn_decode.min,time_stats.attn_decode.max,time_stats.attn_decode.mean,time_stats.attn_decode.median,time_stats.attn_decode.std,time_stats.attn_output_reshape.min,time_stats.attn_output_reshape.max,time_stats.attn_output_reshape.mean,time_stats.attn_output_reshape.median,time_stats.attn_output_reshape.std,n_embd,n_q_head,n_kv_head,block_size,num_tensor_parallel_workers,max_model_len,batch_size,prefill_chunk_size,kv_cache_size,is_prefill,attention_backend,is_mixed_batch,mode,seq_lens,total_tokens,max_seq_len,min_seq_len,avg_seq_len,equal_seq_len,seq_len_variance,seq_len_std,seq_len_cv,is_chunked_prefill_sample,chunk_start_token,chunk_end_token,total_prefill_tokens,profiling_precision,model_arch,quant_signature,measurement_type,is_true_mixed_batch,prefill_seq_lens,prefill_kv_cache_sizes,decode_kv_cache_sizes,num_prefill_seqs,num_decode_seqs,decode_batch_size,total_batch_size,total_decode_tokens,decode_avg_kv_cache_size,batch_composition_ratio,batch_spec,projection_policy +0.0,0.0,0.0,0.0,0.0,0.01414399966597557,0.028863999992609024,0.019705599918961526,0.01771199982613325,0.005157200849836681,0.047968000173568726,0.07046400010585785,0.05810240097343922,0.05810240097343922,0.007477463486041561,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.04947200044989586,0.020412799902260303,0.01635199971497059,0.010107497379722417,0.046560000628232956,0.08323200047016144,0.05587520003318787,0.05587520003318787,0.011126758739503428,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.022207999601960182,0.017033600155264138,0.015312000177800655,0.002819235991970241,0.05104000121355057,0.07692799717187881,0.056396800279617305,0.056396800279617305,0.007481982178637539,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015072000212967396,0.022272000089287758,0.01706880023702979,0.01616000011563301,0.002460889579319197,0.06931199878454208,0.0838719978928566,0.07432000041007995,0.07432000041007995,0.004777766433175866,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.018592000007629395,0.028543999418616295,0.02095999978482723,0.019183999858796597,0.003198175496053494,0.12179200351238251,0.15408000349998474,0.1307712011039257,0.1307712011039257,0.00858807797538298,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.027775999158620834,0.03385600075125694,0.030131200328469276,0.029680000618100166,0.0021152558103575215,0.32678401470184326,0.3450239896774292,0.33396480381488797,0.33396480381488797,0.0045872424917606375,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.04438399896025658,0.05084799975156784,0.046540799736976626,0.04531199857592583,0.002277905811237223,1.0959680080413818,1.1151360273361206,1.0999775886535645,1.0999775886535645,0.005694403246120485,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.078015998005867,0.08691199868917465,0.08114239946007729,0.08019199967384338,0.00292795706334475,4.070400238037109,4.113152027130127,4.087088012695312,4.087088012695312,0.013660567012509554,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.05167999863624573,0.022115200012922286,0.017583999782800674,0.010340094822340818,0.05196800082921982,0.09011200070381165,0.06328320093452933,0.06328320093452933,0.012557341255467452,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01583999954164028,0.026623999699950218,0.018927999772131443,0.017376000061631203,0.003514650316260619,0.06681600213050842,0.07993599772453308,0.0725280001759529,0.0725280001759529,0.004343502558613716,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01648000068962574,0.030880000442266464,0.01945280022919178,0.017967999912798405,0.004096211183007485,0.1311360001564026,0.1546880006790161,0.13908160030841826,0.13908160030841826,0.007511906874366178,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.03558399900794029,0.020851199887692927,0.018559999763965607,0.005235911594130716,0.32950401306152344,0.350271999835968,0.33912960588932034,0.33912960588932034,0.006027400986663648,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.019328000023961067,0.040608000010252,0.022790400311350822,0.020704000256955624,0.006113051965778337,1.1415679454803467,1.1518720388412476,1.144483208656311,1.144483208656311,0.0032332311374389127,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015807999297976494,0.030688000842928886,0.019596799835562707,0.01774400006979704,0.004343384771033462,0.0,0.0,0.0,0.0,0.0,0.049056001007556915,0.07580800354480743,0.05948160067200661,0.05948160067200661,0.009031541471446955,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.02486399933695793,0.01923839971423149,0.018079999834299088,0.0032282528537266424,0.0,0.0,0.0,0.0,0.0,0.05142400041222572,0.07353600114583969,0.059328000620007516,0.059328000620007516,0.0073307807735143084,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016543999314308167,0.03977600112557411,0.021379199624061585,0.018511999398469925,0.006593576176246171,0.0,0.0,0.0,0.0,0.0,0.0488319993019104,0.06435199826955795,0.05479039996862411,0.05479039996862411,0.005672522998491864,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01635199971497059,0.02844800055027008,0.019267200119793416,0.017952000722289085,0.0035068687666949577,0.0,0.0,0.0,0.0,0.0,0.049855999648571014,0.07798399776220322,0.05986879989504815,0.05986879989504815,0.01043914754878828,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016383999958634377,0.026079999282956123,0.01923519968986511,0.017791999503970146,0.0032161974331284568,0.0,0.0,0.0,0.0,0.0,0.058111999183893204,0.1045759990811348,0.06708480007946492,0.06708480007946492,0.013479022462646494,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.04057599976658821,0.019100800156593323,0.015455999877303839,0.007512281243011577,0.0,0.0,0.0,0.0,0.0,0.05363199859857559,0.07782399654388428,0.06090559959411622,0.06090559959411622,0.007544620176348091,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.02191999927163124,0.016684799920767546,0.01563199982047081,0.0024293118621811216,0.0,0.0,0.0,0.0,0.0,0.0629120022058487,0.07891199737787247,0.06891520097851753,0.06891520097851753,0.005472695665695425,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.038943998515605927,0.018313600029796363,0.01561600062996149,0.007127270260115769,0.0,0.0,0.0,0.0,0.0,0.08675199747085571,0.10662399977445602,0.09391999915242194,0.09391999915242194,0.006988099589086635,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.054687999188899994,0.021439999900758268,0.01539199985563755,0.012052764849597775,0.0,0.0,0.0,0.0,0.0,0.13488000631332397,0.1528639942407608,0.1431359991431236,0.1431359991431236,0.005436271464033599,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.041919998824596405,0.01899839974939823,0.015343999955803156,0.007989843526623287,0.0,0.0,0.0,0.0,0.0,0.06176000088453293,0.08374399691820145,0.06747519969940186,0.06747519969940186,0.0066067747128778,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016256000846624374,0.11353600025177002,0.042761600017547606,0.028960000723600388,0.029104301538020762,0.0,0.0,0.0,0.0,0.0,0.09734400361776352,0.14422400295734406,0.11392960175871848,0.11392960175871848,0.013198594600417867,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.034143999218940735,0.018918400071561335,0.01643200032413006,0.005500943993080684,0.0,0.0,0.0,0.0,0.0,0.12918399274349213,0.15087999403476715,0.13807999789714814,0.13807999789714814,0.007658330538677587,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.03641600161790848,0.0198208000510931,0.01780799962580204,0.0057264128169845765,0.0,0.0,0.0,0.0,0.0,0.22099199891090393,0.23904000222682953,0.2293503984808922,0.2293503984808922,0.004861342907006028,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.035840000957250595,0.018908800091594458,0.015792000107467175,0.0064374817924757475,0.0,0.0,0.0,0.0,0.0,0.10134399682283401,0.12201599776744843,0.10896319895982742,0.10896319895982742,0.006336330809165179,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.03417599946260452,0.017846399918198586,0.014800000004470348,0.006495007539635255,0.0,0.0,0.0,0.0,0.0,0.13126400113105774,0.15561600029468536,0.1389280006289482,0.1389280006289482,0.008381472811075022,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03519999980926514,0.019168000388890504,0.01600000075995922,0.005995522477654695,0.0,0.0,0.0,0.0,0.0,0.21728000044822693,0.2343679964542389,0.2231455981731415,0.2231455981731415,0.004720730646739123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.03670400008559227,0.018441599886864425,0.015023999847471714,0.006596535162793127,0.0,0.0,0.0,0.0,0.0,0.39321601390838623,0.4524799883365631,0.4058080047369003,0.4058080047369003,0.01578349755088941,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.022143999114632607,0.016912000067532063,0.015168000012636185,0.0028156089295136347,0.0,0.0,0.0,0.0,0.0,0.15587200224399567,0.3079040050506592,0.17838079929351805,0.17838079929351805,0.04355865575265927,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.02112000063061714,0.015516800060868263,0.01488000014796853,0.001940870731593904,0.0,0.0,0.0,0.0,0.0,0.21587200462818146,0.23561599850654602,0.22250880002975468,0.22250880002975468,0.006181951170646666,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.039264000952243805,0.023609600123018028,0.02131200022995472,0.007236979625711548,0.0,0.0,0.0,0.0,0.0,0.408735990524292,0.470335990190506,0.4336863994598388,0.4336863994598388,0.01844662383160074,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.025407999753952026,0.016227199975401164,0.014960000291466713,0.0031617375441736185,0.0,0.0,0.0,0.0,0.0,0.7412800192832947,0.7627840042114258,0.7464000046253203,0.7464000046253203,0.006112167448837547,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01462399959564209,0.02812799997627735,0.020652799773961304,0.021359999664127827,0.004308706957613102,0.028383498565450627,0.039859687970646644,0.032492258074592426,0.032492258074592426,0.00453597266208597,0.029312501176103633,0.04116431058787463,0.03355574193327539,0.03355574193327539,0.004684436757701648,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.034143999218940735,0.023171199765056372,0.024255999363958836,0.005908614918096971,0.03333159243114438,0.038935341782478095,0.03580428402241854,0.03580428402241854,0.002082270297044095,0.03633240903369937,0.04244065945634484,0.03902771507087562,0.03902771507087562,0.0022697354261490147,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.02611199952661991,0.019635199941694735,0.018112000077962875,0.0038298050749257795,0.04189529417991216,0.057484239920526384,0.04744885718421094,0.04744885718421094,0.004779830748455743,0.051672703037266184,0.07089975418195164,0.05852234134479412,0.05852234134479412,0.005895334539786378,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.029311999678611755,0.018662399891763926,0.01673599984496832,0.0044017112162725355,0.04322973959325901,0.05049827064705393,0.04535414343408448,0.04535414343408448,0.0022944156000240697,0.08733025617719538,0.10201372836398578,0.09162185574241775,0.09162185574241775,0.004635047631846023,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014592000283300877,0.026367999613285065,0.016336000058799982,0.01515199989080429,0.003415353455946402,0.06031842775160765,0.06618323188375198,0.06274415549817247,0.06274415549817247,0.001985647289644255,0.14768157653992678,0.16204076249052324,0.15362064543185072,0.15362064543185072,0.004861590945216247,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.02454400062561035,0.017167999967932702,0.016191999427974224,0.0028685016454498436,0.09128700688359712,0.09689150775996329,0.09360396051475776,0.09360396051475776,0.0013477542744916764,0.2740889887523892,0.2909164873209846,0.2810456356995366,0.2810456356995366,0.004046628526808561,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.01881599985063076,0.03097599931061268,0.024598400108516216,0.024848000146448612,0.0037539437391565975,0.1035249255866932,0.10603132147437412,0.10399797220840973,0.10399797220840973,0.0007025109429056814,0.5652750707893444,0.5789606938874117,0.5678580377517648,0.5678580377517648,0.003835906384194897,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.018624000251293182,0.043487999588251114,0.024460799992084503,0.019600000232458115,0.008922081629781394,0.196169204945307,0.2076140047945799,0.2008444429250931,0.2008444429250931,0.00394350953292805,1.1196708009293268,1.1849940417370974,1.1463555573610091,1.1463555573610091,0.022508285530529783,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.027135999873280525,0.03667199984192848,0.02945920005440712,0.028256000019609928,0.0029446678408169553,0.37757279619664613,0.3898113624476372,0.38075520430942184,0.38075520430942184,0.0036600203831042254,0.2522831942132049,0.2604606493092597,0.25440958703617433,0.25440958703617433,0.0024455194930253126,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.0435199998319149,0.049695998430252075,0.04502719938755036,0.04395199939608574,0.002035785660169308,1.1048984388245497,1.1189053886476108,1.1112371236754128,1.1112371236754128,0.0050220696724624985,0.1395495077239122,0.14131859607071193,0.14035008841031085,0.14035008841031085,0.0006342911944856293,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.02364799939095974,0.01809599995613098,0.016352000646293163,0.0035481127058959038,0.04822399839758873,0.08566399663686752,0.05961279980838299,0.05961279980838299,0.011445665413968877,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.020864000543951988,0.01668160008266568,0.015664000064134598,0.0025769063833097584,0.049695998430252075,0.08057600259780884,0.05882879942655563,0.05882879942655563,0.009515126108519331,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.028704000636935234,0.017430400010198355,0.015056000091135502,0.004349294613335555,0.049855999648571014,0.07366400212049484,0.05459520071744919,0.05459520071744919,0.0069610920757925574,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.03161599859595299,0.017788799852132796,0.015711999963968992,0.005005385723318659,0.06102399900555611,0.08179199695587158,0.06650560013949873,0.06650560013949873,0.006947995595801105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.04416000097990036,0.019200000166893005,0.015488000120967627,0.008561241323364038,0.08054400235414505,0.09196799993515015,0.08607039973139763,0.08607039973139763,0.004145329035483754,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.023391999304294586,0.019667199812829494,0.018463999964296818,0.0022577669687832585,0.18729600310325623,0.20585599541664124,0.19546559900045393,0.19546559900045393,0.006339068824303663,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.026240000501275063,0.03446400165557861,0.028297600522637367,0.027088000439107418,0.0025148441522922374,0.5754240155220032,0.5889919996261597,0.5800191938877105,0.5800191938877105,0.003858869273829596,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.04182400181889534,0.047200001776218414,0.043036799877882004,0.0423360001295805,0.0017073405772076728,2.063199996948242,2.0787200927734375,2.067151999473572,2.067151999473572,0.004959651271127835,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.05648000165820122,0.02270399993285537,0.017935999669134617,0.012377915150727689,0.049536000937223434,0.07196799665689468,0.056015999615192415,0.056015999615192415,0.0070476637552742884,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01408000010997057,0.021344000473618507,0.01611520005390048,0.014928000047802925,0.0025499884993961702,0.07072000205516815,0.2642880082130432,0.1588256008923054,0.1588256008923054,0.053220347086102376,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01360000018030405,0.03014400042593479,0.016975999902933837,0.015056000091135502,0.004715159697364111,0.08393599838018417,0.11036799848079681,0.09160000011324881,0.09160000011324881,0.007911669434472792,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.020767999812960625,0.016233599931001663,0.015392000321298838,0.002202615907995427,0.1998399943113327,0.22070400416851044,0.20855360180139543,0.20855360180139543,0.00689307200230667,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.033440001308918,0.018684800155460833,0.016048000194132328,0.00553991005639174,0.6043199896812439,0.635807991027832,0.6126143991947173,0.6126143991947173,0.008745933953408096,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013824000023305416,0.03359999880194664,0.017811199743300678,0.014944000169634819,0.005926140483565472,0.0,0.0,0.0,0.0,0.0,0.045471999794244766,0.07539200037717819,0.054758400097489356,0.054758400097489356,0.010253548506101549,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02409599907696247,0.01641279999166727,0.01473599998280406,0.003347158638713987,0.0,0.0,0.0,0.0,0.0,0.0461760014295578,0.07529599964618683,0.05460800044238568,0.05460800044238568,0.009748937798340135,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.028416000306606293,0.018611199874430894,0.01598400017246604,0.005209875093863647,0.0,0.0,0.0,0.0,0.0,0.048128001391887665,0.07897599786520004,0.061353600397706036,0.061353600397706036,0.010488153655157845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.025728000327944756,0.016092800162732603,0.01512000011280179,0.003300888123052605,0.0,0.0,0.0,0.0,0.0,0.04864000156521797,0.07648000121116638,0.05810560062527656,0.05810560062527656,0.009473544218404408,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.03551999852061272,0.018588799890130757,0.016064000315964222,0.006071057437992447,0.0,0.0,0.0,0.0,0.0,0.04822399839758873,0.07862400263547897,0.05660480037331582,0.05660480037331582,0.009565394213730401,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.028831999748945236,0.01699519995599985,0.015024000313133001,0.004285000744868188,0.0,0.0,0.0,0.0,0.0,0.04854400083422661,0.06719999760389328,0.05778240002691746,0.05778240002691746,0.006852125554679805,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.0144640002399683,0.024927999824285507,0.0161183999851346,0.01521599991247058,0.0029774808524673907,0.0,0.0,0.0,0.0,0.0,0.05379199981689453,0.08966399729251862,0.06270079985260964,0.06270079985260964,0.009983591277092696,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03001599945127964,0.017648000083863736,0.01547200046479702,0.004585126309484848,0.0,0.0,0.0,0.0,0.0,0.061664000153541565,0.07692799717187881,0.06704320013523103,0.06704320013523103,0.005500191798490629,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.026367999613285065,0.016947199776768684,0.015039999969303608,0.0037951566103550205,0.0,0.0,0.0,0.0,0.0,0.08799999952316284,0.111455999314785,0.0964031994342804,0.0964031994342804,0.007397541615558088,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01369599997997284,0.021247999742627144,0.015359999984502793,0.014640000183135271,0.0020942770950814317,0.0,0.0,0.0,0.0,0.0,0.051711998879909515,0.07065600156784058,0.058054400235414506,0.058054400235414506,0.006633034815910025,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.0307839997112751,0.017148799914866685,0.015039999969303608,0.004882374686312284,0.0,0.0,0.0,0.0,0.0,0.061919998377561576,0.07843200117349625,0.06628479920327664,0.06628479920327664,0.004962852689801192,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013887999579310417,0.02969600073993206,0.017500799987465142,0.015232000034302473,0.00467273319705314,0.0,0.0,0.0,0.0,0.0,0.08819200098514557,0.11097600311040878,0.09493440166115762,0.09493440166115762,0.007509235042985577,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.022112000733613968,0.01751680001616478,0.016047999262809753,0.0030306621792915785,0.0,0.0,0.0,0.0,0.0,0.13065600395202637,0.15110400319099426,0.13857279866933822,0.13857279866933822,0.00750137841249771,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.04012800008058548,0.01892479993402958,0.01550400024279952,0.007459545267816961,0.0,0.0,0.0,0.0,0.0,0.06278400123119354,0.08259200304746628,0.0704512007534504,0.0704512007534504,0.005979055984382744,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.02236800082027912,0.016332800220698118,0.014928000047802925,0.002784044363186731,0.0,0.0,0.0,0.0,0.0,0.1003199964761734,0.1279360055923462,0.10921279862523078,0.10921279862523078,0.00862769617046716,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0225600004196167,0.016128000058233737,0.014800000004470348,0.002958953332547708,0.0,0.0,0.0,0.0,0.0,0.13116799294948578,0.14812800288200378,0.13783999979496003,0.13783999979496003,0.005696384361148053,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.029983999207615852,0.017468800116330386,0.01508800033479929,0.0047379550962483065,0.0,0.0,0.0,0.0,0.0,0.217631995677948,0.2447360008955002,0.22715839892625808,0.22715839892625808,0.008831828221847138,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02304000034928322,0.016512000095099212,0.014512000139802694,0.0035026774828624254,0.0,0.0,0.0,0.0,0.0,0.11020799726247787,0.12307199835777283,0.11600959971547126,0.11600959971547126,0.004667637266950902,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.042080000042915344,0.017510399967432023,0.014607999939471483,0.00823117883530167,0.0,0.0,0.0,0.0,0.0,0.15702399611473083,0.17587199807167053,0.16399359852075576,0.16399359852075576,0.006588299074676393,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0208320003002882,0.015299199987202883,0.01462399959564209,0.001916768086505386,0.0,0.0,0.0,0.0,0.0,0.21779200434684753,0.2415360063314438,0.2260768011212349,0.2260768011212349,0.007251352080685236,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014015999622642994,0.021088000386953354,0.015619200188666582,0.01473599998280406,0.002013227452302141,0.0,0.0,0.0,0.0,0.0,0.3959999978542328,0.4152640104293823,0.40332479774951924,0.40332479774951924,0.006942401431914052,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.03379200026392937,0.020595200080424547,0.019600000232458115,0.006548754248881344,0.026192623739694512,0.040006882507168426,0.029155180178492036,0.029155180178492036,0.00408828748438241,0.024591375524545756,0.037561119537986146,0.027372820355088746,0.027372820355088746,0.0038383559348575944,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.020896000787615776,0.015318400040268899,0.014431999996304512,0.0019640312960926966,0.029349018208693862,0.06236262941356679,0.03714313592014963,0.03714313592014963,0.009618313791872569,0.028826981462526914,0.06125337308649041,0.036482463672774496,0.036482463672774496,0.009447230957011863,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.032127998769283295,0.017286399938166143,0.014479999896138906,0.005536495446636193,0.03273085874558354,0.04394578491937082,0.03563837490653034,0.03563837490653034,0.003429601730256567,0.03488514202593899,0.04683821345079978,0.037984025407085474,0.037984025407085474,0.0036553316361902684,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.021503999829292297,0.015078400075435639,0.01425600005313754,0.002238435975478849,0.042100813549974185,0.051784144690147756,0.045378692890289625,0.045378692890289625,0.003184109940650555,0.05111518843748309,0.06287185663927425,0.05509490773570219,0.05509490773570219,0.0038658725544299132,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.02687999978661537,0.017667199857532977,0.01566399959847331,0.003864811846387173,0.048475323773821306,0.05599956978723733,0.05123499252968345,0.05123499252968345,0.002427379820423941,0.08429268135885387,0.09737642835214408,0.0890914090616175,0.0890914090616175,0.0042209177332077005,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.01462399959564209,0.03359999880194664,0.019804799742996693,0.016032000072300434,0.006750519908145474,0.07121508474579985,0.07292308367136396,0.07194410845270183,0.07194410845270183,0.0005500065915943844,0.14760091249712767,0.15114092373010238,0.14911189243564577,0.14911189243564577,0.0011399477384397005,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.021856000646948814,0.016835200227797033,0.015263999812304974,0.00283854448975065,0.08146338272142935,0.08560865714384096,0.0834932625520734,0.0834932625520734,0.0012259635905347874,0.2782486219401307,0.29240733788179385,0.2851819365989658,0.2851819365989658,0.004187435731481665,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.04495999962091446,0.02143679987639189,0.015856000129133463,0.009709987872683342,0.1194459208702178,0.12302524755181463,0.12053885718696096,0.12053885718696096,0.001005118119341339,0.5597220650459199,0.5764947443228802,0.5648435507167102,0.5648435507167102,0.004709970715400788,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.018112000077962875,0.026496000587940216,0.019318400137126445,0.018432000651955605,0.0024249605112359905,0.19770253574610402,0.222811797868348,0.2054811520619412,0.2054811520619412,0.008000944254868043,0.13941746080159492,0.157124211777114,0.14490284788179197,0.14490284788179197,0.005642170080515992,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.02595200017094612,0.03155200183391571,0.02727359998971224,0.026575999334454536,0.0016260948764843166,0.6014684881116659,0.6095472988212,0.6046757700946376,0.6046757700946376,0.0023537435563177303,0.11325152264578045,0.11477269562841545,0.11385542721485654,0.11385542721485654,0.0004431903698023628,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.022784000262618065,0.016435200069099664,0.015519999898970127,0.0023688301421469523,0.04879999905824661,0.09139200299978256,0.06054079942405224,0.06054079942405224,0.012152608702448775,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014816000126302242,0.02844800055027008,0.017008000146597625,0.015696000307798386,0.003941466294662679,0.047807998955249786,0.07356800138950348,0.05626560002565384,0.05626560002565384,0.00842179125412236,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.053408000618219376,0.019353600218892097,0.01532800029963255,0.01138302400841626,0.048448000103235245,0.0785600021481514,0.0556256003677845,0.0556256003677845,0.009348274502616908,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.03139200061559677,0.01823679991066456,0.016159999649971724,0.004860071238302512,0.055424001067876816,0.08505599945783615,0.0640383992344141,0.0640383992344141,0.00921448636178623,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.026528000831604004,0.017324799951165915,0.01536000007763505,0.0036004822686428305,0.07660800218582153,0.0942080020904541,0.08209280073642732,0.08209280073642732,0.00515895587669752,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.021247999742627144,0.016403199825435876,0.01532800029963255,0.001995897022647934,0.11395200341939926,0.15113599598407745,0.12431039959192276,0.12431039959192276,0.011164431123683732,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.017184000462293625,0.028672000393271446,0.019865600019693376,0.017823999747633934,0.0035798491315929977,0.3171840012073517,0.3341119885444641,0.3261695951223373,0.3261695951223373,0.005111046452878918,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.024639999493956566,0.030400000512599945,0.02656640000641346,0.02556800004094839,0.0020189846603237303,1.0648640394210815,1.0828479528427124,1.0715327858924866,1.0715327858924866,0.005558639263049851,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014976000413298607,0.021695999428629875,0.017305599898099898,0.01593599934130907,0.0025718046517268054,0.04956800118088722,0.07932800054550171,0.06228480041027069,0.06228480041027069,0.01027160349757827,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01539199985563755,0.03580800071358681,0.019686400331556796,0.017136000096797943,0.005918387811304003,0.05158400163054466,0.10713600367307663,0.06364160068333148,0.06364160068333148,0.015832957809696766,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02223999984562397,0.016672000009566545,0.015887999907135963,0.0020934945946034563,0.06521599739789963,0.08902399986982346,0.07520959973335266,0.07520959973335266,0.007913840684102929,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015231999568641186,0.04028800129890442,0.019971200078725816,0.01646399963647127,0.007351305886577286,0.14467200636863708,0.16844800114631653,0.15363519936800005,0.15363519936800005,0.008174435899956223,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015519999898970127,0.02304000034928322,0.01775679988786578,0.016784000210464,0.0025077243712082584,0.33740800619125366,0.35343998670578003,0.3445120006799698,0.3445120006799698,0.00445648463590358,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.02393599972128868,0.01809599995613098,0.01654400024563074,0.002998393102466254,0.0,0.0,0.0,0.0,0.0,0.0504320003092289,0.0843840017914772,0.060083200410008426,0.060083200410008426,0.00986959318572296,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.030047999694943428,0.019510399922728537,0.017680000513792038,0.004065185265963332,0.0,0.0,0.0,0.0,0.0,0.05004800111055374,0.07036799937486649,0.059315200522542,0.059315200522542,0.006537768821329647,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.02393599972128868,0.018035199772566558,0.016512000001966953,0.0030667689116777724,0.0,0.0,0.0,0.0,0.0,0.05100800096988678,0.06735999882221222,0.058387200161814694,0.058387200161814694,0.005832787739241381,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.026208000257611275,0.017500799987465142,0.01646399963647127,0.0030666103306165714,0.0,0.0,0.0,0.0,0.0,0.05023999884724617,0.07254400104284286,0.057254400476813315,0.057254400476813315,0.006890853653068151,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.027327999472618103,0.0179776000790298,0.01648000068962574,0.003783417447531392,0.0,0.0,0.0,0.0,0.0,0.04819199815392494,0.07100799679756165,0.05621119923889638,0.05621119923889638,0.007824391484124725,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.036448001861572266,0.019136000238358975,0.016704000532627106,0.006076530095624803,0.0,0.0,0.0,0.0,0.0,0.047488000243902206,0.08441600203514099,0.0588383998721838,0.0588383998721838,0.011275940726332522,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015135999768972397,0.033952001482248306,0.02119360016658902,0.018000000156462193,0.007136646614305304,0.0,0.0,0.0,0.0,0.0,0.04931199923157692,0.06992000341415405,0.05755840018391609,0.05755840018391609,0.007297587694315226,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.034272000193595886,0.0204415999352932,0.017311999574303627,0.00695404701803013,0.0,0.0,0.0,0.0,0.0,0.05215999856591225,0.0735040009021759,0.06117440015077591,0.06117440015077591,0.007381118436894922,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03743999823927879,0.02012479966506362,0.01775999926030636,0.006181045509079057,0.0,0.0,0.0,0.0,0.0,0.06355199962854385,0.08508799970149994,0.07019200026988984,0.07019200026988984,0.0062246580442117845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.032607998698949814,0.01959999995306134,0.017487999983131886,0.004882475068460749,0.0,0.0,0.0,0.0,0.0,0.04918399825692177,0.0865280032157898,0.05973760038614274,0.05973760038614274,0.011546847942113974,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02675200067460537,0.017430400010198355,0.016560000367462635,0.003243135094239819,0.0,0.0,0.0,0.0,0.0,0.052671998739242554,0.08367999643087387,0.06238719932734965,0.06238719932734965,0.011207824780630104,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.03612799942493439,0.019327999837696553,0.01688000001013279,0.006058251425153085,0.0,0.0,0.0,0.0,0.0,0.06364800035953522,0.07878399640321732,0.06935679838061332,0.06935679838061332,0.004515588328677643,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.03711999952793121,0.019670399930328132,0.017152000218629837,0.006165994646522264,0.0,0.0,0.0,0.0,0.0,0.09071999788284302,0.11507199704647064,0.10252480059862136,0.10252480059862136,0.008782544051535657,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.026528000831604004,0.018396800104528665,0.01601599995046854,0.004279788048986283,0.0,0.0,0.0,0.0,0.0,0.05331199988722801,0.07977599650621414,0.06076480001211167,0.06076480001211167,0.008761579122069606,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.02643200010061264,0.01726400004699826,0.015855999663472176,0.003210008417242029,0.0,0.0,0.0,0.0,0.0,0.062431998550891876,0.08246400207281113,0.06970879957079888,0.06970879957079888,0.007016341676068233,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.02223999984562397,0.0166015999391675,0.015887999907135963,0.00204913969129354,0.0,0.0,0.0,0.0,0.0,0.10127999633550644,0.1141119971871376,0.10621120035648347,0.10621120035648347,0.004029318311754605,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03363199904561043,0.019305599946528675,0.016671999357640743,0.0055445582307981234,0.0,0.0,0.0,0.0,0.0,0.1319040060043335,0.15839999914169312,0.14040640145540234,0.14040640145540234,0.007998041073596942,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014368000440299511,0.04447999969124794,0.018908800091594458,0.0157279996201396,0.0086814937461721,0.0,0.0,0.0,0.0,0.0,0.06428799778223038,0.08982399851083755,0.07349760085344315,0.07349760085344315,0.008605284512197258,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.028672000393271446,0.01890560006722808,0.016080000437796116,0.004797584627815021,0.0,0.0,0.0,0.0,0.0,0.11123199760913849,0.14115199446678162,0.11942399889230729,0.11942399889230729,0.008513394706791027,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.024927999824285507,0.016915200091898442,0.015216000378131866,0.003374147499442635,0.0,0.0,0.0,0.0,0.0,0.15887999534606934,0.18111999332904816,0.16934399753808976,0.16934399753808976,0.007415181260761123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.026944000273942947,0.017737600207328796,0.015199999790638685,0.00415598714375255,0.0,0.0,0.0,0.0,0.0,0.2192319929599762,0.23472000658512115,0.22809920012950893,0.22809920012950893,0.004730841376327335,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.05052800104022026,0.020320000313222408,0.015520000364631414,0.010604522177924678,0.024270629882498958,0.03340247625954076,0.028446344104128624,0.028446344104128624,0.0032330369099793834,0.023697370291069768,0.03261352722995356,0.027774456325453972,0.027774456325453972,0.0031566742680923386,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.033663999289274216,0.019987199828028678,0.018240000121295452,0.005522929139253732,0.0259194055660947,0.03719755183990719,0.029916029687899703,0.029916029687899703,0.003966666590554318,0.027104595853449518,0.03889844644729374,0.03128396954022015,0.03128396954022015,0.004148046317967881,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014592000283300877,0.03299200162291527,0.019840000104159115,0.016207999549806118,0.006546343008726814,0.02587869595769926,0.03763167265431482,0.030174939058162802,0.030174939058162802,0.0037303581782277364,0.026473304070195713,0.03849632587654989,0.03086826083864964,0.03086826083864964,0.003816069654529222,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.013824000023305416,0.021023999899625778,0.016304000187665223,0.015584000386297703,0.0023236165806545476,0.031810621525966996,0.04156949936878106,0.0346749350032807,0.0346749350032807,0.003130209181143985,0.035677378270900374,0.04662250161636451,0.038889864871740294,0.038889864871740294,0.0035107033960828727,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.02175999991595745,0.01569600012153387,0.015008000191301107,0.0020882666168036863,0.038211712107062853,0.07212229256520057,0.04364509673334097,0.04364509673334097,0.009671953074025085,0.0476442859917874,0.08992570455184198,0.05441890342616107,0.05441890342616107,0.01205947791784038,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014368000440299511,0.033824000507593155,0.017628800217062236,0.014864000026136637,0.005791495403857738,0.05214261250030033,0.06266261508706669,0.05677791295527661,0.05677791295527661,0.0030298223079651514,0.08648138506878382,0.10392938682791132,0.09416928531647478,0.09416928531647478,0.005025126612204489,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.031199999153614044,0.017900799959897996,0.015343999955803156,0.004974475661316249,0.06411958891421111,0.07405276123263956,0.06793047918211377,0.06793047918211377,0.0033918658556700006,0.14058441263169497,0.16236323591492058,0.1489399211274489,0.1489399211274489,0.00743678300375353,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.03855999931693077,0.018495999928563833,0.015647999942302704,0.006918530056761627,0.09872985549401277,0.10683454583043753,0.10185316839884552,0.10185316839884552,0.0020802254037042074,0.2743261390166379,0.2968454508984596,0.2830044295482752,0.2830044295482752,0.005780016596065092,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.030400000512599945,0.01977920001372695,0.01726400014013052,0.005350883464206169,0.13918872472233365,0.16279522855335207,0.1501912864839173,0.1501912864839173,0.008992185529011513,0.11892328861766266,0.13909276049083735,0.1283239123428725,0.1283239123428725,0.007682951884956939,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.01727999933063984,0.02223999984562397,0.01819519978016615,0.017680000513792038,0.0014397298270620873,0.3728044181625443,0.38295504353701676,0.3761264483787333,0.3761264483787333,0.0033660272332490977,0.07967557017401883,0.08184495665371805,0.08038555277807365,0.08038555277807365,0.0007193856241088455,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio diff --git a/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/attention_true_mixed_fused.csv b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/attention_true_mixed_fused.csv new file mode 100644 index 0000000..153bb52 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/attention_true_mixed_fused.csv @@ -0,0 +1,31 @@ +num_tensor_parallel_workers,batch_spec,num_prefill_seqs,num_decode_seqs,total_prefill_tokens,total_decode_tokens,decode_avg_kv_cache_size,attention_core_mean_ms,attention_core_mean_as_median_ms,kv_cache_update_median_ms,pure_prefill_reference_mean_ms,pure_decode_reference_mean_ms,projected_prefill_mean_ms,projected_decode_mean_ms,projection_sum_error_ms,representation +1,q64_8q1s512,1,8,64,8,512.0,0.06604800000786781,0.06604800000786781,0.021359999664127827,0.05810240097343922,0.0600041144660541,0.032492258074592426,0.03355574193327539,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q128_8q1s1k,1,8,128,8,1024.0,0.07483199909329416,0.07483199909329416,0.024255999363958836,0.05587520003318787,0.06090559959411622,0.03580428402241854,0.03902771507087562,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q128_16q1s1k,1,16,128,16,1024.0,0.10597119852900506,0.10597119852900506,0.018112000077962875,0.05587520003318787,0.06891520097851753,0.04744885718421094,0.05852234134479412,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q256_16q1s2k,1,16,256,16,2048.0,0.13697599917650224,0.13697599917650224,0.01673599984496832,0.056396800279617305,0.11392960175871848,0.04535414343408448,0.09162185574241775,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q256_32q1s2k,1,32,256,32,2048.0,0.2163648009300232,0.2163648009300232,0.01515199989080429,0.056396800279617305,0.13807999789714814,0.06274415549817247,0.15362064543185072,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q512_32q1s4k,1,32,512,32,4096.0,0.3746495962142944,0.3746495962142944,0.016191999427974224,0.07432000041007995,0.2231455981731415,0.09360396051475776,0.2810456356995366,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q512_64q1s4k,1,64,512,64,4096.0,0.6718560099601746,0.6718560099601746,0.024848000146448612,0.07432000041007995,0.4058080047369003,0.10399797220840973,0.5678580377517648,-1.1102230246251565e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q1k_64q1s8k,1,64,1024,64,8192.0,1.3472000002861022,1.3472000002861022,0.019600000232458115,0.1307712011039257,0.7464000046253203,0.2008444429250931,1.1463555573610091,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q2k_32q1s4k,1,32,2048,32,4096.0,0.6351647913455962,0.6351647913455962,0.028256000019609928,0.33396480381488797,0.2231455981731415,0.38075520430942184,0.25440958703617433,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q4k_16q1s4k,1,16,4096,16,4096.0,1.2515872120857237,1.2515872120857237,0.04395199939608574,1.0999775886535645,0.1389280006289482,1.1112371236754128,0.14035008841031085,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q64_8q1s512,1,8,64,8,512.0,0.05652800053358078,0.05652800053358078,0.019600000232458115,0.05961279980838299,0.055968457407185014,0.029155180178492036,0.027372820355088746,6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q128_8q1s1k,1,8,128,8,1024.0,0.07362559959292413,0.07362559959292413,0.014431999996304512,0.05882879942655563,0.05778240002691746,0.03714313592014963,0.036482463672774496,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q128_16q1s1k,1,16,128,16,1024.0,0.0736224003136158,0.0736224003136158,0.014479999896138906,0.05882879942655563,0.06270079985260964,0.03563837490653034,0.037984025407085474,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q256_16q1s2k,1,16,256,16,2048.0,0.10047360062599181,0.10047360062599181,0.01425600005313754,0.05459520071744919,0.06628479920327664,0.045378692890289625,0.05509490773570219,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q256_32q1s2k,1,32,256,32,2048.0,0.14032640159130094,0.14032640159130094,0.01566399959847331,0.05459520071744919,0.09493440166115762,0.05123499252968345,0.0890914090616175,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q512_32q1s4k,1,32,512,32,4096.0,0.22105600088834762,0.22105600088834762,0.016032000072300434,0.06650560013949873,0.13783999979496003,0.07194410845270183,0.14911189243564577,-2.7755575615628914e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q512_64q1s4k,1,64,512,64,4096.0,0.36867519915103913,0.36867519915103913,0.015263999812304974,0.06650560013949873,0.22715839892625808,0.0834932625520734,0.2851819365989658,5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q1k_64q1s8k,1,64,1024,64,8192.0,0.6853824079036711,0.6853824079036711,0.015856000129133463,0.08607039973139763,0.40332479774951924,0.12053885718696096,0.5648435507167102,1.1102230246251565e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q2k_32q1s4k,1,32,2048,32,4096.0,0.3503839999437332,0.3503839999437332,0.018432000651955605,0.19546559900045393,0.13783999979496003,0.2054811520619412,0.14490284788179197,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q4k_16q1s4k,1,16,4096,16,4096.0,0.7185311973094941,0.7185311973094941,0.026575999334454536,0.5800191938877105,0.10921279862523078,0.6046757700946376,0.11385542721485654,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q64_8q1s512,1,8,64,8,512.0,0.0562208004295826,0.0562208004295826,0.015520000364631414,0.06054079942405224,0.0591108573866742,0.028446344104128624,0.027774456325453972,-6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q128_8q1s1k,1,8,128,8,1024.0,0.061199999228119854,0.061199999228119854,0.018240000121295452,0.05626560002565384,0.0588383998721838,0.029916029687899703,0.03128396954022015,-6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q128_16q1s1k,1,16,128,16,1024.0,0.06104319989681244,0.06104319989681244,0.016207999549806118,0.05626560002565384,0.05755840018391609,0.030174939058162802,0.03086826083864964,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q256_16q1s2k,1,16,256,16,2048.0,0.07356479987502099,0.07356479987502099,0.015584000386297703,0.0556256003677845,0.06238719932734965,0.0346749350032807,0.038889864871740294,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q256_32q1s2k,1,32,256,32,2048.0,0.09806400015950203,0.09806400015950203,0.015008000191301107,0.0556256003677845,0.06935679838061332,0.04364509673334097,0.05441890342616107,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q512_32q1s4k,1,32,512,32,4096.0,0.1509471982717514,0.1509471982717514,0.014864000026136637,0.0640383992344141,0.10621120035648347,0.05677791295527661,0.09416928531647478,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q512_64q1s4k,1,64,512,64,4096.0,0.21687040030956264,0.21687040030956264,0.015343999955803156,0.0640383992344141,0.14040640145540234,0.06793047918211377,0.1489399211274489,2.7755575615628914e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q1k_64q1s8k,1,64,1024,64,8192.0,0.3848575979471207,0.3848575979471207,0.015647999942302704,0.08209280073642732,0.22809920012950893,0.10185316839884552,0.2830044295482752,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q2k_32q1s4k,1,32,2048,32,4096.0,0.2785151988267898,0.2785151988267898,0.01726400014013052,0.12431039959192276,0.10621120035648347,0.1501912864839173,0.1283239123428725,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q4k_16q1s4k,1,16,4096,16,4096.0,0.456512001156807,0.456512001156807,0.017680000513792038,0.3261695951223373,0.06970879957079888,0.3761264483787333,0.08038555277807365,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation diff --git a/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/linear_op.csv b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/linear_op.csv new file mode 100644 index 0000000..2803f02 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/linear_op.csv @@ -0,0 +1,37 @@ +time_stats.emb.min,time_stats.emb.max,time_stats.emb.mean,time_stats.emb.median,time_stats.emb.std,time_stats.input_layernorm.min,time_stats.input_layernorm.max,time_stats.input_layernorm.mean,time_stats.input_layernorm.median,time_stats.input_layernorm.std,time_stats.attn_pre_proj.min,time_stats.attn_pre_proj.max,time_stats.attn_pre_proj.mean,time_stats.attn_pre_proj.median,time_stats.attn_pre_proj.std,time_stats.attn_rope.min,time_stats.attn_rope.max,time_stats.attn_rope.mean,time_stats.attn_rope.median,time_stats.attn_rope.std,time_stats.attn_post_proj.min,time_stats.attn_post_proj.max,time_stats.attn_post_proj.mean,time_stats.attn_post_proj.median,time_stats.attn_post_proj.std,time_stats.post_attention_layernorm.min,time_stats.post_attention_layernorm.max,time_stats.post_attention_layernorm.mean,time_stats.post_attention_layernorm.median,time_stats.post_attention_layernorm.std,n_head,n_kv_head,n_embd,n_expanded_embd,vocab_size,use_gated_mlp,use_qk_norm,attn_output_gate,num_tokens,num_tensor_parallel_workers,padded_n_embd,padded_n_expanded_embd,model_arch,is_step2_mini,share_expert_dim,share_q_dim,measurement_type,profiling_precision,quant_signature +0.029184000566601753,0.06780800223350525,0.03157280012965202,0.030736000277101994,0.005874173435341216,0.033215999603271484,0.04825599864125252,0.03443359974771738,0.0337119996547699,0.0031825678429048183,1.438431978225708,1.505568027496338,1.446228802204132,1.4429279565811157,0.014128607642643025,0.538752019405365,0.5440319776535034,0.5416463971138,0.5420799851417542,0.0016099306164432847,1.0073280334472656,1.0163840055465698,1.0098415970802308,1.0081279873847961,0.0032980457197329728,0.04022400081157684,0.041728001087903976,0.04091359991580248,0.04081599973142147,0.0004728505416767937,32,4,2048,768,151936,True,True,False,8192,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.01360000018030405,0.06784000247716904,0.017755200061947106,0.0179840000346303,0.00837681421401443,0.01836800016462803,0.03651199862360954,0.019606399815529585,0.018719999119639397,0.0038821958848767424,0.7512000203132629,0.8208960294723511,0.7566704005002975,0.75382399559021,0.014793092586577971,0.28995200991630554,0.29337599873542786,0.29135999977588656,0.29150401055812836,0.000998381071258815,0.5149760246276855,0.5169600248336792,0.5160208016633987,0.5158880054950714,0.0005038652783291977,0.02051199972629547,0.021503999829292297,0.021067200042307378,0.021104000508785248,0.00023542855306902367,32,4,2048,768,151936,True,True,False,4096,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.0080960001796484,0.04281599819660187,0.011092799971811474,0.011039999779313803,0.005489135752949302,0.012223999947309494,0.026335999369621277,0.013187199970707298,0.01247999956831336,0.0030236510562153375,0.3928639888763428,0.45372799038887024,0.3976895987987518,0.39528000354766846,0.012905176716136006,0.1547199934720993,0.15884800255298615,0.15712319910526276,0.1573439985513687,0.001165121945135037,0.26633599400520325,0.268095999956131,0.26719200164079665,0.2671840041875839,0.0004242740199415328,0.013024000450968742,0.013887999579310417,0.013489600038155913,0.013520000036805868,0.0002382817554057738,32,4,2048,768,151936,True,True,False,2048,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.00825599953532219,0.04755200073122978,0.02398160002194345,0.01961600035429001,0.00874683840888523,0.018880000337958336,0.03868800029158592,0.021590400114655496,0.0208320003002882,0.004057015751746236,0.24316799640655518,0.2710399925708771,0.2521967992186546,0.25065599381923676,0.007998418528894075,0.09644799679517746,0.19120000302791595,0.10407840013504029,0.09963199868798256,0.020043722414992166,0.13600000739097595,0.18892799317836761,0.15760480016469955,0.15760000050067902,0.0112223677907762,0.009664000011980534,0.010015999898314476,0.009836799977347255,0.009824000298976898,9.016971952948177e-05,32,4,2048,768,151936,True,True,False,1024,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.017376000061631203,0.044704001396894455,0.026318399980664254,0.027312000282108784,0.007565344034680866,0.01836800016462803,0.03014400042593479,0.021276800055056812,0.020655999891459942,0.002632977855097951,0.1438719928264618,0.17132799327373505,0.152497598528862,0.15012799948453903,0.007947150319625347,0.1430719941854477,0.19305600225925446,0.16630879789590836,0.1685439944267273,0.014628024163894684,0.08899199962615967,0.10467199981212616,0.0943599995225668,0.09374399855732918,0.003472669494074113,0.007615999784320593,0.007935999892652035,0.007769599952735007,0.0077760000713169575,7.680004540222077e-05,32,4,2048,768,151936,True,True,False,512,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.016992000862956047,0.0544000007212162,0.02573199989274144,0.026016000658273697,0.00803323401720434,0.018432000651955605,0.02348800003528595,0.020648000109940768,0.02062400057911873,0.0013939985047930988,0.10220800340175629,0.1361600011587143,0.11850560046732425,0.11684799939393997,0.010637754898360304,0.16710400581359863,0.21110400557518005,0.19078560024499894,0.19409599900245667,0.013714352646558832,0.06265600025653839,0.0740479975938797,0.06842879951000214,0.0690080001950264,0.0031292240446560557,0.00979200005531311,0.033440001308918,0.01864320016466081,0.017280000261962414,0.006957998188876383,32,4,2048,768,151936,True,True,False,256,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.017152000218629837,0.04396799951791763,0.025907999789342284,0.02598400041460991,0.007714554737915267,0.018400000408291817,0.03667199984192848,0.024132800102233887,0.02112000063061714,0.006120348733354326,0.10678400099277496,0.1363839954137802,0.1193264003843069,0.11583999916911125,0.009838647443214228,0.17017599940299988,0.22748799622058868,0.18853759989142418,0.18433599919080734,0.015728078443174653,0.04569600149989128,0.06652799993753433,0.05192639995366335,0.05151999928057194,0.004516605694976449,0.021856000646948814,0.026335999369621277,0.02384479995816946,0.023599999956786633,0.0011301153013314744,32,4,2048,768,151936,True,True,False,128,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.017343999817967415,1.0683200359344482,0.05748240072280168,0.029504000209271908,0.16366824814254827,0.018688000738620758,0.3317759931087494,0.03685439983382821,0.021151999942958355,0.06767422466731164,0.10255999863147736,0.9434880018234253,0.16412640027701855,0.11956800147891045,0.17964606281728834,0.1714559942483902,2.1306240558624268,0.3011296011507511,0.1926399990916252,0.42310127734378766,0.03574400022625923,0.6859520077705383,0.08389280084520578,0.04279999993741512,0.14321647071615612,0.020479999482631683,0.1831360012292862,0.033024000097066165,0.023856000043451786,0.03470987082429836,32,4,2048,768,151936,True,True,False,64,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.016095999628305435,0.05142400041222572,0.026363200135529043,0.02676799986511469,0.008637725852473854,0.01849599927663803,0.03577600046992302,0.022193600237369538,0.020848000422120094,0.004615266181181815,0.10540799796581268,0.15014399588108063,0.12211520001292228,0.11896000057458878,0.012772013396624768,0.17315199971199036,0.21478399634361267,0.1881632000207901,0.1873439997434616,0.011761164657572015,0.03481600061058998,0.058079998940229416,0.042200000025331974,0.03969600051641464,0.006461281521769989,0.02143999934196472,0.038816001266241074,0.02466559996828437,0.023856000043451786,0.0035418033748569927,32,4,2048,768,151936,True,True,False,32,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.015904000028967857,0.03753599897027016,0.023127999808639287,0.02527999971061945,0.0054230027890305385,0.018400000408291817,0.03001599945127964,0.020488000102341176,0.020096000283956528,0.002585688394137697,0.10355199873447418,0.1438400000333786,0.11536479964852334,0.11124800145626068,0.011136028294919255,0.16502399742603302,0.2072959989309311,0.18646399974822997,0.19075199961662292,0.013296437761247597,0.03142400085926056,0.05215999856591225,0.03888959977775812,0.03750399872660637,0.0057399302067536314,0.020128000527620316,0.04064000025391579,0.023937600292265417,0.023648000322282314,0.004144197400898248,32,4,2048,768,151936,True,True,False,16,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.015200000256299973,0.037151999771595,0.02144480012357235,0.02195199951529503,0.005576364091933697,0.017952000722289085,0.0226879995316267,0.019934400077909233,0.019952000118792057,0.0011812499470458758,0.10063999891281128,0.13468800485134125,0.11595199964940547,0.1207519993185997,0.011974301621092394,0.16332800686359406,0.20748800039291382,0.18162400051951408,0.1767839938402176,0.01474120743720334,0.03222399950027466,0.043455999344587326,0.03829439990222454,0.03859200142323971,0.0031378131240041122,0.020959999412298203,0.03587200120091438,0.023795200139284135,0.023423999547958374,0.0031339489695198443,32,4,2048,768,151936,True,True,False,8,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.014944000169634819,0.04022400081157684,0.02162720002233982,0.022463999688625336,0.006017219317026815,0.018464000895619392,0.026528000831604004,0.021264000236988066,0.020896000787615776,0.001934095017586082,0.10156799852848053,0.1703999936580658,0.12565439902245998,0.1244799979031086,0.01641776722785409,0.1653759926557541,0.23865599930286407,0.1969360001385212,0.19223999977111816,0.02058903050274278,0.03254399821162224,0.06752000004053116,0.0443536002188921,0.041519999504089355,0.00984829071098989,0.020096000283956528,0.040031999349594116,0.026934400014579297,0.02478400059044361,0.00562071702648593,32,4,2048,768,151936,True,True,False,1,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.7531200051307678,0.8461440205574036,0.7618160009384155,0.7576479911804199,0.019464233617982506,0.2922559976577759,0.2985599935054779,0.2953856036067009,0.29576000571250916,0.001575901077689139,0.510047972202301,0.5140479803085327,0.5121696025133133,0.5123839974403381,0.0012263137313476844,,,,,,32,4,2048,768,151936,True,True,False,8192,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.39529600739479065,0.47279998660087585,0.40216960161924364,0.39825600385665894,0.0162749796240819,0.15625600516796112,0.16211199760437012,0.15959519892930984,0.15988799929618835,0.0011493418942396922,0.2635200023651123,0.2642880082130432,0.2639120012521744,0.26392000913619995,0.0001903593401384135,,,,,,32,4,2048,768,151936,True,True,False,4096,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.2531839907169342,0.32419198751449585,0.26924319565296173,0.26049599051475525,0.01657194262368116,0.10051199793815613,0.2375359982252121,0.12411200068891048,0.10311999917030334,0.039466165428540506,0.15881599485874176,0.19289599359035492,0.16896959990262986,0.1640480011701584,0.009162416086418127,,,,,,32,4,2048,768,151936,True,True,False,2048,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.15302400290966034,0.1780800074338913,0.16284480094909667,0.16113600134849548,0.00740355661356144,0.14521600306034088,0.20233599841594696,0.17807039842009545,0.1796799972653389,0.013908448560094403,0.0907519981265068,0.09750399738550186,0.09460479989647866,0.09478399902582169,0.0017965487667361475,,,,,,32,4,2048,768,151936,True,True,False,1024,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11740799993276596,0.17958399653434753,0.13262080028653145,0.12878400087356567,0.014146060532423056,0.17468799650669098,0.21161599457263947,0.1910431995987892,0.18966399878263474,0.01270903647700971,0.05926400050520897,0.07577600330114365,0.06530559975653887,0.06404799968004227,0.00479458462514625,,,,,,32,4,2048,768,151936,True,True,False,512,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11260800063610077,0.17209599912166595,0.13610880002379416,0.13809599727392197,0.017675922458993677,0.18729600310325623,0.26822400093078613,0.20815680101513861,0.2078079953789711,0.01700718593658771,0.04499199986457825,0.06537599861621857,0.0507551996037364,0.04787199944257736,0.005727123232692158,,,,,,32,4,2048,768,151936,True,True,False,256,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11257600039243698,0.23452800512313843,0.13866880126297473,0.13964799791574478,0.026590047697062115,0.1828799992799759,0.24751999974250793,0.20735519900918006,0.20670399814844131,0.01500438131586336,0.03654399886727333,0.05593600124120712,0.0408239996060729,0.03892800025641918,0.004864409450710354,,,,,,32,4,2048,768,151936,True,True,False,128,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11734399944543839,0.18726399540901184,0.13890240006148816,0.13308800011873245,0.019086167340006257,0.16991999745368958,0.2443840056657791,0.19185120090842248,0.19257599860429764,0.018017536159219673,0.03033600002527237,0.04956800118088722,0.038387199863791466,0.03750400058925152,0.00500897018702887,,,,,,32,4,2048,768,151936,True,True,False,64,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11420799791812897,0.18115200102329254,0.13938880078494548,0.1393439993262291,0.018141313962922536,0.1693439930677414,0.221343994140625,0.19187839925289155,0.19438399374485016,0.01430752121272052,0.03017600066959858,0.06019200012087822,0.03866560012102127,0.0364960003644228,0.0074199790031205266,,,,,,32,4,2048,768,151936,True,True,False,32,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11276800185441971,0.16223999857902527,0.1331360016018152,0.13305599987506866,0.01432493740801146,0.17187200486660004,0.23625600337982178,0.19287680014967917,0.1913280040025711,0.01754628831589704,0.029823999851942062,0.07574400305747986,0.03900959976017475,0.036927999928593636,0.009277465083962879,,,,,,32,4,2048,768,151936,True,True,False,16,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.1130559965968132,0.19814400374889374,0.13379519879817964,0.12531199678778648,0.020074427302248836,0.16841599345207214,0.21139200031757355,0.1893615983426571,0.19092799723148346,0.012418720676762711,0.029503999277949333,0.07558400183916092,0.040144000016152856,0.03728000074625015,0.010223239196498205,,,,,,32,4,2048,768,151936,True,True,False,8,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.115167997777462,0.2640640139579773,0.13805920109152794,0.13118399679660797,0.03132849683515479,0.17103999853134155,0.2977280020713806,0.19899839907884598,0.1976960003376007,0.028928046763067948,0.0297279991209507,0.05990400165319443,0.03834720011800528,0.0363520011305809,0.0070885330713495905,,,,,,32,4,2048,768,151936,True,True,False,1,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.3940800130367279,0.46483200788497925,0.39923040121793746,0.3957759886980057,0.015108903906233569,0.16022400557994843,0.1634880006313324,0.16203359961509706,0.16228799521923065,0.0010220720912414007,0.26073598861694336,0.2627840042114258,0.26138080209493636,0.2613760083913803,0.00041640363494172775,,,,,,32,4,2048,768,151936,True,True,False,8192,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.2536959946155548,0.27161601185798645,0.25960480123758317,0.2577280104160309,0.005142017404245015,0.09849599748849869,0.10281600058078766,0.10057279989123344,0.10063999891281128,0.0008836232237268523,0.13913600146770477,0.17606399953365326,0.15875840038061143,0.15988799929618835,0.010058952112389172,,,,,,32,4,2048,768,151936,True,True,False,4096,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.15142400562763214,0.1780479997396469,0.16139679849147798,0.1602879986166954,0.0070656055731385115,0.14601600170135498,0.23343999683856964,0.17913119941949845,0.179967999458313,0.023065274309176566,0.09388799965381622,0.12108799815177917,0.10317599996924401,0.10073599964380264,0.007364345618470178,,,,,,32,4,2048,768,151936,True,True,False,2048,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.12198399752378464,0.18406400084495544,0.13770400024950505,0.13232000172138214,0.01623164885687898,0.1773120015859604,0.20688000321388245,0.1870912007987499,0.18535999953746796,0.00825628058448234,0.05766399949789047,0.06739199906587601,0.06187200043350458,0.061216000467538834,0.003004312467037863,,,,,,32,4,2048,768,151936,True,True,False,1024,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11737599968910217,0.18614399433135986,0.13109439946711063,0.12771200388669968,0.01372168725934724,0.17587199807167053,0.2699519991874695,0.19926720038056372,0.19075199961662292,0.02327478982490538,0.041728001087903976,0.06224000081419945,0.05000480003654957,0.049375999718904495,0.006185850944273961,,,,,,32,4,2048,768,151936,True,True,False,512,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.1159679964184761,0.1610880047082901,0.12988320142030715,0.12494400516152382,0.011865375783184065,0.1767680048942566,0.27379199862480164,0.21035519987344742,0.21275199949741364,0.023206964017550125,0.037087999284267426,0.062144000083208084,0.04439679980278015,0.04283200018107891,0.006155226392511492,,,,,,32,4,2048,768,151936,True,True,False,256,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.12015999853610992,0.158720001578331,0.13573280088603495,0.1343199983239174,0.010666830836014414,0.1737920045852661,0.22127999365329742,0.20261440128087999,0.20321600139141083,0.011153965849067301,0.03587200120091438,0.04944000020623207,0.038265600241720675,0.037328001111745834,0.0030431580113575636,,,,,,32,4,2048,768,151936,True,True,False,128,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.1212799996137619,0.15139199793338776,0.13183839991688728,0.13014400005340576,0.00851207547022104,0.17257599532604218,0.2250880002975464,0.1872655987739563,0.18193599581718445,0.013638284975248818,0.02969600073993206,0.0525440014898777,0.03840640028938651,0.03444799967110157,0.007598511754254174,,,,,,32,4,2048,768,151936,True,True,False,64,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11270400136709213,0.1624639928340912,0.13508000001311302,0.13180799782276154,0.014243564451606775,0.17052799463272095,0.23715199530124664,0.1953311987221241,0.1966560035943985,0.01500670718978398,0.030239999294281006,0.05270399898290634,0.03807039987295866,0.03742399998009205,0.004906165602021684,,,,,,32,4,2048,768,151936,True,True,False,32,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.12319999933242798,0.17919999361038208,0.14059039913117885,0.14156799763441086,0.016217662982107223,0.1701119989156723,0.21987199783325195,0.18903039917349815,0.1870879977941513,0.014117854507841239,0.030368000268936157,0.06406400352716446,0.04045119984075427,0.03710399940609932,0.00878942205983628,,,,,,32,4,2048,768,151936,True,True,False,16,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11296000331640244,0.16761599481105804,0.13843199908733367,0.13814399391412735,0.014925286935582404,0.1711679995059967,0.21561600267887115,0.1906527981162071,0.1876479983329773,0.011803992622137974,0.0306560005992651,0.09216000139713287,0.041129599791020155,0.036847999319434166,0.013739721468154687,,,,,,32,4,2048,768,151936,True,True,False,8,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11929599940776825,0.2008959949016571,0.14535359852015972,0.13971199840307236,0.024769473061576282,0.16803200542926788,0.25123199820518494,0.19573760256171227,0.19366399943828583,0.023196011081705884,0.03049599938094616,0.08089599758386612,0.04739360017701984,0.0453919991850853,0.01286389846889463,,,,,,32,4,2048,768,151936,True,True,False,1,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none diff --git a/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/manifest.json b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/manifest.json new file mode 100644 index 0000000..3abcf44 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/manifest.json @@ -0,0 +1,53 @@ +{ + "attention_tp_coverage": [ + 1, + 2, + 4 + ], + "environment_contract": { + "dtype": "bfloat16", + "frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "hardware": "NVIDIA H20", + "model": "Qwen3-30B-A3B", + "tensor_parallel_sizes": [ + 1, + 2, + 4 + ], + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + }, + "inputs": { + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931" + }, + "outputs": { + "allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3", + "attention.csv": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d", + "attention_true_mixed_fused.csv": "43ef4be90bddc9aeac6dbbe339feec24162cd1f2129a08fbd959e6ee4eaf5f60", + "linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "moe.csv": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0" + }, + "profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-fused-mixed-total-conserving", + "projection_contract": { + "allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity", + "attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target; true mixed rows use a total-conserving compatibility projection", + "attention_true_mixed": "The directly measured fused total is preserved in diagnostics. Frontier's two targets are projected by the same-TP pure prefill/decode reference ratio, with projected prefill + decode exactly equal to the fused total; the split is a schema compatibility attribution, not an observation", + "linear": "Frontier profiler using vLLM 0.20 CUDA operators", + "moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero" + }, + "row_counts": { + "allreduce": 24, + "attention_frontier_compatible": 132, + "attention_true_mixed_fused_diagnostic": 30, + "linear": 36, + "moe": 72 + }, + "schema_version": "frontier_qwen30_vllm020_frozen_profile.v2" +} diff --git a/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/moe.csv b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/moe.csv new file mode 100644 index 0000000..e5bf507 --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/moe.csv @@ -0,0 +1,73 @@ +time_stats.moe_gating_linear.min,time_stats.moe_gating_linear.max,time_stats.moe_gating_linear.mean,time_stats.moe_gating_linear.median,time_stats.moe_gating_linear.std,time_stats.moe_gating_routing_topk.min,time_stats.moe_gating_routing_topk.max,time_stats.moe_gating_routing_topk.mean,time_stats.moe_gating_routing_topk.median,time_stats.moe_gating_routing_topk.std,time_stats.moe_shuffling.min,time_stats.moe_shuffling.max,time_stats.moe_shuffling.mean,time_stats.moe_shuffling.median,time_stats.moe_shuffling.std,time_stats.moe_grouped_gemm.min,time_stats.moe_grouped_gemm.max,time_stats.moe_grouped_gemm.mean,time_stats.moe_grouped_gemm.median,time_stats.moe_grouped_gemm.std,num_tokens,num_experts,num_experts_per_device,expert_parallel_size,routing_runtime_path,routing_assignment_policy,routing_weight_policy,routing_uses_router_logits,gating_runtime_context,gating_runtime_context_impl,router_topk,hidden_dim,expert_hidden_dim,use_gated,num_tensor_parallel_workers,total_routed_tokens,model_expansion_ratio,tokens_per_expert_avg,tokens_to_experts_ratio,expert_utilization,min_load_ratio,load_imbalance_cv,max_load_ratio,load_entropy,load_gini_coefficient,load_distribution,seed,moe_grouped_gemm_backend,measurement_type,profiling_precision,model_arch,quant_signature,router_median_nonadditivity_ratio,projection_policy +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.33926400542259216,0.405023992061615,0.36780479848384856,0.36507199704647064,0.01690507644474779,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,1.1943039894104004,1.286784052848816,1.228384006023407,1.2273280024528503,0.02832547242381263,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.3984375,0.0,1.346291201783626,4.0,5.59375,0.661865234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,1.679744005203247,1.766144037246704,1.7095808148384095,1.7015680074691772,0.02438921262998535,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.625,0.0,1.015504800579495,5.0,6.15516433212955,0.529052734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,2.1576640605926514,2.2921600341796875,2.2097824096679686,2.188944101333618,0.045572321842012986,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.875,0.0,0.6343057228182637,2.5,6.64370748444639,0.35369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,2.422368049621582,2.5130879878997803,2.4516672134399413,2.434159994125366,0.03278900287381846,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.984375,0.0,0.4921254921257382,2.25,6.817190042344769,0.272369384765625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,2.2217600345611572,2.289599895477295,2.2571327924728393,2.263375997543335,0.021660416089449488,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,1.0,0.125,0.3486861500690843,1.875,6.908192310183997,0.197662353515625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,2.18668794631958,2.318079948425293,2.2218016147613526,2.211087942123413,0.035380897213135316,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,1.0,0.4375,0.2525504668006971,1.875,6.953347743053017,0.1410369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,2.2291839122772217,2.3929600715637207,2.2908096313476562,2.2804640531539917,0.04479348924786221,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,1.0,0.65625,0.15765965680164504,1.5625,6.98229848728205,0.08779525756835938,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,2.0678720474243164,2.1297600269317627,2.0837119817733765,2.0779199600219727,0.017880044357986735,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,1.0,0.625,0.12169081635504074,1.3125,6.9892029662356325,0.06879425048828125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,2.916032075881958,3.0819520950317383,2.9805248022079467,2.9656319618225098,0.05482195799019572,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,1.0,0.796875,0.07935434147688751,1.1796875,6.9954297964750305,0.044734954833984375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,4.421599864959717,4.535359859466553,4.486294317245483,4.497056007385254,0.036990243787549344,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,1.0,0.8203125,0.060849326483103046,1.17578125,6.9973188375859685,0.033740997314453125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,7.302591800689697,7.402751922607422,7.354758310317993,7.3464319705963135,0.032142662400335566,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,1.0,0.890625,0.0412323087266341,1.08984375,6.998772433185578,0.02334284782409668,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.35280001163482666,0.39692801237106323,0.37662720382213594,0.37196800112724304,0.013401318050665304,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.4692479968070984,0.5523840188980103,0.5134752035140991,0.5100640058517456,0.02291433464135784,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.34652799367904663,0.4119040071964264,0.3789471983909607,0.38550400733947754,0.02073945105803335,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.31462401151657104,0.7456960082054138,0.38617280423641204,0.34545600414276123,0.12230201266253077,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.32521599531173706,0.419871985912323,0.36325119733810424,0.34968000650405884,0.03161798848223672,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.289792001247406,0.4663360118865967,0.4091839998960495,0.41655999422073364,0.0446001986506615,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.3761279881000519,0.4416320025920868,0.40686399936676027,0.40540799498558044,0.02257778769899645,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.7172480225563049,0.8663039803504944,0.7723807990550995,0.7591840028762817,0.04164772379451242,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.0195519924163818,1.2216639518737793,1.1253888130187988,1.1453600525856018,0.06548005322243594,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7490559816360474,1.9644800424575806,1.8529024004936219,1.814303994178772,0.08042565617327288,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,3.2479360103607178,3.385279893875122,3.296070408821106,3.2800960540771484,0.04525529026442046,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,6.344799995422363,6.517856121063232,6.464438438415527,6.478623867034912,0.05116674443145098,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.2648000121116638,0.325439989566803,0.28852800130844114,0.28390398621559143,0.01933778377635077,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.7347840070724487,0.862496018409729,0.7769344031810761,0.769216001033783,0.03290485328796285,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.421875,0.0,1.346291201783626,6.0,5.652114648336087,0.636962890625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.9198399782180786,0.9646080136299133,0.9412063956260681,0.9411839842796326,0.014939365085478117,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.5703125,0.0,1.118033988749895,5.0,6.008641773518898,0.580810546875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,1.2796800136566162,1.3484159708023071,1.3006976008415223,1.2929120063781738,0.020807998177176254,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.8828125,0.0,0.6959705453537527,3.0,6.60872850615583,0.38055419921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,1.3630399703979492,1.4430400133132935,1.3909215927124023,1.3892319798469543,0.022335744492366926,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.984375,0.0,0.5201036555341637,3.0,6.798826509158851,0.28302001953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,1.27948796749115,1.3904000520706177,1.3176063895225525,1.309440016746521,0.038060887827312775,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,1.0,0.25,0.3511282039725661,1.875,6.91002266305238,0.1970977783203125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,1.264415979385376,1.3145920038223267,1.2791999936103822,1.2753440141677856,0.014130605249568332,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,1.0,0.375,0.24692938483248605,1.6875,6.955481130775285,0.13909912109375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,1.3081920146942139,1.347648024559021,1.3292255997657776,1.329967975616455,0.014558863679016933,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,1.0,0.625,0.17143053326165383,1.5625,6.9786675275754035,0.09520339965820312,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.242751955986023,1.3112000226974487,1.2747935891151427,1.266207993030548,0.021093073517695057,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,1.0,0.78125,0.11000099875256815,1.296875,6.991308871213679,0.062183380126953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7388160228729248,1.8077759742736816,1.772764801979065,1.772704005241394,0.021056644077284283,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,1.0,0.78125,0.0864630150197678,1.1796875,6.994552526394139,0.048796653747558594,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,2.6563520431518555,2.7063679695129395,2.6785055875778196,2.6791679859161377,0.01639963463052944,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,1.0,0.8671875,0.06127686514721937,1.16015625,6.997291583027146,0.03497934341430664,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,4.386879920959473,4.452256202697754,4.4108480453491214,4.406303882598877,0.019768161791937636,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,1.0,0.884765625,0.041723768525324195,1.1171875,6.998746434318934,0.02298593521118164,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.24208000302314758,0.4028480052947998,0.3041536003351212,0.277103990316391,0.05660721484881584,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.2447039932012558,0.30502399802207947,0.26446720361709597,0.26265600323677063,0.016744548364435372,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.2337920069694519,0.2881599962711334,0.26074880361557007,0.264384001493454,0.016469850143940968,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.23369599878787994,0.28591999411582947,0.25465920120477675,0.25385600328445435,0.01593204652619194,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.2295999974012375,0.26556798815727234,0.24674240052700042,0.2497600018978119,0.010345732066199003,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.21721599996089935,0.29020801186561584,0.2394208014011383,0.2346400022506714,0.018747330357768585,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2717759907245636,0.305184006690979,0.28813759982585907,0.28809599578380585,0.01183422600545854,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.3917759954929352,0.43772798776626587,0.41130879521369934,0.4131519943475723,0.012992473640805227,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.6176639795303345,0.7009919881820679,0.642767995595932,0.6330719888210297,0.024074084919938756,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.0820800065994263,1.1674879789352417,1.1034304022789,1.0977439880371094,0.0233067292981566,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.9809919595718384,2.0415360927581787,2.003715181350708,1.992751955986023,0.022066076434645737,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,3.790112018585205,3.8651199340820312,3.829139161109924,3.8230879306793213,0.025177464160110564,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.212351992726326,0.24383999407291412,0.22760000079870224,0.22723200172185898,0.01050568575837594,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.47494399547576904,0.5184000134468079,0.4920704007148743,0.49169600009918213,0.011991064701471855,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.3984375,0.0,1.3919410907075054,6.0,5.570159765557392,0.667236328125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.6360960006713867,0.7004479765892029,0.6608384013175964,0.6572319865226746,0.020416877242438597,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.625,0.0,1.0307764064044151,4.0,6.138251855282827,0.5382080078125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.780896008014679,0.8301439881324768,0.80346559882164,0.8030399978160858,0.016801230312128875,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.859375,0.0,0.6903350635742038,3.0,6.5943747091218174,0.38067626953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.8607040047645569,0.9195200204849243,0.8783008038997651,0.8751039803028107,0.01719059253115595,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.9765625,0.0,0.49410588440130926,2.75,6.814452474347134,0.271270751953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.833952009677887,0.894752025604248,0.8619967997074127,0.863215982913971,0.018716378851797198,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,1.0,0.25,0.33693529145074724,2.125,6.9186075263155535,0.1867218017578125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.834879994392395,0.8871039748191833,0.8651552021503448,0.8638879954814911,0.015262430894400969,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,1.0,0.375,0.25567294018677456,1.8125,6.952441049154937,0.14349365234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.8518080115318298,0.9097599983215332,0.8810272097587586,0.8751040101051331,0.017005819633271906,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,1.0,0.625,0.1747801353218523,1.53125,6.978069554482723,0.09820938110351562,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.8470079898834229,0.9010239839553833,0.8694015920162201,0.8716959953308105,0.016138881171190216,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,1.0,0.65625,0.1158122428154187,1.3125,6.9901908183358845,0.06445503234863281,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.1698240041732788,1.2311359643936157,1.1888479948043824,1.1890720129013062,0.017978797794492758,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,1.0,0.78125,0.08347181893108634,1.1796875,6.994921772573154,0.046871185302734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.7702720165252686,1.8097599744796753,1.7919103980064393,1.7956640124320984,0.012641295676021557,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,1.0,0.78125,0.06866734477822484,1.20703125,6.996602562938728,0.03801727294921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.968672037124634,3.0278079509735107,2.9899007797241213,2.9824799299240112,0.01816414122631667,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,1.0,0.8984375,0.04399546833977376,1.126953125,6.998607314922362,0.024699926376342773,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.19574399292469025,0.2512960135936737,0.21939200013875962,0.2199999988079071,0.017532156418212565,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.20483200252056122,0.24006399512290955,0.22215040028095245,0.22433599829673767,0.00969639786132892,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.20559999346733093,0.24726399779319763,0.22126719802618028,0.22207999974489212,0.01328093478485499,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.20003199577331543,0.2301120012998581,0.21453119963407516,0.21598400175571442,0.010402239855151332,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.19551999866962433,0.22972799837589264,0.21238719969987868,0.21488000452518463,0.010706095835489097,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.19420799612998962,0.2903999984264374,0.2211231991648674,0.21476799994707108,0.025955008886196004,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2290560007095337,0.289247989654541,0.2543327987194061,0.24939200282096863,0.01663236753503115,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.30831998586654663,0.36953601241111755,0.3324000000953674,0.3288639932870865,0.018617999572156707,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.462911993265152,0.5497919917106628,0.4893856018781662,0.4816960096359253,0.023348152887178286,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,0.7662079930305481,0.8717759847640991,0.788454395532608,0.7744799852371216,0.03174746482604812,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.363935947418213,1.4143040180206299,1.3812703967094422,1.3798720240592957,0.014770450075530007,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.5507519245147705,2.680704116821289,2.579859209060669,2.566223978996277,0.03673283558365955,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize diff --git a/runs/frontier-qwen30-vllm020-profile-v1/plot_profile_ablation.py b/runs/frontier-qwen30-vllm020-profile-v1/plot_profile_ablation.py new file mode 100644 index 0000000..f2a831a --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/plot_profile_ablation.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Render the profile ablation and execution-context diagnostics.""" + +from __future__ import annotations + +import argparse +import json +from collections import Counter +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--s2", type=Path, required=True) + parser.add_argument("--routing", type=Path, required=True) + parser.add_argument("--opprof", type=Path, required=True) + parser.add_argument("--p1", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + s2 = json.loads(args.s2.read_text()) + routing = json.loads(args.routing.read_text()) + opprof = json.loads(args.opprof.read_text()) + p1 = json.loads(args.p1.read_text()) + + plt.rcParams.update({"font.size": 9, "axes.titlesize": 10, "axes.labelsize": 9}) + fig, axes = plt.subplots(2, 2, figsize=(13.2, 8.2), constrained_layout=True) + + real = s2["real_scores"] + cells = sorted(real, key=lambda cell: (-real[cell], cell)) + x = np.arange(len(cells)) + series = ( + ("Real", real, "#111111", "o"), + ( + "Old profile-only", + s2["historical_modes"]["historical-profile-only"]["simulated_scores"], + "#d95f02", + "s", + ), + ( + "vLLM 0.20 profile-only", + s2["vllm020_profile_only"]["SLO-gated"]["simulated_scores"], + "#7570b3", + "^", + ), + ( + "Frozen per-TP calibration", + s2["historical_modes"]["historical-per-tp-calibration"]["simulated_scores"], + "#1b9e77", + "D", + ), + ) + ax = axes[0, 0] + for label, values, color, marker in series: + ax.plot(x, [values[cell] for cell in cells], label=label, color=color, marker=marker, lw=1.7) + ax.set_xticks(x, [cell.replace("_", "\n") for cell in cells]) + ax.set_ylabel("SLO-feasible req/s/GPU") + ax.set_title("(a) Full 92-probe config ranking") + ax.grid(axis="y", alpha=0.25) + ax.legend(fontsize=8, ncol=2, loc="upper right") + + ax = axes[0, 1] + categories = ("Actual prefill", "Actual decode", "Frontier prior") + cv = [ + routing["phase_summary"]["prefill"]["actual"]["load_cv"]["median"], + routing["phase_summary"]["decode"]["actual"]["load_cv"]["median"], + routing["phase_summary"]["prefill"]["frontier_simulation"]["load_cv"]["median"], + ] + max_ratio = [ + routing["phase_summary"]["prefill"]["actual"]["max_load_ratio"]["median"], + routing["phase_summary"]["decode"]["actual"]["max_load_ratio"]["median"], + routing["phase_summary"]["prefill"]["frontier_simulation"]["max_load_ratio"]["median"], + ] + bx = np.arange(len(categories)) + width = 0.34 + bars1 = ax.bar(bx - width / 2, cv, width, label="Load CV", color="#66c2a5") + bars2 = ax.bar(bx + width / 2, max_ratio, width, label="Max/mean load", color="#fc8d62") + ax.bar_label(bars1, fmt="%.2f", fontsize=8) + ax.bar_label(bars2, fmt="%.2f", fontsize=8) + ax.set_xticks(bx, categories) + ax.set_ylim(0, max(max_ratio) * 1.2) + ax.set_title("(b) Per-layer MoE routing skew") + ax.legend(fontsize=8) + ax.grid(axis="y", alpha=0.25) + + graph_counts = {phase: Counter() for phase in ("pure_decode", "pure_prefill", "true_mixed")} + for cell in opprof["cells"]: + for group in cell["groups"]: + graph_counts[group["phase"]][group["cudagraph_runtime_mode"]] += int(group["steps"]) + ax = axes[1, 0] + phases = tuple(graph_counts) + bottoms = np.zeros(len(phases)) + for mode, color in (("FULL", "#1b9e77"), ("PIECEWISE", "#e6ab02"), ("NONE", "#d95f02")): + values = np.asarray( + [100 * graph_counts[phase][mode] / sum(graph_counts[phase].values()) for phase in phases] + ) + ax.bar(np.arange(len(phases)), values, bottom=bottoms, label=mode, color=color) + bottoms += values + ax.set_xticks(np.arange(len(phases)), [phase.replace("_", " ") for phase in phases]) + ax.set_ylabel("Observed scheduler steps (%)") + ax.set_ylim(0, 100) + ax.set_title("(c) Real vLLM execution mode is phase-dependent") + ax.legend(fontsize=8, ncol=3, loc="lower left") + + ax = axes[1, 1] + mode_order = ("historical-calibrated", "historical-profile-only", "vllm020-profile-only") + labels = ("Per-TP\ncalibration", "Old\nprofile-only", "vLLM 0.20\nprofile-only") + accuracy = [100 * p1["summaries"][mode]["probe_classification"]["accuracy"] for mode in mode_order] + mae = [100 * p1["summaries"][mode]["pass_rate_mae"] for mode in mode_order] + px = np.arange(len(labels)) + bars1 = ax.bar(px - width / 2, accuracy, width, label="Label accuracy (higher better)", color="#1b9e77") + bars2 = ax.bar(px + width / 2, mae, width, label="Pass-rate MAE (lower better)", color="#d95f02") + ax.bar_label(bars1, fmt="%.1f", fontsize=8) + ax.bar_label(bars2, fmt="%.1f", fontsize=8) + ax.set_xticks(px, labels) + ax.set_ylabel("Percent") + ax.set_ylim(0, 105) + ax.set_title("(d) Held-out P1 boundary probes (12 labels)") + ax.legend(fontsize=8, loc="upper right") + ax.grid(axis="y", alpha=0.25) + + fig.suptitle( + "Qwen3-30B-A3B / vLLM 0.20 / BF16 / dash0 H20: operator provenance is not execution-context fidelity", + fontsize=12, + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(args.output, dpi=180) + fig.savefig(args.output.with_suffix(".svg")) + print(args.output) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-qwen30-vllm020-profile-v1/prepare_s2_profile_comparison.py b/runs/frontier-qwen30-vllm020-profile-v1/prepare_s2_profile_comparison.py new file mode 100644 index 0000000..962952a --- /dev/null +++ b/runs/frontier-qwen30-vllm020-profile-v1/prepare_s2_profile_comparison.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Prepare the 92 frozen S2-R-b probes with a replacement profile bundle.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + + +PROFILE_KEYS = { + "linear_op_input_file": "linear_op.csv", + "atten_input_file": "attention.csv", + "moe_input_file": "moe.csv", +} + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--resolved-plan", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--profile-root", type=Path, required=True) + parser.add_argument("--cache-root", type=Path) + parser.add_argument("--shards", type=int, default=1) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + if args.shards < 1: + raise SystemExit("--shards must be positive") + source = json.loads(args.resolved_plan.read_text()) + source_rows = [row for row in source["runs"] if row["mode"] == "uncalibrated"] + if len(source_rows) != 92: + raise SystemExit(f"expected 92 uncalibrated probes, found {len(source_rows)}") + + output = args.output.resolve() + profile_root = args.profile_root.resolve() + cache_root = (args.cache_root or (output / "prediction-cache")).resolve() + profile_hashes: dict[str, str] = {} + for filename in PROFILE_KEYS.values(): + path = profile_root / filename + if not path.is_file(): + raise SystemExit(f"missing frozen profile: {path}") + profile_hashes[str(path)] = sha256(path) + + entries: list[dict[str, Any]] = [] + for row in source_rows: + fixture_dir = Path(row["fixture_dir"]).resolve() + fixture_manifest_path = fixture_dir / "fixture_manifest.json" + fixture = json.loads(fixture_manifest_path.read_text()) + config = json.loads(Path(row["config_path"]).read_text()) + config["mode"] = "new-profile-only" + config["config_id"] = f"{row['cell_id']}__new-profile-only" + config["calibration"]["a_tp"] = 1.0 + knobs = config["frontier"]["knobs"] + knobs["cache_dir"] = str(cache_root) + knobs["no_cache"] = False + for key, filename in PROFILE_KEYS.items(): + knobs[key] = str(profile_root / filename) + + target_config = output / "configs" / f"{row['fixture_id']}.json" + write_json(target_config, config) + entries.append( + { + "fixture_id": row["fixture_id"], + "cell": row["cell_id"], + "role": f"probe-{int(row['probe_index']):02d}", + "anchor": float(row["sampling_u"]), + "selected_count": int(row["request_count"]), + "probe_index": int(row["probe_index"]), + "sampling_u": float(row["sampling_u"]), + "tensor_parallel_size": int(row["tensor_parallel_size"]), + "config": str(target_config), + "fixture_manifest": str(fixture_manifest_path), + "frontier_csv": str(fixture_dir / "frontier.csv"), + "sidecar": str(fixture_dir / "sidecar.jsonl"), + "calibration_scale": 1.0, + "request_count": int(fixture["request_count"]), + } + ) + + base = { + "schema": "frontier-qwen30-s2-profile-comparison-prepared.v1", + "status": "PASS", + "mode": "new-profile-only", + "source": { + "resolved_plan": str(args.resolved_plan.resolve()), + "sha256": sha256(args.resolved_plan), + }, + "profile_hashes": profile_hashes, + "isolation": { + "calibration_a_tp": 1.0, + "prediction_cache": str(cache_root), + "all_non_profile_knobs_inherited": True, + }, + "suite_total_runs": len(entries), + } + write_json( + output / "prepared-manifest.json", + {**base, "expected_runs": len(entries), "entries": entries}, + ) + for shard in range(args.shards): + shard_entries = entries[shard :: args.shards] + write_json( + output / f"prepared-manifest-shard-{shard:02d}-of-{args.shards:02d}.json", + { + **base, + "shard": {"index": shard, "count": args.shards}, + "expected_runs": len(shard_entries), + "entries": shard_entries, + }, + ) + print(output / "prepared-manifest.json") + + +if __name__ == "__main__": + main() diff --git a/simulator-fidelity.md b/simulator-fidelity.md new file mode 100644 index 0000000..728ce14 --- /dev/null +++ b/simulator-fidelity.md @@ -0,0 +1,360 @@ +# Frontier simulator fidelity：以 config ranking 为目标的阶段性评测 + +更新日期：2026-07-16。 + +统一实验平台：所有新增与重跑实验只使用 `dash0` 的 8×NVIDIA H20。Qwen30B 的真实 P1 artifacts 也来自 `dash0`；早期文档中的主机 provenance 标注错误，本版已按实验主机和远端 artifact 路径更正。 + +## 结论摘要 + +本文统一使用同一个优化目标：**在给定 workload 和 SLO 下，最大被测试可行 offered request throughput/GPU**。我们关心 simulator 是否保持 config 的相对排序、是否覆盖真机最优集合，以及部署 simulator 所选 config 后的真实 regret；绝对 latency/throughput 误差不是首要判据。 + +当前证据不支持“Frontier 普遍找不到最优 config”这一判断： + +- 在 Qwen3-30B mixed serving artifact 上，未经端到端校准的 Frontier 排序失败，选择 regret 为 `25.63%`；但使用独立 workload 上冻结的 per-TP calibration 后，Frontier 的候选 top set 包含真机最优 config，Kendall τ-b 为 `0.9668`，最坏 tie-break regret 为 `0.76%`。真实 serving 与本轮补测 profiles 均运行在 `dash0`。 +- 将 operator profile 更新为与真实 serving 一致的 community vLLM `0.20.0`、BF16、H20、TP1/2/4 栈后，新的 profile-only Frontier **没有恢复排序**：92 个真实 anchors 在 simulator 中全部 SLO-infeasible，12 个 config 因而全部并列，最坏 tie-break regret 为 `60.91%`。这否证了“旧 profile 版本不一致是主要原因”这一简单解释，并把问题收敛到 execution context、operator composition 与 mixed-state schema。 +- 在 Qwen3-235B-A22B-FP8 prefill-only 上，补齐 FP8/MoE profile 与 serving semantics、但不做端到端 action calibration 后，Frontier 的最优集合与真机完全一致，Spearman ρ 为 `0.9487`，20/20 个可比较非 tie config pair 同序，选择 regret 为 `0`。 +- 在 Qwen3-235B-A22B-FP8 fixed-shape mixed 上，Frontier 同样给出了与真机完全相同的 TP4 top set，Kendall τ-b=`0.8944`、worst tie-break regret=`0`。但这次成功掩盖了一个明确的机制错误：Frontier 认为同一 TP family 内四个 MNS/MBT config 完全等价，真机 TP8 capacity 却形成对角为 0.30、非对角为 0.20 req/s/GPU 的 checkerboard interaction。34 个 config-load labels 中有 10 个 false-infeasible；20 个 real non-tie pairs 中只保持 16 个方向。 +- 现有结果表明：**在已对齐的受控 case 中，绝对 capacity 有 gap、甚至 action differential 错误，都不必然妨碍 Frontier 找到当前 surface 的最优 config。** 统一 dash0 campaign 已完整覆盖 Qwen235 FP8 prefill-only 与 fixed-shape mixed，但仍不能外推到 trace-faithful mixed 或 decode-only，也不能忽略达到这种 fidelity 所需的真机 profile、KV capacity、runtime-semantic patch 或 calibration 成本。 + +因此，现阶段最准确的 research statement 不是“simulator 排序一定错误”，而是： + +> Simulator 的 usefulness 取决于 compatibility envelope。问题是需要多少、什么类型的真机信息，才能使其 config ranking 可被信任；以及当 workload、runtime 或 execution topology 变化时，如何低成本判断该 envelope 是否仍成立。 + +## 评测口径 + +对系统 `s ∈ {real, sim}` 和 config `c`，定义： + +```text +capacity_s(c) = max { tested offered rate r | empirical SLO pass rate(c, r) >= 0.95 } +``` + +所有结果使用 `capacity/实际占用 GPU 数` 作为 primary score。我们只报告实际测试到的最大可行点，不对未测试区间做连续插值，也不因观察到局部非单调就强制单调化。 + +核心指标如下： + +- **Top set**：达到各自最大 capacity 的全部 config，保留并列。 +- **Top-set hit/match**：simulator top set 是否与 real top set 有交集/完全相同。 +- **Worst tie-break regret**：若 simulator 给出多个并列最优，任取其中最差一个在真机上的损失： + + ```text + max[c in SimTop] (RealBest - capacity_real(c)) / RealBest + ``` + +- **Rank correlation**：有并列时优先使用 Kendall τ-b；Qwen235B 已有冻结分析使用 Spearman ρ，同时报告非 tie pair direction。 +- **Pairwise direction**：两两 config 的 `> / = / <` 是否一致。只报告 non-tied accuracy 时，会同时给出可比较 pair 数量。 + +这个定义也修正了之前的口径混用：内部 `throughput proxy` 不是“满足 SLO 的最大 throughput”，不能用来决定部署 config。本文的 Qwen30B 主结论因此取代 [旧 simulator fidelity 总结](docs/simulator-fidelity-frontier-20260711.md) 中基于 throughput-proxy 的 config-selection 结论；该旧读法仍可作为 simulator 内部机制误差的诊断证据。 + +## 实验 setup + +### Case A：Qwen3-30B-A3B mixed serving + +| 项目 | 设置 | +|---|---| +| 真机 | `dash0`，8×NVIDIA H20 | +| model | `Qwen/Qwen3-30B-A3B` | +| serving runtime | community vLLM `0.20.0`，CUDA 12.9 环境 | +| simulator | Frontier commit `d9cfeb6d8791fbf2f295dd9744c56a666171776e` | +| trace | `chat_w20260311_1000`，replay time scale=`0.1`，input 0--8192 tokens，output override=128 tokens，max concurrency=64 | +| SLO | 至少 95% requests 同时满足 TTFT 阶梯阈值（input ≤4096: 2s；≤32768: 4s；其余 6s）和 TPOT ≤50ms | +| config surface | `TP∈{1,2,4} × MNS∈{8,16,32,64}`，MBT=8192，共 12 cells | +| ground truth | 每个 cell 最多 8 个搜索 probes；92 个实测 anchors，共 27,583 个 request observations | +| simulator workload | 对每个 real anchor 重放同一冻结 request cohort 的 token length、arrival、request count 与 prefix block identities；old profile-only、calibrated、vLLM 0.20 profile-only 各 92 runs，合计 276/276 成功 | +| primary objective | SLO-feasible request throughput/GPU | + +我们区分两个 Frontier 条件： + +1. **Profile-only**：直接使用已有 H20 operator profiles，不用 serving E2E 数据修正时间尺度。 +2. **Frozen per-TP calibration**：使用 `TP1=0.723481, TP2=0.468089, TP4=0.352137` 三个 execution-time scales。它们在独立的 `coder_200_ts2` workload 上拟合，并在 `coder_200_ts3` 上冻结验证；本 `chat` surface 不重新拟合。该 scale 由外部 hook 在 Frontier 组成 batch execution time 后注入，并不是 Frontier 原生的 per-TP operator profiling 功能。 + +Frontier 原生 profile schema 可以按 `num_tensor_parallel_workers` 选择 TP-specific kernel rows；这与上面的 calibration 不同。第二个条件不是 zero-measurement prediction：它使用了同 model/hardware/runtime family 的真机 serving 数据，并能同时吸收 CUDA graph、fusion、routing、CPU launch 和 profile provenance 等残差。另一个限制是已有 H20 profiles 的生成栈与 ground-truth vLLM 0.20.0 并非严格同版本；per-TP calibration 吸收了相当一部分 execution-scale mismatch。 + +### Case B：Qwen3-235B-A22B-FP8 prefill-only + +| 项目 | 设置 | +|---|---| +| 真机 | `dash0`，8×NVIDIA H20（97,871 MiB/GPU，全互联 NV18），driver `580.95.05` | +| model | `Qwen/Qwen3-235B-A22B-FP8`，block-wise FP8 weights，BF16 KV cache | +| serving runtime | community vLLM `0.10.2`；Python 3.12.3、torch 2.8.0+cu128、FlashInfer 0.3.1.post1、Transformers 4.55.2；eager execution | +| simulator | Frontier commit `d9cfeb6d8791fbf2f295dd9744c56a666171776e` + 冻结的 FP8/MoE best-effort patches | +| source trace | `thinking_w20260327_1000`；从 15,409 个 eligible requests 中固定选择 64 个 length-stratified prompts | +| fixed cohort | seed=`2026071501`；input p50=1478、p95=18,439、max=29,385、mean=3511.55 tokens；output override=1 token | +| offered rates | 0.15, 0.20, 0.25, 0.30, 0.35, 0.40, 0.45, 0.50, 0.75, 1.00, 1.50 req/s；只缩放同一 cohort 的 arrival timeline | +| SLO | 至少 61/64 requests 满足 `TTFT ≤ 1000ms + input_tokens/8000 tokens/s` | +| config surface | `TP∈{4,8} × MNS∈{64,128} × MBT∈{8192,16384}`；实际 TP4 为 EP off，TP8 为 EP on | +| serving controls | prefix cache/speculative decoding off，chunked prefill on；每个 config fresh server，第二轮反转 config/rate 顺序 | +| primary objective | 最大被测试可行 offered rate / TP | + +这里比较的不是 stock Frontier。为建立公平的 FP8 compatibility envelope，我们先用同一 community-vLLM 0.10.2 栈在 H20 上测量 FP8 linear/attention/MoE/collective profiles，修复了 MoE FP8 tuning-key、Qwen MoE serving plan、TP/EP-aware cache key 与 critical-lane prediction，并输入真机测得的 KV block capacity（TP4=26,101；TP8=62,351）。Frontier refined outputs 在真机 v2/v3 ranking 运行前冻结，**没有使用本 case 的 serving capacity 做 end-to-end calibration**。 + +该 case 是有意设计的 prefill mechanism-isolation workload：output 固定为 1，prefix cache 关闭，并使用固定 64-request cohort。它不保持原 trace 的 output-length distribution 和 prefix reuse，因此不能被称为 trace-faithful workload，也不能代表 mixed serving。 + +## Workload fidelity 修正 + +`thinking_w20260327_1000` 的 600 秒原始窗口包含 15,479 requests，字段包括 exact prompt、arrival、`input_length`、`output_length`、session parent/turn 和 block-size=64 的 `hash_ids`。只读审计得到： + +| 属性 | 数值 | +|---|---:| +| natural offered rate | 25.798 req/s | +| input tokens | mean=3,660.0，p50=1,491，p95=19,610.6，p99=29,944.6，max=51,827 | +| output tokens | mean=3,924.6，p50=3,435，p95=8,945.1，p99=17,705.5，max=74,867 | +| total context | mean=7,584.6，p50=5,317，p95=25,102.1，p99=37,110.5，max=74,973 | +| rows with `hash_ids` | 15,479/15,479 | +| `len(hash_ids) == ceil(input_length/64)` | 15,479/15,479 | +| arrival-order repeated block ratio | 40.88% | +| input+output 超过 community model 40,960 context | 72 requests | +| output length=0 | 6 requests | + +因此，input/output/hash filtering 不是 trace fidelity 的默认要求。此前 fixed output length 是为了分别隔离 prefill 或 decode 机制；如果用它回答 production-trace fidelity，就属于错误 benchmark。新的实验矩阵固定为： + +| 层次 | Workload contract | Prefix contract | 回答的问题 | +|---|---|---|---| +| T0 fixed-shape | 固定 ISL、OSL、uniform QPS；所有 config 使用相同 request IDs/order | prefix off，使用等长但不同 token 的 prompts | 最简单条件下 execution/scheduler composition 是否匹配 | +| T1 trace-faithful mixed | 保留 trace 的 arrival、input、output、session；不使用 completion override | real 由 exact prompt tokens 自然计算 cache key；Frontier 消费等价 `block_hash_ids` | 在真实 joint distribution 下能否保持 config rank | +| T2 decode-only | output 保留 trace 或受控固定值；input length 表示 request 到达时已存在的 KV residency | initial-KV state 单独冻结；不得用一次 1-token prefill 冒充 | KV residency、decode batching、TP/DP/EP coupling 下能否保持 rank | + +T1 中只允许两类有记录的排除：模型 context 上限不支持的 72 requests，以及 API 无法形成 completion 的 6 个 zero-output rows；二者不重叠，最终 eligible universe 为 15,401/15,479 requests（99.50%）。排除后 input/output/total 的均值分别为 3,575.0/3,823.3/7,398.2 tokens，p95 分别为 18,887/8,768/23,697，说明没有把 workload 人为变成 short-request case。其余请求不能因为“跑得慢”或“方便形成 cohort”被选择性移除。capacity search 只通过 trace 已有、同 session 共享的 `sampling_u` 做与长度无关的 session-coherent thinning；每个被选 request 的 arrival、input、output、prompt、hash 和相对次序均保持不变。完整 eligible universe 是抽样母体和最终 workload 声明，不用 length-stratified cohort 代替。 + +全量 tokenizer audit 进一步验证了 15,401/15,401 个 eligible prompts、共 55,057,919 tokens 的实际 community-Qwen token 数与 trace `input_length` 完全一致。在 867,538 个 source blocks 上，`hash_id` 与 `(parent_hash_id, 64-token chunk)` 的双向映射也没有发现冲突。因此 exact prompt 和 source hash 足以保留该 trace 的 prefix-equivalence relation，不需要用 synthetic length-only prompt 替代。 + +trace 的 source hash block size 为 64，而 community vLLM 0.10.2 的 CUDA KV block size 不能直接设置为 64。下一轮实验冻结 real 与 Frontier 的 block size 都为 16：从 exact prompt token IDs 重新生成 block-16 parent/content identities，并用 source `hash_ids` 验证每四个完整 block-16 所对应的 64-token equality/reuse relation。tokenized length、request order、cache query/hit/allocated blocks 仍必须在两侧逐请求核对；上面的离线 audit 不能代替 runtime counter parity。 + +## 总体结果 + +| Case / simulator 条件 | configs | real optimum | simulator top set | top-set 结论 | rank / pairwise | worst regret | 判断 | +|---|---:|---|---|---|---|---:|---| +| Qwen30 mixed / old profile-only | 12 | TP2,MNS32 | TP4,MNS32/64 | miss | τ-b=0.000；exact sign=37.88% | 25.63% | 排序错误 | +| Qwen30 mixed / vLLM 0.20 profile-only | 12 | TP2,MNS32 | 全部 12 configs | 不可辨识 | τ-b=0.000；exact sign=7.58% | 60.91% | 同栈 raw profile 仍不足 | +| Qwen30 mixed / frozen per-TP calibration | 12 | TP2,MNS32 | TP2,MNS32/64 | hit，非 exact | τ-b=0.9668；exact sign=93.94% | 0.76% | dash0 | +| Qwen235 FP8 prefill / best-effort | 8 | TP4,MBT16K,MNS64/128 | 与 real 完全相同 | exact match | ρ=0.9487；non-tied 20/20 | 0 | 足以选最优 config | +| Qwen235 FP8 fixed-shape mixed / frozen full profile | 8 | 四个 TP4 configs | 与 real 完全相同 | exact match | τ-b=0.8944；exact sign=24/28；real non-tie=16/20 | 0 | 选对 topology；漏掉 TP8 MNS×MBT interaction | + +最重要的区别是：Qwen30 的 high-fidelity 结果依赖 per-TP E2E calibration；Qwen235 的 high-fidelity 结果不依赖本 case 的 E2E calibration，但依赖同 runtime/hardware 的 operator profile、真实 KV capacity 和多处 compatibility fixes。二者都不能表述为“拿 stock Frontier 零成本预测即可”。 + +## Case A baseline 结果：Qwen30 mixed serving + +![Qwen3-30B mixed serving simulator-vs-real config ranking](docs/assets/simulator-fidelity/qwen30-mixed-config-ranking.png) + +图中 config 按真机 capacity 从高到低排列。上图显示 profile-only Frontier 选错 TP family；下图显示冻结 per-TP calibration 后，simulator 基本恢复了有用排序。 + +| Config | Real | Frontier profile-only | Frontier calibrated | Calibrated − real | +|---|---:|---:|---:|---:| +| TP1, MNS8 | 2.1000 | 1.1000 | 1.7167 | -0.3833 | +| TP1, MNS16 | 2.3500 | 1.1000 | 2.3833 | +0.0333 | +| TP1, MNS32 | 2.2833 | 1.1000 | 2.3833 | +0.1000 | +| TP1, MNS64 | 2.2833 | 1.1000 | 2.3833 | +0.1000 | +| TP2, MNS8 | 2.2750 | 0.0000 | 1.7417 | -0.5333 | +| TP2, MNS16 | 2.2750 | 1.1917 | 2.3000 | +0.0250 | +| **TP2, MNS32** | **3.2833** | 0.0000 | **3.7500** | +0.4667 | +| TP2, MNS64 | 3.2583 | 0.0000 | **3.7500** | +0.4917 | +| TP4, MNS8 | 1.2833 | 0.0000 | 1.3208 | +0.0375 | +| TP4, MNS16 | 2.4417 | 0.0000 | 2.5000 | +0.0583 | +| TP4, MNS32 | 2.4417 | **1.3208** | 2.5000 | +0.0583 | +| TP4, MNS64 | 2.4417 | **1.3208** | 2.5000 | +0.0583 | + +单位均为 SLO-feasible req/s/GPU。`0.0000` 表示该 config 在复用的 tested anchors 中没有被 Frontier 判为 SLO-feasible，不代表硬件在零负载下也无法运行。 + +相对排序方面： + +- Profile-only 的 top set 为 TP4/MNS32、TP4/MNS64；两者真机 capacity 都是 2.4417，而 real optimum TP2/MNS32 是 3.2833，因此 regret 为 `1−2.4417/3.2833=25.63%`。 +- Calibrated Frontier 把 TP2/MNS32 和 TP2/MNS64 判为并列。前者是真机唯一最优，后者低 `0.025 req/s/GPU`；因此 optimistic regret=0，worst tie-break regret=`0.76%`。 +- Calibrated absolute error 并不是一个全局常数：TP2/MNS8 被低估 0.5333，而 TP2/MNS32/64 被高估约 0.47--0.49。**选对 config 不代表 simulator 已正确解释每个 state/action 的性能。** +- 92 个 anchor-level SLO 判定中，calibrated Frontier 仍有 21 个 false-feasible、7 个 false-infeasible；aggregate top set 正确部分来自 cell boundary 上误差抵消。因此该条件足以做本 surface 的 near-optimal selection，但不能替代真机 SLO certification。 + +### 同栈 vLLM 0.20 per-TP profile 消融 + +为了判断上述差异是否只是旧 operator profile 与真实 serving 版本不一致，我们在 `dash0` 上使用 ground-truth 相同的 community vLLM `0.20.0` source commit `88d34c6409e9fb3c7b8ca0c04756f061d2099eb1`、Qwen3-30B-A3B BF16 与 H20，重新测量 TP1/2/4 profiles。Frontier code、92 个 trace fixtures、SLO、KV capacity、config surface、communication backend 与随机种子保持不变；这轮只替换 compute profiles，不使用本 surface 的 serving latency 拟合 scale。 + +| Profile component | 冻结行数 | 测量与使用方式 | +|---|---:|---| +| attention + KV | 132 | 102 个 pure prefill/decode rows；30 个 true-mixed rows由同 TP pure profile 比例做 total-conserving split | +| fused mixed diagnostic | 30 | 保存 vLLM FA3 实测 fused total；不作为 Frontier 的第三个 target | +| linear / norm / RoPE / router | 36 | vLLM 0.20 CUDA operator | +| MoE | 72 | TP1/2/4 local-shard shapes × 12 token sizes × uniform/random/hotset8 routing | +| all-reduce | 24 | TP2/4 实测；本 base comparison 不注入，用来诊断 communication，保持历史 analytical CC backend 不变 | + +attention、linear 和 MoE 的 `TP` 字段改变的是单张 H20 上的 local-shard shape；它们不是一次 distributed full-engine graph replay。只有 all-reduce 使用真实 TP2/4 ranks 测量。该限制是实验设计的一部分：本消融回答“把相同 runtime 的 isolated operator profiles 换进去是否足够”，而不是预先假设 isolated profile 已等价于真实 execution counterfactual。 + +![Qwen3-30B vLLM 0.20 profile-only ablation and execution-context diagnostics](docs/assets/simulator-fidelity/qwen30-vllm020-profile-ablation.png) + +完整 92-probe 结果如下。`0` 表示复用的 anchors 中没有一个被 simulator 判为 SLO-feasible；它既不表示零负载硬件 capacity，也不能被解释成“因为真实最优 config 位于并列集合中所以 top-set hit”。当所有 config 都并列时 simulator 没有选择信息，故 primary 指标是 worst tie-break regret。 + +| Frontier 条件 | Sim top set | τ-b | Exact pair sign | Worst regret | Anchor labels：agree / false feasible / false infeasible | +|---|---|---:|---:|---:|---:| +| old profile-only | TP4,MNS32/64 | 0.0000 | 37.88% | 25.63% | 37 / 0 / 55 | +| vLLM 0.20 profile-only | 全部 12 configs | 0.0000 | 7.58% | 60.91% | 30 / 0 / 62 | +| frozen per-TP calibration | TP2,MNS32/64 | 0.9668 | 93.94% | 0.76% | 64 / 21 / 7 | + +若忽略 SLO，仅按 simulator 完成 throughput proxy 排序，新 profile 的 top-1 是 TP1/MNS64，真机 regret 仍为 `30.46%`，τ-b=`0.3940`，non-tied pair concordance=`70.49%`。这只是 failure diagnosis，不能替代本文定义的 SLO-feasible capacity objective。两路 CPU shard 都是 46/46 PASS，合计 simulator runtime 为 2,394.9 s（0.665 CPU-hour）；所以结果不是 crash、missing row 或未完成运行造成的。 + +#### 为什么相同 operator provenance 仍不等于 execution fidelity + +证据把 mismatch 收敛到以下三个相互耦合的层面： + +1. **Execution regime 随 scheduler state 改变。** 6 个真实 P1 cells 共记录 161,161 个 model steps。151,471 个 pure-decode steps 全部使用 FULL CUDA graph；224 个 pure-prefill steps 中 192 个为 NONE、30 个 PIECEWISE、仅 2 个 FULL；9,466 个 true-mixed steps 中 8,623 个为 NONE、842 个 PIECEWISE、仅 1 个 FULL。本轮 isolated profiles 则关闭 CUDA graph，Frontier comparison 也固定 `decode_cuda_graph_mode=none`。同一个 nominal operator 在不同 phase/state 下并不是同一个 launch/fusion regime。 +2. **Mixed attention schema 不保真。** vLLM 0.20 的 FA3 对 true-mixed batch 执行一个 fused varlen call；Frontier 必须分别得到 `attn_prefill` 与 `attn_decode_in_mixed`。不做 split 时 simulator 在第一个 mixed batch 明确报 missing prediction；采用 pure-profile ratio split 虽保持 `prefill + decode = fused total`，但 split 是兼容性归因而非真实观测。scheduler 使用这个虚构分解推进后续状态，误差会累积。 +3. **MoE routing 不是固定先验。** 对 exact trace 的 8 个 requests、48 层 native routing capture，实际 median load CV 在 prefill/decode 分别为 `1.076/1.107`，median max/mean expert load 为 `6.12/5.64`；Frontier 固定的 per-layer Uniform prior 分别只有 `0.485` 与 `1.84`。实际与 Frontier expert allocation 的逐层 Pearson median 接近 0（prefill `-0.017`、decode `-0.024`），而 Frontier 在所有 phase 复用相同 allocation，prefill/decode correlation 被固定为 1.0；真实 median correlation 为 `0.667`。8-request capture 是 mechanism probe，不是 workload population estimate，但足以否证“固定 routing vector 等价于该 trace execution”的假设。 + +更直接地，在同一个 `tp1_mns64` fixture、`batch_id=0/layer_id=0`、两条 trajectory 尚未分叉时，新 profile 的 layer component sum 已是旧 profile 的 `2.70×`（1.148→3.101 ms）：MoE grouped GEMM 为 `3.76×`、RoPE `7.39×`、attention prefill `2.67×`。这说明版本更新没有产生一个小而一致的 correction；不同 components 的误差方向和幅度不同，随后又通过 scheduler queue、batch composition、graph mode 与 routing 反馈到下一个 state。 + +因此本实验否证 H1：**Frontier 原生支持按 `num_tensor_parallel_workers` 选择 per-TP profile rows，而且 attention/MoE kernel 也确实可以逐 TP 测量；真正缺失的是 scheduler state 与 execution regime 条件化的 joint counterfactual。** TP 和 MNS 不能拆开：TP 改变 shard、collective、replica capacity、routing placement 与 graph shape，MNS 改变 batching/queue evolution；两者共同决定 phase mixture 和 execution mode。历史 per-TP calibration 是 composition 之后的外部 E2E scale，因此能吸收这些残差，但它不是机制解释，也不是 Frontier 原生 profiling。 + +下一轮不应继续盲目加 isolated rows，而应依次做可证伪的 context ablation：按 FULL pure-decode、NONE/PIECEWISE prefill/mixed 建立 graph-conditioned step profiles；保留 fused mixed layer/step total 而不是要求不可观测的拆分；用 trace-conditioned per-layer/request/phase routing histogram 替换固定先验；再单独注入 measured collective。每次只改变一个条件，最终仍在完整 `TP×MNS` joint surface 上以 SLO ranking 验证，frozen calibration 只作为可达到的上界。 + +本次 accepted profile jobs 使用约 0.118 H20-GPU-hour；包括 smoke 和失败 attempt 的完整 campaign 为 0.410 H20-GPU-hour。冻结 profile、manifest、condensed artifacts 与作图脚本位于 `runs/frontier-qwen30-vllm020-profile-v1/`。 + +## Case B 结果：Qwen235 FP8 prefill-only + +![Qwen3-235B FP8 prefill-only simulator-vs-real config ranking](docs/assets/simulator-fidelity/qwen235-prefill-config-ranking.png) + +| TP | EP | MNS | MBT | Real | Frontier | Frontier − real | +|---:|---:|---:|---:|---:|---:|---:| +| 4 | off | 64 | 8192 | 0.05000 | 0.03750 | -0.01250 | +| 4 | off | 128 | 8192 | 0.05000 | 0.03750 | -0.01250 | +| 4 | off | 64 | 16384 | **0.07500** | **0.06250** | -0.01250 | +| 4 | off | 128 | 16384 | **0.07500** | **0.06250** | -0.01250 | +| 8 | on | 64 | 8192 | 0.05625 | 0.05000 | -0.00625 | +| 8 | on | 128 | 8192 | 0.05625 | 0.05000 | -0.00625 | +| 8 | on | 64 | 16384 | 0.05625 | 0.05625 | 0.00000 | +| 8 | on | 128 | 16384 | 0.05625 | 0.05625 | 0.00000 | + +单位均为 SLO-feasible req/s/GPU。这里 Frontier 对 capacity 有明显、且依赖 topology/action family 的绝对 gap： + +- TP4/MBT8K 与 TP4/MBT16K 都低估 0.0125 req/s/GPU，但正确预测了 MBT8K→16K 的相对增益。 +- TP8/MBT8K 低估 0.00625，而 TP8/MBT16K 恰好匹配；Frontier 因而预测 MBT 在 TP8 上有收益，真机中四个 TP8 config 实际并列。 +- 尽管这个 action differential 是错的，TP4/MBT16K 相对其余 config 有足够 margin，所以 simulator top set 仍与 real top set 完全相同，最终 regret=0。 + +细化边界实验中，Frontier/real 的 33 个 config-load labels 有 6 个 false-infeasible；这些误差在 fresh-server v3 复测中稳定重复。因此这个 case 的结论应严格限定为：**Frontier 是有效的 config ranker / conservative screener，但不是精确的 capacity estimator，也没有完全学对 topology-dependent MBT effect。** + +## Case C 结果：Qwen235 FP8 fixed-shape mixed + +2026-07-16 在 `dash0` GPU 0--3 上完成了 community vLLM 0.10.2、TP4/DP1、MNS64、MBT8192、eager、BF16 KV、prefix/spec/CUDA graph off 的 pilot。所有请求实际 usage 和 streaming token count 都严格为 ISL=2,048、OSL=128： + +| load | requests | TTFT mean/p95 | TPOT mean/p95 | 40 ms joint SLO | +|---|---:|---:|---:|---:| +| single | 1 | 586.2 / 586.2 ms | 136.2 / 136.2 ms | 0/1 | +| concurrency=2 | 2 | 310.2 / 434.3 ms | 124.3 / 124.9 ms | 0/2 | + +Frontier 使用与成功 prefill ranking 相同的 frozen source fingerprint、TP4 KV blocks=26,101 和 block size=16。第一次运行完成 prefill 并提交第一个 decode token后，在纯 decode batch 抛出 `attention decode prediction cache not found for cluster monolithic`。初始化日志给出直接原因：`no standard decode rows`，而 final attention CSV 的 726 rows 全部是 prefill rows。该 crash 没有被记为 SLO fail，也没有使用 dummy prediction。 + +随后在 `dash0` 单张 H20 上用同一 community-vLLM/FlashInfer 栈补测 TP4、batch `{1,2}`、KV `{2048,2176}` 的 4 个 CUDA-event decode points。profile-closed Frontier smoke 精确完成了相同请求： + +| load | Real TTFT | Frontier TTFT | Real TPOT | Frontier TPOT | +|---|---:|---:|---:|---:| +| single | 586.2 ms | 267.7 ms | 136.2 ms | 88.3 ms | +| concurrency=2 | 186.1--434.3 ms | 470.7 ms | 123.8--124.9 ms | 90.5 ms | + +因此 missing-row blocker 已关闭，但 absolute error 明显且不是固定比例；这正是需要 config surface 而不能只看单点 latency 的原因。steady-QPS 下还会出现 prefill+decode true-mixed batches，最小 4-row profile 不足以支持完整 sweep。完整 closure 另外测量了 TP4/TP8 各 81 个 standard-decode points 和 108 个 true-mixed points，覆盖 decode batch 到 128、KV 2,048--2,175；合并 root 共 1,104 rows。Frontier 已确认用 108 个 TP-specific samples 训练 `attn_decode_in_mixed`，没有使用本 case 的 E2E latency calibration。 + +完整 8 configs × 8 rates simulator surface 已在任何真机 surface cell 之前冻结，manifest SHA256 为 `801aa36451c8647f71cc87011144622d2203786e82f189ed1375d964399b106a`。64/64 cells 都有 64 个 finite、非负、shape-exact request records，累计 simulator CPU time 为 11,675.2 s。冻结结果如下，单位为最大被测试可行 req/s/GPU： + +| config family | TPOT 40 ms | TPOT 120 ms | TPOT 150 ms | TPOT 180 ms | +|---|---:|---:|---:|---:| +| TP4，任意 MNS/MBT | none | 0.20 | 0.40 | 0.60 | +| TP8，任意 MNS/MBT | none | 0.05 | 0.15 | 0.20 | + +Frontier 在同一 TP family 内给四个 MNS/MBT config 生成了逐 request 完全相同的结果，即它预测这些 controls 在当前 load range 都不 binding；150 ms sensitivity 下则预测 TP4 family 的 per-GPU capacity 是 TP8 的 2.67×。这两个判断均已用下面的完整真机 surface 检验。 + +第一次真机 surface attempt 还暴露了一个 measurement-contract 问题：一个 fresh server 内连续测多个 rates 会把前一 anchor 的 execution warm state 带入后一 anchor。两个 TP4/MNS64 config 在正向 `0.10→1.60→2.40` 的 1.60 点分别只有 36/64、41/64 requests 通过 150 ms SLO；反向先运行 2.40 后，同一 1.60 点都变成 64/64。逐段看，正向 1.60 的前 24 requests TPOT 从约 234 ms 递减到 169 ms，后半段才稳定到 114--134 ms；反向 1.60 从第一个 request 起就在 113--128 ms。这个 attempt 整体排除，不能用“两个 round 都通过”的保守聚合掩盖 real 与 simulator 的 state contract 不同。 + +修正后的 ground-truth contract 对每个 `(config, rate, round)` 独立启动 server，并在目标 rate 下丢弃 `min(32, max(4, ceil(rate×20)))` 个 exact-shape warmup requests。anchor-isolation smoke 在两个 config、两个 fresh-server repeats 上均为 64/64 completed、0 failure、150 ms SLO 64/64；四次 p95 TPOT 为 127.8--138.0 ms，p95 TTFT 为 295.8--305.5 ms。只有这个通过 gate 的 contract 会进入最终 real capacity/rank。 + +### 完整评测 setup + +| 项目 | 设置 | +|---|---| +| 真机 | `dash0`，8×NVIDIA H20（97,871 MiB/GPU、driver 580.95.05、NV18）；TP4 cells 各独占 4 GPU，TP8 cells 独占全部 8 GPU | +| model/runtime | `Qwen/Qwen3-235B-A22B-FP8`；community vLLM 0.10.2、Python 3.12.3、torch 2.8.0+cu128、FlashInfer 0.3.1.post1；eager、FP8 weights、BF16 KV | +| workload | 64 个不同 token 的固定长度 requests；ISL=2,048、OSL=128、uniform QPS、prefix/speculative decoding/CUDA graph off | +| config surface | `TP∈{4,8} × MNS∈{64,128} × MBT∈{8192,16384}`；TP4 为 EP off，TP8 为 EP on | +| simulator | Frontier commit `d9cfeb6d8791fbf2f295dd9744c56a666171776e` + frozen best-effort patches；1,104-row attention profile；无本 case E2E calibration | +| simulator anchors | 每个 config 都运行 `{0.10,0.20,0.40,0.80,1.20,1.60,2.40,3.20}` req/s；64/64 cells 在真机 surface 前冻结 | +| real anchors | 围绕冻结 simulator boundary 运行并自适应补齐相邻点；共 34 config-rate cells × 2 independent rounds | +| 每个 real anchor | fresh vLLM server；先按目标 rate 丢弃 `min(32,max(4,ceil(rate×20)))` 个 warmup requests，再从空 scheduler queue 测 64 requests | +| primary SLO | ≥61/64 requests 同时满足 TTFT≤1,256 ms、TPOT≤150 ms；两个 rounds 均通过才判定 feasible | +| primary objective | 最大被测试可行 offered req/s / 实际占用 GPU 数 | + +150 ms 是看到最低负载真实 TPOT 已达 136.2 ms 后公开记录的 post-pilot amendment；原 40 ms 阈值仍保留，但所有 config 在最低 anchor 都不可行。120 ms sensitivity 的 lower boundary 尚未补齐，180 ms sensitivity 在最高 3.20 anchor 仍有 censored cells，因此本文只对 150 ms primary SLO 报告完整 ranking。 + +### 完整结果 + +![Qwen3-235B fixed-shape mixed simulator-vs-real config ranking](docs/assets/simulator-fidelity/qwen235-t0-fixed-shape-ranking.png) + +| TP | MNS | MBT | Real | Frontier | Real boundary：pass → fail（两个 rounds） | +|---:|---:|---:|---:|---:|---| +| 4 | 64 | 8192 | **0.60** | **0.40** | 2.40: 64/64,64/64 → 3.20: 57/64,59/64 | +| 4 | 64 | 16384 | **0.60** | **0.40** | 2.40: 64/64,64/64 → 3.20: 62/64,57/64 | +| 4 | 128 | 8192 | **0.60** | **0.40** | 2.40: 64/64,64/64 → 3.20: 53/64,55/64 | +| 4 | 128 | 16384 | **0.60** | **0.40** | 2.40: 64/64,64/64 → 3.20: 52/64,54/64 | +| 8 | 64 | 8192 | 0.30 | 0.15 | 2.40: 64/64,64/64 → 3.20: 46/64,48/64 | +| 8 | 64 | 16384 | 0.20 | 0.15 | 1.60: 64/64,64/64 → 2.40: 45/64,53/64 | +| 8 | 128 | 8192 | 0.20 | 0.15 | 1.60: 64/64,64/64 → 2.40: 40/64,56/64 | +| 8 | 128 | 16384 | 0.30 | 0.15 | 2.40: 64/64,64/64 → 3.20: 50/64,46/64 | + +Real 与 Frontier 单位均为 SLO-feasible req/s/GPU；boundary 的 offered rate 单位为 system req/s。所有 8 个 real boundaries 都由相邻 lattice points 闭合。四个 TP4 configs 是 real 与 simulator 的共同、完全相同的 top set，因此 optimistic/worst tie-break regret 都是 0。Kendall τ-b=`0.8944`；28 个 config pairs 中 exact `>/< /=` sign 命中 24 个，20 个 real non-tie pairs 中命中 16 个方向。34 个实际测量的 config-load labels 中，Frontier 命中 24 个，有 10 个 false-infeasible、0 个 false-feasible。 + +这个结果同时给出一个比“absolute error”更重要的反例。TP8 真机在 2.40 system req/s 上的 feasibility matrix 是： + +```text + MBT 8K MBT 16K +MNS 64 pass fail +MNS 128 fail pass +``` + +换成 capacity，`ΔMBT | MNS64 = -0.10`，而 `ΔMBT | MNS128 = +0.10 req/s/GPU`，difference-in-differences 为 `+0.20 req/s/GPU`。也就是说，**MBT 的作用方向取决于 MNS；它们不能被看成两个独立、可加的 controls。** Frontier 对四个 TP8 cells 都预测 0.15，interaction 恰好为 0。相同四个 configs 在 1.60 及以下都通过，差异只在持续高负载下出现，说明问题不只是单个 operator 在空载下的固定时间偏差，而发生在 batching/scheduling state 与 execution topology 的组合上。现有数据还不能区分具体是 batch composition、MoE EP communication、memory/KV pressure，还是 runtime 的其他动态状态；下一步应对此边界做 scheduler batch trace、per-step kernel/NCCL timeline 与 KV/queue counters，而不是先给出未经验证的根因。 + +因此本 case 的严格结论是：**Frontier 足以在这个 8-config surface 上选择全局最优 topology，但不足以解释或优化 TP8 family 内的 MNS/MBT controls。** 成功来自 TP4-vs-TP8 的真实 margin 足够大，能够容忍 TP4 33.3%、TP8 25%--50% 的 capacity 低估和内部 action-ordering error；它不是 simulator 已获得 scheduling/execution 深层认知的证据。 + +接受的 ground-truth 共 18 个 fleet jobs、68 个 fresh-server anchors、4,352 个 measured requests 和 1,664 个 warmup requests，消耗 36.26 H20-GPU-hours；从第一个 accepted job 到最后一个完成的墙钟跨度为 5.74 小时。增量 full-attention closure 在单张 H20 上运行 12 秒（0.0033 H20-GPU-hours），但它复用了此前已有的 726-row prefill profile，因此这个数字不是构建全部 compatibility envelope 的总成本。另有共享 GPU/endpoint 的污染 attempts 和暴露 cross-anchor warm-state leakage 的 diagnostic attempts 被整目录隔离，不进入上述统计。Frontier 64-cell surface 消耗 11,675.2 CPU-seconds。完整 hash、逐 anchor input 与排除 ledger 见本节末的复现链接。 + +完整证据见 [T0 smoke report](runs/frontier-multicase-sufficiency-v1/t0-smoke-report.md) 与 [预注册协议](runs/frontier-multicase-sufficiency-v1/protocol.md)。 + +原预注册的 TPOT 40 ms 因最低负载已不可行而失去 capacity-ranking 可辨识性；本文已把 150 ms 明确记录为 post-pilot protocol amendment，并保留 40 ms 全部失败的 sensitivity 结果，没有把 SLO change 隐藏成预注册结论。 + +## 尚不能纳入 simulator ranking 的 case + +### 原 internal-runtime Qwen235 prefill-only + +原实验有 8/8 valid real cells，但使用不同的 serving/runtime contract，real top set 是四个 TP8 config。它没有与本文 best-effort Frontier 使用同一个 community-vLLM FP8 profile/serving contract，不能和新 simulator 数值直接拼接。本文的 community-vLLM Case B 是目前可辩护的 aligned prefill comparison。 + +### 原 Qwen235 decode-only + +真实 surface 有 7/8 valid cells，且 capacity 搜索区间大量重叠：经验 top set 为 5/8，而考虑 bracket uncertainty 时 8/8 都可能最优。更关键的是，该 case 依赖 initial KV state、DecodeBenchConnector、EAGLE3、FP8 KV、DeepEP/NVSHMEM 与 decode CUDA graphs；Frontier 当前没有等价的 initial-KV/EP8 execution contract。因此现在给出 simulator-vs-real rank 或 regret 都会把 contract mismatch 误写成 fidelity 结果。 + +下一步不再用 decode-dominant mixed workload 回避问题，而是在 community vLLM 0.10.2 与 Frontier 两侧增加显式、相同的 initial-KV contract：request 到达时分配与 input length 对应的 KV blocks、标记 prefill complete，再从第一步 decode 开始调度。首先关闭 speculative decode、prefix cache 和 decode CUDA graph，建立 fixed-shape decode-only baseline；随后按单变量顺序启用 trace lengths/prefix、EP8、EAGLE3 和 FULL_DECODE_ONLY graph。每一层都重新报告 ranking，不能把后一层的 mismatch 归因给前一层。 + +## 当前判断与下一步 gate + +当前证据的结论是： + +1. **绝对 gap 不是否决 simulator 的理由。** Qwen235 中 11%--50% 的 capacity 低估仍可保持 zero-regret config selection。 +2. **但 rank 成功不能证明模型机制正确。** Qwen235 prefill 的 TP8 MBT differential 错误，fixed-shape mixed 又漏掉 TP8 的 MNS×MBT 非加性交互；Qwen30 还有 28/92 anchor SLO labels 错误。只是当前 decision margin 足以容忍这些 residual。 +3. **alignment 是结果的一部分。** Qwen30 需要 per-TP E2E calibration 才能恢复排序；Qwen235 不需要本 case E2E calibration，但需要同栈 FP8 profiles、真机 KV capacity 和 simulator patches。论文必须报告这部分真机成本，不能把它隐藏在“offline profiles”中。 +4. **目前不能建立“simulator 全局选优普遍失败”的 premise。** 受控 prefill、fixed-shape mixed 和历史 mixed surface 上，经过 alignment 的 Frontier 已经能选到最优或 0.76% 内的近最优 config。更有证据的 research premise 是：aggregate selection success 会掩盖 action interaction 与 state-transition model 的错误；若要声称全局选优失败，仍必须在 dash0 的 trace-faithful mixed、decode-only、prefix cache 或跨 topology case 上得到稳定反例。 + +后续每个 case 建议使用同一 gate：worst selected-config regret ≤5%、tie-aware rank correlation ≥0.8、足够数量的 informative pairs、ground-truth bracket 不足以反转最优决策，并单独报告达到该结果所需的 real-GPU profiling/calibration cost。 + +## 数据与复现 + +- 图表输入：[data.json](docs/assets/simulator-fidelity/data.json) +- 画图脚本：[plot_simulator_fidelity.py](scripts/plot_simulator_fidelity.py) +- Qwen30 audit：[report.md](runs/frontier-multicase-sufficiency-v0/results/qwen30-baseline/report.md) +- Qwen30 aligned metrics：[metrics.json](runs/frontier-slo-alignment-v0/results/metrics.json) +- Qwen235 fixed-cohort comparison：[v2_refined_comparison.json](runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_refined_comparison.json) +- Qwen235 full report：[report.md](runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/report.md) +- Fixed-shape mixed comparison：[comparison.json](runs/frontier-multicase-sufficiency-v1/results/t0-final/comparison.json) +- Fixed-shape mixed capacity table：[capacity.csv](runs/frontier-multicase-sufficiency-v1/results/t0-final/capacity.csv) +- Frozen Frontier surface：[frontier_surface_frozen.json](runs/frontier-multicase-sufficiency-v1/artifacts/frontier-t0-surface-v1/frontier_surface_frozen.json) +- Executed real plan：[real-plan-executed.json](runs/frontier-multicase-sufficiency-v1/artifacts/t0-real-surface-v1/real-plan-executed.json) +- Real exclusions/acceptance ledger：[exclusions.json](runs/frontier-multicase-sufficiency-v1/artifacts/t0-real-surface-v1/exclusions.json) +- Fixed-shape / trace-faithful / decode-only 分阶段协议：[protocol.md](runs/frontier-multicase-sufficiency-v1/protocol.md) +- Trace tokenizer/hash contract audit：[trace-contract-audit.json](runs/frontier-multicase-sufficiency-v1/trace-contract-audit.json) +- Trace contract audit reproducer：[audit_trace_contract.py](runs/frontier-multicase-sufficiency-v1/audit_trace_contract.py) +- T0 real smoke 与 Frontier decode-profile failure：[t0-smoke-report.md](runs/frontier-multicase-sufficiency-v1/t0-smoke-report.md) +- 尚未对齐 case 的 real-only audit：[report.md](runs/frontier-multicase-sufficiency-v0/results/ground-truth/report.md) + +重新生成 PNG/SVG： + +```bash +python3 scripts/plot_simulator_fidelity.py +``` From 97c2f34700d7b8776783508fdc0769ffcbaa5719 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 00:32:01 +0800 Subject: [PATCH 094/148] Add Qwen30 prefill fidelity experiment --- .../experiment-card.md | 54 ++ runs/frontier-phase-factorial-v0/fleet.toml | 24 + .../jobs_full.toml | 241 +++++++ .../jobs_smoke.toml | 21 + .../mock-phase-factorial.png | Bin 0 -> 118977 bytes .../plot_mock_phase_factorial.py | 45 ++ .../qwen30_prefill_client.py | 216 ++++++ .../remote-sync-marker/README.md | 6 + .../run_frontier_qwen30_prefill_surface.py | 400 +++++++++++ .../run_qwen30_prefill_real_config.sh | 135 ++++ .../frontier_surface_frozen.json | 647 ++++++++++++++++++ .../frontier_surface_frozen.json | 647 ++++++++++++++++++ .../frontier_surface_frozen.json | 647 ++++++++++++++++++ .../test_phase_factorial.py | 39 ++ 14 files changed, 3122 insertions(+) create mode 100644 runs/frontier-phase-factorial-v0/experiment-card.md create mode 100644 runs/frontier-phase-factorial-v0/fleet.toml create mode 100644 runs/frontier-phase-factorial-v0/jobs_full.toml create mode 100644 runs/frontier-phase-factorial-v0/jobs_smoke.toml create mode 100644 runs/frontier-phase-factorial-v0/mock-phase-factorial.png create mode 100644 runs/frontier-phase-factorial-v0/plot_mock_phase_factorial.py create mode 100644 runs/frontier-phase-factorial-v0/qwen30_prefill_client.py create mode 100644 runs/frontier-phase-factorial-v0/remote-sync-marker/README.md create mode 100644 runs/frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py create mode 100644 runs/frontier-phase-factorial-v0/run_qwen30_prefill_real_config.sh create mode 100644 runs/frontier-phase-factorial-v0/simulator-tp1/frontier_surface_frozen.json create mode 100644 runs/frontier-phase-factorial-v0/simulator-tp2/frontier_surface_frozen.json create mode 100644 runs/frontier-phase-factorial-v0/simulator-tp4/frontier_surface_frozen.json create mode 100644 runs/frontier-phase-factorial-v0/test_phase_factorial.py diff --git a/runs/frontier-phase-factorial-v0/experiment-card.md b/runs/frontier-phase-factorial-v0/experiment-card.md new file mode 100644 index 0000000..f291ab7 --- /dev/null +++ b/runs/frontier-phase-factorial-v0/experiment-card.md @@ -0,0 +1,54 @@ +# 实验 EXP-SIMFID-PHASE-FACTORIAL：prefill-only 是否是 simulator ranking 的容易区间？ + +> **状态：** 已批准，运行中（用户于 2026-07-17 明确要求先完成 235B mixed 与 30B prefill-only） + +## Claim 与决策 + +- **Parent claim：** Frontier 的 config-ranking fidelity 由 workload execution phase 决定；prefill-only 可能比 decode/mixed 更容易由 isolated operator profiles 组合。 +- **目的：** 用跨 model 的 phase factorial 区分 phase-complexity explanation 与 model/runtime/profile-specific explanation。 +- **Competing hypotheses：** H-phase：prefill-only 是低状态反馈的 compatibility envelope，因此 30B prefill-only 也能达到低-regret ranking，而 decode/mixed 更容易失真；H-stack：30B 的失败主要来自 FA3/CUDA-graph/routing/profile composition 等 stack-specific mismatch，因此即使 prefill-only 也可能失败；H-margin：mixed 的 differential 可以错误，但只要 topology margin 足够仍会选对 top set。 +- **事前预测：** 若 H-phase 成立，30B prefill-only 满足 regret ≤5%、Kendall τ-b ≥0.8，并明显优于同模型 mixed；若 H-stack 成立，30B prefill-only 仍过不了 gate；若 H-margin 成立，235B mixed 可保持 top set，但会漏掉 MNS/MBT pair directions。 +- **判定规则：** 30B prefill-only fail → “prefill-only 是充分条件”被否证，下一步优先做 same-model execution-context ablation；30B prefill-only pass 且新增 decode-heavy mixed fail → 支持 phase hypothesis；30B prefill-only 与 235B mixed 都 pass → phase 不能单独解释，转向 margin-aware compatibility envelope。 + +## Setup + +- **自变量：** model×phase：已有 Qwen3-235B-A22B-FP8 mixed；新增 Qwen3-30B-A3B BF16 prefill-only。 +- **控制变量：** dash0 H20、community vLLM 与各自 frozen Frontier profile、同一 config 内 real/sim 的 request shape、arrival lattice、SLO、prefix policy、MNS/MBT/TP 与随机种子。 +- **30B system context：** community vLLM 0.20.0+cu129，BF16 weights/activation/KV，TP∈{1,2,4}，MNS∈{8,16,32,64}，MBT=8192，chunked prefill on，prefix off；real 保留 runtime 默认 CUDA graph，Frontier profile-only 不做 E2E calibration。 +- **30B workload：** fixed ISL=2048、OSL=1，64 个不同 token-chain prompts，uniform open-loop QPS；fresh server per `(config, rate, round)`，target-rate warmup 与 measured requests 分离。 +- **30B SLO：** TTFT≤1256 ms，至少 61/64 requests 通过；primary score 为最大共同 tested feasible req/s / 实际 TP GPUs。 +- **235B baseline：** 已冻结 `ISL=2048, OSL=128`、8 configs、68 个 fresh-server anchors；primary sensitivity TTFT≤1256 ms、TPOT≤150 ms。 +- **Baselines：** real community vLLM；Frontier same-stack profile-only；historical Qwen30 mixed profile-only；historical frozen per-TP calibration 只作为 upper bound，不参与本 case 拟合。 +- **Metrics：** top set、worst tie-break regret、Kendall τ-b、exact/non-tied pair direction、anchor confusion、absolute capacity、TTFT p50/p95、real trial variance与GPU-hour。 + +## 预期产物与 review + +- **预期数据：** 30B frozen simulator surface；real config-rate anchors；两模型 phase comparison table；failure mechanism breakdown。 +- **Figure prototype：** `mock-phase-factorial.png`；x=model×phase，左轴=worst regret，右侧 annotation=τ-b；虚线是 5% regret gate。mock 只表达可区分趋势，不进入结论。 +- **人工 review：** 已批准。用户要求 smoke 通过后推进实验；先做这两个 case，再根据 hypothesis verdict 扩展。 +- **Review 意见：** 不把 existing 235B mixed top-set match 隐藏掉；不把“全 config 并列”算作成功 hit；30B prefill-only 必须使用相同 primary ranking objective。 + +## Benchmark design audit + +| Risk | Verdict | 处理 | +|---|---|---| +| Selective benchmarking | PASS for initial screen | 同时报已有 235B mixed success和内部 pairwise failure；后续 expansion 由预注册 verdict 触发 | +| Simplified workload | NEEDS EVIDENCE | fixed-shape 只用于 phase isolation，不外推 trace-faithful mixed | +| Calibration=evaluation | PASS | 新 case 不用 serving E2E 数据拟合 scale | +| Missing significance | NEEDS EVIDENCE until run | boundary anchors做独立 fresh-server repeat，保留 disagreement | +| Relative-only result | PASS by design | 同时报 req/s/GPU、TTFT distribution、rank/regret | + +## 复现信息 + +- **Code：** AITuner branch `codex/fidelity-prefix-pilot-20260714`；Frontier upstream `d9cfeb6d8791fbf2f295dd9744c56a666171776e` + frozen known patches。 +- **Environment：** 只使用 dash0 8×H20；Qwen30 venv `/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1`；model `/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B`。 +- **产物路径：** local/remote `runs/frontier-phase-factorial-v0/`；raw GPU artifacts 由 fleet harvest，condensed JSON/CSV 进入结果目录。 +- **已知 deviation：** 235B 为 FP8/vLLM0.10.2/FlashInfer eager，30B 为 BF16/vLLM0.20/FA3/default CUDA graph；因此跨模型只检验 hypothesis consistency，causal phase claim 最终仍需 same-model phase pair。 + +## 结果 + +- **观察事实：** 235B fixed-shape mixed 已完成：real/sim TP4 top set exact match，worst regret=0、τ-b=0.8944；但 20 个 real non-tie pairs 只保持 16 个，10/34 anchors false-infeasible，TP8 MNS×MBT interaction 被漏掉。30B prefill-only 待运行。 +- **异常：** 无。 +- **Interpretation 与剩余 alternatives：** 强版本“mixed 必然失败”已被 235B top-set result 削弱；仍可能存在 phase-dependent error magnitude，由 topology margin 掩盖。 +- **Claim update：** unchanged，等待 30B prefill-only。 +- **下一步：** freeze 30B simulator surface → guided real anchors → joint verdict；只有判定需要时扩展 decode-heavy 235B 或 trace-shaped 30B prefill。 diff --git a/runs/frontier-phase-factorial-v0/fleet.toml b/runs/frontier-phase-factorial-v0/fleet.toml new file mode 100644 index 0000000..76a9e93 --- /dev/null +++ b/runs/frontier-phase-factorial-v0/fleet.toml @@ -0,0 +1,24 @@ +version = 1 + +[paths] +state_dir = "runs/frontier-phase-factorial-v0/fleet-state" +artifacts_dir = "runs/frontier-phase-factorial-v0/fleet-artifacts" + +[ssh] +connect_timeout_sec = 10 + +[scheduler] +gpu_free_memory_mb = 1024 +gpu_free_utilization_pct = 10 +prefer_pack = true + +[sync] +mode = "scp" +local_path = "runs/frontier-phase-factorial-v0/remote-sync-marker" + +[[hosts]] +name = "dash0" +ssh_alias = "dash0" +enabled = true +sync_remote_path = "/home/admin/cpfs/wjh/aituner/phase-factorial-sync-marker" +fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0" diff --git a/runs/frontier-phase-factorial-v0/jobs_full.toml b/runs/frontier-phase-factorial-v0/jobs_full.toml new file mode 100644 index 0000000..c67f255 --- /dev/null +++ b/runs/frontier-phase-factorial-v0/jobs_full.toml @@ -0,0 +1,241 @@ +version = 1 + +[[jobs]] +name = "qwen30-prefill-real-tp1-mns8-20260717-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp1-mns8-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "1" +MNS = "8" +RATES = "4 8 16 32 64" +SERVER_PORT = "8720" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp1-mns8-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp1-mns16-20260717-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp1-mns16-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "1" +MNS = "16" +RATES = "4 8 16 32 64" +SERVER_PORT = "8721" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp1-mns16-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp1-mns32-20260717-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp1-mns32-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "1" +MNS = "32" +RATES = "4 8 16 32 64" +SERVER_PORT = "8722" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp1-mns32-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp1-mns64-20260717-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp1-mns64-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "1" +MNS = "64" +RATES = "4 8 16 32 64" +SERVER_PORT = "8723" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp1-mns64-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp2-mns8-20260717-v1" +gpus = 2 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp2-mns8-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "2" +MNS = "8" +RATES = "4 8 16 32 64" +SERVER_PORT = "8724" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp2-mns8-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp2-mns16-20260717-v1" +gpus = 2 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp2-mns16-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "2" +MNS = "16" +RATES = "4 8 16 32 64" +SERVER_PORT = "8725" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp2-mns16-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp2-mns32-20260717-v1" +gpus = 2 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp2-mns32-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "2" +MNS = "32" +RATES = "4 8 16 32 64" +SERVER_PORT = "8726" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp2-mns32-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp2-mns64-20260717-v1" +gpus = 2 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp2-mns64-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "2" +MNS = "64" +RATES = "4 8 16 32 64" +SERVER_PORT = "8727" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp2-mns64-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp4-mns8-20260717-v1" +gpus = 4 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp4-mns8-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "4" +MNS = "8" +RATES = "4 8 16 32 64" +SERVER_PORT = "8728" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp4-mns8-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp4-mns16-20260717-v1" +gpus = 4 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp4-mns16-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "4" +MNS = "16" +RATES = "4 8 16 32 64" +SERVER_PORT = "8729" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp4-mns16-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp4-mns32-20260717-v1" +gpus = 4 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp4-mns32-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "4" +MNS = "32" +RATES = "4 8 16 32 64" +SERVER_PORT = "8730" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp4-mns32-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp4-mns64-20260717-v1" +gpus = 4 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp4-mns64-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "4" +MNS = "64" +RATES = "4 8 16 32 64" +SERVER_PORT = "8731" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp4-mns64-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" diff --git a/runs/frontier-phase-factorial-v0/jobs_smoke.toml b/runs/frontier-phase-factorial-v0/jobs_smoke.toml new file mode 100644 index 0000000..5741e2d --- /dev/null +++ b/runs/frontier-phase-factorial-v0/jobs_smoke.toml @@ -0,0 +1,21 @@ +version = 1 + +[[jobs]] +name = "qwen30-prefill-real-smoke-tp1-mns8-20260717-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 1800 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-smoke-tp1-mns8-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "1" +MNS = "8" +RATES = "4" +SERVER_PORT = "8718" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-smoke-tp1-mns8-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" diff --git a/runs/frontier-phase-factorial-v0/mock-phase-factorial.png b/runs/frontier-phase-factorial-v0/mock-phase-factorial.png new file mode 100644 index 0000000000000000000000000000000000000000..6c3c2bd88f563c1b9d2cc11c85993338748a9b6a GIT binary patch literal 118977 zcmZU*1z1&Ew?Dj5L}?Td1!<+EK}s4#N(oWv4yC(G8c78SX+a4|0cns1LAnGaHqs?X z=Qq~5_kGX(KOfIY-0Z#9Tyu=^t4XNxR-Wv@BE>X zx&i+Za(tlf_{7H4(Z#^t1ogyL-a ztB?LPA+JeF_Nj39rarXkmb4|9e+Ug_uu;6n6`ICnZrC{G!T)uP)T2oK#WQhtj=!fR z+{KnAH}m#4t7`)idCXFOr6f}J@GRa+q`7h-%2Zr>cn)1e7e1M z_V|HXbpojqN?L9E5+w zm6eqwPL;-4l=q!yR)#;Ld^=+poBR7qWansE*5UHZHYTe=A|rz`GwB(WQniO>6O55- zk@B*85zS#KAt7Pd>;pGFhLVsDd_CQ{^XV@BzN)mb5G-7WF&7ut4SwreU#)-Tm9UG5 zP#PE*T*1X{-kNLYvY8U@Fy?M)ZEb98`*h{Q$J+4{qsz|Uv3^8ENwY^Q)AeQb4$$o*0-c2&V!Ab&6epzPMQx*Et|&fgYSYonCZAF;o%v- zEi2=7gf&_0Nx6thNJ^Spub2)hD&qJa&p{|b^ylQzLA%QSQZONP+hDd5?Uf6d5=`ih z+;oz71I-YertUfJkZF_U#3&rhnXCceHMPD)4!3B)D)@VrPjM=keS zety2biAmu1cX#JfT&5b)tzlOwC@j=;DEKVJ;f`k)7O+05Vk;>rg;R-WvN=ojq>6;e z#V}+%E6}ucbPV|T@s6;tFsJJu{k2~;&o{rqCi7QhfWL@?xvPB6MMV#0E~*u1l8%gw z zBFS9JeV3Ng+S9!W87p{9x9pM{&d(;YPFFJ5$Lj3nu%^8Bb&z-c{vC&%oqcI2mw2Q| zuc)FBsYas(sEY(HBP^(qU~ZG|IqQts2M4a*Dfh|XU9U$GGcq#rT8)#OA5WhXQBheB zt%in#e0#3v6{uNm4qIF%e{O%;=b@qD2NkF8WWgt%ltDo!vPw$rc~3uc+Rli(y1But zu;f2pd&lxrvNu(P3$}i(=bUVn6QgF+`YN5sZK7{s5k`K`T>pD4Gup!}JdHon)EK7cl{;1 zM-BG#I5JUG_^^-FH8nqY9&Rr|G4feYUJ*aO$H^$Q`d4!#VO7%Jp8Gnxc1Tak{Y-d? zGRwQ-^76p=c(V6=Rrb1--G&Lt7m-n>={zJ@M8g2 z>1FiizrT}}l_jd++P$wqgqbYtc8$lhTRfP6a%jV*uC9*DxPxeIs_wqhmYCNO32f8= zXkL?(lP$-GP$l|nVdxbIe~wehS^pFY4^8FpKYueiax1fWD4K!ie1I*r;wnb<9_w--8+;xiUuZu zg6iVg>2aX_{P)n^l_P%lofi?%B(3fiD#^=VJK9_S?BMZHm3eKVLdk}8Yip}%rqLh2 zZ2(q?mXGh51tm;b(k`##^@^rAHZ4rJt%6;6Y6d^QnE26Lku*2Y<23EQk#g+l;6Qxg z;zg&nDd`Pft$5r*ucn(tBa6VtROZ`0v7meqtkXl=bl=HZg-D zXJwRxKNfaqXsFO?C(GzpE3Mht=3q(K_KqX-%gfBd-Y4p2I;p9thEvY>+ulh)V#9w8`YHq1+x zmQOw5o$F8cwCs{#u&qo^!)(E%rl$69*M%|OVuiDveAjwWQbs0-)BD6VQ+Jaw&GQ2f zmChH@hf?Qp9D3~5F~>LRY-jzHOAH^ROz^$v6AuXwAG+xuA0My6<+(fjT(hG0uxjNv z{y3RyB#ewBAPApgC4aQPUq!%iDSF&4cl~d-S4!UkR2Hc*cVgq{K(-Qf!7lkx{NVO{ zSGjk+pq$e!vqR|PmjWd-N+Vxgyo|HIIX&t|fa|xgl^w-pg0DkHCG1M0maE#Dv%e3M z(0w%??cEjacOuHN;vmxX_KpH3)T96IP?4UV86_FJHc@R|-LAEbfdTEw-(E49y3KJI z^AGbaW}5=Bx;WH}#aYjN&i8%HSGPTGk<$wcYrhyT%T+lZtM{s@Ov#<@gq7kmud)7B z9lCFQ)cyXR0t`Z4^=a4xykCE~6$~@f0GO<%V}h^%E63QP*B4{hnbn#UC&`ejE9HE;Oyj3bCq*m_j|&=0ukoy z;on8cu~Ni*m8UqK6cJ0?Tp*B>mS%v8X#A1HkAdZ-*{Ij#u0$Z%ub<1$F){VF~O{^WThTX~#bHTB^yu4pQ0h^HXT1Z<2 zMRQjdZd_cP>)zT;c#MJ2o9Jk1G44Ju41fu+(3A4{>uEc+Pk6o8vEfv$n}GWc6H34T z!vh#uPwy_y&Sva8^ese*oqZe1ebPFuRc;YfrGC#x$@lDk z%UX^VW1#e3zQl%sU1s_78cyH^G7jB-MLbLx~xq-@JJ>-xY7v7JgHx zD^`!b|17aXsS6eNw^&4m@danGEDxU~d$S zZ|Ow;1ANYoRykO8s!32YOG_azKGO}FgKTYW#SIPVfY;b=ObTv!b$o6OBctcyx(Yyo zy{Wl*7LE3U?^wiI4CTBTS(lQMLZ)P``z}s%FyX`Fy>$b?E6^{E8vQOo_i6#W$>p|f zoUd6fZ$T+1Cua-O3cz%Kv4NCYzWP`Cm_A zUqCHf@jTrft#n$wj?#w@AV2QsCrL+8#L`FsMWJ@p1r+S1)`P7%mF&lqw{PE`nf5tv zMb;O>TGKa;-p8LOCP^i0i@8VGkkwC2m!)|yR6La^gqw3Zl5Yv{-Ydm03qBb@FhWLn#HC+$Q9r7%Gn&|XE%`ITnZwf zWMPj#5*RXR3c#+koxOlcmw(4{L&%W{Kr#x2iBWLRnAp)0J-}GKlRDsL z$dh7G%khOtkBNnah5~E`vcB!vaEqM$b)inp>;1QYimdA0^F=%lh+sV*N=sv+zKxEO z)w%Cp+g#_+sSbc!rD0*g5j**F@wa_d+gY{avY(RJDT%{EcO&e*t;WmLFOppl@=5J; zf+0RUX9P3IeeUhHdxiqc_A&oYu)}h6>x2MF`;U#O1LiTBYm2A^YDOjM8M5)KW)6TV zFu5VEMOe!%n4%KU9pT}yX_on_=C;7L-|7`Rr-AMUh_VSjI{@xj*XX#rTM^oHx?~{E z(qt76LSq0$NwCL<`6z9Um*XxkFCPFXSILmPw6i*@R1kSGrLcUBj4T-NmKn># zJ<>$I0)ExFta{Q(r@_7B05d}mG6JU~<>uijkMr7Hc{h^n;UM-#(5MYBLJ0Q4P-?*O z?x&B^2c&K{OOu0{}jx3pXzqBi;S&Ig|(a{Kj+x zhgy!Z1Z#y!*YybQs<|)a<)o3+VucRV(C`fjNeBrEXODJQHM^i3A43Tfp{{IfY#{hl zCJ|=v>tjkD)9|*cigW+i7PuSuL)Gw#B5J+diqKc(uzH+YWovqzl_nj42_g=kv zz307(o||)u z!^gq722+GLZgKbiVM{2nq?%f6&o06oph>n0Ie9&!;I}5}0yRO#rl`$he*>G}da_SF zwcKlivCd|?o{fsbzG^ZjmF@^;n<; z@)k2Iz+#JwPpt)myEuoewBnO}zBw=TNtjgud#;;L74>3q#`RfDaoO46xp(hrj&{`- zlz5J2d0yMmBtK8JwRKEP%q377O0=N}ozAw(K+7qagrY6t7xl6~quqy%^n#7vY0d=d zvvn6xE4QGtl>J#rh7D5y+-xX+D6k~W>A%ToY4!Uaop0}8b=|C9J_7*XDnvjjlvS?i z=vXF@|1?F&X=sGhxJi7P2n#{WUH5FJw2izrD&eI{b;oB*nNck=xO0EwQKX_&9OZ3SUNfINiBzEpQMfLMg3r|12f(V*CT zm?g!;tUJL#O;8F%Jh75gXuuY53r#feY;P@eH8n}DQjcob6C9_ny7CUC>(#kCVfZQ+ z>59al*_Hw%yd&q5@x?dm_EUS@>FDW)gY@sN1T;dSw>klb9jxKd$*P{_ekS2fO&aVV z@)={Qu~EY2Vj8>8$yTuXIc({jI1&5bA?kbCY2M%e)&q|l$=5I%65Ih{Z281H4%y$( zV9=rq-HAr3ly0Yw`DCD04x)!D?dEt#%mBrWK1MYG2T>eX8rc)1u!h-vS3poObQ+_Y zeE+W@6d$^N;YU(-#af-mev9!=05)-1T$RUup=;Xe%nVKaCMW`5E2(vSDz5mIjp9mh zT8B^?8RLtr%*`38O(J5S#^|5f<_H$EjxQ!%E$)v~g>%cpW3l}~40^rq3-Gk5)U4NW zB87T z1(;n1prn&?S8qoL5hGSvX=yVEZ0|P5L=S%xr>y3w<@y0Wz(83~*B6v{4}N-t4$u<} zChu#ToV?|KnGh@eeg5NVV|$Q&WIz}yFMPuC?!`b>=)o}*eugqrS={dKE~(GR^@b8e z^x)^`M=`~+$05a9ObiI84Zs_&u^33+Xk`kg)sYAL8^4_5tYG&Dgg>&i=?2^chCmOJU{>Jq6DUifVxkUfcv zdM=~3CGBZY#)k5q;-ZL%h|*nDtwEp&r{I6Q=q^EoNTDckP;2oCLb(BwCnV;|Jnt6K zmg=F^76lNauCD%~K#C#80N%`f-zYr4a^7Ch`(&%#b2MJBbyvIi>sPGwYEyYGyx1Xa zZEa4=G0v3Lj{>6@=_Xxq=-$BLAE&@S(8d$+;AMvMo;t;yu9f!g5R{sQ=l2S+vf_c1 zf0(CWekc+NY=ikMe>1hYvlE;CVI(h?{IZD`1FB`VaGPFiU+Yu!_Hj;S@~TFfoVNBY zAd!Ey&jD>a(VTK_(}InT|pMm)qJ@wnDUTP){*XV7uHQ(8zz} zJynEuPoZI97YN3%c)>Jz;9=0Y=WS-C_?V z$iy&NMzm~}3$nq*K=*3!4+edI878IxtPJ*77|5x!KzD(Uh)YWPjeG&e#oZ`IvghzR zQ08}=K}h_VR17Z@?ro)MsS6Uc|5TZ+CCG-T; zqT%2Tfzu^v)V-KrGR?-uhFfqo4+=ntVxrU$MnWxi1L#3LGGTy~K)shS6REKN)dW}@ zm2;NE=Xi#Z*A@a($c}Z2Rb%eWg}mSXqCq1R+bhF&w{}N?%Rp`QJoi#%DqKWJR|)WA z;DDE*uQvhPMm#EY4UL=OC7ir32c*rKu5+Gg`y6iH0Yf7Ysv5nwJ^{?-T8vi}Y1_RU z9TS2o^`34}MGDEEJ4gUPwZYr}fzK+lnz#-aRDoOwq zBN76@(~z*R!Ji$cpjg%|R&b3DvmO;bf|jUWWq${tX%OIHIw2wQ&dyHFuIp`Hvbq}z z1qB7OtE*vPlF@@+0Gs_fu~pUl;weksDyE8DV*tk0`J~cP9_W-IAZ}bhrHc7Pf^c+8 z*p&z|%!n_*td;~$0lYdaGcz**At6|V@uk!GSROge=yn%fl2M1*Q9Ym5+$Y(8c9x$c z3)o$NX-VXvQEonrLn-8lsQ>z~3pp43Xj-f4?dD{F()BN8ui(y+Y3uuhj z^9~j9*qebZ54evU$_*<}@)hVR>Co$5OxKH4BZgV$hZD~p*no!q#ax|@=xFh7^zYvR z=f@4_%~^6W(W$ACf({G%J;{QIAJY}b)?e*p;kY^VYiU*8d!s^;c79*QPH?n6f+84j z>@4tP*OLP-2E_zlmuW9DWC$Za1Hw4yT^9)m2*@Xbf`aIHc!+)TX89JSID&pNhVp<2 zw$-@}Q1ZoXye|y*dsvc^5LX0*FaUaz0A1g_fBzaL47 zfVb_bLI64OBKb8^ z(vN?S8erfi19@6MRj+j|8MElTp{mQ_2&#M|vQvhP+WTDUNk%^bsz!HlH1HmcYu(tV z{WKX5)J$*k+4CkEvCZcy+KeIc*FOCSk+p{>1EYxp@tVLw$i>g(@z}G3<`CMjSS!)# zzQWwF{a0GtJLpjd`FGi*+LqX~gNqk0BAOGIhBBZ)#Z=*&xYt>~0oFtM#a3Ea7%pmQ zeS!I%-$8PDXjmAr&&dAmny>2i)=63hA|{9=UJ_>g%*KyZFqoNpXkAObDh+RlVbOyU z3cfTsBS?a|w;^AkySqEw+>>v?wf|#r&1q1>X4W!-Ft{0he3Y#x+%9U0hi-)$O%- zf|#j5hr#{~8oSR8n$4uo=%4pDH zVT5EJh=P%M?Y{HcH+UoE&(GM(%!lRIUn=Y19AhgY3R72=HR!$BPgE*9u|oqx)pQ@mBWG0o?+voC&gv8U?{^M&>9f z9T;?he;0c*fu5ng#2p=Z_co`aZt|F>gQ%RNRZ*y-)b`Ktaah9{{0ari?+!PU0 z$G=+PqRgd;6YMh?%JY+m6BUDYc6P3fQrbcPH;&EG??V|tMW=p4>dL4H)g%!Y7b*?T^GV4*>q?e9wtSc1)v}Ph*$|8i-6M# z3Br{Sr4H&hLD-EOk>7!Zlf7V7O%inYGcG5-l@9XfG1~j6E_KBeSD<%(;b>UVN( zFmEHOZpi!PQ#^++m&2w`=sF7!!4L*%pwo|DAD=1o)r_s z1KG+gZBpDpeFF@)t;Qg>Y1Bg-wtYX<92ufP9UZbU!Sol@?95UuT4#l@V#6I8fyIHw ziS&?S=%E3@!A3AV4EMM0D8$VG!`X~j ztF$BGX`n%LXn}wKLAT;sPS;aDe*75a3w=i)c*)!KA{s#ffe@uaQTLq=eK#o+dxCiV zob14iPVT?ofBw7)(7Ls#;p{bFlYoGL5Mqq%j~@+Sx4hlpXu$b-4Ds1#pI{li$nHA(JIoOu|={$hSNM9$%Jf8RT%Y@_udXN_e-|)nIG3Rej`}-Vo zD%Zi$szsroE3JCvtpaeuN>l5u^70PG>J(%hP8IRE?IF)ykxX1pCFIzlf0&$BQs4Or zHwMx}Ac0|_#%kSo3=IuiWAQq@^gcZzadx`E!<1{OZVB4o z)@zsR>HQ0W{Gb+}#TUINf}mL_t^NYV?8JzzC`))z9$kf4hrps|GNVMV#U~!A>5h14 zBs7yhkdb-aECR8fB{mO$x&FIyZo(+H`;2gf&(b?NU|)dehlc7Ui=R(kp1GhI30a8G^AF3N^;xC=)$%$HQlaANXGF%R@Ut;O~A6xjsN=JbO3()1BpAv z{H~?tx($iNb>{x$U;6)J(%^{i4tb0a-+5rt5lw4oKLrRn>*UXTyVbdS558zamwm1(@?_40Xq4wCH<_6#EPwrkQ}-!KAhry zw6lyP7*WHwE+3`pvs{7JH*hRHiu6}8A+53(v}dnuZ~`T1op+($dq(ZvI&9ZL^ISd2 z1e|y>eSVtOdEEJS_cn|>22;@PR4G+c9qt@zceua5?%O%XW>yy;=!V}bsJjh)(J3x0 zER1I*A!geZ|>AxO67N`eZ43vS+|3;S2rgz?eVMCe%AZrtgQw56wLPy;M<_P0_||+7?00~` z7kycNGR-$DqJb20b<)v$xyY&8#Pz3u`y7JX`+;1n}aUP0?hWZ z_&voRFs#Dg`)5KsXw$t6mub`K5xi_@shE+GpGp0+4I(=q>;jg&r&b)qd$$O@+X#Vm{FM1UwHQZnD*WA&i;hOKK!G zAz?dc0QW|a77OMdmKeYIumWut43xi=6EN9YDe9F(4>6&;4QT8^!(>o$+C5yiXhInaSy!IpF?gxK8kq;8RYGq&B;_yNG}EvrM26fLp- zB^1F8*Mh(dmF9DkT^0q{g#fxgmxuzkBxZcv?NStl1h+sda8DZ{xN_wi$R{Wi{Db}a z-{aqE3DXa&W!(yu61iJvT$FpHRW|@Y6<6h(A@h=z`RS2+4fOuXF8t zR1jyEOEB(U`z!W9yIWKOwkV&o!|bV0H)_c7FbrB*S=~Dm?d-b8w0fpRgo%J+^3_Wc z-}t9J{Xu(LDtO2clY560OzsaA@dF587+N}C&Q5zR$I2YUNY1J6doVs)q!$IrkyT1f zHi$DqQu-UrdEODoC*lE@z(VnYE5L@q<-Dd2Iw6rSOG)0kQ0^lMT@L#_*#mfjt#UyQ z<8>bV^(Wh1!lPDsATzwK<3dtYyEeHn>(YTv$n37fZ|i<N=k8hrs~3%ubBa5BV% zKDhh}6cWujv+FiCHV4PYMoWDeNcQD-XH1FZPj0*$oG6guqPymi2m)fEgOFl0^X=6o zq2*N=^tQVzPhmu2!q|KO_1X_w1@MRqApIap2JoQPZ09skR6v|DKfRby?6uSX5dG+f zS(;Z6fQ<}LSZtl0WgsJf=xJc>l-W#EL2bIX-&#kg+g~z0&sX4}L+(`PR(!c(>uW<7 zMuj3h8Ok%TIU#LHW}{jIemMXkvEE~5Sx^fgiSqo|KL?y@ z!;7v4EB%?5!TJe6S~Fl{5MF{nZU+N#W^$U-Ax z7A_gPKB$g4dJUow2l((T7_pb`lc}nz{(;QV48%voL6}5jn4x?PVqopa)qxIQ4CQKf zmH>|EXMD3bGN=TSUkUUq%n0ZC-Z>Mjvq{`9{%HU9YXWfhd|k%-Q7T-PzbL8 zVmt$V4DKSf(O4b)gVt;oMA|_z2}lZ2G8-%t2z<$>qL0B^D*??~yUv{y{F>|S3Pp(3 zf9={okqPjI5TOPP7<_{4%wEXY6@$hyRH#D%EB6q@+8Gd;0s=6F-FL14GbgIqdk*?L zLDcxDNtHOmjK%x4KXGuxRPOVf>QEvTtxdj+dMxbgglc3Oz9v(>PTcsK{o$S zdIbVsu)Pgn(;Lz!6lPoyJ3BC9g~En(4f0%(xT?#P+hw5Su774Qkc|vpfdLEvO^Wew z(i=BIAQnVG?X`1N$YmoOMAr{+zYrF71r-@#h_J9Box`w7Nwr0O%45?Hk7S(+Z9oqN zA^%lH1wWEx15p7%1K8vZ*Bv6?$e1Wh;Bm@76@T3o520p}Y-1M24slqQ%1OyOx#=sqZfWiH>o#o{RvGkJZ0g-UPhG38yQN$GBPh zmjIH(M37$rZDSUcl$@uZufq`nTL7#nC4|rksl5+q7kbm8a&nlE(uXZB1!xS3fxtzg zp?;Ap9IO&H^f7qa6HEK1kjRMcG)A~Qxb6M*ULGeVF0>QVT3Nl5mGGCN;=~h+Zcvdxc%MHU!KgB&Gsc8GlU?=sD;^W6(}u`j~fLECEpf z6BCn$mzM--GyoXkoKK@)y)J}P2w_vVz~b1RoghL#E)=X3tj$$Q$_U_S24KrU53`dO#u ztL<#lr8+0romxl)x3L;DnamO-uH3pCbmIvhn8pW?xPg?b!O$uN6sXKU`Q%X_2xht) zZU6mC+$9QWVVj@%_V7c))n}iJ>IWz+Kn)LoYsP`E0y|1vaRkYVOdX=-yJll&riw_l@TpMlxxjr{Dk_;9EH=&TR`(6jPa(6xVW!+e7z6SU#sho&L>0$&@22%xO6HFwzB{6 zz#Vq>$2s7~_Z-1}vB&-B;xf;%G&TLi>3LvVS5yGFQxeWiH16|?i0D9T8`g-**d*1_ zt#zeE7@}Dnbj&7*AmVv*+&qfp*()u4UXm=XtY z8fd4DAEdfjs4Mg)aHo=<>Ns{CmX3M*6sHJkF09(s=F+eu12|?6|>i8%Eh9yqK(epy> zTo(%FNU102fjuRL80<99?NCU0WR#~u&2ng0DsKvEmzqAJ<>c&J`uh(Nh3$%0k7cz9 z_DT`NIG?aCqS17dm3GZD$9Y|_)PXR|$#+xllHeFXECT%%+Mpyzm@T`oH3q8EU}lDG z+I^IN7XVq0+=BRRciJmK)9uJ}Gx;9duh9U|r5OBD|NUsU!=!%YpFs68V6oiA{6VxO z-}QPM@4;*|#UVk2q&KkvP>xch0Y*niT+#{E4XtO0ar3g3HWZFDT_&WGhH?gZgA1bb z$O#xg9?0nwNc$!yC-*^u5-e;2kd~T&DYatzJ1!4@CPVVAU_zt=0%3YycneM<7?_w` zf><et*)d47BcBe;9 z<+Qx7$D!YZ!(7jV3x}Fxg*QgrKrksWpeh={77M4k>;oJ700=+gVZ^d&J)3(;{Un@P zj0)+`2)u{+5xV&qPBfSg<-9`9Yrt?mTxqfZ1qP9tV8$b7+8}Lk0Zt)-HQ5L!E$C&! zzkys>?DluT@Hk605Qnryt{&8RI8Ts37iNNFqd|W}h!XVR=7E80pgaN2yNFn}FzK5H z22=;U_yAQw^J)fs!3oe3?yq@df;8BuGQuS^G&$%A)2AxCG2IVG)u38W$Vn6pz+d z%l`iqCwv=8ha!?Ecms$5KRYvn3Y5fvB}dQ`1h!ithPVa!)yAJcfByFR3FD2128Enq zffF7xGc)NfbgsE@t^&|07QhB6&_ZUHmf+wYF8B{fDs`;Vj^24~%wWi)K$*!Ff`ov% zIbQUCxCG$^`c7Mw!oorXXCcX71Zjew5Ag*6-@{t~`q2<3c)Mfj4rmzJb#MfP*L+Ae zT+E{T46#=s=7v<{_`x?pr_Cu!#QXrpJF~v7vsO3?B62H?U_=ywOKFDf1#?3RadE-) zfHb3qRmUYb$OUZx_AKJtgIlBw8wgGjHvjm658hh~=tcm1-g;6u!wr9f9rXtyxak0+ z|7>sjy?aLr4_*0_;!R$Q5j@y8&CvFSgLv;gevLksrO55s1)gjOm*@&9puT(!o~`;v zaN_OPFG>;joo`U<{`Xxrkbtw$YA$3zk<-qWnZUb3z0$xYFSOmAOKZU6~27IzY`cJUQ4xd}e`QRb7NF8*+RF#(*RPHaYE&kYhv0(1fKzhA@H{p<#oP z07>(Qa4rTpi-;%;2(E&0JFIIqqMR!ggx7!n`~hH`Kf}-UFq9*j(`wouS%YK+$1DWF zCJaIhF5pE-tAT@R$U!(@-^f8VFr9)scaxx`u0eBZfzwKmdD=_0VJ255vn0w)1 zpR6jt6&P9MsNn#;ptibq18{dOwBE?M8aUlo+PC*eTJ8M_1Xv*|1Oxv`QgjO5-2T<8 zH^WA8>FAa@AIL~ctKpqOM@9Dn>t=0|4GHNW0dS&T_TnmPIBO6Vh6N2~2YlfnA-#~C z1h=Cd=Ib1~c;feNzEF9z1_wIETie@LQ!06Z;y@>D1wMqzVd!vla$?K>^Usee#+^6j z!1XdsL%cK+)CNH4i1P~|>@wD>CKMS`4+v}(5D=gd6{P}V z-40D<3mkuxI3z?6&O&0>^3NgZHiLgI?Z%(Kf2C|~i^so+dL1DI?t|7bVkWbLxhOJ1_iPn2`2A$7VC(u@Jo$3aPDz*E?@cMSy35oG`gU zNca#^F>CAVaMBSUajT(+`avwQ?JHz*TM+nWON~57U;`2XO8=hvR$Ghhpu&Ro+U$1; zw*?8jz;p_PzBP+DTCiK-yhs@2Ju)7IU5C>*{QRTul{*YgpV3W||Mwg(9bHUp$FnUs z($xqjPY?+XS~MKw!9an5hX{QLb%rY0S)U+-(@k*L(qD!WsX5q4IQ+K%zRw54JZT0z z(4s#&Rl#u<_>W6aq{#VZM2>`9kdff?f`}NV5^-PSIF_-EB#<7#c;(>c}U`SF!Yj_Y3&COYm6bf*;IXF>-@`WKq2?{xv5J93M!w)FE|9-L19I0No zBWe&m2zaB!eb@X1@Gl&6Zv*u@0%U-xrFGaU$To1BJ z&JKHEB8ch&C}|8dBp{0>J71yc$v`P#ptLHiTOr;1Xp zP23U4Ms^CEBtXGgCbY2C1hlNJML;)llCyK8Pz^@k7Dn z^gLS5Mjt^(ONUX5*xyY}O)c}#*97c;U(u|vj)19yfjZl7IB$gCG0>;`e{W9ZEp`x9 z4TmYPgP|atFVNUw;6ao;Iy#z$eSdGS4fYapBFFA0)JPxD&Mg;+Xu;LeFfd@jN1#w3 ztRXN_>~tq4%Z`HQ;=TW~bP}AOBC1>N;DIVvVlm1N(Q!OvLIRwI7hwRrAtNJ0D5}@d zP9p#YIAj0us})$t&+^q@Ar>;UKv)TJXejxRXIfv6fU7|S*9QQQ?nEVFU;KBOSy(=D z8z`&(Kfh$+L2Clv$~At72>$ye7xjOCip~(}rT_i<|NnA{e+y0j|9K?}`GpV0wTxb( zlG}-&$-7cMDTya4JzSpLSoT$V<9PC+ZJRqS>eV2bk&|_NVzS>sLGT+#G@WYU)B_n~ zUT)$TT9@KL4M8(?zJiOZ>;?M2as}aBW`7a=uEZ5|1*Ob()vBz=jo0Bh0#{zXjGmbI zR%?Gpov-jxl68Ht@nBD{N7E4K7%Pkxip&`W7EaIA*l zXq^#$AAcJDnrC^3RCxWkWC4fglJWJ0=m~Wxm65BaRP!16&+)%{-T(ICeY~AblLa&X zg{vbHe|eNgiU{_b6^7$pdYrm_TtCIJ;>XIZpn4JOBvx|m`}=o)%Nr&K(rU$R{oz-N z9QS+8Y)`0VejVJ^s4O_a<`Wrz!8*F9>!v-9=@7LowEyP%4vAe7AJY6|(S>`IwRmmT5YI=KFt#CUM?*t6C2 z&#p#-l(iYYLoZITrRrs8{~8I6qW3*$LLJS;m**qDyi~m3?HP&9etX$sb3Z)xv&)+^ zx@(f=XH|AaK|$7E})if;2S+h|sYPiS;H3-=tB_}?86OF@kFQGD%titUN3fH6*BTlLVfgl5d8pcbnW zvNzLg(lV~GZPnc&FS2B$>32t@S_;Ro;WKBq7vqa9)t}g;EOXFZ<+38UwwHp5DN#Dr zaU)B*jb^^-QOijwHecO%lht9ATcn8dE_;B|p^g>xYx|y@Vk_%Zik))#Uf<}{SK%$j z#sLkID=80Ug>A`I8rXuLPB7RWTIG$%m{_6t)<&y^=f_2|Cf#nk`Pa31TVt9(x{k^l zi=FnOSw6eAXTqO6?%1o9S;F_GG_I*SiNn}_@V>c{$JKU-D z(%ZRRA@56ZoG;d-ujO|}PiQVOI?COl63R?uMfnc#w%CYZ25+6>^7#&xt^M8KrfgyT zd7+&!xbe_ulXQ%oXYSe`&mFwmDV14O>~!RbSNml>i0JsHlH5i9GZ)=sY-K;F(wufG zln0F^8^&4xaPYc%o)i^NeU^1kBmEKFr0c-y#gyi&#O;hHNJVp!N7N-rK=e-X3A z(w1W{*vk4VFOfB8<~7d!^S2Bd`r-pyVOLfK2(W`M;>D2;3|y^PkM7y6ev7R-P!V;& zc9}LSTcNrk;52Ve&D!kZ-D#P_LXPzfBY1eDzt;zwGl@p#by^Chv*FWe=`Y$*?9u0U z)yVR1pD^K-xQn(o2}JT&+qhhjSw)<@~?fr8@oUyrRc zWGsczd{iA;B3%{q3!4$db0=4&1e804$HjJZoX3Ro#YRIq!i7|6ux*pCdaqj2k zaFNJqE;$HO?*+`Ou~hd0BeXM*yVDQ0a_foMThInz~6uQiE7Nr2BcC$O}OEC7*3um+wFVSHGf(1 zWH~8^1(`khA>GlfmYr+0=?teVlry!PHjK!NCfpW|i#wf!QNB%`5+^^Eo9O1|@7_Lh zA>h-|TxN|SyV-9iE98NXQfpS%NKjC<8Co+b_Q)nSmJbuQVl+0M37mG!x`g$ajlXD( zpj}Bu{II%R@%#M!kTVDT0jl6olYtBVTTM1XENiKRiINy5bDDoD%jArxJL9cq=WC{O z*}8;+>|^eaEij+4-n^sSRi`A`AC1 z2>h1s7dA3?=k%pO>fW=>VWYv%Y{}(2J8ve`N!307x8ipe44cw#gImeC9Mh{k{DLH0 ztAb3@@U4u74!0rt&q{Xd3hoaQriLw-@*s0uYC?4vSKK0Rt;n38T`lvHsyK9K{M4i_ z-A_K#ZBJYx@4V1;eRuZj0vXfVFooQ^-OtV5F9@^5++H1sFzz&*-TH8*9{Ri?C+g5< zV?7dQVzDXTP%v|qOL{{;Nra$2>T0u^6kz$Lti&9u8`ed^tL z4J-7r(#7Sx?1PUtBAFE2$8cL%rwQ8ku@+y}R6e>K?K%5toIA{YSlMn-&wR7=i>b-j zj&)eJjZYSz=acQgq>u?uqCKfcmOa5~%# zgI-+PMe^;Bs-Rx5n~4k4>hp)n<+|dgpVS%5-aepC9H3L$l(YZlR6hNqmL+|1n|gRr zHFx{y$>mH3u8L>~)53ZBw z=)}+)=j-UrXX{Bf^Eu!qjoVs2vpL-O^?dcoI!f8}H8rcfNO`j75ss?0DSzH47)P+_ z%}stkCa~2y%ar*-!&v%yekNz0+0sh)hbGcsRzh)V&ffBBq=!waVvkk^$~baur`De!*yCgJMiAH#921ss*L#`cV?`88z$&6O@gad&PB`-kVK%k9M%EA2{G z!uqfFux;z4b#vD|lo*;E?g&@t7~JCu-Gb)EfuKTySq^=q9TFx?|3obfxjh5C#6ZDYj06MlLrdW!+f`{e z@$pUpPQj+dlzk5hl4J4r1YatskKwQ2tVaGA_F%0|<>hbSj(HO8s^+Ave#rMWi}t6t zV#C@Iy%ABJeqH~>r(meeJ*je&>*ZeQ`Fc8wd~x^|=C?akZA6#`s_@DlX7s+e$;P>NR!MBqi_C7mmQsrAPH?_0sC?b|FU&_-`*1(({Oxi0>)< zZxySJa~UW%35I)JR?Z>E3T!cMDVV@Tg=oC}1(@nfOO)yam z_2n-}H9FduBuXKjOQzl`VcwDg{pL zu3{!8-n0~|P^RwE{ea~Wak}_eT|n?#wUI+d_YeJv?7AuSM{gY0`~-yK9ud9_qW`iy zjuZD|<#ofu^U}_xw&OC7sCz+61J10ApHyUs)SO7)Or}M^^)OjHf0d|kQ@z3}Pxl#m ziY1Y2OBOIn*+TRC@$@+WQP-sb&Bcbv#Wzmv6}$`ivNhD< z&&AlA2B)ZJN|amz4kD!whNj;fObcESME@5%|4QIwn(Y?(-u%fzVQW9ZWa5qcCf8eg zyg(zd!iJ-nx0BW}tt zuyKjjdH%65g)nJ`)kFIE@wyA4f&W`IIlKl<=_JLyT$9S~eNSo=IG=dBYnoc%A7{L8 z#(F0sDbJ!?d#uxs|L}v=z{F?A^62 zIR(8+1Ckvb;!Ud=iLntDjrqx?uh_!PhHFzbFVH#fT4!AndzXKPX5G&kj_xv;Xed4S z{Ahn9`lq@{5l8wIOO?h@;f4K+Vfw=#ykBj!8#JLR3l1+W4Aw~mxAtNnXrK1YML znP@KgLaf8z4x?L+GiDtj5u+s%jh(Ka4y)}Oyw{ZiKQmro)fx)cxWiv=qPoZZ|FQSo z;aK)@ypKIIvn6EjQC7&xo*}!Ckv+>SGkc4SQdWp$gorXjMkPdMc}j$kk%;rXEApOm zu5(@II{%*My56hz?d7@e`*;6--}(7`zOPk{4X#Go&b*%&{&d+FcU9cMb+w+sx~CnX)LLgK74=je0H6ps|x}6 z<-wOnmE6+C;fTu_&-x_$zx$D!g)Gy|0LPs@K5sT0=Z4zQ&XS>G-^&EIpZSpvPXy+= zzma7$v2&Krqj4QHw^I7Xdk4Kk$o^*s=M8j}6=N|wNW^_hT;_gWB6zcVTBv2&rqhJ) z0id@RZv1$floB&&MhfYU)~pz}(bDB%{kNyLXB&os%=;w$iT#zSGZ^jeKQ4I~hnl!xSO%3cJ#MlUit#%gvK7{fJ}n`hZ%~ z@$ihoF|wzDPfhF5MB*oF)5b=bMO8xeRMmzW5_{Wi;%x~7W>PBmw-Ylx0O4+N6Og4=vcH=&x zj|ht^z6)l2a&*?L_ovaB+ES|eMax=npT+-;UHqrvJA)U4mWo>A(}pbXUVKkM{X*o* zz<79&4h9OInQA0sL;GSvPY$-YC}kz@)i3;RpCN5Cou~9Kn)EOBN^z7M8p+9Wy37^( zj*^(Bc0Bd`tb+ zcaPH(cb^ZbHm&QrWZL%N>JFE%?z&zWJto#PR+82o7uWLLWAE+hcXB^dX-pZWYlJts z9;ws&8XB18_@5xt*}fL%&ELkM#w{v|BY5Tqxx;4YU^*|uu)$X~pp`E-3e~j8a_Uo3 z2Ro#GzCRuq4Cw)i@V<{c^>=mGisKV1I~Jl}i2SXP=tVcPM5~i`5@5AQ``D@z8T&Zq6fIJbTy_8X z(?Tgwm@~l2R4V7lPhyg9B;knFx>c#-|HZ5Q;Y8RE^COoC5g+EtAAzw>+>)s69P z@bhU^*(-D`50zW36oQOvib>DX;n}=kXZ{4?IFnRt_%BYY(Z?hZQ=H_v>b2=v{N$^k z^x`JL08>OvkG|WP*0;}&=eE|KVb63H7v*4bV0pz{P{=Z4do<;hh04PGges}u{qx#( zE(EuxiXQOy{4zkItj(Do7SVUIHX1t#d&~2YLG4r3^b1tB3tBN2X05GT+oYCns~--p zf>~D>OXyG7S}&+4nTbFCna65RZDlqz<2iGY$lU~mh4)QlaktmrySj_Fmyd?sBogni z}pTK3jg;<^YX1S*)Q6u~LwwYyG98(f+$0X_GxSc(^ydJ8tIf};|sL!ahi7#cK{ zm06R_+4yC=BXg!rW9~#yGs*LlWY)+JA}w;)-}J-tvMPmC*MG z!E5&xSgw2={&q{+3RUcX>a5^6K8x^EBl+Gbh7UsJXTUy4)oIjwGqA+t&6x|<)!o=s zb;v3MT03vCQkn|#5GV~bbPE8)GTB?uVF z*b1c589r+5Zh`toW}27sjblMAdFek3${k1*SI~=zN!0fP@KXa*yDf*F(T(qYkJKuk zJoY|^ux@GKoZA&Us-Bj-+Mg2JTXZoTo3)TQ5pOhf9lJDKQ2zo6DhIP4NMblQY$@2G zXLF7xd^-}sO6oEui|M$%5RO(c#F;|ht&n&i9l#;!74f47xczw?uBp>s|2 zfk0*ryZP=8#noR@bWK5j!_mByP#I!X-ffl%tFa=ik9AB611m(T^;eJ6CBNy?)C?V; zq_%VINA~xyg;`637g0WaSl{@|R+mq&RF4oV@6@)b%K98DAyr7Gqr_u5b;J|TEF<7) zTa^d@i1+s!w!D2G6yvNupTni9ebx4M_ioIQVN?>+n$pl)%Q5qefb&8nuBc!IjLaK! z=vpz|n^(d!Rs0}ljdXdQP*I%M)EQhGab1xBOYW3|XTKk775zfmDc-N6>Zu#yH$I0C z(rdjKmQ@vuh5*5XEYlR25d)5`DD>Kn=b5N7lTj@M6&6;AaY(V@nP>e9xLs9UcTCewJ zCQ0c=xFW^D-;nF+eMs8B59CjqqyJHhj`}}T_nU!V^Y0V#pP$71e^H?RcR&Aa68!%Q z{#%*>LTD}{8nOtT901OtBRKzF zIk}o@Rk%E8*a2e?8=#c{wZKJzZaL~6Gy=~pM(l4{_>)lX86`r$MZO9!i$M7&l}qD7 z`pJOUhypl6p7tH7Es~pueB%VPK*C26`yw`Q;>5Gkp zN7@h-XXt0cb-DEK6XTLH#Q{Dm5Aa#^wxexw;v0vw+w=0QH5qDJ!sUT04D4Afcpc_X zuWFS&0cbQJ8`^-eH~rxQMlb=^wZmPxygXL5R^}q;f?LiF6auwMCNCe+xdsI2YiJwC zF-lSZGWAlc9NuBGP+Y8iXN698^&M+pj< z4jTdk142G%Rs**;ad440>tCqon-5D`U? zXHW*-E^RQVUV;Qs;fPjr$6-@MMWsf8>sikTM4L`P*v3tIfW#dFs0OY+f|rEl?CR3Z^hi{!@;E5hT@ z%EJB)4{P*Y!Gxyp3xHpcQBYvQ89`u8a7F+bX#OmP?O>-W8FXy~5lBBzel9gWk2w*P z%#lOURm_A7up{4}YYtW&Hm^Y~C<=#&I_kNZsVRa-_VDodSOv>x4KQrL#NBwY0r#T- z|3MBi(@t>-LAy3TL<9tQ z#MmH$B~PA6fOMrLsLle|q7zJykl!jg&kr`q!RPEfr41n=H1Mkt#cmYBIz-q;PnJLc zw*{yU_^?tKr~*>+M4uVD?JYr&b@!ydnVk7eMpO@ z%LLd{6beCzsZyhwRgPc7HZwDWH*AS0+%a(d+eLc9dRzoyphH9b!c_^-=tqM#R#wP^ zo`L#`f`S4H1y=E018{Hn0hn+^Zb%;l{`Ws$v~(>A!T=M6z#st(s0`2{LPzmDL>dwY z2a(;S1zPHFJxYYdX8ai#0UuvonIC*(Vl>({`)W}?*+(>=xHKhSw0kB1Z zOF$pjHu8>^oE&t>4(g_?A~}lcn7!XkfR#^gJvl9A9!7wGs>0HMgkl8_VKW@%fcBqrg-L~)TQCN76StV6_+|m0NsP|8i34nG=LHT=>T&}8Bu@|>(yWZ z4(ZMom>C2lf~Z6yfJs38o_Yo>1~@3x*F-h2MrFuAPS}k0!#~fOCQFdq+;&24T7J64 zikVw|V7{-fZ|aNKxHKoR>-ZB4u-#=APpQF(QZT#(;uoSMgK7puEupmDy?Bspy|0-W z3eb&wz-}P|2?z)i+2km|GIFMCX=;Xp0{gH(faE*wdbuk8dxtl;L@9l*AHkO{P^SpD zz`nQl!A0bb9s#1!S!Py80^)KATn#luxelnQR(R|Wv$K(~QxyaRDSXB0cjrHar1`IwX3gD2l?*Fc*bc0(G*Voq|jm_5`KPpIXki1El7=U02 zjL`s(10`s>Q08W!oq(Z=ynFZfdmwSs3TLr7ef$Wh$EY3vM4s3U{Qg-BC`I~TYsK9r zu~ER4C`FxM`?qQEF_%{p9^lDOvoNFim~D<;F>qrGBM1Wqh5|<_AnHkwP5yK98z}=r zc54*>MSyK%%Ux5=1#)6bTN@Ur0sR2x2clpbzX&`b_f@nSN)D_QA*b=dMe@Ye)TNhy zC|ylq7VCZN-}?~bTdmwsfAi3l=G54M7h4g+{v=@t9S$@h02#(V5H7E*j0@VS644c6 z|HUZv*oc7iCn2zZPKY^=1iL-k!UY*M$1wpoqkyUdNGVtvU-?GtLl@UUD;kDQ?_x0j zIGL2}Xey&ED>o$YfGxzHw|V?r&CX}BIqPeR-Qi^>u~#z0T<8(4IbeE$kQykX<^zTa zp??AF5DzeImColCuY-*Y(eh_JdNdr7iU#B?a6UvqUAzM=fB7;O0_O%o0XdL)0kho- zfJ{W2(e*ebWf+M6PXpWvVUr;K6yQWBPMxBHZIb~!8UQ5;1E&kL^L4A9R9l~1K7nZ8Vfsc*hT` zg0KYusY1E8`Inh>aa0|$u&bh?Cq zY4+zB%;rP=sW>@)}sg{K3l=JA$#7}-#^RrTFK#9CXLYJRdpnt zzc;aJ^r?qTW1gb~p)VIv8hG6K1?Gg(R!Km0ApyE?C!&tn9Z`LC5|L8}wjf{>yT7@r zpFF&zN-uj+tpawNtiIPL+GA{j1Kuvw*(C_6lVv-#(^a+$Y#6sT|;L zfD#FydYU!Ym{W%z!&of_m+9EGJ8TlzY>g)$G8M{9zrB#gTs%Nm!OQ88!4{9HK?`tE z;7y}QkIS|rvT-O-@mgM4kq1sG1i4n$U4=A;CD$eNP~e^EC?&O# zKap;R1_Q5Je}={6+`X&8v3L;R2r$st8kK}zhQf&mZN|K2{RGH_CH!>NNb zLrG2!1!NFlr`Q2Q$G{Ej&?|&O43PU9Fk1vvVUjyvYaBoH)>K8_u#C15%Se$FaAuxQ zR4jZ)5GoMt7D<1eqWMiPsbVbsiWt zlO6frFYdCTbCByAU0{>{Vexi8IU9qCzlxLZY6(W8>(hb(trWGyI+rnA6v9`1(~XG& zy(R&~;B;XG78VL<4<0$ZuikT%k{tXl>q{0EOka2O6~WLGa_4S3tQ$*W7)q5;U>izd z4r>p*P;PuhR(Ie7Q57a5JnJ_6XWSvhO zU|ODAZlICpY%afDU`0WvNn21<6kE-i4Q`!9MrH!Ncu*3HQa>t>x-f}>NeTvn1tU1g z%G9E~rfQb8c#Zm=5G+6*BP_svSPUeE;G4eFE79*LsgGk?w`rDkj?)*9Yp1cnadt0b zbI4HoNt-5w@RxzwuJvACQ!`F&>|aDqWfqE3k`27WEP;=G7gyw*R%uT+Uh%MD8r$+^ zf($ct2Hxakn^5L2ck~nmZp}KWvdWV|Hl37gWqm9A9P3w^!;VqUBQ2d6`_4;k3MCn_ zKYsC=jFpFd^Qg*uiRi}w_kC|kibYDA7HoNoEr9drB`-asncf9}9nY88fPPp7wq7WKs3od9mRBd3F^XDO>Pm-ZIv}04T1%@zWGqldCrX2c7r9#sMQ6wx~ zMlCPXCehoQ%G7=`P$V{7%*%Q*G9dqwE1Hn;kRe;M>;7%wU!7gvX>eHYLL(-gO}@S% z*j&J+OzFnq6r1N41#2*#iUC1rq)KoB*W39@Ps|72m=CVt1WKWPZ?%+I4Q<7)QYE$m zn`Y&b;K#>{Lx6by6cjd(1vpykLo+NUP z2NlXZ?Rfd*c|7b(uh#8VwO$>4XOhwV=II5TxXVi+xM_?XmA|4>nP}*>{WbSDut9T5 zUWkf{p;~COqFk%S+j+hBrT(M_i+S@}mPXnGLM4_RMFp2uBX}hOJC|ptO78Q1CUL+w zvz-g7odj#mK}<++Xjv#Iv8O*RzVjky#8kjUv9vvIFn4N#JLJ_Zp}RMobc$UsP+cm5 zpP+fYf;PXdWOga|3T4) zB(tKda$}JPY9&lLT>#V`xFfQU?PASK(IN}(iUfzp2st$tUCI5SPv)Q2&9U!bygDD8 zdG5V&iK@kscrk96br|`rvAw2(_GB@d%SD%+_|VN5G#0R9$QMPL7;3rjS3f(QtaW}P^ZaTY-Nx&25=T?(1t^7m7qVj#3U zP3FFL&#^0ajqUBcNAVgQcN)X0Kla%;2DZY?anci~WZt5s_YG;k#~ey>5#ktR*RO*b%@nkAqn!=Ba>4%45gKrJZ}A@@No5eg3iPgfyx`v}r5 z7$_JrCv{Q&peV0La5EDl+@d!pc6#dy_?K>s&HJNjn05XOXXPB7ZxTApSNc{L{W|qy z9^6a>u$87kxe5^(gfgs6MQ1ZI0|xXYFi{XjBBByn|FU?4Ms_6>X+q=DS$AyJ7>LAq zix#j$bnu$rtAR^2SaGk4xREJ+v5kVe4_%D5VYf11Pi#?3#6aky$Q59-feiOrzY5e$ zY0{(YM?Zo0VFI35z{bL@oTaqjqigJ zO?0wj-c(}(XN3K>cjL!bI`93YP&Q~(6`2!85!0W<2lda7*e`(X+k$BC=qfWpNm)sesqXX&^>pIQ=`tEr4SUUN|skD6{FR5JwOh@4H>cg98sXj<-n(zv)+nrCK4)k2LaBmITzaCz$V*$Eza<1+H9 z(R1iId(!{xKNGJ)PQc-E@zn&zSB{gB539U&e4T-ZlLOougrdl2QiTnCNhl9>!cl~w zsCXppKy$pO>+W1R{;!&uXZ^=8P-5eSOuv;=e&BFL(nxQex%fTZvNKKwcVT?mmzrKp zjBn-7qS|~eWIetHMkeEQfmMkPH3}y~2y{%Lh}8m>Bcy1Ik2&c#dkpBk$=#U84qt?m zp8#L#&K{?`{NEXYy(zHAGR|BBy=y;`;z0qSm$QI<`45frIG)1HMp?prXEEmn}FnzZ{-cer%aYT zB6&2a(P1s&EwBtkC;r8BB8JLR%V=wO(pPR7=Myzao`^%f{bcBpASTaw5m6k?QST$9 zZIma6YaT6ycHenGC2vp|sg}y5Kk1=*5E!NyC}1y7QMD4{7sT@@k7g?+-2Z-z>+``2Oq zI9g#$5FaynhV>Wu2Gq@UBrV$p#~2yBpP0QL^&&TL#O3<2b}CJCg?sLST0-h6PRZ-f z?~pdUe|btMZ?E1agIaM)iJLs2z7-ErU^B0_e8p~&XZ}Hq zl#nd?4XY@za~`5Z3m;cfP*pxpJTsWZx9nc^qLJlDCSOtpY$=dUfin&Ey-V6i_K%gP z096S4f7#2Vxr3VA>p2Mb1}{Op#D(!~7}iPxNaCD&!k!D08ar2~b6$Azdf`wfIuheK zkyNvv3R9#o1PL?H!#E3#!<#P~ZMpQ%odc#Cj*pKIoNP4j>y(6l4eBPVyilYV5s0a( z1MUis^qwh=VM=o;M7sPC>2~P4+hF?dT_LPA{mh?nri{gZs8ko;h%@ZKqtI%=!;6&W zBtjt*S>L=t%&Xs_RZ&udBinDsNx*vYZk$-mIPvo3760N~fnbd*axbimzVsz$WFPgOW{7CDFf_fLrtsbq(oc*w%cqHZ;uCxSHC8VWs3tm9V z`7j$C-~$T-N*c5G8g%S>(J-m^*@)*cpSw;u8V^g;J7_GNj>;Rjkw9Vu)f85!0Kq+i z=!0_(Pa+WYV1*#%)g(#!cZn`lyd3l2p}~>;(4YwlDpfQX+68CPObQ~wl| z1OXAG&j~e2J&-Pd21iSOzxF`02K^*PdIpc-olJ>VZ7 zV+BEj>46U?FyN7C$H=(0*?Sti|1xXBOlDLv=><-F63iC+;5dV~^Pc5D{RKk%gY7k% zHMt=jbh{@?$|~eJ>Su$_dLrk*(K)QfuUEu6%b%2jMo>VRE3kkb=q_@ky9PZw6bjTk zk+vkrTPeW+uDBe=EiFWC|IbH{MO6bcR2wS<4+>qy%ztgr12*|I?cnM}*tbn{t$7qH z8#`}()9z)qXNwlgPCB&boIb~!6poP@ao}R|Fe#{R%k;jUp_KDaaafn|MGZAT!gT?n z1Sg0RSW$AGrntB}Uby0SFKc^B-%q1?wrKGbcEvAbSMZiu?w`*nw&Wbb@Y~>PmBvLM zGS$;fzft%d^m3UU@twG*tb~J%ftXztD6(y0=$3x-Djz*9b7u!Z+nY)@%`ZjeN3NQ9wXUp^ zq#Fx0lJ`=rvPQ*Jp}DP+va$$a!y*kppQX$F=0{K<>DG?SNQTn1LJ1lzxF2=cgvNPK zcXz-(ygMcIkSUYV-7WnNn|o=`1#G3WY}0yd*GEc6%bl$_I?tS-&oAR2OO4HX<3)=M zr2|2xW<<7uHiK54LlHzDkf9-n)?SvXhD85COV>sY2`4Ie?1R-jo4^(rASBKfH}C}dVD z=zku8p@ICh&ZDa@I1dVGleGJ3$_OVEE?uz!jEf>o>D8r_H(wvYn0lQz#+y!i4YcEf`S^MM)oLPa z$-uqGmItZz+iLRAJd%gNT}*7aU{QnUz-AR2s38`_mXjbaRP%6210pbkALj-{g?#l; zXyS;9U|CX_V$J=Sp6(p1)K>T9Y=x&Oo#jP=}<99a*WDtR%|}7 zCN@wc8KSSKNI$8`%3~_(Gcl0GcX`1CcWR^D;s-|Z8z@e~p(kf#j05ek)}d2w)qT%l z*Z?%EL61k|laK|DR2c zTM|tr1lnygCpjf|OD-NG4(dTHuxh;vOK;k5j+Xx8(S*w?)N3K;9hI_JFM=?1bE(j0Ve43Tp4N$52k3YHe(JDJ-XC zC>5#ic6>^@9DhYI>wIgO&wR{czJWL7ekepn7C}X^ad2RwK(`|j1X}WuNv>C~;-X-R z`bWsZ5aB@3sUeXkL&kPOwx16Ouyc($6!8b%hDQWCQPsTAiIS-&Zf+Ngf#?Tn)le4U zWjm4~au^tiA zf@XT3;^799L2*i6a3oU9#ET3LcL&EPg))?a>bRj4Mr}FN^p)7(M|@}b;hFte*Ur`@ zZRC5h;5eg%o4f~$Ln{geMs@{8iXh0Y%F0U99;EXM`bbD~+1z3aC*{V!XVDEsH9jev zmnZYZ$Ctswr-Q9ZKBeO(4-t{k_lem(7bGZ@=; zlZrEileh}X`ZItmL@IAcpBCwxAr0m%F&9En(d*IZ7YD{vO@N)r*PHLO8rkOHe*B_i zJWl8CeoI81mFLEV7wj{O&$Va#oU2|m9RE4^qKo>%jpIk>3@r(;UYmhfC~v=XhZswB5p;R?r0%f{^$L;Q%=qZBKJz zy0^-G<%qla{pm0fg7T-F_p1P4>tM=$5A_2-xk?S;+;AHZcbX%gI+$(q}RR* zndpZ8lhZI5SHdZYpe2;XmxjWG6Umm;nYxhwKANc1}Zy zqMn*Qp6;}(BcD;1EfOd2NSH$6&&D3+MGLLwn#%aq_jlNwvZ34lHouo$M1%tAVFTh2 z={CU(S7dSr2;;%HIRdCGB6Cw7j+I0B4VtIIPZFU&l?zydl#bqmGp?$&@OI$>c47iX z$qG0Kquq6z4hz(`S}0_IgCNI{Qwk(RlDAjZCj+FfjXkA)@px4rw3thf?M_=W%%ui( z$gMgdn})jfWU%-;r}NhkoJEJ69?%Vg7Oa7vg?PKy?cfV>1?gZGkR30mpQ z-xtJ|8|Tfu(E9ZPm23BlT4coMwj_0xV)_BxP8bcVa zM}Q?yhG60VO8_&_a3R16E-bW_@4TRIu#kO3t!?>D9lIT{ zqkhTnPRz!S`imB@>0)RZQ@;EW7e+*^K;-jCrY3GcKmgKohsAF(b89j*u8ct@ML~o2 ztN9*`fwBU@FjR0-5*Y}FtC`M30Y%2KOm1CPuQMWoBx#&U^>+$iv(m|sQ0NN&gG$yadf0j`|?PYiRe%Fv0zd6z1Z=ZNxHtotx&~ty;?@p@MM=yK3Z<<{VnWtZQZH98EZXPE*|ErEh zzEWyBW6A6kD=VqtWlc=Q&r5Y&)TqXH_w!J1v~X$x!tB>?vt2BNU;P&6XO#R9n7 zB@m65-W>O;eJQGl%!;|-_lXJxFzK_P%-CBwNQVm(q+qglGRTlef`Bz4jesW9e*8Y% zpg|@=_IV8uWBrLjlNtGB3H#lC)v1zQIpx*qu@Nq!r5}TBYZmOLH};vE)f!8!Tlwd& z9h#^86ID@H%JjksWE_j#)&5~qjEesBcqGKt+54q0L~s?hzUqZ zi$K*HaK`ZfE=5#fpMEUC2{QmZB?|`!=+|R^AA!O{CcF@B?YT$uMVmBi5XVn+B2^Cf zB=cE|e4xF~G~3Mo1ly%$YkrD0E;>`nAmyS1g@zal%RPeVvrKp@Mhu+1tk!0fwb;UL zQ&(bFR<~q+)En83-yYIu=X`nh%o)pkxXn+Xhi3y?r%(`R4V;;Pf*W;YKUw}8RX~H+ z!|)QYBFH#0>ulDuF+wo<1|SbGMw$vT1$IfvOwg|sfxm9jmY1?|qWR6)kP4Sqjvnw`JDz5uYTD$eSD%WUuc#4kI8o$Gg(pe>@oGG0MV}9x% z4U#K{ot`|@XwS?XE3%_U+d)YZNt+?;&Cuk4zLmp7!zs}6%LCj!oxj2qHB5E15}-hK z0cfn-UJim=2N)VOBB((05ax=+z*3L0iSw)d1;3>nxq-7ARQ>UYyA=alhwYNYb@qGa zr8h5NV;pJeXuIza6E)JdQp&BBFhlWe$!yv(cruiT?WNIqozFquZjpk~5|HLHfM!Y$$YP zvOs5Sf@7=1lKD@FnsS7Vy}4szZz|RjEBtoQd_VfX-L@5h>)W*^TGWEvKwq9k(GiO zF16hfsfYx+SEVHcB$)!#C{RC558PTMr=UQDsadV7?-n+G<%GM~id8slZn^;Tk6b19;#lrs0JPejfTOl`l9MKs}&BoNs9wO5TV)*7Fmc{-s5 zBW5UMb?;2%!6{_{Z!4KJ3VB8;7tHr%GZ!YKfshJD4e@42_9y>xpQBN?Q|4L zM0NJL80^{I;3rYN$2w04-gHS8Nxi2jy1ZQHlEcb6+M9L4D6?V6vykedkNy3Phw+gx zY;#b3xWSDS3u>i!<;--%xPa?t`a-*R#s2pi&t;idiexVN32>xFLqb{wg20T7j4-Yq z8&p^;)Sk^-{eAh?ipnRa?}1g!{WNj^udThaq48u+^?}L_Pqq{? zi}VSxZSUef!mRDnS7O7%Pt>wpzWs~312eeJ)KIr_n*7QX9qwo!fH%2Ks@Rf5wpoI9 z1NPdkeeVVvN z$de_zPpC^fPI<;+Y6|RY&X|gdO7?R>;|gZb7RLh4!)KW53{e;*%qIc3Ge{#NNGoWR zM#1bv>Bl>N*<@&@kNML(!$AzxDi(d?9=Bz3HQ(N3l^>;^HXE$=~#I$ zP+sua{6!8+v&@X&)ndGBd?0Z*2KFVFrzh z5x@If^)=&;#@I_T*V5&(f%|XyTme#1ZVC3qp^?1$C-paNrKQUH^T_n1*_A z?9*L-iocd97oCcnMJhD!tyQkFsHlSMHBP@Grf`#-6&X9}o59&mGjC{`}ptJ|;@~BZsq8 z-{&B{Y{pys(;e?7oIpodp{zi+4%VQ5GVWu1N4i!5#4%MT3H6S9a2?MFs)YA3M_oMk+ zo`*u0H)Z{*O6cU;mG&dB?3aX>=00{kzFxE9;ng2n##U6}kU3=|GP=4iLz-f-bu>gf zHMc!=8l5#X3R0f#0uov8XpU9UZ{Jrr@{)SF)=eR0=JJJm2Wy-&pxdB!-X7AcJwYCU z%rTRcC-$wmV%$JOztakU0r|imU@Os zAVKr%ro^))Se&_{(7(2(D<8eAfWOOGS`Fuh0iQVfL)r~I;Y_|u7td_dej<9%U1<|X zi2JvAlo)eO)T-lkSwGOEotZggqg%W-$*Jr|(LX+M*k0Cu=uU0JE_vtQkB2ttTlJ|n zzr0k;%9@f7{%F$FKjW=<9=~;b&+Ke`I#m^%wj&3pt$XpRX7Cj^`zzF}tkx$|>vCr5 z^55~l(D)IseC-vz1adDa`sX?K%`|CrH@4aHaQ=%nS0FjEK4G?7-);1jKj6)i)2Bsh z++22~&5`ft|M&gznj9l1>91;6Fl~0>ZS|zib>I;i_Sjg%-I>VOS^L_CtaEf;lrg)b ze&gE0-J58hg9Sy4aZv8;3M|~Qmf1BH{2~E$w#97N?lz{*3KS5Fa}KDd&Oj9MIdlFj za#5Fm7uAYWHr}RRxN9qOA%OYuHZJ4fu&xqY5yVQ3O>6i?V@xGyLDDX8F zA=@=^l9^n?y1bJfgTu-y*<1M9I6_b&_zgUq;M4O(>f{%I@$-vsZM|2hOrgJBu3!Ku(&^6 z_sxEEZ^?4|Gekh7y*45%ORFbtKcRLYFTKbi_^nni^^%{&zqO41$ZEM;?e6d>?HP`Bkn1qsdkjedLCp#q_*o9j$2+=1xUV^;>S!f& zB>a0*<*v6?=S;}F-imzA!TVVr%zEnYwT70*h_i#59aCpsoi_Yb`SJROjyjXWzYD4u zk+zKYyPqBu`J7Dp^I@oh;bIPe5Jr^EtwjX;#$NIBY9(yTn55Vr7tFyi>CV^QH;c&& zp9iLd9`UxT7{6#Fl;0aldLD|bFnKl{wZO-#w2^{l&t9D&Pl_ZRGX(3d^zUXbVEVKQ zPOz}N7x`=^ZEGh#DYEejw91_ACUcn67oIx{UwGY_rq<;2;bQ@q*;D_1QJ~)?0_TnrHmOwr7Ea& z4Sn4*lC=w%OV>C0)#3XWZZ*-EEWcxqUBR3z*F2wksau(tBenZwQOeUhXM%DFC^?WB zKJX=~odVo=;85wue-4Oe{qyUgQO76ogQ z05vlGI#&zF;WD*nfm;aUd}87b$UZA5D%JyY9L6fy0)Y=mLTNw+g;EQEW^4f$4+EWF zz9`E2bzsHqo^rHCkd}46JQ>FH9a3t3}?`m)NcW%f?#z*E<52nCo7 zNzH}2vc&~~>w)gMl?bc{AcY=xg!#O=Y5LN;TN|Kx_-+GeC}7Poz%fK1`wzz=WIKWZiltZh@IAX8OZsXF|70R$30 zO;0z&fTb2deExt@VWQ@IsO60fJ77laK;8sY;B4r>01Y_j!^~fEhQ16AamD0)N9fCA zx!a6XL`q7Y(K^>s^*-9b%)gdE1H%IA29YMSU07td5DFe*3qcl&q4` zjm;n%K0*1gvve~!;Y?_L>NUh3Mp}QkNaN?Tr5C#rc%i(E?oE=yGai!IUxl+w>oRAe zrOUE^CYo0MjhC3LlrO1!&0G^L>vUQVv*!p7%eQ!oe96pohmh=rE92JV-7U?DO-?1o z%%fRM7JqXCKjXOWil0;xCtvwD_FG>5#V#@K)nQ3a-p%5w+_5syh{eXnz;B)Z-srwI z_a3|#wqIb;!HO_SxCaAc1WUKU+X1M5g*8w9>^tCSRv1#Uv?B^~FemWd#hV2e4H?B0!#cr*d6g}{-F!c-%5R=hPgb;b4 z*2N4stR=uwhIt+{P{>7Y84mx)FJHP~AfYlSmOH0;B^2E6{BtZ5quM2m$4UFk>X?Fl zUb?&%am@=^KF9878jUq;P-M8ELv#Junl@Xb^>NM1?Dlo zm(IC)vmFe< z-@3eXrAS$V+R_(G%Uz-?B}P(i)V3U+;@L-ufwiXtkh`~W0sj62CHkB&oy(0JnbX+W zi3cB;nmdr~Q(o}bG^z^F&=5RdgLJK|bL<+HjN;|zaLLk;OG^s&)(tF zsp-Oa_P$p+3>@9|8jUc->i!TpoqJnTkC9s^n9|!-CTQiJdgDKEnRfuxaNdP9%KF)VcXkB^Rq12y5>)q z=Y98)OpDH#pmCfL&qCYkt@)Xyj~VdI1yhQdR_;YWYOVr7f8;BH;O+o`W+()nU}Hne zlk7X;U*@M%2CAD)E1ORUu3gOkz1%hPuc=BqFwcboMMVVrT%S z4Dc8f%Rz%=3mhc4uHV3RC^e=)8BYWpGg9n@qBN7;rxPxZuyCc<2lX}YNTopyTc%O8i{)VRl3ej1?fr^NpqDOuH}AFEDKxLs zpUq1Yt9Gxg;FvjK&4StTh3pt&zXzI-qozoTnWTJyX|2<6fDFpbGr@a%K<6Kv+*zb< z3nQ0c8sGDwUio)r3aT)Lzwp9LJuHlle$&EGbRo#^>sHrnX`W=|vLrbUKp!p^nxZ2J67~TL> zXAzGJsJ_m}4sV5Mi2*R#$pm%C-yi9~$m=17ZOt~1si7r@gSW?lT_|x9A z@!Ga)%}O4=#GVs=A-1Kjvz#0L$1gSpE#jQ0gRp`Et!2)avwPpg>R;OE1Bm>BAE!~R4JcN zHcI#LQ1`+vuS#fZ3I&giw6MhjD#7cBUf=PB=~2$rm;Syjo2WPVUnkX`LX;@(y`|QH zNuF%pfBB@a=LdO#@OI;N)AIsd758nComF<_RcH%%XN1fS&`mUejyo@(w3|N*b5UqT z?TDdz{qc^^Bnt?!=y(3^1wua0tbC%3jvz>@hAC*r?~4#pr^!On$Ek3wqR}0liVok* zot(glU&4%x#7HbQ++hE9280yC#3y~jU=F9H5N#K917FE*^MMyf1f^j3rtyTJm{>aW z*eYRKAoS(?!1$ruJv^*noL%5@3*L{PKV@%HzAPVHGyb#DW7#=d9$2~Cej6 z2V!uJi_ZcP`H=Z}bz@^%fF(ioKHL5TgbtuGrI5`Da-ZXsxGF>cy%2jCO~LN6W^ z4}jJ1Fu?WemoKXDCjHTpz?M#0;HJGV(|7tDmJ{3Mmd1n(lQFS3sa*GCH4J&-qjg0Xc-3xbA5WwOz)8x|XS z?)RY$;p_K5*!m8zp4;~C%#4zXlu;x?OGSGaAv6dnTG}bvdm$1zYA*qxTr#LwWsC)mWF&hi7 zAMku{EC-nG0FYvr-Vwd}@?8wcJ^q`Hz8mcX(2@1?G#8_E$$wEWO}+82?_aR5y_>;o zEXq8PqfaV9wn~SLeyX;+B6oPzL?YQa9t(adtrjMgU?h%8tCZ_}|GB}e=y$A|{0L}0 zVJ-ES4*S;)wIc@P%guInQRAYN-*GN>DjpYQ+4BG3S!RJhE(J9+CGJV_VA}n1a@*_H zmVf_^5Xi^xE(j!}Kse3H$ZLK`OZenqN#Qfzc!*FbHLxPC9dh0oA;w z(=W3CYWBNI9+X{E8mDSQghUttg?bNFYka(ZvCF8esVO)*+Bt4(R#q0KAWFj#ZJPjf zA4%iFSTy$~mj@4sEC^X~6bRk%3Ja5Qa1bL}<1VBn?M2}#P%SFH=k>p0nT`HbZQ;2@mg>PROd z86fD@pQ5T*vz3vt43pvUzx5Q5!rp?(-Fi6G&rb>}WHR#?O)>awVKRU*>T4N{^ESAn z!Qj1uRsj(^-+OFN_iS1Z-~j49GYBs{&PK_ilRb~7`d)UnC?u(wnVH&_D^7`idIu35 zrdO{G19-2i_oZ%o+L-LDhIOs8`m`otv@e?mIBq^TlXxcd_M??EFpy*OdRUKQQS2I& zJl?XW0=~ahnKt9BXdDGgF;uf)Gfxi1CmcvNxGhJ2S{FQ6M`C>L;mMApWNYRet=dK< zbC4b!FuR!8FI`)ZRwO3})I49lHM!}uFA6@7J0|pe;|Gtd-urnRFQuvN^?>my9|FQ( zBRSIbhq^Lv9gpDQ!sd4zfJa%IPA76Q{;GxVQ8rxUu)HUydR`bV=TI$lpt_h#hG+#p zjBu>iRQD~H!k|17wTDuaOyd-&5@D zUgoWvkS3R2oCmYTmlr2z=jO(tJBT)1$Y^s4MM6k$;3m5tI5n%<5M36U@A>SR4oa01 z7d+P(Vo^YRPUblq{dnKeanOzhud_$@%x-@|82$F72hX;s~%VspUu;wM*Ch@urEp!t_pW7 zsaX333{gF5_&vcRPbT(GQ%dooBg`OGZINQObvb{XG3T=Wm*jCPj1EK}$0Fy`zmAM_!nrPynFWQjC)8a(4my0`^nbiSZy6$#ra;5bBt@t8s{s3O1fy% zrYw$jevbtFZ*8Rw_6@0O7FpRzG37mt+Qr{_94IvlX5DKJ3PZPWH6L0YIsk2^c} zR=;2V$Zv%K-xx4jKlQUeH<*Qv&L?pgpXfZMykj~h2>vhkfOELRcov`3h7{l>oVTsm`qg&d*-=(yuVAeCnC6Y; zIaw3OyiLm)lJn%Z#p)F2kb0nI>WeNTKKW6{c?(S~Ees!>FSKJyh0oU?ClLi1IG>%< z6Fy>ng=;F-8PH&zWe(!u(1T*2A3`Z4O`KR+nQ*?jAy^P{p?;?};##uRf1dQvlIK2e zIiKSDBXW+{e~ z(x3Y55~OyIX{d~PYXD+Af>7&dBme7 z3;r+im-hYr?zf7!9%bUvdSrN_ZF{?SUW_&h66-mG1>5htvvhgYI6Xztl%e|emXU5W zepar{=gdUqn*Db{O&wZVq7cBC$eEYnpP=BWhGK{M6R;j8I0qYspKhD8zCD2Jq)x0g zJXsXOJy-q`6+KEBK=T2`zS@rYB*m3=nBFO~VfiKq!yna6_un`g`j%dieSU_oGmoEY zJr&~#WXs+2`+O3{Bmk~@O({ACBb_)zJw0hi3}Z`6i*EVf#{XA56Vj{*FCg$Byo1u` zd!lggDP{nHVx+(kcfq<)>)*#oPW}%zyjg#3EwEvp6>PZa^M5I>icKBrR`2BeS@S?h zumxrs7rel9Thga3wjH?O_d=uK0qH+!jwde=Jx4YFWX- zwyDSHvgwoE`8x~LcbpP-I{x6bP^Dn!&|QdSfI6FZ_S?QilggsWLR`Fd!)lD z8yg|Oi&lv|3}UbG2w2c^O8lMegk_yP{qlehPX1QOFHSZ41+QBv7JP7&0^|Ao#L$}| zd0uB?;%?|uvi(LTK7~u--yu;=CCBC~!`!dNaPiG~b3LK<+^7Gj15}E`_<2;nAFYNf z5HPk1N*=tX137S(QJ^_=C2-_)dGA&Tja+sV+BlByyY1N|l|5?x#c|;7Pse2HX+i@~z-<5rzd zv_bU_llv{i0>Ia|aG!o1xy02Sr?z|5v3EcxAB`M51E-~#v%$W;^gwJ6?%hk~ihcnW zMFvs-`4f%W85M>CMx+xU)YrlMRo^U|T3D(l?r6=q`y6a+EiEdX`^w8w)&%x1c5;({ z)i)c>S#g#5K%2uPH7?jcqirlSG~%)W%)rsswF9zcM|t9&N?uYHjqk?q`83V>ADEL` z=|9#`oYPzAK^f3gl4!oqV`(_LI5YkVh~ONxBZu67oQmDF9x!8fIeVlw7P8d2b6R}C zoDl-GzU(x(RXgz>G;xIqod2$WtaWM$mBGl|_C0&+N&iSOs>DT&uX|7$p>X+r#2ndz4 z=3p<$yg^~d6pH#{2$Xu=P=bUNM~L)E}-Da;uFT#l{j zLSQG)$E#7dFCQ0XNcN}NL51X}0(YJ~X-!Smq@0OOC}Q@b@lwnf#vZ{^f;U9mAD{~HeYpgle*?{{)3b0JD} zb#-;<#Y=!eM)XMR;8tlT%LN~a43w6amxn>^wMrHrn%A7siCaMUU*QPc#;LRwwntJ- zVXpih5ep4O^c58sE4T0n2?b$hv=@HM--CnhL0g5mz!xh(or21_4toDB_%Lne!pL0#tQQ3=4~7x}Jqkm~+)+kKaS zAhfwxiw_*+w`g@brA5c7xn_X!ON?SN;c#Xu#`f9r27nmZnIKZ>*0EpWagV(VKNxc~L#U$r03cbnipLHQpBex|dl>miK8D42|Xrai%f z=Dq#x^)dGw6fo}{yZb8$8sOg;(2RUdK;9Un9*op6QcHj`(N1oIEVc>W9tt=h@U5Z3 zD=GJ7CyxSGo+bifkt6m+vklT+Uh^KANoRXoTcVbRR94u2nlu0JJO^6x{d@PeCoU0@ zF~UqP;@~A_7jzgV9iP`Bn)VZHg|mQe`5I)z(uL}{dj{_LoxI#%NJR^&_~u=Qm?#WR$*F2s{2}%38lMoHdRZmaw*6o z<^P9Ue8D!(#h0I}CL5Wl#Q90Gq{Wl|xa%&HlZ+N*@K`kn5eZ+odox;^_E$?wmPNlp ztPlM{a5j5i!F1b_#zOB|lAh_jG_?bXgZuDA5ZcGBmP1s!6ti8v@h^aT ztwiI>sg@H2^*ws`liJ#BI68YUXPO;~^+0u}rr+n6A^a!zDeSzsvUGsT;5eseX0E{- zw&@AhL;X!)VnsbqnFz69)PXx7D@QLX1e_Y%R6xBGu+bNAJG<+~yMEy`(f{#&AF$w8 zxT--mHQ!FAcwsgCQ%G3YJ1ndUo#LqdN?DM7{^{cbYwEZe#HJDn7W^4H6i!K#V$GcQ zNxLMc6Cb6L3pK33uOM9;FVWP>>LA}lTZLN3=SNw#lLrBjGc?1+76}<19UW&JwfF_9 z<>ixtxq+fkJRlpiB+VgoB1Vx=KzZf z1<|nITeXI7(M~qx7_Vf9q*e))yE4VuC~qiqQy_78d-aW`lcxB!f5lwp$B$-xU-r3& zf_p0PL%jeU0L;DTpbDrDzlmgOG;_bfkyeF^&Ll=q!2I3`u<|8fW+;v&T1v@)=sAqN zw;j!$VHCAWXt+Zb6heRPMXZMOlaGDL4TJmvObEq$1Rp{IaK|!R)VTy!oemHenH&mduL)qA@ZJk)cjm+$+r7Kig6W#GCYg}ZbWiDJk3lVjKhxO=C75h()S+(0kpBcV7i zZ$>RH8RYkFNvAO2GCv%i)$JD_JxOY$U>}=Z5Vi2{N+=EWbahQ?I(`7PpVM?${+ZE! zd#-tgOd;X-+GW1*zH>|Qae{y4S8+9mcLLR(s$gDcXD7@c@{M%6Sf@?$&o;d&UVZZr zm~o^Ir7>EYn6wwtNUkL&X6qJ}Jcu(|5+sBpB%PfiU_~V*h0n{d|2QJN_@dh7-sH#O z%_zD%e_y+P{R9Bj-(6ir$lea_v3hfn_Ri0=!!4!j5zhE_ZULolo3%Zr$s=LWfOn*` zQ!7swr_<4-Vdpr!fuPOnmva+wv94}MxswLl4VE&_{X-NOv2N8^i*Sk<5M~`iLsf!p z4!((+_m5-e#=)~m&JHA7V2rlw0>fqsbI)MWq!=`0(gEk)?yG-{Q{~iOk|ezT!}yL2 zCV}O94F^jjt>3-{xOm#H2i83!wJxXwjVK|J#3f{`^;-0o~FaS@^Ag*_& zC68R*c6K+O#nf=y;-su!%!hZ#RV*=GQQNm+G#TvGK8?@?8C)S>FI#X|`xm14W9?-w1cko1R;83_f zT3LS6C*?WEj3qM?jtP&Ib!l}u{g-(?h0HEPO&CTy!V>WcE)!mr3&ht z%t2lIfh&VJLv{(?jSe0S2c%G+zK6KU=NA6_+ACjsyt2wtc8RHFR4;@_UBW%GA7#}l zW_{&I^w@ju;nW<*a0gv|b==GCBZh~u%QUSs3KgENyFPuhglfx|@pG@o!&V4ZqhhEh zzgB4N&NFIC9e;99%;86(8F@1L<$!tMz(loIi56+LFUoTR4NX3W6R=8H-av6!!l|+i z^_S*Yd%=f_>ui<4=WnW(9C{bGnEDxQZguek+!Vn$H|bbd*g)9sK6w->%5FG~}~c+P-|SZgZ>k^x1*yaH)K(WjAik z=^0&K_~tC<-E_ch#P6r!ZvYuuh)`qtDLB>E8{VjZGOs96^OT0w!m+38o`13YNbZT% zpK_oXOIc1SwmHdi{oqvMBX?ii6k*_B6(a4LyUEUL8 z3o3eSuR)6rt{w`48{@uLZaFyhWF`&d48L~SRye9xM;}`gA>k;3qx1mTbGUdNfD}+{ zJ92dcF;QX<8U4+6(rbHwTRK!N-s8}v7Zhj{Q1p-bdG(~cnE*hCoMPWMP7b; z^{~Xk5Wkw!oDMWSq&59GEz5-jQ_6<~)Tutxat|tjSwEesuj?Yh*KNNuWBm-Enu;@} z>qPlLC(#~mkr}xXy1Ml)TI;eX_A5GBxS>$!F~i7zN>lSy)(I29i?HfI+jPvJ?jfqe z&|S|iOn=W%>j}KLs&ApoWjJ!z9eB*Dd)o1p*pZJdt$d|TQ6W&~t#WE?LuR@5Z{(P*bYm4a7{}CCI_N+iu$71dv&S3oK>pmgl-~XlKoi}>55fns{gBnVy$k)e*C=pk^#w`s|!tv4w zWQ!rFMEb&o-|3BN7gOE*m@$My6Ar%<6%!_J_{HnYa0!^Fc0zP{q&|rVJ$!E6x_`L% z!58xYUOk$=bjFI~8TL2u(@RC{2UZ%RS6rhP|Gd5VAj0wjRMM=TiD-aHdqDo4^sNFc z9S!_92cS#NWQ{1Q(@5k>{S>=wn+cqp-tK#}-FoB2>pVS`4Ns+U^+liCqB2R9S zyyTMhT_Jm9HAUEu2n!guQfcJP8FBzxUr)jv9HyJ}dP+)A;Ck#44WQkBHaaU{39*y? zaA2>reFnzpJ5i6{0^M@rLT8KU5)=wHwzj0S25VRfh0Dn8_x9Zt+k0xlE3rnZA<^;e zFu%Yhje1+KaN*|#ghk`}<0d05dU0m)3ZBga`+xx<#|59{rFI;PUKEwya}&nEa8YYw zhU&2~GU5cX9D`EFdg`%sC30<6nsSO(n@(!?#_THt$2s7if~i)fw~3J0VlTW>kEU`x zR8RBBvN_$h{n8B@e%$C!zs7r-_bFr++q|Q5^0=LJu#|N=pyNRS(6P^;mZ}ELE{=3` z4;1n6e7`s&;{xt-tkQjI%&gjk!{gj%bo5o&5;bVXaNrcQwMEh$xIpG3P)IC}offek zF{*DIYD{9m9lg4pjQeSB-i1sawBqCgp8#cR+aD8{FcOM(5q&n9P=rw`Ma|8jVzv|Q z!^72hJyNjIV3Bo^^3gCK&vF9z&z0Ag_7OWU`WiIvtN=aGxLrz6;>GbE*b(EUAU>mV zVNDKRVeFvmd?6aol+Io_$ z>h@-D#d+JTxs%J+F1?CqA@a_l{{JN!?!U6@I(sR4{I0Kf;wOn-7tgS%Y3^3ne&wpl zh9{Bp;x!ewsQYfj2c4O*z0OYgv4Oh;KQa5Jt@8-OFBi}}==3PD1Y%(4g}jg~XtKO; z7!kpb(ccjC*r0~)qo%u@=U^Ka9u5Wy?bl=U@2S8=cJFz&Tss}*De)W6G!DQRskT5f zLGE`Lr##n!h$y^KI57XGzYyhU@X=eR zTx;LJ5l#LO9KETqr&5g?a{wUHff<=xEZ^d<(~x#uOY1N$T&NRtAiKz!sNgUdA?AJ4 z*|A${4&!GgITK)_hR~(dyjHvvUbou;rDpQi<5a=X=L5rG5BUV;DGJVvJH`NoBOQI(9Tmo@>+W>+)o2#iK!d znJ=vH?MO$Hg$p3$VyIipw*&Td_F15HG0vhGuiocCYT`>1P}_m0jOl&79>%%T7caGp zTU|-+Q^Sq)8T~R7<}#qns2Q*@dn9b$^CjQe84}5m{@_iXA(XkgP2J+sC<)BmDi?04 z>A9uW)mOyt{d1FgU7FM@@S2%SOr`viL+!eLI&BYXMX$$Lt1j(1GI2Dzm)R1O5)~8Q z+NiHKZMva8qBhyWSF+q_fPiGpVbHi)87FU4ZYLb+ z51B7_gX+IRrx?dbr4mQPe?bS>m~0LHu8J>y5Fx)UE)W%n_8Xj}q?3l%xrQKn#AS>L zJIrUn@#=yg>ePES zpWQvm2w8|EGv^gr+h|Zl&jHRu1+xLmaeQK;V!$uuT~Uz~R#!0)sZ^2eq~5NsF?#qY zcVE-Q#>*1{vdXSy8L$7nbHlG~JAyNMUlrR(@Bl9@=jKy6ioE|y)#Y%s^B{T)@wab} z#9OMTZd?e@;fudgMvHaq+i)09PWrC(#C2GUJe~w3FZ#b|J2J-okEw4Ku{a#&smPqc zXl-_MOl$PUwh5SLyodG{u5-aQ6Za64GJCen(17=iH|`&G@z$Ki(E~S2HeGw1$5Fta z^zS4X)V;dt?xfw(LLbf9Ntd^6=z1d z+=l_>a#5`~6f52fKnZ2{xW0D>sg`_0DM-^ll=Zp*pyAo^p`0;6SHJvB0?B`Bvl%BOjs^u$aeBp z_Jwa8;ezB3`*Rd!*p;(ZCMIQz(QDC4-Cw;WH%w$LbiU921n1$PKgFe~dYhqpaKy9UJOv>N@N7^ye{9A$3xE|lf9D5_N=oW- z?X5{si@=gqnP%OakbZe<{C6?B(+Ow*+B-W7p;o9y4hs<@6EPXKAEeSkzHzV)vT8bi z9Y61(V&f&tZhYezI~$X-RXbnh7L{1rnAI|U%Jgu~%Fui7F42G3)^BBR@K0E}N_IP<22;s*3WowHPMMQ})Tok6f9TPMXri_yBI-yO<9EVPM(BpGfeb6H zI4BVB%{02Nl_*U95g9TwYY?qV(h3x*WV4mVx=i$E(n)Wo*^taZo0C2Cx;WEi)xoK^ zF`Dyd9lNQ`?Q5* zs9bB8$@r8Lf#?T~R(1!qUvz!TPHeDle?&Lr_! z`7mJztK`~kXOH_MA8&18(S_TuXW7csw4k)~@$BrZM_w8^nF50T2M%_36TAAv^nE5v zn39fo>(FVN`TYczDS8{Ny}!^|_>=!wGehzg&+Q#^t}*B-Xv-6D(t}(97C@@VnU-n% zlVCuDgNDo(<>auc9-z|E{r8QAxZ(VTepdVouamf$|LaVtT{>*6nx{7=L!}m`rYGJy zVGhsbiuVE&2ixS62Y+$7kHdvJn__55&fHr#b1`>=7cT-LVR~Bc=|A&0&&f2$0*x2w zMtjCTeL8_Q@$ii_{a88%+J#M+J*aV%=t~|???iR?8S*PUZ_Q&jX><=$Q~ANG#9?k5sr-poieImW zkOF{dNZe?p;m-0?%jn zM_(~f#UP`xN)8bgTMw#o#(z@+P<(W{U{nX z32I*Vz)ET3&KHGJuP0ySrlN-2-Sz*|qx;au(b{gYXbp^Vxnh_Z)OtBTt_(^0L;KSDdz3(%&h+RN6xT32x^Go`^TsY0YkD+t>X4}bY zf@Ev;8>J}OpnV~2BM#?l$tTt-p+5vb{1W^C$mvepuUTf@VD8sIpu&p5RkIm$!K0&tPo2eL&Qn(ZV z&Rs(hbli{uv7u~vd3gXk9}{Q?EXCwBcz&YnCyBQ#EG!O=jt0#ixG1;m&eY?^VNQ4l z%)i@=e+5qEFwTFT+&^@nQSwK#2x$e`g93~x;7-s)y#&Hbv_AkP}JvXK5My*cq?Pu8C6+g8sHkSPAw@=?uQ}%r3yxT5S zh&I@_$XaEjm8;a=wI!E#W2DbpiAenXZaI9-tx=RwrR?#nZ$?Z?zPI`ebqjBSR9iGTXt| zf~-aYVVJ1(b2MhWK!1pSA}@GReV2gI3&1bP4h{Ci$z6%vxN*}aO@M}^Hc)lk21yZ- z1K_oP#@&U@@d&$0ukXeZu5==nJEYo7!+7)_2wK7mL+puL4L~>xIqQHJe0gZ7@4Te+0oE}H6 zy&Z!rzWIcL0N7ufc;88P5<1ZyI%v$oa~ScrC}5>%Dt9fgOVO=EDE zdE4;ycQ=X=lNT&vc?1MLQS=UcI}G+sRzZPo|5*=m$7tjCB-l2z15jO(3k>r`5DFUr zkzm8`Ob}|sPl6$3SJLz^V-O-7i&Xf%c7vw1n;O8EkOxOtJ=fa5G}9)ygP19c&_JCA z^aXi8)?%_*_XP}M4No*gY(6c*n`+-&iPHyPh4{PF8=Nbnz$l212+%;kNS<4C?cWzi zMPz;%)N2-m>ke;Jbme{4qGuG-lgyy^ z<hYp*fR1RFC-z|7Nf1d8Nf&crJGt~AVL)-%4SeLok7@twYWv_U$GMY?&6=&tM{p6SgW~1emak(3 zGhVbd(5`JGD`Le!y73GO7S07z^;H8nN;R$;au;>AsL0=_T&_XId2 zQ+#`7aqBZlI*C!BtNxI6-~0#!Iv{POOP|je?*>h#NPd9NN=zj95|9rcuO6=S=Xea^ z2xKAz?1kjMb>_HK*WZAl>>^5z=iXjMX**ed?7ionp*h$2t$KkuUd_Nc@mvwD4aav@ zt9DJH;k>p2n%aiwlgfh^Q;?bSey~(Qm!arJBLA~&N%-~8++N86C|CA*c2G^##?Ad@ zK%`;wRj#5+*$RU4PQ2H)aV-GxL9jvG17wZjM?UQG%<;`zFe(R^Dc~*^q06*4apRYW z=bt*2u*mDb3f2z_@IjI$WT)BHh30CVyUPEr^&GkfyBb=)31;~eH+xbsW8BCos7BbD&oeQ5eQ z9v{~zaGyfz(Egu_scH#Zo}GU&>X9q6I7wAPY4nzpRkLH@WYCfaEPi@UjPDf*(QrP{ zSm1Br_xb{c&hE{2ynmZYzjZ0G1rnMK$6^Sy9mHA&m+C5vI|^#Jb2I&dLC6$jk&-8# zvVQOt*?*AhAI*NGzSJ4lv8lOvCxBlfzQ!hqKy!pfVR;b9IB`2-#lxWhMOIp|ZA@Zu zcj^xd=aZ8fx zGs5Qg#&}btf)P;_YU-+`T@)BtTea>Xc4FMzPXLt^3iATQH4fFo>SeV?>>rG7wQ4he zuL}hkffqpylcKWKd3niVbZ~SO*`;1{H=auONEIDK7|N;-k;!VuxZ8&-(wsc6APc~Y zFFpmm{_SluS{9EhR5snt%Z}nCwMQwU0L%s(ZHi0Z717wq!p43!HbXj&x=#DNbHtAoTD49MA1Gn zvSrbp>kL~b61sYdg%>in(rF$4t7Iw(`>yOsmccV8jRp0N?8ustbwZ&yNquo#iC5iq z$?V=P?{S9aMcaXc!%jbH{3O!f+&C7>C47igxbo5P*hL!Yw28c)o2)TP?Z(-Um4Q6Y zh&b+lDj1+NJ;Be|o*i-gkUuhB&X`3}@tn6nKKDxX$n7W}4T47bSI)NbCxATOacgbc zxzn!E3OcxhP~kvzM1(=a9}1|$eFqQG^bsAk)LGpe?;fD%(2%oXgX={)20-BE4iycA zDOJ=y`ExzHKA9bS#1%`4*Oy8JZXoB??D}IU)yxiC3WtkE&ej?jnW`@Q>0k2+%u^BB zdoLZYEDy)1U8F4!s$TvTJsBNtz_9t_H_nPz1ygmL4BFp1SN%n%gd6+Co6F(3yRAaG zeXWznlH6y)Y1x5YG)dx2RpMZn4RpPqzfzy})0v8T?=zpF57n5(XXV`WbClV{V-P^wXzL7RJ~qwO;#9 zGxBL~pumlz1%%XLt~-$2NZ7%l#v^f!a3jH15sT)5QzdgNC1s_Z`T+gtvj5K^VK%mh zSY9&KeEKNnktO^Bet*nP^F)ORF3cUi09Vg-hf}%t>L^Mf!8kVXkL^|*zPLSo!orM_ zJ?g_pJhmUY^l8x%k`6NT4z*)&%`G&j_$7SMX#=*t&CMBe?smc6DQxC>V17m(ba$CyfkyQsy;eD=XG9eNQujNP%_Qr-Nxd zm`64Vi3lu{R_kWCs~~<18fn_Vnsp7$5SblPC2Nnu$L6}&RL^p5#35PLE&W8IilXR8 znt^Y_I10%%VEL=y8s!UE+=Cehm8cL*%+0m_*`hGG~8M#kN};SpFqTHe^gQdn@3?7mds zP71qRs07Z-y)mp=o^f>^;0zlZ8#^{LH6^QD zKaVn6ar0)BPFjS7nt?2g6UgKGb`+tt-+ZJY3%gwm>dn;JS`WY}Bw>?@c@+KEP(f{C zfJ$^0dK4XzNXiCeI_%Y1Uy3@M^EZkak6nUWqz?y0|J_Xi2gf~JOb}*iLJ}EVgSP%Y zDkXjl*ufds4{a}AY+kPfl#9@0K#e7ZJXXG?pRArfySz>R1bukiO_`+hEU==t{OK)f z{Ts>m8ZzIdzgF=&zJI4e*88d8Aoip)bxyyx2<*;~u$Pfqd#}(P6K;>*uyu=)mhD^S zurUxIEKu!A@S!I^L#G;o4-t9SdUs}|?EnCA*Ujsba;JAfee19^)d&u~a6kZ!F)38h z`jWZ>VvFvT=7mwhA#-lyxM*nS*>mUkbt<-@AzO0-W$nn%Bmh7ZWZU8>Qx>!*DT(cvOJPKiq+Q%*a%}r?xA9Bafp(H;1ClbLtmbL94maW zLk@Q?DJ#LdzDJZUN(Z@95B%AGj�>i&(9JUV*8#Q4t5<0aV{)-W6{CTPTitV(Xye zi-Af2hLKgc*`VDrZ_FcY6J(#OH2_@YjD6l5COkPx8IR+M6v8-nMnfMLXlOuVle0Kc z2_*V!{B<4%P9?9N={JoDfTC!A-Mi6s*<@BhXwSis_OTd~`0=yd6Uh_Z>;1Si+S=k^ z`&a&J=J-)kt?IVe5pEIzI9dGBm+io=;gC?bE{X-c!l(N*QYQXh%DG#?C&m2tAMz5N zDi~#>m8|2dbDP}6`)mLFFNG$XH=MLv>H~#IJO9)S<*(jLsCY zktevy>9=kr@wk`?zp5Q|zknpQBqmUj=7jtfP`(1Mud2D$>(vj_Cn z?fv~E3ko5=fKx?CY1Y@*XAy1)?e^?&M%&n*Kg0=PfpO4@oJruu)gpMIP%4 zrg2$t6GWJcbFKKFs2{rj8h|HAk0zR1vFRUL3=5u2{H{<7t~v_!X+TB`HQ_yn9Izgp$q?BVy9f#XX*2#K=9Q68q`A|qcP`Fv(&*;D zdSZWH>h50-Tw0(<6s1ZI@Y#$wz3SxBOURwK5WaC0IbGi8&R{=L;G);XY9x^_PzIuw zgq8@{*kiPGP-$Zf87J1dD70|zEo)etz5^F7LM1GZmH0*}nH7Krw8mFnoLB|ix`0TD z0Htw;Z0QA{g$G5x^iu{dyJ@c!~{ydFE+5*;;bny8(m9v)YJ#%!pfis zCHMr?qkDOIY2kmxk|5@-`Uc;2IqZLNasWZOOwbGERNa>Ck^n#v%v*ww2}z>PblMrS zTc7};%p5p;2^|Yemqml_Tg)YXq&O>*gw-j7se{{X z{TO9v#*~s^R|B>{k zd#(S%-d)}#Hii$#7$UFBh$6kAziY#ErmmGU4PA0_S*Dbls)<+i^4govdMC|vZnxd| z#vQyAFyQ7U+x6_g;~y4vDPMQr1KBzA^J*8k7UmmTy0qo7$n6x_^1&mWIqA!mi13a7 z39m3q8x;ueXqJsbMriM(cK6u&+1JMRr4zj)?7x8licTzOeS3|=yig%4-$?!F)^b*$ zFlavs5l-~d$(r9A#{;%uL@Y4at+2fVg_-=)%JPSqO}Oci7o%Fxl3;Iw*uHg_(qGfm z&r8VBN)F0T=ux76DZg1eKIQ6sGVZ>Fi}rmK^v!O33pRu2Pje-pJ-a$O29NR&i<@(;T>}`sOQ=r?NIyFeAAebb)p@G zBq27S_xIQnT2gPODS%n5kP+B)`(L-%uU_>T1p|EhbAK~R|^Ya7Y!n2d_<5MMlY`NED!B^0p zWw(b0M~)*S!d6OH$AZ#%+3(E)JRSn_XQh~BVOrZKCbs*ZzjDhK<@^8}tM2~(a?}$j z-L)0tSbg51Rs`;(UG>wKEpA-;FUeXl^W_u1%@henE_vGjEb(fS@VVc;r#1VI8L(?* zUyU)Bt^Pt+?oZNDMTQ~`kd8``w!rXb4v{8&dqq~VW$8&$uA8oofol3CQY1E=bPZ&? z4RZmka0fU!hw`WuyVY;_NK2W~2MkN=rBFvbf3Ny6!zb@Dkv z<%L8=#!MF~WQMNM%kLOmci+~YyUwxVE+RvJ3X9U(9`d2T)wCHhft?;#NuC{_&ct_f zIF(n~iU)m3jRl$NnTcXg&8A}=k(7`yDpH~<_;i1RtR5Pj;UE4R0Q=pte8YT?{bH|9 z!~6AD5Bb@DS96TDqs20l&RYFl&z3oFcL6jp8kLfQ!(qX0Kdvf>aWYYq2Hx}(w5190 zB)(iJKBLJup&o5xp2eAu-S?9oIw^a;!%b{}FNf_be8o7@Id6XOz%dG`S0Ut@Cm{ha zu}J${6pC5`b%(%tp}0C9DOMlfK5^-kY*3vOJwAm+2X?+IS&&rrp^1Fsk*YPKQZI@` zftdgnovDs^#m@C>a&VWZ87OX{6yJByCzY+$^^9Zt(80m+p2!o0-d3_$31*K!c`^Q! z|8??eSa)GQJRxVASAr2}lj9KDQDai@sh|L^Ag{z-ZEu=M?7Xip{{&ywb0pOM`z#zd z_%cG}CJQ6=-j#&!rgHe1d?a@u;Jvx-v>snD2w=C`?_qm+o927nOS&idU!J306jvqt zMf<`jD$c%*J3c6TX8ffapiW4=mQ4dt)8|2cr?qJYapP;u(vA36|8NW@2aa@1C>OA!a%E+ zT>YWGfMk;S+BLocT(ZD^n6)2?c9W5hF4`lBWUMM^A92d>^SKQ9g+-FSMkK!fqp)loX z6AqCGZ}%<9OEPW^nW<6tJ)5t;ly9N48#~P=`jjN)b_}fgmz3*TYPS*qRlfb8v2bZ% zK0*X|Iz|ypOg%RF#po)k0K1XPy)};<0m!j6{;=eJxP{(2TA5S-O8VAHwYSS<@I6e} zW$_tm>`}A*7}j8;neqgxpU$b`$4_M6tE4JQrCgHPP&J|fR;Ycgs@`kD3I%^y2`sqD zu=!W`p5UJuAFvxVPp*W|lz-?M*%CeKk-x!R7LaLE&Te2wpX-ko#&hRe z@S2EEjV(NM#*l~7Yn;z|NNIGk!Uim$eF|` zO!->V^kl<7`R^p;zw&*#Ybyl)$$v$0 zN+W;G?@A+A;2$F;0T$zQ1z#Czjb;{GB-dg2Fqr814Et6jFkQDQ+eR>_e0$cCK92yu{;~04QP07EK z|5<-C&)8rGQkjbSe&4LvS=(^9CTGWh zPino7s^Jc;;!5fbWrn$j|Fqq|mA2p{nqAwHVy?^T`C=D4P#P)a)}n%Hk`PQc zHAE7EyO>359*&_EVd%VDvju_op<)q8m}8y!7q_E*7yvgA;9Gq;OGO9SZWis?Lk@#Y zTB6qri_f#D@M+y6z(;U zS;(Wf0vq@ef==j>T!Ba&Q|AY~gCAdp?iNXHEZ^UVD-aJ0?pR@v@QQIh1`s&tLKFcj zldk0v)~2(F9WQhe{v^0d{^qj&6Z2}frX0Ff}s$U3i@`g&h5ePp^1QJunnp*f`l zf$4gJ1po`3oz)M&b2A7r!aA3jEVbHmSd zI4mFPR|d)R_T}%K2>2JRV@THhO{a(SiW1iwd3*y=S+2P&yv9CKe+&9Pm3u9EsHr;Rku`pTcQ)Nkn9jgK>1a0GC}r;lJ&s~Q=$Kw+Zx zOjqM9qo?>6(jvwDqgHf{vTa$4q}*jm2otP214#K~yBd!&$tAe`b0z-XfBr2s2s z(l`y%2-O~uZRju2dg!+y(!Jj^kZe@;PtrU~lr!^CFIE{rFGq+6mrL80r#Se+>`23>{ zL2ub2F~@{!26eJR9=Mg|YG-O7l&_!4@fz|I(Xvkho=X9K;p00+8P~NN94}%(&1Zbo z(EzR4Dgg2g) zJK*Tb19?~g*}z8QhVCi_9FcvVA~;nnqE~=QGqbZv|46Vt5(t5%R59OymhPirJ|t1* z&q1{i>Yv~Xkey&gCRac!@6Kmcfd3#XQ2^!uyp&|V0TR;&Ovq3z09@r+fd+E3H{639 z;>MrIve)qMsM!x6zYxg%432r^_w>M`(+%}3MNj9A(32@J)xdGQ2R1w@pc7zpV0ZQe zZ!iv1G~J#%b89;w*%+xvMUInoxA7g@R~~BY-eG$WHf|VXKw&*4MNlpvqMdNiY0Qpw2-NC; zSS7>7pw0N*+3Dh2-;$q4GzkzKqhVfiINt^762PHc(5n-bJn_jvs>1c$5mSnx>#lf0 z&rX3?evp^f8`2w+B|Zt+ax6zF3>&;~RS~`N*ciyzxkMY-&amHf(Wv(Df_6ba^dr-B zKYVrCpa40QQ`SQDL5B4n0^b7Y5iAw@D1bHc%a0a30WuLHCs9o{II$mDdN6$eK-U4W zi`a-i5et%YUA{e|5Oo1D-3uWBP>mF1{Q>kV1iAtV|0mU6B`g(W@T?+12UWe4N`JnW zRaw%Nj}Sh6x^k6OM6S+pwX)J@a2v>_YtBa<@T@bg3g@UjE_8$Rn<$ImC2>J)wa%2r}+*oV>gMDT$vS2#XZ925NR&(CCrL>u~vWWRK>h3R%IlKt2``*I{fK$W!n?BpZb|cp9#&rw7U2ss?XDEGJ+b z1<|epF$xX5a{){V-XR7U1F5?`2FX3K=tE3^L%S+MJaqNCO(d~`?8-@AP^Dd5itbt784)VVlZShFgQRo%MqpeZF%Cb{{9B_jZ{+% zFDGjUbYUEvBE+K)5U4d>{&vk9i0quM|HVhl5DsV>Wb+9R&!atqf~uaA;oFgp(g9lB z31ylw6SgV8yVqO1x%Tb07$5hOld03=$o^D})TF0vZx>UVelVqZGMOC-uOug@Yz+j| zIr+a2olBBZDuTAX*Tu?C`Z{LBXjHU+WWci^_7G(Fq-dAZxgWT!0L<(e4#>+uu&hQ~ zpFxpCvZetvk@u6Vew=U8Wm(94IYuFq7jyZO z@but}-MDdCu>%l!85}*F5kF}VG!2Znr`rUXG%&}mdI5a{Y_z-YL88KkQl%BoId6V| zBeWYp^vd|v6R?;5K-vS`4-Cyjee3{%6kz~yv3>nr?sSjnq+%}aAcQmtwd(zd zO?Mx+?R}dmqMVm;{yS-zX9ww`uCowpf@SD>oyiH++K{gLkWAL)d3%ur$3nXV?KOSr z8&qu$M0aTPJ}yUUi&V*goJh+M%2T3J)~4YE3xF%F;GJF&AB0*0c9OKei``WUtjop_(uFgWoSnuBnB?Nlik*aZE{3&+y zyA}R&k!*F?n#}Ki4p68~hh7Df*eKxOhcNh^3`;&t-2_iZ>4%T)|3*vu@gSS#W*n9* z8DZVUSqljan|D)JZJQRC6oV zrf(zXR}38(fpQI5sOb(zDqYU8~WY#31m8c)|)b*gLyrUBn(A-gf9l2JL;9+ z0|ONhlvNA5z{Tb1;<5&dx~A`5 zANE$-o0Z|2)95i|P$Sv{jIlcIC=J|Ph#go5+J}gZ5H5uyNsoZIZTW4#-%S3w{pH2| zhH+?+gY-_2)DQ!P?G)yxTVc|M^mMbv-9OC``5fD|0bVo;_|$xS5+wIIK=tNa+S@>f^oohHrZ3;o^?Ts$SCFhKm&mL1mD)pf}lyo0g;s8Cj#ur|Jz< zP4q2ud3sT%XE^4ZlH=^#hD^^;dV2eYSEh4|ik&{-$l_J{Z_nOwt(MfVv@L_SaX<=A!@>xI0NFsIjOsl;ohC=EcX-o?UKYyon zGlu&5Bx)6xl!WBV56glq@IIDdjT8$oR5QanycC~9%mEMKnym~0d2yFVtG+7MnJ1oc zzoDKjdOS`q!nP+dZeCSI1rc|GwL^Ho2XLchTOXvL__FW3)XA$~b1bf%)eanFo7kIp z=9eyp3@SEWS$8PB&9{GvInKgqFx8vQzPz*lbL~>3#_yg~hwT1i%i-xE+Q-Jrlh}W- zY%2*=jqmIa>F^9#@z#V&${%J_y7Fd?MffT??v>PMvzW{xZMBe$$eS25GxP4c&xq#^ z;+kR;tQb?b#Kh!>uC8X74+)mgy!H+r5Q!NA4b{U7LgVVyZ>m}o8fQJ3mrhj9AvCyp zK;dNn)QifCEm>l6iiGjOT0rjnL)g9J?DG-C0#;QAY9!JCdYN(Kbwy)WT8kI|#gs-q zI7G=y@i?m$8^RiQ+^6@*=AGLDcVeC81Z#YG-O2 zib|{J>kNFvg@qVxy8F$(adwn)Y~Di05UQVC9kRbs$d=0o?emqy7Xhl?=kERAF(u-g z!(;D7J=JFgMvoGR<(*hbM`Zv_w-{y$c}BbmgY;hdu75{%05<0uh3`k?WzrU1c(46a zj>3#FhFqbLQXYh`(F-J)-9IRA2l! zF)sKWx^>whvyIckzMQ-9JR5(&NWM(%8+&MBs0>s>n3f_hCGVv)yvi+UsAeGb>{#r_ z$j#4Rc0xA8edd6VA>*^L=1?4bx2NydT={qK1!v5#UYx$kZGZE>Gm-qpAJd(Ys`Kfv z_WS1gf@rF7gdz4O5Qhy)vh%R6{~xm6I;zUG>lX$C3_=tMDMbVkB&ACf6a+!RO?OIn zw@8$3eq4Y(kwd9yzxEnIOF@CF?{~m+YKwOdtP(?YC-`-oD)T||MR{ej?;)BBp?7C+~B?KMqL(nO)VAxZlCb;m|)Mt8p)*BMjxb+eO5LeqF+6% z*e3tHCGaSKG7X^>^_uYhXR0hJDnf+yNbn#SH}H_Y z9Xea1v11rP92b8@(~G(PT4tB(wQWr8gcV6zXB;3DC)qTkdWAdU(M^9s(yySnfLb=X z`P$9Ac@l;F>W6{+42GXU7#%`!Z*bfF#F* z?%|jD!Kll4{oF#%94-BZaL6y^mL(|y>{#0d9c95+*n(B&+8*mzq~6t?j6qMs#R+y+ z=!&jYiD)*m3R|dN7t3Cn)o4MpCWIMRc~>BJ#!i(?a#|mR z{z$qzLgPc7Nt2C9fjUHUvO>Aw|7^p@GGefQO@YL$S*r)oZL`|5e>4l2#;3VKx}O1?n@y8 z%Wqc@^8UIwYjWc33JcSaJ*G#;N$y*@LI?tO)1M{i^DkJz2i3bBfIzsKMdTHL0Cv01 zZ0mN-xDwOx)&FNiG4mQ~9MRR&o6zngLX58<{(v;d-E6@oIP0(>JjLyBrvj03gWpfN z%2ppjplBn&kVk~ zFW&je7VU(lBdd0}zD|1T_J%)C8&rP;U1&czNK#kY}|#e2TR(5%7~4 zNHv*-!AuO?d1j(~aIqrxEjV!Cs*jWAU5AeTJIF<@&s?Lt{S4{)U{-?LQ^e9f-BNl* z(w-C9pc#CYFeJ3R;e~q6HR&DQfxlHB*EXuBIlv%WSH5`8B1)_LngG>A1jUS7wX8aA z1$+Fo)IZ~-x(yg7Z@xUp{@1Uq)XzMIy>8NBYZ|u>)VQkXixl<>?lMv%;|%FRt^

z zQ;MgQ;`LZs8bLjfK>~C%Ww25U$pVA90m-2Rb=KlgCELT+SK)Uc3Jd;AuydUS`+^^| zu2;Yz0?#dsgNP0uq9BpF5$1LA>YWx4Xk{`AX2xp2-=>G1m zvG8NCPa$0k(kp^*MKa{_t>CCA7BGQ8*kLdDxf*tUUqmu{!@K~Y>Vk@41`jew=@B&S z?c29R0)#~%e|a29JBc_Ru3yKC#k>Y!5kL7F6PLTH(g~-BTnna*mJI{gs4X9Bsstykj=(Mu6Re7``Sw zUzgz*)5G*Y_Pf4ztrq59t4{GekT*^%AU`9|Ow~AfkF#s=v8&|+3ZvWWc$W?IB%>M1D%J`-Q#x&*UHg zbbzY1fOiT6Xld!`<-#A?-)&#tEr$M6T+J@rn2<+kSL0Ird$Vwmt<1#K` z?|iRhqSalLej-yk8M-mOL}~(c#G3<;2RE$W0>t_O;Im+11Cw3Yt@}q7K&s4It!z4h zIDS7ZT!v(|l~Nb)8TkFQwAo>ZM+XLd&8Xctr z4h-50L|x(ofhW*`K;V{0zVcNJ1&Qtj)723#GjZ;*TrF{Can*~uq*s42NsDu;&Wh`v zI3BXF37N;2VO>I?sl%?QQ@)1&Mp5?3D#SK^#*PB=y zq|n1KYziL;e~%m!OK4aa#2!)uJa`%pkYDI2Ln0#hY;YZH#Yr?KleL4 zFiXRzU)KiT*nUb_@9~dgWWLP#rHkIyuF(U#eAcV+)!t}f577JpV-{=> zrtZ|bw%+HNH@&__I!3wBKu-ED#t5Mk!PHM~}dNX6v;10(=fs z0@kzuxAjcLF}iim^SG`A`OE8AGzfp|M1M^cqxoJeJvsVb-u_X}p7E};C*jww5WVq1 zSJa*#?)=C$#(ivI;57&F1Kjm@?U^l^0o0?Dz53fJL8Kd*#FZw<}M z5F5*gMMkUMglMi^&P=J4RMzUS+nogOOe@Awx5H05inS5dq^N9TKCcyu=ro7Uhf$z1 z!nQ{bkm7_vFAq0A#?KWX?wHK9VA=NrIW)Xh6!&g3w3OK`{bYY*iQ9ynrG^N%97RJ< zQEa2Pg1Sq1v##Z&!Glk&^I7e+j^x>+7GvzIB);#)bJd}oRx=9Ui1=R4IFfP}k8@O2 z7ki2fbH!(v3t93BQTwA#ZJ$&M5s=7%FYYRUc8@_|yoSR;{I4&F6;}ZnD+dEsT_h-w z462DA8~oumJN=l0Ii%^|=Ic`9OUxpSH8&T2}&iV;jfXDvH}5 z7Iwsv`vfpqOv&CL|9hI3;OErGm8kXM?`kM_#|PI==U5vYSjAJ1h`u;|#5CaAa-gtU zrvFQMgysdu`0tj@B)>k?oj)s*Aagsra^(4{kY(hUTo24r1VUDeDTmkN5g7MXbB;xvW<%ffwvLgQ(Yuz#A5P$ zd9~`+&~{_{usdR6&Umgqqmt@-%`1+fCy^I6Hm46WDyawFU3QK|CG`28Ur<=@T1PWw z8>=xqyCBwcvG%+U&oB7zR70zD9BdP;-weQ#ts;t@{KwH4X3rXfCOMqjpuhEN5QsQ>z5@z;OJ!=RIp;)pnMQ!y{3Ty zfzC`#rIHq=?6qG7=RQBy<}Osd^E|vKh%d+b<(pPa)@`>>7G^{JcY6$R&X;17mQ{2e zb}q0K(dw8Exi=^%S$bBawBHh9hsBa5%H7yAtw0%gD zVDU{^l3^GA$RJb8Im@(N+rj>ya;x(bwoi6EG}GimKjG~1!qaJFrI?A?bvZ?jmwpD1 z>0iC8z#P!MrmMenHEw}!N# ze7waEpV}{P;R!i_MWdb@-yb6aX+Rnwb>45(8UixD?D`i%bsjj{1=F-!!Uvc@5l+=s zP)1G;4PQe~S+LCan1lppE9cP6%w0Vt)~be4ugDyVWyaP?@ek+pUM}3JbJX>!6RjZ9 zD7bQMv2);q2Ci|1baiHL${LD#7z^bhBjeu-P-wt^93?8ldUk9wQ}0Nxoy#mCA(sTL zdf#^PAmSz?quWZxYtm`r+pDMG=Ojx$3C^A3Xqo|X(6zfN?q?pL9>eta0N>%BJ9$6A5PSxX^I2Cto`uwZy->(6g- zmV&Ey**)>nmNj*s*KiPN68ODKXoDb+_fv%{BqcLd`-^D=GyXd^E01c%ZZhtEC8i;e z#sYFueCSYV?uH^q9ttFd)6h^>Q4z_y2UW-9{Cr*`Xp!2e10^3l`dS%(77v6At{xV0 zDPOL$-t~u1&1h46yV)G?)}KbT6}?ijq+^xB>mtnn(# zIDeRG|2Un;?ai|bY4zhx0;K+(Mqxk7BJ_SKtq#=DLnRWEb55JhCtem9qu9n7bbPb* ztgirAV?D}JfJlo&)q1+5C1RE{>*L2Yz`gJ$z*`A9riGysZ^3)lh%diMLA4(zA~$~etA8U)I#CoF{wX*C65V7$LS4z&`UZ9WMv?DNa3%0e!&6e$LP{+dDOczYh8AO|A%_x3Jdx`fvR4!o1-GvI5g zAvv-bt8Xhvi=25xBmg)Jv2UV|dkB09VMkV7gO>d8vmQe?E-NE*T}Y_S0Ahrmx+Q>jGQhpw%5%_ z<%}8i@_>CtBZ}V2!aM-XD-!%uD~}k*eLH72}{t-Zh2FzhNMMGpGcoZ#mmz+fq-?vS}u1Vx9w&RRBNOjYAJRHc2*~cFT zb{?#x`#9IRn|{`IF_FVBnVOE54?1%yj&y6L@5bc4LUIEuE=(i84?+Hsl}`Q;K7lu} zIDWqg9COB7=y`*kib6|4drV<}y{wz~wQnyp6~cmN%IqWdvaMM~VQXoR*fuRdN{qK| z-I9~!;gR1Ws#F6lfGf<3V34~4&X*g^%z=Q&0~(KPW&m4;9`uVyl11?c2)7&|59({p zXWQcTz{&Qxb90i8tqi>OpouRROm4?os8kEGF)YB+zp-YUAj6Trrc+AQ$K?@qKMKVk zYN&pXT6u4M7_{2QHd~pJ{`S~%^^S=d6B!w+`Y8B#vTF9 zKr|ssYGK~R_K&Vhd$G|Ka$O>e9&r|8O@BNmRkHibxX(S{je$HJg@XTjGX={P47EaH z^J58y*;rkY*g@4#8BaTL<6{x4xPBE)=}7&i8kGK$=Io}h_T-V-#Jr&u6RZ?Ghp^a!mZlU(!3%C^{EmRY@zZxAb&wq2>HOh~{`-y8Mvw~{u01Rg?HM}~3Q@A-;bX{{ZN9JX4;G?|!F1j{reE?p&nuKYLL1&9_2 zxae7xq6d%Jh@c*!3dUSASY}t4ghtIY7{0x~T)g=*7-jl-<;o=tNhoW|!=npQoI4zEyt@FW2);gp*zQhXjT3u(@RNSy7zXb>}gDS(S2@QR%G_oN?4|9H4QOjgw@tncszNZUx3kmvPe6!E&rI5b9%G&i?nNGCaRv>>XU7!%o%Zo(e z@PFc6GXY`}s+;1gXxZSMP-FdoblHAJQ;w)_UdEqn% z2jBV~Lnq3^_N7oAZ}rEh(+{A-Pq7ZO5kZbn*)bzO4K+3OptqL(`>O%HJXR?7VG8q9dB$q&g^QW z$!ATp+t#>l!>Fz9!rtewCs5pyFLrHWA^tGF!oY93ldD!BZFEwneDK|z>9?%LlQz12 z&$~wm#&1Uv`acVpb-SfJl@19~rI6qeHF&Y~BXmKAt$r~e66d`ddyp@XYddw$KIb8ercp*;XE=F;%f4_CiEFxqyQnZUEfy2i z5+nEP%^}skj9kV+M^WYw?(&<0w&Kkd^wSF%!o!r_yaw+-a~Id<^h`*Dh5qQ^Rs+3x z-*t=i&mor{8%L{Tk1UNKeeAT#zUtgi_=*}E@&=94jfqd^^kHT?Rks80K7Zfxl=u z(M9u)ndD!}T;(EK{DV(A`X~vy{8G~Ywe$&{#>n^OEoA$>(Anj%%#)vqwzfeeSVlY% z+WNsvy@O-$0^z`MiiADEwTck>_wJEPNJwaWw#2{&BpC(%aR7V74)dUW`7$ENs^t4EU0)r^YS+YcK@V3tT}oj+4G#FXS*>{i~? z$w0uVR1U?NU0`w!osE5U?cTm#UhaH3?xg8h7Ku z=3n?MiU}=ALUT=t%LvZt;Lg{;%OB}D0SdAXOIAeTr&ZCBuC5skRyqH; zP4|W{40{>$1|Ksj`A$DU zPry>!Ie?n7foU($s*ZuclavhD_|AdYt&r6>OO*qz5ZzXW;|YjFp5Ap+s5Qg+BwU-S zv3CXVY)D0HHm#*HS3{vSVRhHy@RzlSTy0hik`IbF?P#SHiQ2!%+|<@LWGme|n9h~Y zlBH!=yc_>Yzw>&*ZaMN|LWQ9L#%I)(S%LrC>5NMi z=)J^FJt*|vdxx(y3k^qkJjDyDXQ5s3i-ng27DHN+#|Z@2XdoV=Q37P6Ev?B1c()%Q z4<4DX;rEd6wCd3fj$6K8ulgqoR|vgpZh+5*%(uLh6((s+WNc}+lY|ZF*x%7$`loDa zGzBHXrTDX3{u@%;8nz$v*)BfmFF64S-53rXlSqApr=eG7^+|}dc>Jb#HY`M@8D7R| zc(!0i6kx#E-dz~ivT|}v^O~O}`okzxp{XZHGn^nFGCRbChew4nXD+tO(Cg>tyDdG6)2tW_89K*-zdKp}+&yA=E zbPt@v33IHVDfnf%y5jT07={(NDyL76WjO21kNHLs@}prLLL-Jc_5B|K<140yPdS`B zBj~iwIuIraTzv8~kg!n|dEnPuO8-a!-kD>zbm;vQuKef+oWsQafR7kZ5dbiLgO87| z52vK8>;}j}>7b?K-yib{58!quf00MRqIh=%OEZ2H*_q|XFfoOpM5pl1MqV2bK4grM zaI{g%fHL&_Lq(&R*Ib?&GQ^IPr1sp5mS5B*zy)TQktX=l4#ox%f4L>q^0s2xUyI zzQ?P5E#2t}u632d_0NhM8S8|-aUgyZ#0F3aJKiPA0u7Ho`NSZo5-pN{wX=DK;@T-p5MQJ2Zx7GgGvWH8_I#O3JqXF2PQrJ zcSySyklAUxdK5*NTrrz#HVPa7jTlXDV*r6$D7g`Lh?ywA4IxdG4o{(n$1k^P4=;ks zf`UONJQ@iE$T$P7!id#upH^xf;X`)}pZISK^Xf!VSXnMSH_4t!>c zSyYDVpJfYGakb?j{a|BXy3aii&qkc)vI6eYP^xAYmu(G^_`%wxD+9EcjWj^xT+2;w z!Ud|9#Ga|X%~9Znu2b2kuksOgO)mZgB?)6CYev0sild09Ziy zR?M7g`ldMqeUs}vPmG}wG875o1n_(z@#1gXB=#S8&KNW7eROzAv1{DSU6t|v=(qdK zq~~*B8VHM>;YR!F>I99C2^0@3ka#tmQMAG`1A2z$BKq1LF929&fPcf0qyW zuG8VWE`dc}a1CO8j$$!oza`v23_TB70f@YMxf6P5@^dtiE#JI;2g;|O)uxFz{GP-Q z1%q6PZXw=yW`_jG1|tm#TXDbT%mUM&I@@7n0SWp|e<^uvDDM3w^vZ#50$#x1a;G-< z@8h44PsRFSkv^P}4XOp4@KSK+0Epp^c0ia$2ylmNwJod>tf<4Fh|qdpzg{8QH)peB zCFyfG`dQ6?g^&;GQ{Qh#MuU$XaBfAj@1E9p!BfbRWz6O7e8$FWTLi}O)_L|>`)1Vf zD{O4nfp3LNqzKq_1I%c{`cg9IZip20Ya2wAoSfsGb%i5PD zjh#}6uEglxqPCHajR?JPBMHCN3i2hf2p#@?Gj)1yG>BHhUehU~7Nvt6%2I`*E-ug! zwDs{W$OVIt1KBkG9+}f5%T_FD#h9(z0g3peaXul+qS(YwhHT)iCl}jSRE~0eS@!0i z*Y^m#N(1s5>F?4C#2RIxzYxOF7&)>*22`dV>cHkk$`x)aii%?O(45*I805LzRs2jt z05QyrK#>F4F}gYe|4&9q`50O=n>%Uno4waYGaNS3>q2--S+h596teMp7_mLsv1n8# z^P4cxk*B-X-FW<*vOGBinj@b^k*1`<=$4$69!b_qw0T@UT(oLtIx9PuM|SIAC}xbA z10TL1n{&F8h)(Iw&l_zS^g}#~QfI8+>m&U$9;5<$qaAXdrUvR$f%3ytBnf#56ZX`e zt5Y}V)33;eqUTi*?6}_a;n<(SbDW3mm!c&~hpob^KM<5Z{`~%huN$x3&jy_1v5{C~ zW*ACPn?y$ORrTS-D(UPdhmB;Ak9w*s%T3#iqZ>CF?Vo-w zwj`nzyoBa*g)Da1XyTMT2Vqd5fLor2iVG=yHmq6jl5}LAbWVI1iuOFf>G!$YNpPA( zylnl}{~tRff1i~dzV>lvtCZwku|dP%t(K2Q?p`&1HjIjzX^RwDU(t&Vww~#3^$UdG zW}jQ|LX(js=p>Z9hGVo#UOctqocR?VbW5+>WYM1N!w;i~(G!s(%M-wXqODPVl@_Wn z#y^U`qso~8apT~ay8u1Eh!zJWYSftqGFPw>i)TXv>BEA_*>gBpaI0NQ*&b=9ez4NC zQ=(Iq^IpSrCzbWqUQoO4@DRU2%}2D)fuQnJm)z*zsf8JR{W8lZ)%KO@wkEY6Sd*aT zEANu6sbCJPON?30>Y0vM6i~+qy!YbMSyVZhGpEx5N9%S*;}vx?!`wj#T9>*lMjvA; z0wCc{0osNA**vwJ&;KBksFJOvkaGy!%++ae)*6kq-s<9i#^dl96ya8t(O-pQCR+Yd z051FN%|};X?K;}39a+jSYrj$#n>e7~PQjjbR}6UMbwZ7=QwHiIR|Na$yg2Hu)8^$a z`tMI1U=nJ_R0iVv_RSlCkqM;92=g^*t3}s83kznU>QQqWoBs zbM)xT*`&eK??<EMR`hq@R!92hmX^eYDBNDhp>_OkD4fZsTYcCV#6|ZxgFXp2B_iLAT?hY?vRqp=VW6yOm?Q zdLZB-zv6kOWKi!Qop0APXUKKhDVArMsB4iVF|ukv)DzIwTrU)Y2}sFXie5v1<3#;F z=zMx#Jj0nH!dmV_Xo}`_M(J3XnX9>xH@#lmDLXGJ%<>A`#*x|&uaw70&EcBLH`0q_ z7zP8~7=afhdje=Sm5Jg9Il- zbwIfPz9ngiKLfAfW=1Nj%#ofMS`XBU(4spGTcKuJ`mPEickZ+zVGj$;)%<{Eau*5! zgpYang_SP%RwrYOhjT`#@nV`xcT0*7MLg-2aqk3I%6}ZzRwZ(ei-E1~sT!y|4Pbt5 z@7E!D;Nz2JoCE=ee!3`hWlqHdS|n($uOk@2G(!SE5kfJ5vVod<7Cjs)eDdBi8Me~r zLUu)DpSYy-?*q|51`^ zjRxLcd*y)@pbBL73!SXa1h*K|MI!Ws?nRfir&%7cwZ@f~yQl{*a1w)mm$7?2uo;$z z*$HDgYsj5W-Zuk$xMozxBr$kQY#8f6JZjCbtvuI%m z*OObm%gqBXQ3uZU>QNXVnRM-8fPDQW~8>kDy3%oPUjm#8%2?e3}z2w zpWJY}A}__1iRs`oFKk6Xn<-1n3)_l=NoM75O_K2vBCG&451y_T`5UJ{v$J(C@1?lq z;VMk!@>)HpWY3JzirlAMAA;P4sAVD-S;e<;7ECl5aP=TI)2z-625R7-s1ZGQ2>R}-}*1?Lg3Z-+( zL4nH5GA?v5`5VGWT_n-1InK~z#xp_oR=J+1&c{dj)nOfv&--)-{XsP@`OiY1E|EN> zc*w173m@;njHlL}QE_!efYLOVeClbq-+tU(WPmsB7uXEuSqFCcnGWgw)0!vmvp~o; z8gP{BH}0Y<#zYEE0pEX4my;AOr{L*=!ZuC>R-n8x!kL8k0yGd9f3%s}jbHUuhiq!Q zuXnGgTA%i(@U?kHl8rnlPG-~!1oER@0C-qeN*3P1jq5}dklq5^X~Z7gguZFYkqvVm z4^J282>I2M!oY#%N?Kpw2C59NJWAt#QPbM3WOhcSZ`$%Q#s?#GmO|cx70XLzbDQ+U zf5v|+3anUW5anm^`Ft@)H8&ME4@*$?A}n6YT*Dx`u5eiZ6$lXXSZ@E0>)-6_WYLWd zoy4bMG|9%)YB9^Dhh0MIp}iu=rpm)y${pv_``q6avt$xkex-BjP)mcSdfON}g>Cc= zKy>c`D9u;gZXtH|b@=n>K$02O$+ybOHFv0+Y5nK2KX=f~^bE?5i^m5q-kUs8j+( z!4$-6^n&RV^5a{O)B*6dKkcan6EYA*zrl0~`fnv@0~hP$|KS7Oh;r;XFUqvWXWOb$ z4u|nI@sZe2h6hK8EVkv(AQ`P-=4#Fdf*$v7!-}~lI6}U~)AiZDLE>z%qvyDq%Ezg_ zbkOBUz@6VnX5qb;|EFLXXQP>GPM^1F-GL_xiHe$oq!6GT@t$SNQzL21FrtH^N}z0? z2d+FH3ru9+Mrs*rpGkB3*3TVtR|fe{3{RBf zUIsvu(hZ~cn7WfMYa=}`GV1Sw&|-iiAtabwI#9nvZ{fV<^7u zj&DqW8In2L$o%M$R6ls=t^{{Q7sJnMy2$lNhKbY~jAYPbY|7sGUnqXzZ#A8sH6l z#^0VO>;I)Q;Mn9-dOHEV(N{F`NG+lG@iF% z3eJ~{VE>Eb&=#)cgd4#=9gxh8l!_aDCXyd6G_x}dC?GWM3FB9x&c*$Ku-KFc-%=}B z(Xi4*-b(4NN3N{ zfzcM2L6BKhK~AJKGEtCQHPVM>&t(zQMkw$Ux&Sa2@L~8*kbiLS&8xGd0f%x=J6B9c zdOWBP@rNr7sX6=yRF_RhM(7YdS*$XAw^x8L3(vSJS<^S8;xXT_S5c7tpB(@ztT1n< zi+LnKN&u`y2!lLHB#7K))QJ+*I(QSH+6G5LAb7@NQc|A4)CWt9w=3PBe?;c*vK}s? zh9TQm0*#|lozqi&CjKxt>US;i6?t`s0`)eJK?Xdm(JkA#0d-xIOHp3+{?J%v=e*!t z>H$$^o;E)hr9V`pBnLrlq9f0Xnu=P^`J2SW?X_Ms3m%QZ?ROD277udyARj&qSW@FX zNh>M>Bt;&E2V?-;zIA~PsZwPuchMN1ghgZh7PG#ZMie0f@k>O23cMg=4C%*PU+VB} ze+%ID>Q^K?gYX3@6Tkctik8X<=M$?;N+dj9=mWqxY>QX|#FF)@2`h$sU$0oV(Wnxc zG~cvBz_@9O84WZZWHv@4E_raFAZv1Yc#1f5w^mo*g1;Ej05ZYbafvFoHugST7}1MW zNpCv-sFNk@bbV6ljRH~J09^?qBVL{zS1o7Sn7LAg3zRC#~A%d5c+wMpfsMtL$<9!^qB{m;2A*f&4cs2-1QC#i?2iMg=f5 z)cnk0cRcCpQtC~AySmOgcej2_=^y^dj>(eLf4$=dh`$sQ!;=Q0a`)!y^ku$O!i_zm zYulY0lL&-3*jg*A01a>Iiwl!|&H69CNnb-c01t(>?{no5S*BVNs3x?v;a0FpMc#9s zs2F&#ITHAYroSS;vbywYZTI@dd+jXY5yCWl(Qlvca5@$SB?gV2Jc-W??sYBVQ*Kue zEM|P$X3HI)3RHBMHaPXc^s)W4!F#j%s!U4wzP>`K95*)rzFi>ka$nU->}DEYwf>(I zshn4(s0`eXg6h3ofUhFDONgO_=op)sybFBD0pOXlr>;Aml4LFvGS}E%FIm%w3M8`3 zYXqhrFIa+*xAtMz4Nw^iY^0udTE?vR-TG*}3X%);>ismcQ4{|*k$#f{ZNAcBK1;BX z*$0A{=1-#LFV8NocDrJet2=LXubdPmNx=5K`cN+)J$lp&E_A>_S;6)L`6Z1--mrI; z-ffx;oUc!xJn6tTgu`QN{|^w4hF&+TQNZERXBB|#r%6j$^oPV|v8Q?rxL$gH28HjD zc!noW33c&^DuW_0>iSJV5_G@cQ4d$e#Cexvpgk(W^}7%Pa7WxhxCY$<0C!}V(}2$E z!Nw?9s(3jFz2>86R6gmb)3|L0;zqS{6Jqey>%%G_RK{n#K15dF!oqor7p5u_ZLV>G z4?5M?#Th6RT7P>Z_ar8VgTbGA^u}_W2UcXly!@^CzNULfa4uVT`W4KsD^8u~>Vr^3 zZjtihLiNW1sAuo3;Fb~P9*0i|01kkh6hB_xBZfx2&)zto8Me2$?Mw#Q3DP1)dn67S z)TGH7BAz?8Fc5)Xt_Q(1Wp1F>UJ{M66vAC*3+y~alo^6Xw==vaZLaXZrk)j-VeuNXzdGZIR`q8e7E-yGKJ9a_>4TPkYrXhvt&FTo zN(MZ*2DA1`!ckqJ*YBdC+#Zd&R}`9gD}M?OR=WxH7P6%LudpaUrl>R!u=vQ5Jup2x z4#%8p8XsqX`wnh%duBWQlOlUv4Bg5Gk}H=^*}j@gC~O-R9l)x9Bmkafk=oNHo3G;7 zmidl*xh$#Utln5~9(Vd(6y^4zv}2upI#%<}PoDm=LZ#ACbpm9KUI-$}37hQuTsGN$ z3#9=Q|M7e5**ZM2|*OD_d7AA0-3wruagbC#7`8<+GI zFg#1KN%0>bi%{=p=yRHdmh7Csgrq3 z>|0x`%Zg2x#I=OZO;C9&*&agSOWs@HMon#YoL$2veJ028k2gPnNAM9~ov>o+2Ed$h zlns5oy|5+r784WG$rT*XepJ5Zm|2Bf!2s>5i5H#Bg+9&(VqJ@7W9;XO&X9VW%9i83 zRXQ&L5JzWILcILKc_61U=rF0{AP00DH-O(_v|CgTT=^vl&TJB5giT}0YabNB4Kr^+(+;*-8C`fQ>DZ72c+vF#Pxa+j~@1 zDFkCiEL!uM{l_1M{7rbY4$A++u?RPm4Wj(_dE|LCY)9bJ5D>POmtX>``VOp)2VFDN zD>|2|MhT#~M&v1syD|X0f=z=H00I3kuN%cUfk}iV2owMfLBPBXe&muQN%w zO%#||Nw)7Uh%WN$tPH+6-ZFpf!c3U~bZwUZIJyr1jehUTyZ`N*mtO}Jybs&Rc->_= zdaK9rF|(8|pEIhu>KTenaopR7dGIv+sg8;3zFCfn{cEf1x78)_xA&dOJW_iZlQCOF z{t(>*@#fN?%Yy6!(c`^=#$46nu}W$JP?~!lA8sR&;ggMnO*tY@a9E+R|= zTzz2-+qztZ9>1(a`#zF~O@CkDILiDL1Vh)M^f!I9<+%ZMKYas8HUZgd|fKK-2 z&SBcFUYD@(>0M7XhdZYBrqFvwS9;*HyPYA5rN)~{`SR`7gzA@dH6PIXXCJtOkcF-( z9y2gfyO20j@BQ@L`eqqF0OQlu$6h|ZJ^gfmWan5S@-hk^hMtp4%n@lN^uWd^dbBfz z_&qDU+vr?YVSmj_@8;tifc(hMf&;?@kVTGLA5;591~@x|4pT3`WdOEqAS~^CC&Ov^ z1~wJT^A9QfY)WQI%gp<}%Y|p6{F7Sl^|}9y@rFmfXmSfar}mCfQ)&U1py9Rax{4~{ zHj42v(jF`q0B(GPz>Nq*42FAHnikeK&je5ghv}aERk##{Z8BwdOqKZ)Iu$ffy^6qNIMn&Fj9GlYu`3DOFMh z&Y+d7XSQ-oH8v@4t{~K~MWoPi@={qZ%d_;at0TNBxe7^6n1IT%>DTPreKp02=`Q>H zlMT!>NE4@oXSqY=?fJb0Js(_<4yS7=Y212nWJ@U^mV ziJs0P5S4s^_!5l-3ldz#11sn2B?uL5^STLxsWT0t^(U<; z{PgiF46`%#%`V5UmUNQ`(1DVus3x7N<#}!*e<6V7L1~&LqMbeGnEPKSsOyz@?njuA zMZfy98gK^D^eENrznS6;5|{%R5QoSe57aY?xzws`Z-6Q3MUj?}fG)gW_BS!h)1}2O z>&xaa4ZkU7;iWgn0C)cHO#@fRkO)%7nVSJ+>UgR?aYSa?0Wf<#yOEVNFVsjE`EHnH zAw<-z6^Kwq`+QJ9g8pF=OqQxVqITdntT9e29@ywj#k&;FY-s&q3$u>wd_rhaP1nnUi=h)f?mqtPdGNkJl0z=z`57@ zJjj8%8>k-4mq-2uuoO2>P(9JvGX%PEaKs-e^RHzVZBqAVho~b)3$HCVLHNXIO*{BVA>fNeA+2gTsc+sIzDF z&;ev`iY&y_-~>K!suU}X2UL^Brdv$YW_(*>vMZ~1q;Pt6~p;BX%%+^ zb=86-2bKtKmx}$paJ(>j&S%NxE@Mpb)16(eM%t=B&UKH>e8m3@oT;sUd2k3-B57om z#uZ8{>?^rr>on8!kW@?xHVa*0mT%wScCJkaH_A1BOBslSv%kZJ2TKEI#=xmL-}YEa z3P~A*7_c*laSF}{vVss?rbvPnsAD5hF3k-0Sl}C`u$Dl&J(HTbjhiUdxnvHfwQr>O z&|F87CrOY;9=rrPlgH7ucSwHMchq_+Gu`}qh5b2DSrLN00_C8xDBQU+_j}h;hKjHA z%&g=SC=SsG;{eEhE$HN`g{K9d36R5hB~vZFA(&IBU7Ijg?{S5K!g;9&qP3f0_Oz+` zavhf70n=xH1Phz`3p?Q!L6RWm=jV|Hp`vRoSD=4Eq8va*m-3;S53xqZKb6P& z73_HV0QnaKKgVHFFX9J;zx%7vM*BpFymQibJ4CPLKcDKfm*L~ScWowW$}cB$?1srMqUet7uj>7zl!ekj4G4b^n4K@ z*8oCO_J4(RJ$H9bgmgl4MIT}Q=U!l5dobn{2LVwzgvHyI z?XQoXvd$VGY5NQXULi`V330GvrO1JPPr{*^C_}5H6R7%9YwDxHQk9cuF)eN>uo?VF z=PzC~gjL%DqAt_z(a5?-ul=pm3yHBL^rEg-g6m@2BZlq@a73Xw#hZY9krMV)pukIJ zC#2n<4mGV&ZF3&u)u;Zaca+;7N4#G7@7pCm4Xhk5*~!qqdn$#bOk5TJd%61Agh;E$ zn@UI7o9(fsj{B}+G?7UHmB^m(r& zbP!`ez>WfJnDS7bwC{y;3T`T7(T&(j)DTR74q~zJ+JMDS_7UrN*oTVND-Af=!4)<@or z&b#s&4rdbOG`&KD)#m5MIaenYgGM63;($mRC9i?oU+K21$u>R&|D2ux7i7a^b-nv0 zvTO(*59WQ0{jbcRd#bt5ZuH@fa$DQlVqsYmJnRV>$5Drw)=bQgT%je9C4sV03MW?FCjMohKp5hMkB4q9}m5;JSnpn2a0&*q%~!Se}`wn>}|}y*1Yc zBLM&RkooXEcOMvHI$FJwxvTJ(@`Kc9psm6yZOdfbN%cEH8Cu4-&qm%sX)+Y}mJeDI zS?ffy9xf<%j2}e)9FJ_}aqQP2(c&6^qKM30N;R=LiFC}Yqw^{rxlV4&pS-3P9wM`v z(Y|w9TpT%QE6Y6_^r@tOf7u^@nhGurWl5D@>el66_F`(viVWIquT{-plSw8s{ z!a%ac9OG6PA3}vr$q_z$g~4)<`TYuHXRNZfSa7(?nEoXG_x`mRY6;F5V4G`y_OyIl z`&M^YH}(PlQ1eB%|7$pRsc?aPNjn#*ZO^+>9 z$XL&%armPej?>%bR+z*9DA?XAL97l3IhtOgVr7Q|{-89pv)=xR-zG~rYTWrE7{N}46a(m`-VWO#Qa%!^BAReG zxOtYkWY<980MZD!w7H!A{*doUD(y*h(Fnd!8U%ZRvXQ(%p4RqmBS-wJa_`EfeWti(bzGbUTE|-Sd z_q3#?=Jt%$g&M?Qu21gwk)TvWDfZ*g4LAgBV$}kn-x>h-@~~2ixPfDmz1ZuZ`z0(*YDc3(2*Zh{Zu<2kafy zhMMAe2(*yzK|;kq5_3`pJ|hLvT9`go3QRC3Ou<;mgG<1L->eG_j1PfB3;c39Vb<5` zfme6Fz|fvt(?55|;8Wa&DdvM!wM5-}GqD?d{?E%9cgD8~!G%$r3st-^F)pF`W;tW9 zBGxuP0*Kyi{yT{XE!=WvklvjLNwEZtJ#?xdIl4sgDj7) zG&Xh3usID2;hi$j`4@{I2M4kCPEIz1mviLU!IFR+7d?*z#k(Rca~lX zFr^Kn4&6(xA1}A=7`+Xo1-;W*el*u`G>1<*w{NPiV@c%wX*s%g^~fc@#~@)zVqMvfy#+?e*#!_wJ}zURi2> zjgS4K)mQ+^DX9x@$5orY5H`=)7RMI>04DiXToI?CRfaf)r{_*H`{&}5)<(=N=XC~e z{Vek0q$G=8ICQ_ALYQsUx|!HX(OM_EGyDB>a(5H}tgmi@1<%GjKvI$df{{TE?fN@} zB^MG1;Ke~Y)ey?22ZDdPfv?G&LI|jQDV8kmKe)Rvx7|rxH3YX2V!bkI&5$n9s{;8% z1-IyGVanNEsgg`eZ~=GsY5&!8#^nQ)@)gy{>x`)4nUA;0$yk3Y=sYnwsH3ySp7KTQ z^COUT*177Ls{{}f2ao|k?_E}?(FEn{3y38%dgL>Bfk}Htx~r>R^tq@0y#?AVs@B8$ ze`EVn(&^4NKYgBis0R*IW#>Y#f>M0sTi{J@L>QbHb7DN$<#m%0bb4_5gsqVPNz=Fe zz6e^u6>*f_538SUfS;b6O#6o_MbF0_(RvUdiHsnS-?T6kW^8mYEBNFyky2#A29lu`-^h?IbYbcaX?N=mmPprmv+ z(jg6-ly~Zw^o_boBk_DF2EcJ1ySPWYjxU^ z?qVXrNX1OPFlbYO#sFqmK``tn>IU0k!>6;Ve}9ZzgcIsvy3yl}^5a84P^?2TwG4iB z!Payel4}aLq8*?uD(AfU(^3K>fkEUpc0_mS?oq-0v;Y%RD`ozpc4xK9#FtU0ty|x@cl_txzBFCX{8II??`CZ$v3aegkg; zFSsMwWtbTPw|nS($I_Oatp%3mb@aeE_v8(mJScuxU|;tW+B@e0<4j-YqxIM?*PnsK z0`jq~P*w|l?}Aog-)#QT)UdH@aaDM!{FO=u6=gjX*l!d|6EN=;J;^sv_a1S=}Zy+P;+)I~-5_X5bhsHY(DSC(RoTw&SJgK+!rQFuU?N#+* zXu*!=MVzHD*VHR89Xk!$*3)soaei51LzL9%c*o?`Xt(BTT->EAYc7X8$NS z=~w&Iax>9JNcI0q=1EbIP!NCC@yilbBk;#UtHTMY|B2WTLn7ddfG7vnl~SvPWlD_l z7W%$9IXV9tc~5+RQN>E5_u=gpW9-wk%L(UcAdLzn*4u3rd2KE8(w_IX{VI*iqCXSg zgVwRRJh6#GF&zl?mOIdfp}OEPA?{V8W{^7va^K7A)>#%w_Uk$SBajrs`}LB-ZUMjo zK%LV!AI&&{DIKDXd>|VBHv%-t)F$NOMC%Q(U!R(+oT${iH~a$zSnxxPnN@k$A!2j7 zQ$xbfvg7>0Yl3g08ocY56aO-EmsB1U-<^v=n+tbI42^CR5uKMpn^wWv73zbjJThYy z1C{U7BpDhw`XTWSs&^Rk)lQlLA+a;E0WO^t-)nAqI2F1>=O?0v?E|saYFOT2XL1x~ zfn3+aVW%Jv&<*cJe{xG-c2gsh#!Ja;tVK$YMZ*xDwAhc*W8jmxVQ>CL>70*5{IA&= zrs0ED$NFFSNzp8#k@7qh%!n*d!YOD`Lj*u;2+sv*(dB~68?&AFAY&NLY>@~F$-s+cI}o@x4>`^{;W9+3kPKc$$>oDu2Et|%EEV=37GoI*7q8Lz}s z;YmT5JMpX7Afl%;n)4{zf=iaOPu$GT1G!h2^@R5kL&x!^TH1k9#>1OGlaU*jnPQ$E zIlR7d8R~-2rEP+mz;-HJ5Ao^3#0Wvqz#J(PRK5r@6)+o6BV9n!0wGK~|A+?f46=e3 zRC2fP!6!X7E7~PB#Zpz8>RuvP+3G^3(aL`%+}*3k+F954qK{qCv3Yz!6X&6^QUPm& zDD*FPS}!PrC$4BJaM8R%HU-ttIa9M^J-}OEYfam_kN#QGQ7VGDzLxkSop*4gJqD+W zm#=TxiOv_{j>9392spBS$B&q%asw-L`zAuKB9iUSBF~}UY}?&>67#I#PB=bZ^%@Dx z)xB<`eU$5aLnimpt3A4b4wGGz+92-Mw-X_tuYd{CK3@;13pl7jGf_^u^f$7e*Elz% z`w2JfUL`m4;jpT&500-x0Wk%+sovWeb%QIxEAro#&VaD78Ff9|SLRrZtl575sF=#W z!T>li9~_i~Z9bLs$oqkm91=?3nJS|8VJjZokBPv3@coh#JN1x@ zMD$QSn{_+dtG*??kfjk;^3pNk9ptbSJDN*WB0o?DzzLKVQG)iQNu7_O!wabi6>V)? zNj#)@h})9iZ0`vVVcVzCZF6Y$4d`I)ynX>Ech__Z7_YzAR7I7U+C@J8gEJ)Tni&rv zKMuyA22g*I<%w}L`255;+_OY57P^Kajyld8J3buR2Rg>Cu%g}Jt$Y04zXTUv9NTOE zcF!EO=3Ne@5M+BV2|A2TnaxHX&HM7F9qwVyY8NIA4`sozK~9XCTH5vZwkoY8o(C*- zUR|p*%!nsh5_^aH-BVtb=Un>8dY}>jC52;bD)Y5JhugIRj#Ql)w%;++BHMq7_8OXd zRb|)yW|iKKnu;YHUrKEViZ+B9!g;uJ?j!o=*WKVb5c z=|i{m&*TS6LBByIHC1eHiL!t>(+=l3U8`ts%M=15HFvOJ=Lmg)dCU2K3_JB2h$d}y z5wY=A<-j>KJya%*)Xj!&@M>DD>S;c-F0kk;of-pinn<)W_!{vTL0tz!$YEv2i%Crf z#SU;C3wUc-qVB)`sp|3l#2M6gXt8R{UzPOv(IHfne780zo+AzO|n=FK0+ zzq=uHVr#ECv9*KlY{F@gfRn-Y)&9pO`~!z;zHu4z!wECqcHi()0BAqNoCP{4i=WK?t=og#e(y7b% zAjAAl+5^y8n8mIx&ACY(ZoWvI#(%5)F?89`B_s4mh>XY(gni$kzb^e;q!j8f*t=R0 zS4!Z>8xIkouO6-ve{{)lf_iBthq}++*eB0KkGiJ=S4mBgK$;kS*>(Gjban~fcNrvX zr$j^V?v}2~U5v$QiK*?-tZX21$pmJAs?+(8Ut}a)GbgD#AsyIE!xc9r&IDZk?p5%5r>=X zaS)6Gfi{I8Ns%04z-PfQmm0{cDuU2uh9V7y&cA`6Ycj=zM*^$cJElGe)7AfjyR`3h z8XONd=?LCF^v?*zoW!o<<&}d+qY`Er*v~!2n&QMPVT}cVC<=>&TR$%z6p{KB1>ZBxUD=Nti#;5x-v-k9rwjc?4YHWm^-Ng>Jh@ zUCoWm8W2-^9i8(;dht@@QWG&NbdHHRd>CZb!4kn!`evnmvD-FLQf)~8I)fRn@4*=j zLXS@h+bSRDU`qc%AxkKf*tS@%UjPk|gzFbr;J+IW zVbO&`!H@22g<AY6?>!&69cw#>~I3zRP34XUw5VCH0-_^$RMd_8T8;mu620-hYR zM26*!6&N#K3WyAe8T$*iBIx!VwXoeNP@Kf0+n?pm9P1DW zfc)?nG06v)C{XukE?lomhnXj(j?i@1I zp;8A}NpQAMXnid4_O|W-76+gZjNV?`#?daoS6BNLySi@W|0P7z@jemDin>N&?BcWM zgTDOG76&TD%VK+&WliX6gXi~}bVMd3B|%cR3=Ec0|7cxGj3pTgigGG@dqW|F>-Fo` zW}KF%5+wU$@) z{G&;w5lqgzn@MxLI;VF|Mk~nF8Sf`HT4uHR?~rkwB(_a9qF~349BF4^ zU^rcPvznfp+yB!ie8>dND7~)>2s6O`T?TKPT~R~yD{S;%1V}2zVwu{Ap2HHs9gW!E z;G-UfxrN;5!aWkn@;1IgBgZ-Y{2NfNs3>xUBMUtt4iP#6{!mu7jh*8iFTPt%jo88A zVK$=Y-Hp*CcQ~5AK^sxlk^^G~h*a-FNLQHkqpX@(BA)3j+3a?f60 z{b8Idq)EbSB!NiNKS2j|sbKs93TO*f)#kuuSA;eiLjSko2C2;4YX?@WL}T4|6Z#mT z13LveL|{JT7jogdqDh{)Wwf)(v~Cw|uQ_^er9-GiE)zWdt~%ob&q{N4@fP4xkKirm z3CR|BO&H8d`O6QZ)0jDnX-pKZgz?+o>LPdor^~l2!Rl8dWsW!XJI+9#1p%PLD?X@8 z4xDaR1sJv#kd z^U!l@b5Ah7jSO6{$}9z!I(c@szYaMRPh&)g&X(yd->kc8jCT199bAkh=p=xIg92&M<025a45`Fm#D^+@b)wSJAGs&w~d9SOM=% z`dfJs51oGa0@uOQVgtZqs6nH-)J9TF3?zTAy3I3nvTRFNQ8PDi@7+slVG2adODP1* zHuJ?J4czFFiv=mh6haHV3j7RS#LxcGh8nL+gYZb#?~QmQCDM~;=npc4mkAJzfk`oT z-atK@S6}f8lf1f!qLV^voG zM|H1DJ(~drS^H+WnkfIUuzE1B_h)KifyknE*5VSxbV0UJ!ST^P?uDMv0b%eT%5gI# z+_gEFk3MjyaA>#xx38QD@odU@6+~serCx|IR5!Urd9+%dvY+Ec`~Wo7NEa*8R}HbZ z57?&JW9{W9CzLjdd%M82^@@hECg`dPv0=ZQU7MC@L~_n2RUv)o(& zgJq>sI`?|6eQJPTL2N?lHqA=Q^}g-TwxvVk?Xx7e&L^}Z#-7nSQHO{I?}sjBVoWr$#-EG{HwiV zZ1iK0YtfmytEBTT$vycW+4Xy zIY_FJvivqL#r>VGp^!UxvE2gVOPBCJ0nz0+Q)@9 z;5qSQUfwpW8X|aroXUmz%i*{8q=qF1G(4z5?rA8-2CcYY%r2T)AM8;H33Ao)vVyh1 zLumoUoR!)lD61yE`3ac`Y{67ecbW;0396X2@*I3-E{>tPgGDfpAAxziul$Eol#kco z&m6HZew56CuKIBuy4@}(H(YEjT){IaG-Oeya;q~uW*sRj_LlD2*u5bykPxv3Al<%@ zavNOW^9Rmv_l%FBe@-LN3zoOQ-n)w)VF&)Q43t$cvMpG$(bi@JIYI?DKDbH5K!^bH zQ$(f;qi@}99gw4t1cm(`ZOhk&EnVR&xeuLD3ep2ld-B<>-;qb3v@Z^Q28K;2@E#4% zqCWt?qzc`6pojiqSx3G$Ny>RG0Vs1~W^x{#wL7**rLZL3$eSgK(N8moD?+7!&NIS_ zgEFC)=6Eh812xl~LIxPgaY$A9L?b{7mvhK`G^YpM+fpCUPZ64ceHWP8p|PvXmnUc` z>V|nE>KQdW!Ue~=bm6jiL6BK={VazcQ()|-$+K{i-#OpSuI-1I2w8<2gSHU;u?Z^V zfT*?BIX8q)xM)*qCIa$y5-#HI|FseT|50v-O$=wP8a zayY?Y2h!YrMM4ZUoO0AMoqL&liZ;Jg9ZPb$G{h!)7;9e|k-qFh z2FeeX+u}7%0MD3G!_;Ne~*hspt124<^&K4dr&lvf{;0Y6Zr@JEz3#24!aqn^Sl2a zzmzE8!S%6w%`+EnoEIdZJ$3;j{1K0os;KNzHfW_{4Da0Lksjtxcq0Y~;up@POfFUuedo z+$jLcu5^0o^x4?6@3CXI*PL>RfZ?kO^6I2P|y!*OdVfZ)}>#E%u(J_gm!%REn7ENCO^#DImUd2KTN)HSTOcT3KO`R+3k80sv`+S9+D~G6k=|`)abw((_lGPIQTLUy%Ck zU<~Z3AYqa49^adaX~EGDY=1AXIg$j=E(=6OWj{PzwPT?s?uKS$bC++DLN-i8F(;&cRL)W_g{2=3@xY0uBG{7$IUJb}grFwFdy?33mcrdT8W=XYe> zhKWFphA)tOJf-FVVajEh(gwS##aTbtGw$S2P7z}3McAilOOuzKe&47HG=l~M$NEj>UZc~G|2ih7No7zZ~z&06s@<8QI9w?(Vf-TsnI;ngG z3&O}i5Bp86MGo`J)i`OKI!|{dv>M=2N8RDrrzl6S2Zn-H!p}*l834Ym8&Us5c2#R- zxHJ3z(|Zyt{?inkhylyUr-;I6N4;_B`P{}cE8tpCVEIvO4W0Mq7;&h!z#Z&--Uweb z^u6ntZSZmanoc5}xCy0bumNbI(?Ii|1kdQX;IZl>Y21wy<>Qq+Ek1UtvtbSyKia81*gY^Hr~oPi^8sMn ztZAtzeLW@<2%VGu>4mkbW-bx~ zy%3zoMp2D<$uLc=a{0!aF>+iEF3P*N{Pyckm?l4!+{{3v?tOpfc zq9vWI0HO@^OC(7JWuRQ2QN)>)SpE#fax9!?EE8eLtANB(rhXSe7P54u8+d*B$tqDF z-KwrrO&9{%`~9M)<21!BwyH%N7e8hMrm9O}v=2QuTV~K{L&{%Gb+sEFNtV@>B>>?f zG(gZA{4)%bXoZsrF*&9RU(!>{33@A}J=g{hO#Vh(Jt!D) z43+^IqQmVt7ONm#3|0uSIKo+Z3FOiZfdlJuz>Mr3n}EBAt$p{YW>Qw{Jv*Mo%8*YB zaZGCr40jqQKah-TqRFNd~6D;=i~{xy61nBcI}|b@?fSE^p!| zI`19A1+qw%(}n84X!sR&ck;Xh1P4DNl>TmymA3MQVey3pgP(J0=f+ebs3hZ+`tQ$+ zf^=r^E7b)-r_Ju?w(uu|%FDl7z!vay&_aW&17u1ysmiLl=+FF;=vlmN>;6br9B8#I zIZpPA9S-+0DJp9%K*=rP+bH2#OIW$g);ymtNMKzShhU<987$kzs2BL02{O8Jit-EL z>$Oo+*3b>0dP@vTN;M%~0P3=^86_%kyT#1OVaB9h2c^iVtO;PSPZWrZX5iQvY6i#= zggCNDDLSouG!mf)fKPlnp(ub->hju+fyqOn=l{YxJ$Ii9ff@nZSK$RW{)G&_jFHXQ z6vTca=Uc~k14d70TbaXp+#CdVGo9t{_OKxvM+s;#fVY)pFk+iOKm!!N$raMQQZT>o z%K6^JKWa42zf4Kl2n=x(4mBe9-1-r7Xg0vY|8Y%npO8m%i0Xd9T0FmoxB50EW<_=GeYy6CCKoPKS

?{_cM@IHAgIRoj(~T?AeR8ISxH)wjn1x6NAwh%2x8!Ugcb3OwW1Cz20hB=D-49v zBU60&6>VrY#LX|03;<5>(FcQ*r0@=WV3+`DbUo}xcSxhPRZL#q2>T%#37HS@)If?s z&qO-4;ogas^5&blK@>so>u#{ugw}sH!Q(~Bq$mfW3669=0{8y|$!CWUOL~7#V%`&p z9_3+f)4S0S4j2w>bfV%R82~tt-*_!1Bk1Uw2(zGKOV6)}9q3FD*C0wTB4m0|==194 zS|-x0Bl5Sv)!qECUOZ&TM>r!$Lb;g>pr8q7fG^(@a%MIpjL5jK2^_C@LqXa%s+$AL z^a1ZVA#q_i)_7tRd)t=wn@L7x_I6hbG7o}|6B~gWL;-n&9y$z!QasK_2lEDpdz)I| zSnpNu(#?Jc-n4Z9y6W-UBZ!9jV|bgndg-RiM-F|PCWLV#31Vh3+i^eaafX8CU;*<^_;khpLDZaZ48{gDg6TYx$r_o@0a#f#LY6Ow+LkAsnh9 zL!oe6WQDB`G)uPQs^Cb0b5R4|ur*A;0LcXkr25-dg|Wuc3UR;kh0B=tJ$RHgH1PqJ{-hwu(e~9aIY{j)r6-FR@JXyR)ESe>|H7v2f)Ue z(J@cDDNsy~!xJ3sey#z`cZ9XpBqE+LS`66DiZ=f2+iS3XVKNzVZy>~Xvk_^qCwLWH z!tPvhmGm4!vXj~!l80&m{VVs%GFdSKxx_&-RpREiKaV%Ua$F~iTUDr}7(EX_@CLpu z(3Uqhxgl%w@ASzTXC|AM&j?)+VlbXyM1&<^BSaLvpvLF~a3~5QHIaC>A5Y^?Lr1o; zunXHCu#jJkw`X5GZ22_m;P6ww)UicQin>cmqd;8Dtm*qV?r#D$kvEM>v0foas2=2p zyPlv|V6%=sZ0;itp1o6zRv3e+;p$fNmZKqq?ca&-+DnlIF$Kh)s*B^UEi=Oj4ktHE z+d`h|3J3`SR{$quIWP9X{W2MT5^xqy4i?Ub&O+e0KR^-;e(s7_eT9ZAOJm6L-KNg- z)*vb8NfQQ70s>5zB|Nl&;Uo#*qN0`kky$7ezG#`NcMC!;b8m$p4jZdgTBNCAhtv}A z#_jsQhPt>RW?}=TQG`*}BXWlSsfwHEnP|jE*t>A;PFK~;gxqS>K;Er#wiZZ+K^g(> zh~jK%4hve9h~f)ip-~jES6Mdb?sHeX8aCYo=E_c+gp5toxsyIdEU=8Zg|~ zO;0UkJFtWc|H5BEXe(LWl3bGtJ&}yoQ1+KS=*`p=MM18m>0%$Kt3doS2 zFGJXfM7&b*TY8XrB^i#JC2j&h0{_t9eC&Y(d8i~?Ce!im-~91JbAHwj2+>lrs2@sI zyjyJxLa*TZirANaY1P5uNZJ}J+y*TquoTB@lsolC7wdOkLxGnym{(ss;l5YVml^~t zrmdL>mkJfLkGkcv=ZJ z90D_T3z3BQI=MbLpqizhmE@TTw@QVC9H*EY{T@q~sij1K?KjI1p@6iIthEBnRy~A3(;WEpt4779WkIa`^wmb(E$G`8- zVY;UQQAjcpe>b;3!hOZd1{MDQVtw|Nv#3$Ni2=n{`rVW0pV_6{4oe6sJJ+T6%WHxF zaA|IIgt~Xf@!}Aw{Y<|Zj--FO!AE^9Z#CU+^FPKj+ zhE5EkIEMt@L6s^7Ku@uxLJ|WxG%hgKRyIkw}qHW`idtPCDgXG5R! z;J`kXL)hZGDd3|V{fI5_SmM@h$>}){oVAloKR}TYD{7<(w?ckjF%1{pLX$VZkSJc2 zeFs#Ygb0GLhPER}l_Z`#Day4FhkKVC`Jw<-9cTffK1$z6%m1nLzCrXRIS>GP7o*NL z(PCFmsxS3}2|XE0{x-)Wk&X;4;fx8QR>pEQlY-!sH!AJV4Jx`0|YTMzJ`ObwHqI%`Gd+ zx)1Zi20bVi5i#C|iF!j$j6X&fZ!^C?(qJ0lQ1)@c1-W}iQt97yTTDA4SDYw9k?1jq zh`s>zA(bsS@4*)(w6l5+L3BqCieG^O@y;aRMrx?TX|nq?U^8vC5C+Z?lJ9+R4`H(wjJ66 z{crEX^JYxVV9Nz0hQ+>LZb)zySpVSgwKAe`I}ikpo4bgd3eGV|kQDb@F=llDUziSb zUB_si7~cHf6q|s^T+l$nPd+g*soiM@0t+BWeABs=78ag-J&5`_mNC}^=CJ@4|4C%9E7FS*-_zOTg0FD5ELjXn=Kc7RO0StHHtX|*EsM|ox8~4MD3VU@sy~Z_y3y3@z zUd6W=+>qW@f8^8m7Vgv9`|sR8Dv?Ww_nG;jE?E(0J4&L+>AnMcat2hGA<2yYp8>7BG;D(N5bee9D5#b83D`_ z0uHZ?XNdX=F0hjic_aouSi1(mRxfNb9d}nva{f(b;cy|xyL^$09?sy|PlDLIz-fdM zHp-O;;}4ruEe=pYVXzGO6ga7Nd{7MqxwcOuuAPMY+5bs(&7Zvkfhb9JH3X1$V)E`T z%M8$OxUGy6RI^c|pJND+XGe4@%hBBg=B0%JS7rMatsquCT7RgnuCD4P!>$kXG;&Gy z!*lO|s2@aQhx?Hox$o{^igXc9GKc*$Vp9AWp2FwUgi*zLPcH^k#T$K=OQjxheP@2P zpVqQA%?ZLQVuJWqfX*^QlQ@=RrUiI)XoFn}nsK_?<=+!L+xt}xQ%W>AL-)J@ffUU z>4~ke40&8*=)GejfdjJmKAyEHFz>-*=Yk{tsP#5Ie|p{)THKgt??$n z&hed1);%hsIG7Q#W>itsTX^Evkwmuli!iGI4B*PFKX-|sr9OPb1mT^v{I9?lxO+DO zP>E48dNTLEdri24={0FP3F&&(<_wqHGELnZ^zLvSNSB0NO(&u9{aB{+58FfE>G<&` zz`hq#Fcw3`IIY z3lwNPvB#VG3Ezhq7?dPrJ#4YYM)RjOTQe$Yj*(`I_m_VX)ZlW%pH_EGz{Yfs581+G zg5vY!HUI#o(;Ov1EwjB{4OqNON`*EjU;oHk4^8d;$Sx2N6fT$T-?rPh9xgbjn+@m% zX$YVff#5KcJ8xtn;N5`9AU(uFccrO@LRjlxi@PiUiven_4wyp+-__CF*^9_KK6X30 zb~H`epa@54jCzhwaB&+Z9qEStQ}RW_V2sD zls34swNT_&*z4o%pf|4WldxM?!8T4h_Jqtp{IS%7zzcrRW zr=&f_v@zUizwF`s#DYT4A~Lh*w)xU~q3w-fHvf}6ej*ovspCw?_w!dDetLzJY3RJq@E6S2CLRmz&3s!ZlS*gRD7%N` zU4o#20R-Vqp=&;fVZOxKUgDCowFe`JoSA5UPgO>*44m`GrJ|NGy8PI-8}^SWNV z4k{ zMa-u3a-}4y)?L5fTZN`+$rjn6tQTVf8q(w_hi(GJlkJSmZEI49+gCV_9 z%TMqj;1Z2@qJ9(PQD*6w`)Pvl;Yl#-q!oJ#WaJFeE-yxJP8I;7!`N?oqj?X_bp3k$ z;;NN>Px%+PWfj1;47sfDhQ>QPJF_=;%FC=IBqT!U4D-=Lv~n98q1V%c|BQCdsMmGaiTmDNt0jw##fJ+r0O>&t?UOS7fhdg4o^(}opQW2Wy+ zLg=>7mckdcZX{fW%_E8-C%QK}(+rt{`+KSJdmUZksM1Vp7krI+vpP^?5lqU!th5H)t<89~a z1FrT)oPiOgkgU+|1cz{vF1du0KZpiLPUejN*<*)07=zr0+E}-6LHemU`<7~|rPx!J z%ej!gA#v(|5qqurF~r`7v#G}~Lf=neA-P&7O|e4Bboqi-z*x_0XG71RP&?0%hI6Q2sS&l1Xvgy?6}->eS>1P0QMc>NuM zkq*_@bzPIV@S_lruLSAPvm|`By8NVf;_DpVz{XD4XP)0RZbQye5^#_JSGohd@37;0 zFf;%Y6_yW^ z;{$&RUVG(&ZQ47lsqWQ4>GR`}^0hV3N}o7gW+bAL_!WuI{x4GtDvun#^4n2gmL^~@ zR+9Hx_(D<>>~4|&En3vJk9>tt%@7aj|2I3O7YCP&_>mti)3vEN_ing7PQIfIP|cpI zK+N~`A0?;IUYi$EDj;1Iu(Z{m#l_e!o9_U3X(_?Jh4~!B4jIrOxlwZGc5Lic9=1J% zzyjPd^(kT7@&VH*S9yMOAwhY?5wE#J%-qUW(_r*`6dz2esz=A!9wzjsd}RIz;0+FMf>yc!nl>wMt6oP`V}p|HjOA4)MVWSd_U8 zaNvYD+Yu)9-MS*~+{l=W_#rYI1pzJ0!ldTEpDsZXqXv2kc3nRO<^!tx`-ono#Bq~1 z5#(pzxvC1o{{{rB$`E*rS$9-8sPm-kuV9yl-Cv69CZ&gKXeb#dMZdx&9ULPs&HvWm z?rqN30UQ%?y>$OnD}6C%R@hf$$%f>Ar##QVc?HSu4p2VcSu29>XE5NJW7J9_A$2jL z2g^}#SNj$3QnLJS%uQ|ZBxtF?B-}-V1<=(Gs8ph#h$TElB1K>vnL1!SoMW6{DX%ui zy_ZMXw^$UANdHkrV!TYY$6RRG@q2yB5EXZcks_ucKcQ9F!;M0cQQXPY5fp=|{VC{z z!iJ5YSl{%d_P5+~OuzH)Z;d4ye|Kl+B3Mgdf$QrZ^Ha}4BA7O8oi=uMQ}y#dHR+Y^ z(1k*{OB@ozaP;sZGMu*Wa6m!&C9C*bfR2N0)YxOj3ip5I$@G0)MKhttFzM zXaEBFsq6BI_?nuUfEpNS?3{{#JAWDEbb}J02DH~7eSN0@lfJXJC->{M_&}af69QF% z)5dJWvYY!N8;Dw)lEcx3Vg9-*LKJ;c!tW8{(`ziCbj*HS${l?Idh0K#n$ScI zV9=dkZG!7y`0V!Nzw`Unryqf|KV zEAXZ_okmDV=uTvo*VF3zX6QhgB#3hpgr1(%JfU^+Z)gfvi#DZ#quBB!vq=ttu~n^* zT$Jkv%?*QickKs--)%#cq(K6mryTAr0%_pNp^5@jN!s0Cy*d$@HhT5a(QqpHNDSVU z+LLDYQRr2K)8w@H5%32{EUERdEi>}RXdtJuNz$5C4+8uGRbfaCoy}K`8FfX<|HAJT}8L9y&KDh+p@p~75PSD!cuG!^&4x3|w+yIyF)s;87`jv`WbLw(q{2Tp;=~r0aGhN>eSBtjjDF&2vJm z+(%E(GO2JFO`3_qsL{Cz19Ax&WIlmJ65qVrkTab8fExrH z@|-W4-T@4{f4i+}S5R44c$T4tnkOUJ@H7nMH9k^*ge@a~N=)h!T&PTuC}Y9;-XOxg zTi@jmvmky^7}Sx7`i+myeB{Z`=h=9$v3^8s-(G|F(+LDy>+4?u6(ki{I*@vo37tO> zeGjSM1_3xg{!Dw-tbv##tG%s*_!}6R5l1%M>ORWeD!=^GK4`6^GyhkJEIger`f67U z4%38XYe~6nh=|V@ZXzj!)Iotu6T*_)S-00LfIVaPh#+R+C6z4U5OF9Oa>~G*!CGJ# z0u+57=6?^%ciM+5@SsHcEtpb`nPx(%2MEt6>*%V14D0NZmUeLxQM7<&3Ia3+5U&+j^#{{TXn~xXi6qzOMZgA^CBw!jcU<2KG<_+txABoiu6?FaE&P7TMQ%tnCp=NsGdvU zUJ4xv!lce09;Xb*>(eB`?jFlzuS~A#)GH6e2q3jSsk2jI1rF1=88|iz3GU^v>a9WR zD!u=|5(0s?{6<|-wMkhM(QG>91t`>QN>djTKpKj8aRfqaVF*+N@Z@6nWiE?@omE5h zWrTMG1BsX0Rd7$cv&zFD-UII|!rx!hPQ{ELAonXuMaQxi1d3%kol3aC0}}b>T?tol zBobG~b{=f|u)%^~%ZFoim)S`xtsVS8LkP$cX~~ zbC>su<%h7#$|OTSL_M=JFKvSWn3a~8|4xt$FLj2YVyO$sWRDmDDtk}@!4*Xr8;Bw8 zp;dCoN%>RG?wE4Nc5Nb;XqC!eyd4V{1Yt{5@?f!6f3K*b{ zH0-x&=oEvu-yI0>&X%^To>P;mK`|)4ddE`ywP5d@g!xddx{_FLonqzi3muY$d){$z zoO7>F5tBY83RXW85OPt49(PtMFubsh?uu4mI7*+lVPf5>bggkB#(w!gm;TOn^K#63 zalEtr;kX}JeQcE>>S^8JF%1rChcmlBLBOe3nJoWvkD5>Zb>8xNomj3sy3Qr7J9b8| zxhysFOw#O(YiZNap<`L4sy?QI_R4;Hrd2K>n`jhq@p~ZXD6sR5UV(CU9MjdK@o^45redm3rxmPW1 zbo@{E*2!)O6QU}{eRZRI7MD$WLY{=2BiPEH2{*iC*@vsW3QcA zE>x@)H&81#L5^~nOAh{8r<<*OE$A`WGTR-uhmzn@_$(-nQi6QcVUlEEBfnNP*UUNs zz17!_D&_kT${AlziBgEJ=G39%47zxGMfLTlg7?Jr^y2AjnkW>3N|=XS*$peW=UVr-h1ACFQxqaZvZGpBv)3Wic=Gq($No z(xzS!^_a+5Md6f$_Zre#BQ35hUw zykvPE)=9j|V{(NJ0F1nWUS06Di+8OSUU0g$L(gFD_p-bMc^r?2=9@7>mWbVc&VHJu zV`r&-N3x;aVwsopSZ`k9i#EzyN3-}BYBb)vPEr{ASsqEI=kbTjT56#w-lLi-Mp=qht)V; zP5;(>G?S&Jqsu_PsI>gypADTiZ5Z)KW#F3+k3CG?cD^@}C&*awl`OsZ#nsC%_RhNy zvemO|NYZPKgx8(@UiG%^_oYj0wH9Aq3&iXW$4~5Ml}r8*?phdR#79+3lY7MPjsQ!L z%c@DKp+#79Yy%4*VN(HVVJSAb9U2~7rih?DXe3D+|36HDAW$EA0s|1 zt2BM;%R)K0pa%~bhc;*j4lOx3*8P6};S_6QJzH1WdH(sN390d%g@8#p!(T zyo%*=_f?A(2hYpB!wnYr*@#(ct;}M{&-t1;H=~`u$d#q^uLmN}E|-N#KEFI)yv{t; z7_IsUd%pXlpyjj3>uoqasD(Q*DJdGt({h)qv;&WpJtS}D81f>^+hzGqY+JISkxVQY z%%LL4h2kJhy;KEuxS;B>vWK^%9R zw4f`mFj+zRqnAmitDGfI-wryhSLRvs#R2aW>ORLT3E8){Wy<7YGLt_=juf1WkFsRu z2cGvD{th0pbd0rPzh(E(O5~TZO6DJnKwIf9SB1^t(23LC)urOm}0djz8z0okF>+%aFrT03EZyJc$Dw zM2#&i9KD>6L2o{X&d-3(*w1)T&z~kEWoz5p8He?*IKRiZH#(fR4YnFfjbW;Nt)*b? zf%F{JH2k!2r&To{AEH8ESD#IYx*GGjmP^9hm+SFqg(#Mb!|xMNs5tdsVbT#oUsD1H z#Ajm&^H}Q)%=@RuTOQ@izwuR%?vq^oY-4mpR(_;txD{p5Km0>EOjoVfO_>~pvT3+u zNRGgnTF;2c$oQJu&uT)$rf4EiG79%SGy~bV{dE*rzj}y_D?QcjETuE<%!*!GF8h38 zwRaF#u)AFAGVEk=f3ip(E3>3;p5eJ5(f=dxUY~~Y>4cdroxoExPLgNh^{kO7N3`T7;N#z$Qsri4526_iU0| zc#-AUHJu5vLZf5H?J2u=3$=LXQ*-ymQIu4kYK=k*DzfzE#rOUF;0?)d99jzPT@+w*wm!`!p-ghUH>tpgaE;%v=Xj%?ge-T>R2wyStJ;69W3Y?3Peab7&uSoh^+Uc{ z8uLZ;FJR)8n~WTuL)}+v5K>FE7amLuF76!6Q*FL>=&1c%nx?CA+WV+%XES53_h%_5 zTHq!@x3e)_+zP!e!)eshUJ3H9tMK8T^1I?ihleeDPR<_aEu{bvJ1sL4rX1f5q3v#3z;^v+(`A3g5z!-N?s#pTmn~Y~A@n^Z2iAcC4INwL#t&O6E&A zo9A$`M)J&_M=YtAi0_J9IZ4)*FC^}IuifeI zA6-YUxBZ6QuvJyXSJAqpT$w6xNJHSvZsg`fn3!5;%4g53DKv#WuaQsPmn|Z_Q)c)= zPa4i~q$asPE?yN!dqs0J62H2g&2Y=G=ehmFQf$xLK-#je#l|xu!_z(V*p%*7C3x9q zE*xx`ixlZ;;hy6vw|M$CX@u6Z#)gj+g?eh}y?8;ec!8{RbExvkcHvQnmEDB*65g80 zZGQSY)E>s_1&=kB&C+_gQ&o9O?-cTj`@%!w{^u7qFRD-94}IIKhxRQp@;kTI347M9 zWOI3B&wApG_e2-BKoWnVkW1o)-hpUi+lA`_hV(f<#QwrEjoqB|)haq3K@V{bU48s9 z?doRZhZqB@%kZ#%PJdUl->m8_FM8`|a_Oh630++MtC8e36zaa{gOlG`6)9$=&+{2f z6Kxv?SR0Tg@-uHMIb8PkIKBDcm*U;vg|fu^tweC{kSWLs(ikibKs+P!FQ zhbO6$8TZ8Gyv$z=s62`g+m`Y7o(-Y^##_S?iSD|fz^=T|r zT)@?nwM7#tCo?;R9m&cevsG-|a?^gW*8ALFUy6!vZ^h-5yd#lAWI^FE?ZLhwuK!nb zsIpM9)NA^ZU~N9wYT)DJh>80*{A8`>f;fUlx=eeQ@6IkY71zrVhHl(3<&j0UmY4jz zOzM$+@>egb*dgnSK7c#1xE4WswkL4=V z;>S`I8ca28N@T(g6Os$ydv9%~^m@w{PSr|T4ab<%4Sdn_&i4}Pgwcw#eOgbaukG~~ zFxGy+vAJE;2-UpB=Ah+(Jww~~Mc?@7n~q}7k#&XwEobs03G$n7`CvQY+81AEWf3DC z)$1|ueVK6Mahf1C#i$##70bTqu8!N2pwmf#OrHCej`JTH*u`_rVN2$I_6!Jc8WIH}qf^u7s$~&>_v4$>XK&a-~H}7RHxz zjkIYS8?l#2C}*boM;X)l%^x^B>A6Y1y>g1w(F~2UG!hBWf_)I{pI={ZGv;#9{2_<2 zkLBZz*g%!}AHqzm^|6+Xf4nF>zQitFV=+zaGyS!%ts0GM=yZ*!J^p~_!Hre9`^kD* zr{KPzW9TgS1B$?BN+Q3|Tdgfs8*?@4b6xDsS~tFHPZ(}Ajj+_s-8GJ|ca%L`W$>oP zLOs>+Kl$RRJv3)pObBOYb;=7$?dxEJA@7!!KWn)%_eG=P8y*j`kvEfGZ(e=Ne&rX5 z0iMNmWufbdZp8INhEPi8(+DZXp-0fEMxNP2dB|(EkV`h}?W59d%f9Q*cQ_looXgD7 z`+X4>^d{W7ENQV&sN2qr_;W;;a0uLT`_j(}56AoEQ@=ygLZK3xF5A(b-U4OD=&7EC zKt`A18kbv5*}u++D_-Y$3Du_Qmy=&u56{R-aCO(1W9Q2E4o73^iy6mDOE$c4Q96ttKOThoGmh^W{HE z+CH6|&b^)gx9WQnPVhBsyg{<9bY52?sLP&||FfUE;LHSZcy5-{E&J#8jpda-f0;9- zJ|fS-U|6o+X=u1T9}W-ktqIlT)V3w^>!DPuHuTw1M;xI3m+|o15wi-xm%CAaag8dj) zm&@G{Lyn%qTJ-PwVb9H-W~N)jIDl=n%izh+6*QIFP%tCOIBtq2pJS z5sgBjetQ1%xPJ-Y3+DeND=ph{L2sdalx1H=$IaX2Z<=aXP~USyJE>%{bVFO z68v>W;?JtyrBfuAu*T9Dd2w&SKV$;C$9kDSa#H9agMF>)aT|G0S~wu^{~EMKab9O- zF*|5_*UM1Huht_%cr^CjAesy+a~xbZyAK~k4g<9~dWL-R-fYGM8KWaVV9#wnF70AN zHl_GKztB$tPfyO_MeW8|O7IM}IsCY+E6l>0Ud8Up+u&NC7uf%|5DGQaQx0L8<}MoO zu6%sC-ISL18Mi0pvK>pjrFL+uA-M+>1&WgSg{`_Y?JP>`$Ag~A{U}tI`9EKJUFC+u zq2&?IqVTNK=eC-j04SBXazEj3nQE^nU$AfZ1&Se`DoeC%q8$y zlH5hM$CK}3SpLobtL!_&qDX^oM_qJR5qFIUih)51Du@Kh2#YcVMFf<*WF%)KV}eyc zaRf1dpyVtdS-=&MJOm{UhzW-vNs{F6H0tj6-23O={^6rI^mMeP9wPUQ^VI_1~8 zhW*U4u@dsqqY~&kkW;?IlvUk#_MYkU(yvzoj%-k@7>>wJIbu}bKwimPxApI95i8}S z*g?&Z*y6&1y}Pp=a@=;+I@0Y911p5--D6+#a`0ADw>76EMs3fJuX7a4SIa-p6{pIR z*p-)$A<{kLnW{YQ005QASLFiML0mVcaX~hPcID+m%4(OCP}aWPOke#r;cJ8~4(1I$ zI^{ml^MIx8n_k4zo2_$a=4zt%R+30U+>G(sy5(7^FnB9oCPLRO zNu2TZNKP1Ltn}~gt2tgUuWFd}o;$6&WceViVwnF@;zZDE`LiyJ8?1(EcfAq{My1)_+MShi&gERb^k!P-TkG4W>bmVHoktca7fdsS`-@0(|ct|3}NGq;_FV)!|1g2tIc5 z8sp@MXL37llz1OvLz6OuW;t2kn8NG+mysh0Sz<*`dK#OB+~Ha~dc^v*d<36lRoPZf zI4sv|+gDo(sUKN4(Kf?Q_%mCt{H^ajt(EL^ud$3dm~vNYkGfd=J_ch5Z`Z`k+d zzzTJjdHag1wfk1T(50B)NhS8Ny?B_mTCC|s@|KPrO7)WI!|BUUuM9%(AKfw;`TUZ6 zO5)n<-FY{aEFWnvy_SD;ao&CU*Wz?CkD9nd%ojm9?Lcy@`KsHzhvz7#9eZ!Dh`-56 zS2Pb|7-Miq1J~x|yoMqZtF-tS`1j2NTQk20#>Tn%JI}-jUb}dv`X^S)8xC53qJrYP zf+Lt%ewOW>6<4^QH|WKhU0bH1J3#lp&30($MJlHA>q_Aq4sGtGPVS4R0NGyj<%F@ITdd5gY_eQ|l-wL;y2_oZ2w?I&n(bL95I29D zAYzEnbsSdc-1w24RmA@9-_4{J#)fAfy+ZN%XcZ=K_YHr>)>vYcnSNU(#9Hgl(e`vh zeo9Q0&&7-}pU(a!K7I}V=#KJREt{C`_1<2&eNKr3_;IbXB)hWWZNb{ruH1%acMEiV z-!u-6j&yE)-(yG8yL=wkR2txKla`yb*I-mg_D8k{yf4)`U=?1iq3#lqUp#=25Su4) zTj^qpZE=N7b6r!naF%`KIMi?wb86~XlAZ*I{)d|OiT%E#w}bFwBn@qYN2+aQGneyr z(WFOA4q0tVo=%+7Jhyv#*Gf`jV`PKH8#1#b+~y?W7@>t&+gYbd$NZtwdlo%=XCDG> zTtmz%f3|w#$4RH9AqUGl_##n45)Vc)GauP>)#LTPp^(h(zjWqH(k1nf{rM!30GzQbk;}7 zA>>$oqUFiT{{> z#e}BErMh zTldA^r+3pT5QXoLPK#Lub)33xXK3C*Y`7|spNNIH;zl+m)r~`HOAT&a3a{Q*de@`0UYr z_+EUH+j6X>bZdybt}KtZH!iDP${RftV|A#O9jlthc2ddm0bkeJD%J969QS=xQ|f)2 za((@J4UH#d>Zi6OJy9`A*S~R8%KX6r3PM7qOx1yS^Y_PugKFQ0I^8H}-Bi_On4K2d z?Nf54$cVa^Uc@i5W6S;-rV9dys+X_Q8(i1k?W}ZHSDm_PbD#S|! z?XQS~e*CVQDw$brka_!|O~0k2;FNDt^?E3ay+brBcTIZuxdvTq^PFGA!}!H_unvFU zfB3~1mQVrN^jf~B$qBn2orvlpOZ7a``XFZiPZQROv3Qs_(+LlPx>>#mDkjPGyx3LY zldM%w+KzK)tmN!!+x+(s_tlqg>Yd?Ih)Z^r>{0r*J%C{eO4d|STneejQ zlx(c}%l8kFNWZOmlOcbPNsu}Dl>#%Zf{PCzOL50%hor{Oq1{GR#v(k8 z4F_bLxx`YHXoROEt|az-F-qNw=l)XRxNy6yh`2nwo4=s8_moN4e&NlL1szyEprcc(+L1CA*vT@PU1{uqid z6m><#zrVN@uU)r}i-L=2ivhV}>mjjZqo21Q-D<^JV3gA#gr}^2Q$N? zqcbmET6xgm*}3=c-*;x41)(0PZ%FOuSB>619wBg7#;+7Y;(I7M*{0VHmCZX*s2w(~ z?O7(1xPvm4$Yw&qX^pkxk5xl@M+p%$+D(xUWYHvATy$23uqPs)nbM5B)xoCrf1l~j z@o)idZth0(S|U>*rsh<9v)Z&78Jk=bXw!vP8vh)(=MUI4U;G7S7pJ2;sBu@WTv3|l zvz73jFAzL(L~0{pyO*!GpBO_V$^O2kSXyPeZhnSKS+*_m2{lzz>ckrKb&QJgrvAO- zZMGKX%UxZ1jE|wAp|~AgYYK4h`{k_}d*4Vcg1LOu?br2k{mkp$@{{~CW9(OvzOl`U zko@@{`$@=zSOpxraG_w_@z}9r@pIv}d|v-9pVE}mT+En-UT<$?lKcc#K$YT*_O-cCcutYf(^P zN7YK#iAU)H|MC6@D*JB7QRAb5Wtb*xNhCx0p4FsLg$a8`;^}?NWFAq`vq(N{uH5C9 zhJ5=P8_%AH4ke)X}G(XPe@1u2ZC0w6GBja zAA3rACa&2;)c$SSiCsP#XAL~!v0f9PRxmR+$0s79g`QT8w@*nY*VWZ2-HaA z<$K($K1wfM75#Z+g#SVP=Xo)1ZpE!Z3Nte^G+S(|X0%SQM`m}J#MBLm;YKKa)$SY| z8ObD8e;y=6E&H3}2ih{DYn>Y+#m-)_7nwgp^QeV7y zVVbKGX_?-9TKKhzat^3HGYZj(4j-q$4=tSkv9AIbnNk#d8Sn)Zw{Xa z(*or&f1s(-!0W`D1|GdzhI+~jAyunoVDJzvyOPT0d-CvkY{O*o-0M8MkY#LQ;)(=f)ySh+dK-6<{A(KX zJ#Jk*K8V>F(WWEC-&~%$rO3P04Q^b{qw#sbo@!Rf3l8BlUEV{7XrZPm z_XI&ra`??VS}-G3G^KSMClvBl#FcNwQI!bOO*TDazpj6o6={w6$lC_PsPnCd!Plkvu!=$*|Al{^GVM>?r5$V>gIx3o3wsa8+wZeEz zhZ)h@RouFfRkJk^#U>{LS$0z6l-dkg)~!RG`eicV8#HqtmOWjf#%1UiYYpxok-`-4 z0&wH%)~#ud(kE3^BpHkFp-LxDwqjPGHA<$aBP|NvuTSab%nTzsJTx>06}k9KGL%T} zi@T=e*>tE66^wuVDrEmwQVPvWsY=89q;$t9I59g0J7#%+mbigl=$tt22*VuD;I$>h zf5T0_O_wdDdkq(!pbDY=w!TY~f~GhKuJ}H>B)VL$i1DqdscA;_%s~npHnsKq+#1jAdJ_+L}pM7tP{3>tB34Cnl0RW3uJ>@P+6H@=`!4<-5@m`iIO4C z)Rq(||MoH%*554y9O<%wBzlur_}JmUpFS00ob6x3OAWhsFZuTVQzWx@!1idKn&rtpV3Ho6@qHCXwF+Wf+OWA;)yIy!3V>NMQk+` z4VTW@`C#)?;~*XI&{;kK+J4WK|ht$7S-S+B8iQqkO@)>N<`-EG``Wt~eg2NQ43oTBh* z=Xk(xGknsjP1RLYf{)+iNXJL{_U@IFaQo8Ov$QzPTLB0B-4GAx`J5v_%2W&0DO*{`+yTefXWeDg-5+k3oT(|ZevG#^l!K732S|MkLj&xqbWBMjLry=PHt z$a%8=^@&w$Si;C9sG`@DE=BnOtr7quNsAdN+D*%d4HDUpY(#W(X{$(<1$}g&;Ab1G zp#U&Gc8l`_uT<$%^!I)-#MZ#r$ia~T*#IRf^w86xl&LxM-v*YYy7c%=_hW-bkkO3M zxd;}-+MCw=VGjx|QpXBY;3`g1!=j@bnqeWT2oE~S?(TC=PVUn97=fjv@}8!nyrso= zv3XIMS6gQ#$^ZUi`&4!^a}1m&tb;aHNm`9447BQf7Vmya%S`7~G9LY(nWNIurJaX& zNPD->6xbMT2Bj7Yz!xKlyQAnJvSKfEMu(>g=I6bAkT*n)!&*5zH`8B6BISmZrrU)I zSbl=2hk;#k!1#=8gK*9Pt8s_deDE#U^xM*^$QW^SKL1jMlJRl%Q?=ZeH%6;J(8nBK8MWJpOId zJ`KL(*iJWP7SB;PZ{3=(Yz`&ifG?KT7vt#F*sRUf7p;nDUdaKNX)Z3Vepr^ai>ubu zEUZjSNEomPyPk$&VC}{S(0s|7>51r&rqaciY9xcBK(-&j(7y~CHB0N-%0m*<8OHxdXtl#{{ zUAf2&r>LJkeaiqATxx!_oErDQu85+KJ>}#LM@IONrxpzyl;)xQo{Za1wv9=B%A@mpT*Pf1+_9?=597;A-&B#{fLF;jD>;uB>`}&qTdwF|5 zBLGEQT$~d_C&xSl_;p}j?C8;RuwOManwFzu){%rY92aB>h)2Hwpp%K}3??Qf#9qyX z-2%+LpFUMt<%;K{*-8D(`#(s}-IwjtGG;^LoP8mzhvi@ts7iO-p6I00q;1gkjIa>& z4e~^6hPv|_m=-i|>+J*x(Nt4=H}3(JRh=h)T?E9ko_f53BprgOLOU?hv-0;2VP27U z?zpuszzap*)XlYuCy+T7Z4+iVDfYxoUMVSk06Ubhp1H{(<gwjUwbS-ASzLNpUX z`u%})!!iRH0yWG?c6RpWjT;{W(ONgg94qQico25??gd(#QLav{kN;i@tm{B?e9&s_ zdlBkyiWT+P(4L z2wfH#n?El9{})sregPY@CDRb?PXDYj6|6QDJP$WSjbpbfIED;l(JqI}wT>;e*1uwc z`EOUpp|Yfs{s^RYBP@BOkDKeR_GCeqtv+K;UZPh}qnf(SYxIClT2K5X8+8C`!A2`|tfJ(u?N>(`ngT^({9Vgi~iW}Nn7 zk6FG~G3gG9_6S4(sdR-CC-|@&prLzT(jz<1u3M+y(X6NJ@#DQx7^{J%*g!dX`7PVG zC-*?$MW@u;BivZK)VmM?BuZ3WCiEM%vI7AhUx#oW!V<`99{0d9}!ZujC4K185V z67F|kQW1%mvK}v-!V@7ipN~!4CF#D<`N#hLq+H7;xwHDNn(+&M$?(pDDDYJS^{{2f zjx?-+{tbTLE+gc1EwzWkUyCW+VIoQ*ndGlN>UQMaX~eZ6w0wXw>O8EP)LtPbG?nBUn@E zZlx$sv>}tpM@=TCr$vc~P3A$p`N#MO11B$={<&#*fA$Ci&*brMK4%*ZMc=^^_{XkY_KJcCF~*cWqVsV6xEKK@huSufxKDRRqu~GYEA%pr^f9)yv@9&O z2WfB|7Ux!2n5m*5ph_?llNi!~GDpA=)I@oqi4Se+IuR&X(9yAKxLMlq!=DU~n~1+M z4&wa^km*`GDP{(f^Xg!J*N3S|b5ql!hVm8GYi?JY672}>=6Fc5lZV|*zPxH*PqMm^ zrCZKkd`n>TNwXmpYm-wzpz*>`Zd)*Bj2%M62pR-4pe)GnOIn&|$n@SgxX=I(X^uvL zFTQ=ceJU994vvmfSh%ICp+S_nT!6!~Zx6%o8^9wQOge6H9F8i>bJB|8*NF^%Hn-xE zF6SMXsG0frZCP791|PpR&k^I_%E`$Y0#y+PbFdn9k;Pb8*4hTYdf58K>2$CXnO3Ag+B}+$NDWTzJjcn?Nj@nw~)uXYLri3 zU)*h%rdQZ(+4(eiPxV~^Be_E7=%L521(;0$>S-G+Zg5BqbV(yDwvd7BaJEzBr{3P} z*z#DM@Lh)iluXX0YY{`l$5(p+7$S1lrMibw^juYjfoRupOb6$c|F9<0ug? zDee2kKL8d8`$4w;u$%P-6VZ*UIkaY#_f`*e|L#`YUGOElmYMPj1)b+I(kC8Q8fURS z5Vd#nTEwnl+Zt5yD+`MVOvQj(G$NK*INbp9M=&dnVx^@9E{6=BwQr}Ren_t~C@3gu zTm0~$VWpn3=g(zLusaV8LFPjMD-ZXF$P1rT=?DJtYn%25V?)bRv%F`VB0i>RrH&6r z*9^z3AZe&}EKbnv%UsVJf-KFEI(oFdz1TgoRvMeiIWNNZS0h$NX>q3Vlwi!gckG)IH|JtZY76TU8pHDhh#yQbND21x&*&D@nm#tn#vpl33Rk0M4o9r z1A_|tNkT%w-K@nch?PsXXRB!sVOwp`j1rv%(7SFOb|McZ8|61psD!hHZce#bG=Z9V zc&djydeDvn!8F{dI@$&~6rTd12QS4*(Lu$iu;E394()yt=t76R{V8IkCJQ#mo`QQsd6QPN?qb=^65<1he+{-@nEuCR#&9fi?tm zdq-m?_i}}C6Pjvzza3jeN(so)Z6W-f#>@8f_rcvjNwOxII-})L$1bm~ZfH0q@<0`; z-x{FH3a3`ATCL&v7FLo%Eu9_UgAsq+yr8tsB0eroL)ay=qrfqi+z+#t=@N|${~F`0 zD!cq*pAm9@voe4B#5Y)U3KeW}>Sj9Z&tuTB{W%TNXLNM#-xswXw(%OT7v-X$lraiD zKvG-`NY6Xq4J(J-0K+j@eTaaF-2ATLGgJ1H5U@>|c!uui&FGX6PlO!^t*LRqO^gal z?Grb4I8NpJ&D0v2W}V}evhP?dd|Jjkv_x|oa_BIN5X;;NP!aUM-uzE_R>IIx2?N}@ z6|NW_4iRnt%WIns4Z*~wPP79WDp9|VkAu$lqa9tlb}fywvuH9=W`WYLr z^@u9AS@JGjFzn9-{&73(Hn*Oz46cXW_?muyWp{w`h7@(M*esvUMe$pj>oE&^lXP^j z4UkN7vuiJ=BK*#s&tF}k#bF<)#g*UX6h!VvT@clXwU=W*JX>PjN|HBTiX9plB8&zi zYN0PE&6!g=;SfD08vy&{^#eDMMoF*iCIRk9mU^tkn(KdC#V-a4`*>T}YY=>)E>jZf zT|7}WouLV-8QM9ohWPcIoSX_5lzM{712>%YJe1m^Gc=u%u+{1Og%F`sa^UhAu?V|?IO#-VcF%If;I``IuXX~U3ugiYOkeF?<=qLZHMRY;aq7@eKO1o`~ao<5++dZ8NoLA^&z5fxn*Qzh|LR2WYL|E^C|6^*gdSGc9G=skOrMa?+d+7 zTagDGUSAGV)DfP;n@O@oPtf1ALJUTOpo++A%~$wm-oxS2TfZeA3|f7(j3L2(VB@6% z=Yg_1kFr%zV!ZBkP*LEbz&O7KuBLe|Uaw$B#X4XAmo^qazduzCHj>^Q`1b{#x2JpJ z1t_Mbadst>S|u}|AJB>^>SxXngJ$t4eqeshKzmj!hwnEdO0#o=R3ca$YgkK~+k^T9 zcqyPY%;I5jg0Q2>?MG|0H^Oatv6F*s%GB{o4X-_? zhf#jznVgiQD6{x2mYnc(r9Xd1zI`;&Z;8?jE+ZDAgo;$aJwTAXR|5iYyr4>Od$l{= zM@pu74TdMdgJ0y0Pe9`j3^7@oVw>tx$<@#Dx6n(km5Q!2k!KLR% ze=ad8si>_aJFJ8zg96xSFP-EcYCcb^Vd3LS~(c?|2sk`W}mH|CI z)opqX7xD5q_#mDKNN*5=)vwVad_+$)=aFT1QLczxR}zGJjsf2h2RX3BZ{Xu$Zz(8= zYX-LOZDAz@uNf7Rgy9&ufCZ52`0+Tg9-1C2to`*@hYkk3@f+7R{r%+v#Oy~Mb$rp$ znxe^vDq44_Y~3OjK`zK@@D$h@>LA91aj0AWP)e7ufwdkcCMJ_$@CJbuH6a>oD)InS zY$D*q02t*~^cL4jgpXl6^2)BI98mh*!QgJISEd?T7gx9%BWTnqc1t&}yZZ<|oINFW zyt&Awe5bF|I+J)EV8$A@y9mu?SG{bnpC(-M^DRf1{dK0`Se2>p_zg_^iOBuUnc!|D zo`9B3Qm}tE7691-zMPIUOooR9$UbDhurGh00R%Z|eK(|gV^AN=gOe3NVye>GSbZSQ zx9gW_fOjwPkp9k`Mx-~PEmoR1_Qn*5yY$5n%r}HSCfv>~-%*g5T?f0(QvdP@Z~i+5 n{QqPWh#L~u|2iiOomgsCY|gYEzXktHB2g4h{+S{7*QNgf%S~~} literal 0 HcmV?d00001 diff --git a/runs/frontier-phase-factorial-v0/plot_mock_phase_factorial.py b/runs/frontier-phase-factorial-v0/plot_mock_phase_factorial.py new file mode 100644 index 0000000..fdfc37d --- /dev/null +++ b/runs/frontier-phase-factorial-v0/plot_mock_phase_factorial.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Render the preregistered phase-factorial hypothesis schematic (mock data).""" + +from __future__ import annotations + +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt + + +def main() -> None: + labels = ("235B\nprefill-only", "235B\nmixed", "30B\nprefill-only", "30B\nmixed") + x = range(len(labels)) + phase_hypothesis = (1.0, 18.0, 2.0, 28.0) + stack_hypothesis = (1.0, 4.0, 24.0, 30.0) + + fig, ax = plt.subplots(figsize=(8.6, 4.8), constrained_layout=True) + ax.plot(x, phase_hypothesis, marker="o", lw=2, label="H-phase (mock)") + ax.plot(x, stack_hypothesis, marker="s", lw=2, label="H-stack (mock)") + ax.axhline(5, color="black", ls="--", lw=1.2, label="5% regret gate") + ax.set_xticks(list(x), labels) + ax.set_ylabel("Worst selected-config regret (%) — MOCK DATA") + ax.set_title("Schematic only: predictions that distinguish phase vs stack explanations") + ax.set_ylim(0, 35) + ax.grid(axis="y", alpha=0.25) + ax.legend() + ax.text( + 0.01, + 0.98, + "MOCK DATA / NOT A RESULT", + transform=ax.transAxes, + va="top", + color="crimson", + weight="bold", + ) + output = Path(__file__).with_name("mock-phase-factorial.png") + fig.savefig(output, dpi=180) + print(output) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-phase-factorial-v0/qwen30_prefill_client.py b/runs/frontier-phase-factorial-v0/qwen30_prefill_client.py new file mode 100644 index 0000000..de40676 --- /dev/null +++ b/runs/frontier-phase-factorial-v0/qwen30_prefill_client.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +"""Open-loop fixed-shape prefill-only workload for one real offered-load anchor.""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import hashlib +import http.client +import json +import math +import time +from pathlib import Path +from typing import Any + + +TTFT_SLO_MS = 1256.0 +TARGET_PASS_RATE = 0.95 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, required=True) + parser.add_argument("--served-model", required=True) + parser.add_argument("--model-path", type=Path, required=True) + parser.add_argument("--rate", type=float, required=True) + parser.add_argument("--requests", type=int, default=64) + parser.add_argument("--input-tokens", type=int, default=2048) + parser.add_argument("--timeout-seconds", type=float, default=900.0) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def percentile(values: list[float], fraction: float) -> float | None: + if not values: + return None + ordered = sorted(values) + index = min(len(ordered) - 1, max(0, math.ceil(fraction * len(ordered)) - 1)) + return ordered[index] + + +def run_request( + *, + request_index: int, + scheduled_at: float, + benchmark_start: float, + args: argparse.Namespace, + prompt_ids: list[int], +) -> dict[str, Any]: + delay = scheduled_at - time.perf_counter() + if delay > 0: + time.sleep(delay) + admitted = time.perf_counter() + record: dict[str, Any] = { + "request_index": request_index, + "scheduled_s": scheduled_at - benchmark_start, + "admitted_s": admitted - benchmark_start, + "admission_lag_ms": (admitted - scheduled_at) * 1000.0, + "success": False, + } + connection = http.client.HTTPConnection(args.host, args.port, timeout=args.timeout_seconds) + body = { + "model": args.served_model, + "prompt": prompt_ids, + "min_tokens": 1, + "max_tokens": 1, + "ignore_eos": True, + "temperature": 0, + "stream": True, + "stream_options": {"include_usage": True}, + "return_token_ids": True, + } + try: + started = time.perf_counter() + connection.request( + "POST", + "/v1/completions", + body=json.dumps(body, separators=(",", ":")).encode(), + headers={"Content-Type": "application/json"}, + ) + response = connection.getresponse() + if response.status != 200: + raise RuntimeError( + f"HTTP {response.status}: {response.read().decode(errors='replace')}" + ) + first_token_at = None + streamed_tokens = 0 + usage = None + while True: + raw = response.readline() + if not raw: + break + line = raw.decode(errors="replace").strip() + if not line.startswith("data:"): + continue + data = line[5:].strip() + if data == "[DONE]": + break + payload = json.loads(data) + if payload.get("usage"): + usage = payload["usage"] + emitted = 0 + for choice in payload.get("choices") or []: + token_ids = choice.get("token_ids") or [] + emitted += len(token_ids) if token_ids else int(bool(choice.get("text"))) + if emitted: + first_token_at = first_token_at or time.perf_counter() + streamed_tokens += emitted + finished = time.perf_counter() + if first_token_at is None or usage is None: + raise RuntimeError("missing streaming token or usage") + prompt_tokens = int(usage["prompt_tokens"]) + completion_tokens = int(usage["completion_tokens"]) + if prompt_tokens != args.input_tokens or completion_tokens != 1: + raise RuntimeError(f"usage mismatch: {prompt_tokens}+{completion_tokens}") + ttft = (first_token_at - started) * 1000.0 + record.update( + { + "success": True, + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "streamed_token_count": streamed_tokens, + "ttft_ms": ttft, + "e2e_ms": (finished - started) * 1000.0, + "slo_pass": ttft <= TTFT_SLO_MS, + } + ) + except Exception as error: # Failed requests remain in the SLO denominator. + record["error"] = f"{type(error).__name__}: {error}" + record["slo_pass"] = False + finally: + connection.close() + return record + + +def main() -> None: + args = parse_args() + if args.rate <= 0 or args.requests <= 0 or args.input_tokens <= 0: + raise ValueError("rate, requests, and input tokens must be positive") + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True) + excluded = set(tokenizer.all_special_ids) + candidates = [ + token_id for token_id in range(tokenizer.vocab_size) if token_id not in excluded + ] + if len(candidates) < args.requests + 1: + raise RuntimeError("tokenizer has too few non-special token IDs") + base_id = candidates[0] + prompts = [ + [candidates[index + 1], *([base_id] * (args.input_tokens - 1))] + for index in range(args.requests) + ] + prompt_hash = hashlib.sha256( + "\n".join(",".join(map(str, prompt)) for prompt in prompts).encode() + ).hexdigest() + + benchmark_start = time.perf_counter() + 2.0 + with concurrent.futures.ThreadPoolExecutor(max_workers=args.requests) as pool: + futures = [ + pool.submit( + run_request, + request_index=index, + scheduled_at=benchmark_start + index / args.rate, + benchmark_start=benchmark_start, + args=args, + prompt_ids=prompts[index], + ) + for index in range(args.requests) + ] + requests = [future.result() for future in futures] + requests.sort(key=lambda row: int(row["request_index"])) + completed = [row for row in requests if row["success"]] + passed = sum(bool(row["slo_pass"]) for row in requests) + ttfts = [float(row["ttft_ms"]) for row in completed] + pass_rate = passed / len(requests) + payload = { + "schema": "qwen30-prefill-rate-anchor-v1", + "workload": { + "offered_request_rate": args.rate, + "request_count": args.requests, + "input_tokens": args.input_tokens, + "output_tokens": 1, + "prefix_caching": False, + "arrival": "open_loop_uniform", + "last_scheduled_arrival_s": (args.requests - 1) / args.rate, + "prompt_vector_sha256": prompt_hash, + }, + "summary": { + "completed": len(completed), + "failed": len(requests) - len(completed), + "ttft_p50_ms": percentile(ttfts, 0.50), + "ttft_p95_ms": percentile(ttfts, 0.95), + "ttft_max_ms": max(ttfts) if ttfts else None, + "admission_lag_max_ms": max( + float(row["admission_lag_ms"]) for row in requests + ), + "slo": { + "ttft_threshold_ms": TTFT_SLO_MS, + "passed": passed, + "pass_rate": pass_rate, + "feasible": pass_rate >= TARGET_PASS_RATE, + }, + }, + "requests": requests, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + print(json.dumps(payload["summary"], sort_keys=True), flush=True) + if len(completed) != args.requests: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-phase-factorial-v0/remote-sync-marker/README.md b/runs/frontier-phase-factorial-v0/remote-sync-marker/README.md new file mode 100644 index 0000000..9dfd754 --- /dev/null +++ b/runs/frontier-phase-factorial-v0/remote-sync-marker/README.md @@ -0,0 +1,6 @@ +# Phase-factorial fleet sync marker + +Experiment source is synchronized through Git into the clean dash0 checkout +`/home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1`. This small +directory exists only to satisfy the fleet orchestrator's explicit sync phase +without copying local simulator caches or raw metrics to the GPU host. diff --git a/runs/frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py b/runs/frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py new file mode 100644 index 0000000..10249d7 --- /dev/null +++ b/runs/frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py @@ -0,0 +1,400 @@ +#!/usr/bin/env python3 +"""Freeze the Qwen30 fixed-shape prefill-only Frontier surface.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import importlib.util +import json +import math +import os +import subprocess +import sys +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + + +MODEL = "qwen3-a3b-30b-moe" +RATES = (4.0, 8.0, 16.0, 32.0, 64.0) +TTFT_SLO_MS = 1256.0 +TARGET_PASS_RATE = 0.95 +NUM_BLOCKS = {1: 20080, 2: 76537, 4: 191727} + + +@dataclass(frozen=True) +class Config: + tp: int + mns: int + + @property + def name(self) -> str: + return f"tp{self.tp}_mns{self.mns}" + + +GRID = tuple(Config(tp, mns) for tp in (1, 2, 4) for mns in (8, 16, 32, 64)) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--frontier-source", type=Path, required=True) + parser.add_argument("--replayserve-root", type=Path, required=True) + parser.add_argument("--profile-root", type=Path, required=True) + parser.add_argument("--python-deps", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument("--requests", type=int, default=64) + parser.add_argument("--rate", type=float, action="append") + parser.add_argument("--config", action="append") + parser.add_argument("--timeout-seconds", type=float, default=900.0) + parser.add_argument("--resume", action="store_true") + return parser.parse_args() + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + os.replace(temporary, path) + + +def load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise ImportError(path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +def write_trace(path: Path, *, requests: int, rate: float) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fields = [ + "arrived_at", + "num_prefill_tokens", + "num_decode_tokens", + "session_id", + "block_hash_ids", + ] + with path.open("w", newline="") as output: + writer = csv.DictWriter(output, fieldnames=fields) + writer.writeheader() + for request_id in range(requests): + writer.writerow( + { + "arrived_at": f"{request_id / rate:.12f}", + "num_prefill_tokens": 2048, + "num_decode_tokens": 1, + "session_id": request_id, + "block_hash_ids": "|".join( + str(request_id * 128 + block + 1) for block in range(128) + ), + } + ) + + +def profile_paths(root: Path) -> dict[str, Path]: + paths = { + "linear": root / "linear_op.csv", + "attention": root / "attention.csv", + "moe": root / "moe.csv", + "manifest": root / "manifest.json", + } + missing = [str(path) for path in paths.values() if not path.is_file()] + if missing: + raise FileNotFoundError(missing) + return paths + + +def validate_profile(paths: dict[str, Path]) -> dict[str, Any]: + manifest = json.loads(paths["manifest"].read_text()) + expected = manifest["outputs"] + for filename in ("linear_op.csv", "attention.csv", "moe.csv"): + actual = sha256(paths[{"linear_op.csv": "linear", "attention.csv": "attention", "moe.csv": "moe"}[filename]]) + if actual != expected[filename]: + raise ValueError(f"profile hash mismatch for {filename}") + with paths["attention"].open(newline="") as source: + rows = list(csv.DictReader(source)) + coverage = {} + for tp in (1, 2, 4): + exact = [ + row + for row in rows + if int(row["num_tensor_parallel_workers"]) == tp + and row["is_prefill"].lower() == "true" + and row.get("is_true_mixed_batch", "").lower() != "true" + and int(float(row["total_tokens"])) == 2048 + ] + if len(exact) != 1: + raise ValueError(f"expected one exact TP{tp} 2048-token prefill row, got {len(exact)}") + coverage[str(tp)] = {"exact_prefill_2048_rows": len(exact), "profile_batch_size": int(exact[0]["batch_size"])} + return {"manifest": manifest, "attention": coverage} + + +def knobs(config: Config, paths: dict[str, Path], cache: Path) -> dict[str, Any]: + return { + "simulation_mode": "online", + "sys_arch": "co-location", + "num_replicas": 1, + "cluster_scheduler": "sticky_round_robin", + "model_name": MODEL, + "device": "h20", + "network_device": "h20_dgx", + "attn_tensor_parallel_size": config.tp, + "attn_data_parallel_size": 1, + "moe_tensor_parallel_size": config.tp, + "moe_expert_parallel_size": 1, + "num_pipeline_stages": 1, + "replica_scheduler": "vllm_v1", + "decode_cuda_graph_mode": "none", + "batch_size_cap": config.mns, + "max_tokens_in_batch": 8192, + "long_prefill_token_threshold": 0, + "block_size": 16, + "num_blocks_mode": "explicit", + "num_blocks": NUM_BLOCKS[config.tp], + "gpu_memory_utilization": 0.92, + "non_kv_cache_overhead_bytes": 0, + "trace_max_tokens": 40960, + "cache_dir": str(cache), + "enable_dummy_mode": False, + "linear_op_input_file": str(paths["linear"]), + "atten_input_file": str(paths["attention"]), + "moe_input_file": str(paths["moe"]), + "prediction_max_prefill_chunk_size": 18000, + "prediction_max_batch_size": 128, + "prediction_max_tokens_per_request": 32768, + "no_cache": False, + "skip_cpu_overhead_modeling": True, + "enable_prefix_caching": False, + "enable_chunked_prefill": True, + } + + +def find_metrics(run_dir: Path) -> tuple[Path, Path]: + systems = list((run_dir / "metrics").rglob("system_metrics.json")) + requests = list((run_dir / "metrics").rglob("request_metrics.csv")) + if len(systems) != 1 or len(requests) != 1: + raise RuntimeError(f"metric pair mismatch: {len(systems)}/{len(requests)}") + return systems[0], requests[0] + + +def score(system_path: Path, request_path: Path, expected_requests: int) -> dict[str, Any]: + system = json.loads(system_path.read_text()) + metadata = system["simulation_metadata"] + if int(metadata["completed_requests"]) != expected_requests: + raise ValueError("Frontier completion count mismatch") + with request_path.open(newline="") as source: + rows = list(csv.DictReader(source)) + if len(rows) != expected_requests: + raise ValueError("request metric count mismatch") + values = [] + passed = 0 + for row in rows: + if int(float(row["request_num_prefill_tokens"])) != 2048: + raise ValueError("prefill shape drift") + if int(float(row["request_num_decode_tokens"])) != 1: + raise ValueError("decode shape drift") + ttft = float(row["ttft"]) + if not math.isfinite(ttft) or ttft < 0: + raise ValueError("invalid TTFT") + values.append(ttft) + passed += int(ttft <= TTFT_SLO_MS) + ordered = sorted(values) + pass_rate = passed / expected_requests + return { + "ttft_p50_ms": ordered[math.ceil(0.50 * len(ordered)) - 1], + "ttft_p95_ms": ordered[math.ceil(0.95 * len(ordered)) - 1], + "ttft_max_ms": max(ordered), + "passed": passed, + "pass_rate": pass_rate, + "feasible": pass_rate >= TARGET_PASS_RATE, + "throughput_requests_per_second": float(system["throughput_metrics"]["requests_per_second"]), + } + + +def main() -> None: + args = parse_args() + args.frontier_source = args.frontier_source.resolve() + args.replayserve_root = args.replayserve_root.resolve() + args.profile_root = args.profile_root.resolve() + args.python_deps = args.python_deps.resolve() + args.output_root = args.output_root.resolve() + rates = tuple(args.rate or RATES) + selected = list(GRID) + if args.config: + wanted = set(args.config) + selected = [config for config in GRID if config.name in wanted] + if {config.name for config in selected} != wanted: + raise ValueError(f"unknown configs: {wanted - {config.name for config in selected}}") + paths = profile_paths(args.profile_root) + coverage = validate_profile(paths) + builder = load_module( + "qwen30_prefill_frontier_builder", + args.replayserve_root / "tools/run_frontier_sweep.py", + ) + frontier_head = subprocess.run( + ["git", "-C", str(args.frontier_source), "rev-parse", "HEAD"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + traces = {} + for rate in rates: + trace = args.output_root / "traces" / f"r{rate:g}.csv" + write_trace(trace, requests=args.requests, rate=rate) + traces[rate] = trace + + config_results = [] + for config in selected: + loads = [] + config_knobs = knobs(config, paths, args.output_root / "cache") + for rate in rates: + run_dir = args.output_root / "runs" / config.name / f"r{rate:g}" + result_path = run_dir / "result.json" + if args.resume and result_path.is_file(): + loads.append(json.loads(result_path.read_text())) + continue + run_dir.mkdir(parents=True, exist_ok=True) + command = builder.build_frontier_command( + python_bin="/usr/bin/python3", + trace_file=traces[rate], + metrics_root=run_dir / "metrics", + run_id=f"qwen30_prefill_{config.name}_r{rate:g}", + knobs=config_knobs, + ) + write_json(run_dir / "command.json", command) + environment = os.environ.copy() + pythonpath = [str(args.python_deps), str(args.frontier_source)] + if environment.get("PYTHONPATH"): + pythonpath.append(environment["PYTHONPATH"]) + environment.update( + { + "PYTHONPATH": ":".join(pythonpath), + "CUDA_VISIBLE_DEVICES": "", + "NVIDIA_VISIBLE_DEVICES": "void", + "WANDB_DISABLED": "true", + "VIDUR_DISABLE_WANDB": "1", + "FRONTIER_LOG_LEVEL": "WARNING", + "PYTHONDONTWRITEBYTECODE": "1", + } + ) + started = time.time() + with (run_dir / "stdout.log").open("w") as stdout, ( + run_dir / "stderr.log" + ).open("w") as stderr: + completed = subprocess.run( + command, + cwd=args.frontier_source, + env=environment, + stdout=stdout, + stderr=stderr, + timeout=args.timeout_seconds, + check=False, + ) + if completed.returncode != 0: + raise RuntimeError( + f"Frontier failed for {config.name} rate={rate}: {completed.returncode}" + ) + system_path, request_path = find_metrics(run_dir) + result = { + "status": "completed", + "config": asdict(config) | {"name": config.name}, + "offered_request_rate": rate, + "offered_request_rate_per_gpu": rate / config.tp, + "request_count": args.requests, + "elapsed_seconds": time.time() - started, + "trace_sha256": sha256(traces[rate]), + "request_metrics_sha256": sha256(request_path), + "score": score(system_path, request_path, args.requests), + } + write_json(result_path, result) + loads.append(result) + print( + json.dumps( + { + "config": config.name, + "rate": rate, + "pass_rate": result["score"]["pass_rate"], + "feasible": result["score"]["feasible"], + }, + sort_keys=True, + ), + flush=True, + ) + config_results.append({"config": asdict(config) | {"name": config.name}, "loads": loads}) + + capacities = [] + for item in config_results: + feasible = [ + load["offered_request_rate"] + for load in item["loads"] + if load["score"]["feasible"] + ] + capacity = max(feasible) if feasible else None + capacities.append( + { + "config": item["config"], + "maximum_tested_feasible_request_rate": capacity, + "maximum_tested_feasible_request_rate_per_gpu": ( + capacity / item["config"]["tp"] if capacity is not None else None + ), + "lower_censored": capacity is None, + "upper_censored": capacity == max(rates) if capacity is not None else False, + } + ) + capacities.sort( + key=lambda row: ( + -(row["maximum_tested_feasible_request_rate_per_gpu"] or -1), + row["config"]["name"], + ) + ) + full = selected == list(GRID) and rates == RATES and args.requests == 64 + manifest = { + "schema": "frontier-qwen30-prefill-surface-v1", + "status": "frozen_before_real" if full else "partial_not_decision_bearing", + "contract": { + "rates": rates, + "requests_per_anchor": args.requests, + "input_tokens": 2048, + "output_tokens": 1, + "ttft_slo_ms": TTFT_SLO_MS, + "target_pass_rate": TARGET_PASS_RATE, + "prefix_caching": False, + "arrival": "open_loop_uniform", + }, + "frontier": { + "source": str(args.frontier_source), + "git_head": frontier_head, + "git_status_short": subprocess.run( + ["git", "-C", str(args.frontier_source), "status", "--short"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout, + }, + "profiles": { + "root": str(args.profile_root), + "coverage": coverage, + "sha256": {name: sha256(path) for name, path in paths.items()}, + }, + "config_results": config_results, + "capacity": capacities, + } + write_json(args.output_root / "frontier_surface_frozen.json", manifest) + print(args.output_root / "frontier_surface_frozen.json") + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-phase-factorial-v0/run_qwen30_prefill_real_config.sh b/runs/frontier-phase-factorial-v0/run_qwen30_prefill_real_config.sh new file mode 100644 index 0000000..a406d51 --- /dev/null +++ b/runs/frontier-phase-factorial-v0/run_qwen30_prefill_real_config.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash + +set -euo pipefail + +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}" +TP="${TP:?TP is required}" +MNS="${MNS:?MNS is required}" +RATES="${RATES:-4 8 16 32 64}" +SERVER_PORT="${SERVER_PORT:?SERVER_PORT is required}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +SERVED_MODEL="qwen3-30b-prefill-only" +SERVER_PID="" + +mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" +exec > >(tee -a "${OUTPUT_ROOT}/logs/controller.log") 2>&1 + +cleanup() { + if [[ -n "${SERVER_PID}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then + kill -TERM -- "-${SERVER_PID}" 2>/dev/null || true + for _ in $(seq 1 30); do + kill -0 "${SERVER_PID}" 2>/dev/null || break + sleep 1 + done + kill -KILL -- "-${SERVER_PID}" 2>/dev/null || true + fi + SERVER_PID="" +} +trap cleanup EXIT INT TERM + +IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:-}" +if [[ "${#GPU_IDS[@]}" -ne "${TP}" ]]; then + echo "ERROR: expected ${TP} allocated GPUs, got ${CUDA_VISIBLE_DEVICES:-unset}" >&2 + exit 1 +fi +read -r -a RATE_ARRAY <<< "${RATES}" + +echo "QWEN30_PREFILL_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 rates=${RATES// /,} rounds=2 requests=64 shape=ISL2048_OSL1 arrivals=uniform prefix=off cuda_graph=runtime_default isolation=fresh_server_per_anchor output=${OUTPUT_ROOT}" +date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" +sha256sum run_qwen30_prefill_real_config.sh qwen30_prefill_client.py \ + > "${OUTPUT_ROOT}/provenance/source.sha256" +"${VENV_ROOT}/bin/python" - "${TP}" "${MNS}" "${RATES}" \ + > "${OUTPUT_ROOT}/provenance/contract.json" <<'PY' +import importlib.metadata as metadata +import json +import platform +import sys + +tp, mns, rates = sys.argv[1:] +print(json.dumps({ + "python": platform.python_version(), + "torch": metadata.version("torch"), + "transformers": metadata.version("transformers"), + "vllm": metadata.version("vllm"), + "config": {"tp": int(tp), "mns": int(mns), "mbt": 8192}, + "rates": [float(value) for value in rates.split()], + "rounds": 2, + "requests_per_anchor": 64, + "anchor_isolation": "fresh_server_per_rate_per_round", + "target_rate_warmup_requests": "min(32, max(4, ceil(rate * 2)))", + "input_tokens": 2048, + "output_tokens": 1, + "ttft_slo_ms": 1256.0, + "target_pass_rate": 0.95, +}, indent=2, sort_keys=True)) +PY +nvidia-smi --query-gpu=index,name,uuid,driver_version --format=csv,noheader \ + > "${OUTPUT_ROOT}/provenance/gpus.csv" +sha256sum "${MODEL_ROOT}/config.json" > "${OUTPUT_ROOT}/provenance/model.sha256" + +export TOKENIZERS_PARALLELISM=false +export VLLM_USE_V1=1 +export TORCH_CUDA_ARCH_LIST=9.0 +export HF_HUB_OFFLINE=1 +export TRANSFORMERS_OFFLINE=1 + +for ROUND in 1 2; do + ROUND_ROOT="${OUTPUT_ROOT}/round${ROUND}" + mkdir -p "${ROUND_ROOT}/logs" "${ROUND_ROOT}/results" + ORDERED_RATES=("${RATE_ARRAY[@]}") + if [[ "${ROUND}" -eq 2 ]]; then + ORDERED_RATES=() + for ((index=${#RATE_ARRAY[@]}-1; index>=0; index--)); do + ORDERED_RATES+=("${RATE_ARRAY[index]}") + done + fi + for RATE in "${ORDERED_RATES[@]}"; do + KEY="$(printf 'r%.2f' "${RATE}" | tr '.' 'p')" + SERVER_LOG="${ROUND_ROOT}/logs/server_${KEY}.log" + setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \ + --host 127.0.0.1 --port "${SERVER_PORT}" --served-model-name "${SERVED_MODEL}" \ + --tensor-parallel-size "${TP}" --gpu-memory-utilization 0.92 \ + --max-model-len 40960 --max-num-batched-tokens 8192 --max-num-seqs "${MNS}" \ + --no-enable-prefix-caching --enable-chunked-prefill --no-enable-log-requests \ + > "${SERVER_LOG}" 2>&1 & + SERVER_PID=$! + READY=0 + for _ in $(seq 1 120); do + if curl -fsS --max-time 2 "http://127.0.0.1:${SERVER_PORT}/v1/models" \ + > "${ROUND_ROOT}/results/models_${KEY}.json" 2>/dev/null; then + READY=1 + break + fi + if ! kill -0 "${SERVER_PID}" 2>/dev/null; then + tail -200 "${SERVER_LOG}" + exit 1 + fi + sleep 3 + done + if [[ "${READY}" -ne 1 ]]; then + tail -200 "${SERVER_LOG}" + exit 1 + fi + + WARMUP_REQUESTS="$("${VENV_ROOT}/bin/python" - "${RATE}" <<'PY' +import math +import sys +print(min(32, max(4, math.ceil(float(sys.argv[1]) * 2.0)))) +PY +)" + "${VENV_ROOT}/bin/python" qwen30_prefill_client.py --port "${SERVER_PORT}" \ + --served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" --rate "${RATE}" \ + --requests "${WARMUP_REQUESTS}" \ + --output "${ROUND_ROOT}/results/warmup_${KEY}.json" + "${VENV_ROOT}/bin/python" qwen30_prefill_client.py --port "${SERVER_PORT}" \ + --served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" --rate "${RATE}" \ + --requests 64 --output "${ROUND_ROOT}/results/${KEY}.json" + cleanup + done +done + +find "${OUTPUT_ROOT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 \ + | sort -z | xargs -0 sha256sum > "${OUTPUT_ROOT}/provenance/artifacts.sha256" +date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ" +echo "QWEN30_PREFILL_REAL_CONFIG_COMPLETE" diff --git a/runs/frontier-phase-factorial-v0/simulator-tp1/frontier_surface_frozen.json b/runs/frontier-phase-factorial-v0/simulator-tp1/frontier_surface_frozen.json new file mode 100644 index 0000000..0dafcdd --- /dev/null +++ b/runs/frontier-phase-factorial-v0/simulator-tp1/frontier_surface_frozen.json @@ -0,0 +1,647 @@ +{ + "capacity": [ + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 8.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 8.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 8.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 8.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + } + ], + "config_results": [ + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "loads": [ + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 20.451046466827393, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "f0cfb072afcc2fc82a92c59ddb4e8cfc1e3201e15848f37785eeb9fdd975e779", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.0197002243768605, + "ttft_max_ms": 171.58529929215405, + "ttft_p50_ms": 171.58529929215405, + "ttft_p95_ms": 171.58529929215405 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 22.0544171333313, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "cfebfbd8aa5d9603158d8ff4e786119433b4c0d705f733ba242da46a6a6fdfb0", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 7.742175153637853, + "ttft_max_ms": 570.3133207139617, + "ttft_p50_ms": 391.4107605377559, + "ttft_p95_ms": 516.4107605377559 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 18.859471321105957, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "03578ba406b3299e70ba7975e6b9872d381a5f4bd4fbc3c8f6a8ba49e9b53f2e", + "score": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "throughput_requests_per_second": 8.916224898348922, + "ttft_max_ms": 3318.8404181099845, + "ttft_p50_ms": 1814.2305264490028, + "ttft_p95_ms": 3193.8404181099845 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 16.640390872955322, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "0b761b5f42e65b694eb34735433d43b473ed67415fb4e17b89894fc840b645fc", + "score": { + "feasible": false, + "pass_rate": 0.203125, + "passed": 13, + "throughput_requests_per_second": 9.07297385348011, + "ttft_max_ms": 5147.666503402199, + "ttft_p50_ms": 2739.3537469047046, + "ttft_p95_ms": 4962.713638565687 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 18.142091274261475, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "078209f2f56f21243384bdc70add105bf4942ee44ec8cc9573cad399a4a10e03", + "score": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "throughput_requests_per_second": 9.07297385348011, + "ttft_max_ms": 6100.791503402199, + "ttft_p50_ms": 3208.1037469047046, + "ttft_p95_ms": 5853.338638565687 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "loads": [ + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 19.96323037147522, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "f0cfb072afcc2fc82a92c59ddb4e8cfc1e3201e15848f37785eeb9fdd975e779", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.0197002243768605, + "ttft_max_ms": 171.58529929215405, + "ttft_p50_ms": 171.58529929215405, + "ttft_p95_ms": 171.58529929215405 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 18.554351568222046, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "cfebfbd8aa5d9603158d8ff4e786119433b4c0d705f733ba242da46a6a6fdfb0", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 7.742175153637853, + "ttft_max_ms": 570.3133207139617, + "ttft_p50_ms": 391.4107605377559, + "ttft_p95_ms": 516.4107605377559 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 25.70284128189087, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "03578ba406b3299e70ba7975e6b9872d381a5f4bd4fbc3c8f6a8ba49e9b53f2e", + "score": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "throughput_requests_per_second": 8.916224898348922, + "ttft_max_ms": 3318.8404181099845, + "ttft_p50_ms": 1814.2305264490028, + "ttft_p95_ms": 3193.8404181099845 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 20.056548833847046, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "0b761b5f42e65b694eb34735433d43b473ed67415fb4e17b89894fc840b645fc", + "score": { + "feasible": false, + "pass_rate": 0.203125, + "passed": 13, + "throughput_requests_per_second": 9.07297385348011, + "ttft_max_ms": 5147.666503402199, + "ttft_p50_ms": 2739.3537469047046, + "ttft_p95_ms": 4962.713638565687 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 11.15293574333191, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "078209f2f56f21243384bdc70add105bf4942ee44ec8cc9573cad399a4a10e03", + "score": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "throughput_requests_per_second": 9.07297385348011, + "ttft_max_ms": 6100.791503402199, + "ttft_p50_ms": 3208.1037469047046, + "ttft_p95_ms": 5853.338638565687 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "loads": [ + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 10.411258935928345, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "f0cfb072afcc2fc82a92c59ddb4e8cfc1e3201e15848f37785eeb9fdd975e779", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.0197002243768605, + "ttft_max_ms": 171.58529929215405, + "ttft_p50_ms": 171.58529929215405, + "ttft_p95_ms": 171.58529929215405 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 10.355234384536743, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "cfebfbd8aa5d9603158d8ff4e786119433b4c0d705f733ba242da46a6a6fdfb0", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 7.742175153637853, + "ttft_max_ms": 570.3133207139617, + "ttft_p50_ms": 391.4107605377559, + "ttft_p95_ms": 516.4107605377559 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 10.098335266113281, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "03578ba406b3299e70ba7975e6b9872d381a5f4bd4fbc3c8f6a8ba49e9b53f2e", + "score": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "throughput_requests_per_second": 8.916224898348922, + "ttft_max_ms": 3318.8404181099845, + "ttft_p50_ms": 1814.2305264490028, + "ttft_p95_ms": 3193.8404181099845 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 9.8981294631958, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "0b761b5f42e65b694eb34735433d43b473ed67415fb4e17b89894fc840b645fc", + "score": { + "feasible": false, + "pass_rate": 0.203125, + "passed": 13, + "throughput_requests_per_second": 9.07297385348011, + "ttft_max_ms": 5147.666503402199, + "ttft_p50_ms": 2739.3537469047046, + "ttft_p95_ms": 4962.713638565687 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 9.754717826843262, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "078209f2f56f21243384bdc70add105bf4942ee44ec8cc9573cad399a4a10e03", + "score": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "throughput_requests_per_second": 9.07297385348011, + "ttft_max_ms": 6100.791503402199, + "ttft_p50_ms": 3208.1037469047046, + "ttft_p95_ms": 5853.338638565687 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "loads": [ + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 9.84917163848877, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "f0cfb072afcc2fc82a92c59ddb4e8cfc1e3201e15848f37785eeb9fdd975e779", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.0197002243768605, + "ttft_max_ms": 171.58529929215405, + "ttft_p50_ms": 171.58529929215405, + "ttft_p95_ms": 171.58529929215405 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 10.197595357894897, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "cfebfbd8aa5d9603158d8ff4e786119433b4c0d705f733ba242da46a6a6fdfb0", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 7.742175153637853, + "ttft_max_ms": 570.3133207139617, + "ttft_p50_ms": 391.4107605377559, + "ttft_p95_ms": 516.4107605377559 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 10.00463604927063, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "03578ba406b3299e70ba7975e6b9872d381a5f4bd4fbc3c8f6a8ba49e9b53f2e", + "score": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "throughput_requests_per_second": 8.916224898348922, + "ttft_max_ms": 3318.8404181099845, + "ttft_p50_ms": 1814.2305264490028, + "ttft_p95_ms": 3193.8404181099845 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 10.153925895690918, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "0b761b5f42e65b694eb34735433d43b473ed67415fb4e17b89894fc840b645fc", + "score": { + "feasible": false, + "pass_rate": 0.203125, + "passed": 13, + "throughput_requests_per_second": 9.07297385348011, + "ttft_max_ms": 5147.666503402199, + "ttft_p50_ms": 2739.3537469047046, + "ttft_p95_ms": 4962.713638565687 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 10.055675268173218, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "078209f2f56f21243384bdc70add105bf4942ee44ec8cc9573cad399a4a10e03", + "score": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "throughput_requests_per_second": 9.07297385348011, + "ttft_max_ms": 6100.791503402199, + "ttft_p50_ms": 3208.1037469047046, + "ttft_p95_ms": 5853.338638565687 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + } + ], + "contract": { + "arrival": "open_loop_uniform", + "input_tokens": 2048, + "output_tokens": 1, + "prefix_caching": false, + "rates": [ + 4.0, + 8.0, + 16.0, + 32.0, + 64.0 + ], + "requests_per_anchor": 64, + "target_pass_rate": 0.95, + "ttft_slo_ms": 1256.0 + }, + "frontier": { + "git_head": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "git_status_short": " M frontier/config/config.py\n M frontier/entities/request.py\n M frontier/events/cluster_schedule_event.py\n M frontier/execution_time_predictor/sklearn_execution_time_predictor.py\n M frontier/metrics/constants.py\n M frontier/metrics/metrics_store.py\n M frontier/profiling/common/layers/rotary_embedding.py\n M frontier/profiling/moe/moe_impl.py\n M frontier/profiling/moe/moe_vllm_kernel.py\n M frontier/scheduler/cluster_scheduler/__init__.py\n M frontier/scheduler/cluster_scheduler/base_cluster_scheduler.py\n M frontier/scheduler/cluster_scheduler/cluster_scheduler_registry.py\n M frontier/scheduler/cluster_scheduler/sticky_lor_cluster_scheduler.py\n M frontier/scheduler/replica_scheduler/base_replica_scheduler.py\n M frontier/scheduler/replica_scheduler/vllm_v1_engine_replica_scheduler.py\n M frontier/scheduler/replica_stage_scheduler/replica_stage_schduler.py\n M frontier/simulator.py\n M frontier/types/cluster_scheduler_type.py\n?? data/profiling/compute/h20/\n?? frontier/scheduler/cluster_scheduler/prefix_lor_cluster_scheduler.py\n?? runs/\n?? tests/unit/test_attn_prefill_prediction_fallback.py\n", + "source": "/tmp/replayserve-frontier-rs1b" + }, + "profiles": { + "coverage": { + "attention": { + "1": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + }, + "2": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + }, + "4": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + } + }, + "manifest": { + "attention_tp_coverage": [ + 1, + 2, + 4 + ], + "environment_contract": { + "dtype": "bfloat16", + "frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "hardware": "NVIDIA H20", + "model": "Qwen3-30B-A3B", + "tensor_parallel_sizes": [ + 1, + 2, + 4 + ], + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + }, + "inputs": { + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931" + }, + "outputs": { + "allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3", + "attention.csv": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d", + "attention_true_mixed_fused.csv": "43ef4be90bddc9aeac6dbbe339feec24162cd1f2129a08fbd959e6ee4eaf5f60", + "linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "moe.csv": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0" + }, + "profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-fused-mixed-total-conserving", + "projection_contract": { + "allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity", + "attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target; true mixed rows use a total-conserving compatibility projection", + "attention_true_mixed": "The directly measured fused total is preserved in diagnostics. Frontier's two targets are projected by the same-TP pure prefill/decode reference ratio, with projected prefill + decode exactly equal to the fused total; the split is a schema compatibility attribution, not an observation", + "linear": "Frontier profiler using vLLM 0.20 CUDA operators", + "moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero" + }, + "row_counts": { + "allreduce": 24, + "attention_frontier_compatible": 132, + "attention_true_mixed_fused_diagnostic": 30, + "linear": 36, + "moe": 72 + }, + "schema_version": "frontier_qwen30_vllm020_frozen_profile.v2" + } + }, + "root": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2", + "sha256": { + "attention": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d", + "linear": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "manifest": "af40545e75aff55c6333cd2d5379ccf042a5a0b7d7fc7df4f745ce256cb290eb", + "moe": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0" + } + }, + "schema": "frontier-qwen30-prefill-surface-v1", + "status": "partial_not_decision_bearing" +} diff --git a/runs/frontier-phase-factorial-v0/simulator-tp2/frontier_surface_frozen.json b/runs/frontier-phase-factorial-v0/simulator-tp2/frontier_surface_frozen.json new file mode 100644 index 0000000..34e2063 --- /dev/null +++ b/runs/frontier-phase-factorial-v0/simulator-tp2/frontier_surface_frozen.json @@ -0,0 +1,647 @@ +{ + "capacity": [ + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 16.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 16.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 16.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 16.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + } + ], + "config_results": [ + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "loads": [ + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 264.1678559780121, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "e5f6527532682ee5aca3f423367d14eba063f30e63f46d2ba0ed602db4f6eb17", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.032196934357252, + "ttft_max_ms": 122.24062760264154, + "ttft_p50_ms": 122.24062760264154, + "ttft_p95_ms": 122.24062760264154 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 10.15904188156128, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "8dd3bedcbf76d30bf4285e10b886f95bef9ae8e5d41f4bb50a7650b322b8eb83", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.00276032449276, + "ttft_max_ms": 122.24062760264154, + "ttft_p50_ms": 122.24062760264154, + "ttft_p95_ms": 122.24062760264154 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 10.447325944900513, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "38c1b13c1f5fbdabd7f79b82ccb1fb37fce6840bcf03614c988d7e4882814506", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 13.174042777348715, + "ttft_max_ms": 1048.2973400621365, + "ttft_p50_ms": 643.9195307403329, + "ttft_p95_ms": 962.8278877316862 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 10.053350687026978, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "cb3b7b5a2fcf63cd01304b3b530bb21c0a9bb1c8f5e06d489e3863afd0b4d2f3", + "score": { + "feasible": false, + "pass_rate": 0.390625, + "passed": 25, + "throughput_requests_per_second": 13.539443740959392, + "ttft_max_ms": 2851.9297930156335, + "ttft_p50_ms": 1510.0519836938302, + "ttft_p95_ms": 2758.1797930156335 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 9.744416952133179, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "a4474c98a68171ada0fb007c36fcf69e9e2dc13f06c74889591e23f93418bd5a", + "score": { + "feasible": false, + "pass_rate": 0.265625, + "passed": 17, + "throughput_requests_per_second": 13.539443740959392, + "ttft_max_ms": 3773.8047930156335, + "ttft_p50_ms": 1995.3684369244447, + "ttft_p95_ms": 3622.636520081023 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "loads": [ + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 9.402071952819824, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "e5f6527532682ee5aca3f423367d14eba063f30e63f46d2ba0ed602db4f6eb17", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.032196934357252, + "ttft_max_ms": 122.24062760264154, + "ttft_p50_ms": 122.24062760264154, + "ttft_p95_ms": 122.24062760264154 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 9.398229360580444, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "8dd3bedcbf76d30bf4285e10b886f95bef9ae8e5d41f4bb50a7650b322b8eb83", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.00276032449276, + "ttft_max_ms": 122.24062760264154, + "ttft_p50_ms": 122.24062760264154, + "ttft_p95_ms": 122.24062760264154 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 9.70562195777893, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "38c1b13c1f5fbdabd7f79b82ccb1fb37fce6840bcf03614c988d7e4882814506", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 13.174042777348715, + "ttft_max_ms": 1048.2973400621365, + "ttft_p50_ms": 643.9195307403329, + "ttft_p95_ms": 962.8278877316862 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 9.598989009857178, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "cb3b7b5a2fcf63cd01304b3b530bb21c0a9bb1c8f5e06d489e3863afd0b4d2f3", + "score": { + "feasible": false, + "pass_rate": 0.390625, + "passed": 25, + "throughput_requests_per_second": 13.539443740959392, + "ttft_max_ms": 2851.9297930156335, + "ttft_p50_ms": 1510.0519836938302, + "ttft_p95_ms": 2758.1797930156335 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 9.697861671447754, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "a4474c98a68171ada0fb007c36fcf69e9e2dc13f06c74889591e23f93418bd5a", + "score": { + "feasible": false, + "pass_rate": 0.265625, + "passed": 17, + "throughput_requests_per_second": 13.539443740959392, + "ttft_max_ms": 3773.8047930156335, + "ttft_p50_ms": 1995.3684369244447, + "ttft_p95_ms": 3622.636520081023 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "loads": [ + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 9.397430419921875, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "e5f6527532682ee5aca3f423367d14eba063f30e63f46d2ba0ed602db4f6eb17", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.032196934357252, + "ttft_max_ms": 122.24062760264154, + "ttft_p50_ms": 122.24062760264154, + "ttft_p95_ms": 122.24062760264154 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 9.3956880569458, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "8dd3bedcbf76d30bf4285e10b886f95bef9ae8e5d41f4bb50a7650b322b8eb83", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.00276032449276, + "ttft_max_ms": 122.24062760264154, + "ttft_p50_ms": 122.24062760264154, + "ttft_p95_ms": 122.24062760264154 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 9.647802591323853, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "38c1b13c1f5fbdabd7f79b82ccb1fb37fce6840bcf03614c988d7e4882814506", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 13.174042777348715, + "ttft_max_ms": 1048.2973400621365, + "ttft_p50_ms": 643.9195307403329, + "ttft_p95_ms": 962.8278877316862 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 9.694743156433105, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "cb3b7b5a2fcf63cd01304b3b530bb21c0a9bb1c8f5e06d489e3863afd0b4d2f3", + "score": { + "feasible": false, + "pass_rate": 0.390625, + "passed": 25, + "throughput_requests_per_second": 13.539443740959392, + "ttft_max_ms": 2851.9297930156335, + "ttft_p50_ms": 1510.0519836938302, + "ttft_p95_ms": 2758.1797930156335 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 9.594672203063965, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "a4474c98a68171ada0fb007c36fcf69e9e2dc13f06c74889591e23f93418bd5a", + "score": { + "feasible": false, + "pass_rate": 0.265625, + "passed": 17, + "throughput_requests_per_second": 13.539443740959392, + "ttft_max_ms": 3773.8047930156335, + "ttft_p50_ms": 1995.3684369244447, + "ttft_p95_ms": 3622.636520081023 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "loads": [ + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 9.398636817932129, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "e5f6527532682ee5aca3f423367d14eba063f30e63f46d2ba0ed602db4f6eb17", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.032196934357252, + "ttft_max_ms": 122.24062760264154, + "ttft_p50_ms": 122.24062760264154, + "ttft_p95_ms": 122.24062760264154 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 9.495105981826782, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "8dd3bedcbf76d30bf4285e10b886f95bef9ae8e5d41f4bb50a7650b322b8eb83", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.00276032449276, + "ttft_max_ms": 122.24062760264154, + "ttft_p50_ms": 122.24062760264154, + "ttft_p95_ms": 122.24062760264154 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 9.54387378692627, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "38c1b13c1f5fbdabd7f79b82ccb1fb37fce6840bcf03614c988d7e4882814506", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 13.174042777348715, + "ttft_max_ms": 1048.2973400621365, + "ttft_p50_ms": 643.9195307403329, + "ttft_p95_ms": 962.8278877316862 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 9.192227840423584, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "cb3b7b5a2fcf63cd01304b3b530bb21c0a9bb1c8f5e06d489e3863afd0b4d2f3", + "score": { + "feasible": false, + "pass_rate": 0.390625, + "passed": 25, + "throughput_requests_per_second": 13.539443740959392, + "ttft_max_ms": 2851.9297930156335, + "ttft_p50_ms": 1510.0519836938302, + "ttft_p95_ms": 2758.1797930156335 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 9.294084072113037, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "a4474c98a68171ada0fb007c36fcf69e9e2dc13f06c74889591e23f93418bd5a", + "score": { + "feasible": false, + "pass_rate": 0.265625, + "passed": 17, + "throughput_requests_per_second": 13.539443740959392, + "ttft_max_ms": 3773.8047930156335, + "ttft_p50_ms": 1995.3684369244447, + "ttft_p95_ms": 3622.636520081023 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + } + ], + "contract": { + "arrival": "open_loop_uniform", + "input_tokens": 2048, + "output_tokens": 1, + "prefix_caching": false, + "rates": [ + 4.0, + 8.0, + 16.0, + 32.0, + 64.0 + ], + "requests_per_anchor": 64, + "target_pass_rate": 0.95, + "ttft_slo_ms": 1256.0 + }, + "frontier": { + "git_head": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "git_status_short": " M frontier/config/config.py\n M frontier/entities/request.py\n M frontier/events/cluster_schedule_event.py\n M frontier/execution_time_predictor/sklearn_execution_time_predictor.py\n M frontier/metrics/constants.py\n M frontier/metrics/metrics_store.py\n M frontier/profiling/common/layers/rotary_embedding.py\n M frontier/profiling/moe/moe_impl.py\n M frontier/profiling/moe/moe_vllm_kernel.py\n M frontier/scheduler/cluster_scheduler/__init__.py\n M frontier/scheduler/cluster_scheduler/base_cluster_scheduler.py\n M frontier/scheduler/cluster_scheduler/cluster_scheduler_registry.py\n M frontier/scheduler/cluster_scheduler/sticky_lor_cluster_scheduler.py\n M frontier/scheduler/replica_scheduler/base_replica_scheduler.py\n M frontier/scheduler/replica_scheduler/vllm_v1_engine_replica_scheduler.py\n M frontier/scheduler/replica_stage_scheduler/replica_stage_schduler.py\n M frontier/simulator.py\n M frontier/types/cluster_scheduler_type.py\n?? data/profiling/compute/h20/\n?? frontier/scheduler/cluster_scheduler/prefix_lor_cluster_scheduler.py\n?? runs/\n?? tests/unit/test_attn_prefill_prediction_fallback.py\n", + "source": "/tmp/replayserve-frontier-rs1b" + }, + "profiles": { + "coverage": { + "attention": { + "1": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + }, + "2": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + }, + "4": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + } + }, + "manifest": { + "attention_tp_coverage": [ + 1, + 2, + 4 + ], + "environment_contract": { + "dtype": "bfloat16", + "frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "hardware": "NVIDIA H20", + "model": "Qwen3-30B-A3B", + "tensor_parallel_sizes": [ + 1, + 2, + 4 + ], + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + }, + "inputs": { + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931" + }, + "outputs": { + "allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3", + "attention.csv": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d", + "attention_true_mixed_fused.csv": "43ef4be90bddc9aeac6dbbe339feec24162cd1f2129a08fbd959e6ee4eaf5f60", + "linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "moe.csv": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0" + }, + "profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-fused-mixed-total-conserving", + "projection_contract": { + "allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity", + "attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target; true mixed rows use a total-conserving compatibility projection", + "attention_true_mixed": "The directly measured fused total is preserved in diagnostics. Frontier's two targets are projected by the same-TP pure prefill/decode reference ratio, with projected prefill + decode exactly equal to the fused total; the split is a schema compatibility attribution, not an observation", + "linear": "Frontier profiler using vLLM 0.20 CUDA operators", + "moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero" + }, + "row_counts": { + "allreduce": 24, + "attention_frontier_compatible": 132, + "attention_true_mixed_fused_diagnostic": 30, + "linear": 36, + "moe": 72 + }, + "schema_version": "frontier_qwen30_vllm020_frozen_profile.v2" + } + }, + "root": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2", + "sha256": { + "attention": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d", + "linear": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "manifest": "af40545e75aff55c6333cd2d5379ccf042a5a0b7d7fc7df4f745ce256cb290eb", + "moe": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0" + } + }, + "schema": "frontier-qwen30-prefill-surface-v1", + "status": "partial_not_decision_bearing" +} diff --git a/runs/frontier-phase-factorial-v0/simulator-tp4/frontier_surface_frozen.json b/runs/frontier-phase-factorial-v0/simulator-tp4/frontier_surface_frozen.json new file mode 100644 index 0000000..82a1227 --- /dev/null +++ b/runs/frontier-phase-factorial-v0/simulator-tp4/frontier_surface_frozen.json @@ -0,0 +1,647 @@ +{ + "capacity": [ + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 16.0, + "maximum_tested_feasible_request_rate_per_gpu": 4.0, + "upper_censored": false + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 16.0, + "maximum_tested_feasible_request_rate_per_gpu": 4.0, + "upper_censored": false + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 16.0, + "maximum_tested_feasible_request_rate_per_gpu": 4.0, + "upper_censored": false + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 16.0, + "maximum_tested_feasible_request_rate_per_gpu": 4.0, + "upper_censored": false + } + ], + "config_results": [ + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "loads": [ + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 244.95408463478088, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 1.0, + "request_count": 64, + "request_metrics_sha256": "c0c3c34bea43dae2477b10152b07632e5d03968f47cdfd70ad5b676e2502af4c", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.039607508408214, + "ttft_max_ms": 93.12333977685495, + "ttft_p50_ms": 93.12333977685405, + "ttft_p95_ms": 93.12333977685495 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 9.85124945640564, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "b5c6ba0cd3b2a81a12fb93b2cad6f80341f9352c110385046490f59d7933a5d5", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.032004183533672, + "ttft_max_ms": 93.12333977685495, + "ttft_p50_ms": 93.12333977685495, + "ttft_p95_ms": 93.12333977685495 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 9.70260500907898, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "c40f10c895f2fc45671f7ff7c6c08f30c20854f0ff4fe3e6719934a7b08b0112", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 15.412041282697286, + "ttft_max_ms": 314.4210260695206, + "ttft_p50_ms": 239.37774488651664, + "ttft_p95_ms": 309.403713596319 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 9.855336904525757, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "9f77ee40d102bdf14d213132ef10b4a7f4175908ffee411ef30ad3a5f1e4b16f", + "score": { + "feasible": false, + "pass_rate": 0.625, + "passed": 40, + "throughput_requests_per_second": 16.898601280793688, + "ttft_max_ms": 1850.42236218991, + "ttft_p50_ms": 1017.6783445024347, + "ttft_p95_ms": 1787.92236218991 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 9.947782754898071, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "efb43cc32f2f7129082e47d89239f08b10c6e7e6dd4a4f0b299d2364cb781af0", + "score": { + "feasible": false, + "pass_rate": 0.390625, + "passed": 25, + "throughput_requests_per_second": 17.184897883039092, + "ttft_max_ms": 2771.0751922610093, + "ttft_p50_ms": 1468.937931419684, + "ttft_p95_ms": 2661.056949107159 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "loads": [ + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 9.648890733718872, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 1.0, + "request_count": 64, + "request_metrics_sha256": "c0c3c34bea43dae2477b10152b07632e5d03968f47cdfd70ad5b676e2502af4c", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.039607508408214, + "ttft_max_ms": 93.12333977685495, + "ttft_p50_ms": 93.12333977685405, + "ttft_p95_ms": 93.12333977685495 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 9.353798151016235, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "b5c6ba0cd3b2a81a12fb93b2cad6f80341f9352c110385046490f59d7933a5d5", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.032004183533672, + "ttft_max_ms": 93.12333977685495, + "ttft_p50_ms": 93.12333977685495, + "ttft_p95_ms": 93.12333977685495 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 9.39468502998352, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "c40f10c895f2fc45671f7ff7c6c08f30c20854f0ff4fe3e6719934a7b08b0112", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 15.412041282697286, + "ttft_max_ms": 314.4210260695206, + "ttft_p50_ms": 239.37774488651664, + "ttft_p95_ms": 309.403713596319 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 9.29756498336792, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "9f77ee40d102bdf14d213132ef10b4a7f4175908ffee411ef30ad3a5f1e4b16f", + "score": { + "feasible": false, + "pass_rate": 0.625, + "passed": 40, + "throughput_requests_per_second": 16.898601280793688, + "ttft_max_ms": 1850.42236218991, + "ttft_p50_ms": 1017.6783445024347, + "ttft_p95_ms": 1787.92236218991 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 9.34343934059143, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "efb43cc32f2f7129082e47d89239f08b10c6e7e6dd4a4f0b299d2364cb781af0", + "score": { + "feasible": false, + "pass_rate": 0.390625, + "passed": 25, + "throughput_requests_per_second": 17.184897883039092, + "ttft_max_ms": 2771.0751922610093, + "ttft_p50_ms": 1468.937931419684, + "ttft_p95_ms": 2661.056949107159 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "loads": [ + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.296925783157349, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 1.0, + "request_count": 64, + "request_metrics_sha256": "c0c3c34bea43dae2477b10152b07632e5d03968f47cdfd70ad5b676e2502af4c", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.039607508408214, + "ttft_max_ms": 93.12333977685495, + "ttft_p50_ms": 93.12333977685405, + "ttft_p95_ms": 93.12333977685495 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.345365285873413, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "b5c6ba0cd3b2a81a12fb93b2cad6f80341f9352c110385046490f59d7933a5d5", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.032004183533672, + "ttft_max_ms": 93.12333977685495, + "ttft_p50_ms": 93.12333977685495, + "ttft_p95_ms": 93.12333977685495 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.350512504577637, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "c40f10c895f2fc45671f7ff7c6c08f30c20854f0ff4fe3e6719934a7b08b0112", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 15.412041282697286, + "ttft_max_ms": 314.4210260695206, + "ttft_p50_ms": 239.37774488651664, + "ttft_p95_ms": 309.403713596319 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.300485372543335, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "9f77ee40d102bdf14d213132ef10b4a7f4175908ffee411ef30ad3a5f1e4b16f", + "score": { + "feasible": false, + "pass_rate": 0.625, + "passed": 40, + "throughput_requests_per_second": 16.898601280793688, + "ttft_max_ms": 1850.42236218991, + "ttft_p50_ms": 1017.6783445024347, + "ttft_p95_ms": 1787.92236218991 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.292456150054932, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "efb43cc32f2f7129082e47d89239f08b10c6e7e6dd4a4f0b299d2364cb781af0", + "score": { + "feasible": false, + "pass_rate": 0.390625, + "passed": 25, + "throughput_requests_per_second": 17.184897883039092, + "ttft_max_ms": 2771.0751922610093, + "ttft_p50_ms": 1468.937931419684, + "ttft_p95_ms": 2661.056949107159 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "loads": [ + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.294773817062378, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 1.0, + "request_count": 64, + "request_metrics_sha256": "c0c3c34bea43dae2477b10152b07632e5d03968f47cdfd70ad5b676e2502af4c", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.039607508408214, + "ttft_max_ms": 93.12333977685495, + "ttft_p50_ms": 93.12333977685405, + "ttft_p95_ms": 93.12333977685495 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.34990382194519, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "b5c6ba0cd3b2a81a12fb93b2cad6f80341f9352c110385046490f59d7933a5d5", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.032004183533672, + "ttft_max_ms": 93.12333977685495, + "ttft_p50_ms": 93.12333977685495, + "ttft_p95_ms": 93.12333977685495 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.346213579177856, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "c40f10c895f2fc45671f7ff7c6c08f30c20854f0ff4fe3e6719934a7b08b0112", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 15.412041282697286, + "ttft_max_ms": 314.4210260695206, + "ttft_p50_ms": 239.37774488651664, + "ttft_p95_ms": 309.403713596319 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.498731136322021, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "9f77ee40d102bdf14d213132ef10b4a7f4175908ffee411ef30ad3a5f1e4b16f", + "score": { + "feasible": false, + "pass_rate": 0.625, + "passed": 40, + "throughput_requests_per_second": 16.898601280793688, + "ttft_max_ms": 1850.42236218991, + "ttft_p50_ms": 1017.6783445024347, + "ttft_p95_ms": 1787.92236218991 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.300755977630615, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "efb43cc32f2f7129082e47d89239f08b10c6e7e6dd4a4f0b299d2364cb781af0", + "score": { + "feasible": false, + "pass_rate": 0.390625, + "passed": 25, + "throughput_requests_per_second": 17.184897883039092, + "ttft_max_ms": 2771.0751922610093, + "ttft_p50_ms": 1468.937931419684, + "ttft_p95_ms": 2661.056949107159 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + } + ], + "contract": { + "arrival": "open_loop_uniform", + "input_tokens": 2048, + "output_tokens": 1, + "prefix_caching": false, + "rates": [ + 4.0, + 8.0, + 16.0, + 32.0, + 64.0 + ], + "requests_per_anchor": 64, + "target_pass_rate": 0.95, + "ttft_slo_ms": 1256.0 + }, + "frontier": { + "git_head": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "git_status_short": " M frontier/config/config.py\n M frontier/entities/request.py\n M frontier/events/cluster_schedule_event.py\n M frontier/execution_time_predictor/sklearn_execution_time_predictor.py\n M frontier/metrics/constants.py\n M frontier/metrics/metrics_store.py\n M frontier/profiling/common/layers/rotary_embedding.py\n M frontier/profiling/moe/moe_impl.py\n M frontier/profiling/moe/moe_vllm_kernel.py\n M frontier/scheduler/cluster_scheduler/__init__.py\n M frontier/scheduler/cluster_scheduler/base_cluster_scheduler.py\n M frontier/scheduler/cluster_scheduler/cluster_scheduler_registry.py\n M frontier/scheduler/cluster_scheduler/sticky_lor_cluster_scheduler.py\n M frontier/scheduler/replica_scheduler/base_replica_scheduler.py\n M frontier/scheduler/replica_scheduler/vllm_v1_engine_replica_scheduler.py\n M frontier/scheduler/replica_stage_scheduler/replica_stage_schduler.py\n M frontier/simulator.py\n M frontier/types/cluster_scheduler_type.py\n?? data/profiling/compute/h20/\n?? frontier/scheduler/cluster_scheduler/prefix_lor_cluster_scheduler.py\n?? runs/\n?? tests/unit/test_attn_prefill_prediction_fallback.py\n", + "source": "/tmp/replayserve-frontier-rs1b" + }, + "profiles": { + "coverage": { + "attention": { + "1": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + }, + "2": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + }, + "4": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + } + }, + "manifest": { + "attention_tp_coverage": [ + 1, + 2, + 4 + ], + "environment_contract": { + "dtype": "bfloat16", + "frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "hardware": "NVIDIA H20", + "model": "Qwen3-30B-A3B", + "tensor_parallel_sizes": [ + 1, + 2, + 4 + ], + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + }, + "inputs": { + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931" + }, + "outputs": { + "allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3", + "attention.csv": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d", + "attention_true_mixed_fused.csv": "43ef4be90bddc9aeac6dbbe339feec24162cd1f2129a08fbd959e6ee4eaf5f60", + "linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "moe.csv": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0" + }, + "profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-fused-mixed-total-conserving", + "projection_contract": { + "allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity", + "attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target; true mixed rows use a total-conserving compatibility projection", + "attention_true_mixed": "The directly measured fused total is preserved in diagnostics. Frontier's two targets are projected by the same-TP pure prefill/decode reference ratio, with projected prefill + decode exactly equal to the fused total; the split is a schema compatibility attribution, not an observation", + "linear": "Frontier profiler using vLLM 0.20 CUDA operators", + "moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero" + }, + "row_counts": { + "allreduce": 24, + "attention_frontier_compatible": 132, + "attention_true_mixed_fused_diagnostic": 30, + "linear": 36, + "moe": 72 + }, + "schema_version": "frontier_qwen30_vllm020_frozen_profile.v2" + } + }, + "root": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2", + "sha256": { + "attention": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d", + "linear": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "manifest": "af40545e75aff55c6333cd2d5379ccf042a5a0b7d7fc7df4f745ce256cb290eb", + "moe": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0" + } + }, + "schema": "frontier-qwen30-prefill-surface-v1", + "status": "partial_not_decision_bearing" +} diff --git a/runs/frontier-phase-factorial-v0/test_phase_factorial.py b/runs/frontier-phase-factorial-v0/test_phase_factorial.py new file mode 100644 index 0000000..6e30aa5 --- /dev/null +++ b/runs/frontier-phase-factorial-v0/test_phase_factorial.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + + +ROOT = Path(__file__).parent + + +def load(name: str): + path = ROOT / name + spec = importlib.util.spec_from_file_location(path.stem, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_percentile() -> None: + client = load("qwen30_prefill_client.py") + assert client.percentile([4.0, 1.0, 3.0, 2.0], 0.50) == 2.0 + assert client.percentile([4.0, 1.0, 3.0, 2.0], 0.95) == 4.0 + assert client.percentile([], 0.95) is None + + +def test_grid_and_trace(tmp_path: Path) -> None: + surface = load("run_frontier_qwen30_prefill_surface.py") + assert len(surface.GRID) == 12 + assert {config.tp for config in surface.GRID} == {1, 2, 4} + trace = tmp_path / "trace.csv" + surface.write_trace(trace, requests=3, rate=4.0) + lines = trace.read_text().splitlines() + assert len(lines) == 4 + assert lines[1].split(",")[:3] == ["0.000000000000", "2048", "1"] + assert lines[3].split(",")[:3] == ["0.500000000000", "2048", "1"] From 3a59d5df96489ab7c5061a6e9cc7a904454569f7 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 03:11:45 +0800 Subject: [PATCH 095/148] Evaluate Qwen30 prefill simulator fidelity --- .../qwen30-prefill-ranking.png | Bin 0 -> 108856 bytes .../analyze_qwen30_prefill_fidelity.py | 462 ++ .../experiment-card.md | 18 +- .../fleet-base-rerun.toml | 24 + .../fleet-exclusive.toml | 24 + .../fleet-refine-tp4-wave3.toml | 24 + .../fleet-refine-tp4-wave4.toml | 24 + .../fleet-refine.toml | 24 + .../jobs_base_rerun.toml | 21 + .../jobs_full.toml | 24 +- .../jobs_refine.toml | 240 + .../jobs_refine_tp4_wave3.toml | 41 + .../jobs_refine_tp4_wave4.toml | 21 + .../results/final/capacity.csv | 13 + .../results/final/comparison.json | 5860 +++++++++++++++++ .../frontier_surface_frozen.json | 461 ++ .../frontier_surface_frozen.json | 461 ++ .../frontier_surface_frozen.json | 461 ++ .../test_phase_factorial.py | 6 + simulator-fidelity.md | 99 +- 20 files changed, 8280 insertions(+), 28 deletions(-) create mode 100644 docs/assets/simulator-fidelity/qwen30-prefill-ranking.png create mode 100644 runs/frontier-phase-factorial-v0/analyze_qwen30_prefill_fidelity.py create mode 100644 runs/frontier-phase-factorial-v0/fleet-base-rerun.toml create mode 100644 runs/frontier-phase-factorial-v0/fleet-exclusive.toml create mode 100644 runs/frontier-phase-factorial-v0/fleet-refine-tp4-wave3.toml create mode 100644 runs/frontier-phase-factorial-v0/fleet-refine-tp4-wave4.toml create mode 100644 runs/frontier-phase-factorial-v0/fleet-refine.toml create mode 100644 runs/frontier-phase-factorial-v0/jobs_base_rerun.toml create mode 100644 runs/frontier-phase-factorial-v0/jobs_refine.toml create mode 100644 runs/frontier-phase-factorial-v0/jobs_refine_tp4_wave3.toml create mode 100644 runs/frontier-phase-factorial-v0/jobs_refine_tp4_wave4.toml create mode 100644 runs/frontier-phase-factorial-v0/results/final/capacity.csv create mode 100644 runs/frontier-phase-factorial-v0/results/final/comparison.json create mode 100644 runs/frontier-phase-factorial-v0/simulator-refine-tp1/frontier_surface_frozen.json create mode 100644 runs/frontier-phase-factorial-v0/simulator-refine-tp2/frontier_surface_frozen.json create mode 100644 runs/frontier-phase-factorial-v0/simulator-refine-tp4/frontier_surface_frozen.json diff --git a/docs/assets/simulator-fidelity/qwen30-prefill-ranking.png b/docs/assets/simulator-fidelity/qwen30-prefill-ranking.png new file mode 100644 index 0000000000000000000000000000000000000000..afe8d37a3c8a7b34748750cad34675713fa9306b GIT binary patch literal 108856 zcmeEucU;fy|L?mL4I?y+mJkw^GSW^%QAz_1sf4DchL#a3RFpzWk@nP1N!mtf(4sBU z)Sk}sa)0mNd7N_|=kYjyoIiga_xJwZl24z{^}epx>$zT+uZFtv3Z@NA6h*C2IjnG; zq8KeGYKaHKQhbFvQI~{&A9Pea>8N$_oTIa`y&0uu>}YFs(b3A>g!ht}y@UBh8`0e& z!n?%MTd}53pEtF(9^wls{`RdX^UzF9mpCt3{Aw9~ z3-ZW5dEHRpTW!>DkZViRU4G{f{%34#Ok-$hxNiOWDEr>(k&%&N=XQ^qToRW$z7{wu z?m2Dlck5Phe`BJ^g<9SN2M?B%meRSoxotNplGMw#@?9@_j{E>4k8@|wo*8V(y3m@v zD=9fSDm67gHAJA(U#O_~;}eNZ4DwMXL$y)Y+D0eF#_BH=Y&#awQNiPNJuz|T`wt(Y z5)+x%u3g)ZVz}|zvQ_0lo0XZ^P8j?6_!xhC<$9t1#fpJ}fiq{%(&2lwQ%`eAxs4Y; z+H1+e!xNb*t)-=vZuX9SyuXpTF+t~0bB0;evu8{2)rMq!j<5MnZwH!Ru4ZMue*5-v zACKvo8Q+SQii+3~3BB8=Uwhv1TAW!{R8;h%C2J?%by?Gq-^^QhczE(GGoK3m3^nrc zsEXKcWYq3{5nnVCOb-s88gAdbeLDj)$L>;Znk5=9H7H}@6sxp~vF?S!VZ=$OM% zDM`uSSv65nQS0{GaB^~S8Qa*bJap(#Q*(2a*}Df}`>Z#}%gg_6_XtWfG1D5 z%I|Xde@Qj6%sjSP>9)55pIof^6ETZpf22G;Jx3-cbT04m9C#pjiq^=;Nbpp~^UOn= z$S&ZE{#lG68LjqGZA3vs)nylY*_Evd?q2YVQPXPn2kHsC%73Bo@`|o^kQdQpF z{Z&g}+spGcV*QD6kq>9YUknWmZ77=_`YI9SJp9dK&cWWkE>7n7P<6=8tA&My)!hbB z^7Nfe>81j6u9E|&iiV;dik=r3{Dr-7H9bAOWDpPEM>&8y{&w&ChoV<;M3%2u;ggn@ zc9oNTc9pHOGe>ITv4`X94emv*in}n%`9L~i$=~&yuc!H2327?hX&a*sEi-uKvYL9D znCLI`dVb9={p{;mS$pZ^WVN+fQ;c5Uy}#{PS7YJ{7P_d~#wgYLClb!3(}o5H9Y25i zVn36UC1g}o`mONt>J1zGGBd@FMo6wVH#ZNH_QPva=F{|!)dvnzW>zuPy zYv#x6k9E}~>*pwi?l#oOvo|X(E9;z{_|=qQ##~odXFEG??)bBge$%E+wVywKK9GIY zZRYpZe8-b`!sLagFj?T_2E4dCnJszW4Sy@?JAA8Jhq(iPP--#KE876E@yW24T z`kjsY<+1&YUvmhZR+yU{47zuZmP*UYQcOKvSo>6NN!X)Dd^>h9c6N5=+IG@jy?Qn9 z!Gjw~Nl7N{?OnkVo+wGEHeGmITe;umw0rsweJ$8&C}ZDqIGwK@WM_W&reir`x^?1@D|XH;%uXEov$zmqvrXs4(e+8EUcNYKx?^`< zWc?anIy?$xM@O-)uCCvWCp2Y5V;ZY?be#d`!CjJTkE$KUgnVfD4KhDYAEj1 zOa9w;?yO$B_L_wAP{)r?&o&I~inpH{GR!ouUpn0WdKuM_cw!ammlEpGY95||XUYLx z9dE9=%uVic8~#Q1nAK2TzY8tNrmQDjii6}f9mH(@PiXJ^Uk^?y$G zcj)_QJkj6ivM{^X-QAsK!-fRMxlhlO53$g_4-;9Ulc=k+QRv?u+^hAo_txLvga}q@ zhiII|Y3p*keED+c;=-KjT=mC~?5H?&7cXAy#2%pEFl?%kdRnUU@x`Lm+>$O&Ge1Nt z`j;-$e)_bUTiQJ?a}d=>D0{Fe{aN6>dwe<4{QOEOEuS;Y-u-67k@cXbuD*QvvSV;? z*W|N==+8G-@$@A4CmFpyZCsF+Cj92}XI-ltuRn|KWx6ap((WvHRNL;JO4Qxe*;VGR z+KN{&+^(OU)a2QeW~|zJP+Z)15FhY&IP)oAU*D4QI~#q(9lmcj_!F<)z_eloThZdI zr7;$?r1xE@4~~Q!%c8T8=TWbpLi#Ey^%m-RRl1#3HAPjr4aapJ#%?j$!HAcU#}oIF z`7^I|z+*NOZ9<6Qs=MMkwHi4*Wd95%MO32Nd?!m9@2APhy95pMwJWQukGDvpKGvEp zhUVqx`}H-qw(4VVSJt1yfx7zO!Gme{6Ael%J;RYDlBQ?^LX$5ozwFL)dc8QCm^*~G z&6E?GZRhC7T327s%(={C@ua4|G`r-@n1H^&plJJ_Oh5jw{(p;aeU5eOK0DXm>JZhP zx|BHEDZR`KS5Pd9D=Xu+l^C`@`EZ`fwa zbzoOFWtwZHmpgTq&{N(aA#83_L%VW@3Wkf;dQGXeR=LfN@9RCZYP?!$)z`hzZKOzLzk2Oj<6Z2y%eT2C*ePL~wgY13bqddNuY}X>iN|urS?L2N*qy33 zXJhW48S6Xw+EdCq{O8L3)-7~9ckaw;&CAO({{H@9SB>bo_p;cV1{<3#n?HQGW0YHe zew&EMY8216UyQu&08qWDs$#skl1pIH&j0trQ^9m{#U(z>G}OwKD=lI_?qrgdmJU3; z1^+OtTD6QC_*%g6Quf3Nju$Up#GOdxF2B8gGr!x;uFg&xz@@2XwSzAP-r;j}PBu3;H)UCJpjq8a z&`B(*tYiSvQ$%?eIrlznjq7RJKLyGWGF~fr`$Qi`F9dQco3DAM!1o5=k9E?uqnIYo z+Nj7qKJ6<`ed#_L8XBfnhO6F&1(yh#i8~&{w0!w;jT8fMu?w}-+HLBk-4y}MUMwT8 z$98y}K5J(7rlw}q>C>mtTYc`{T{$o~$YyhSph;wUditr-oeikAf@qT+9UT-v?Dgx{ zk52V0s@Xj44!`Fas|~PTJxEb@c#Wb zi;ASJtgT1OIE;#GYK{%^Jn!Mo5w>=SJVuR7P2I#+5ztLy0A2*heN$1PJk-vVprEYW z9UJK>kH@Ep)*JTtG1)=P?1CBxxw82H0!nVL7w!1|eJSwakN*CW_wSeQwP=vVE)=%w zDqSvmPRPG{9QJM6&*q)KjxAs~a1nLq`2&I3Bkh-ir zIn@{GWC{(XRKAGDe_HERIf2y%Fj3&7oJ1dc^Hei+Ffd!?B*8e%Uhr?uFm)gnassXl2xN z;D<9@Te_m!LH%s1@5qR)91C41kUfgXwYqGN+40pGFJH3Vo$0As8|609ubLC$Vffm! zuxZj`peeoUz|IpX+Rgj+?bEeNOioU=&F_hRG&|CI@iiO805ntca`(=iM{4Wq*HR4r z96}HDNu#0hK4WIK6%dcj#a-uYq{NjgSL`cK0m8k?@VzfJz_jTJgJu`w|^d^N(&?QarJ*LAx9dXE<3+3g(Aa{HrVgg&C% zl!9(YOUdD|9adGXekm;=pla*jptU|UB_&1Jd8kdEXIb;u`n78}XIeEM?;O+r45a6? z&$=Z}>JqBaiGw~CnqS{yrC%$@nC2H0FrzE5p*MwZi)gFq15T=1nA7}}=(#W}AY)p2 z_i^Mpwjim<9Xoc&PYrzyDUV4_jqGb*l?N!UPLN-Va-rMAbv%}=zK(XUKOv8AoNj*C z(4aji@|Q->8uTQ;Q&+&Rah~3e4sReqih8iy&<|kaW$@-%m%{OrFCKc4yUXfJv;OYkmzUF%mF==d~v0OXAP?tRykYq$BvO0Gw-hAu7#zJGke zOo4iM=j1pSl~<+Wz~6e8!FEE-_{ZI+JXZA%(^xEM_Zm~zSA#VCQX4CRiawDM~^OV$+El& zRKH1Dnwu0ky(|GqDJcfe) zpHhuvGA+NXM42oDC`fakv3H*yp($`4R>BL4n3OW)iGVkuy^P=_P_o!fp9-8WYA5Qh zK=;1o@BemTZff>1N0n-w?jTI*30kOBWmDZl zNY>k%Y2C_>A563ooN7D@LIS$bIHyNCm&{HL2h`STa}upay2kbwaw zkjs)?yLQFs6VX1|d-w?;Ag{Ri`pwFBOYvAWplpo3e368PL{sUSvSI!D zP5486sxV1c`-tqk)ceK7vdg(8PYy}Rv6MD}6>Ferj!MhUE>AOl zOUTQXZQHs_ebs7gc+sgN@OGt_GY^0g4@}1=LC|5bhW2! z?SN+pJQ?}jd&F_5ZM|W>)xs%W~4nE(qw`nRBcX^=&*1(yS> zE4_QL%Wh%TDLuhhW!*ljbt^d!c!MoG14AN>J^I5gRpJ z7sLGO)vGh6rq{49UwO>2@4xsBE43{!Fi=Bs*|om}#8f-q)r3?rg{{l>c%>&+UVOd4 z|K$Y=K?w@kW?fr>{?Ue+kS&tloe7#sSDHbHy?mG+P*Z?q^F@Vs*D21cOZn59Vvk#a4rhl%52?dzvxm!_3ihw;s~lSY}Y6mv$iz zzYltn0M}0R!&etKh+W`C70^74MNtnwut_^XTLsgtZ?ddyvl*arsuZcI>fGVcde4*{8H5Ig;AQh`+k z7M2Un&1)01aG@bC$oo%f7h%OdgVE8^rGEN)dU_@J5Dj;ehsGig&cv5P;FSOzv9b#9 zTUT^>O5NR4CD9m6yYFZB_YaRv2FUq+={f%6Q`hW+UHVL;qob}ZbaZqKAi01D%F4>Q zcHK9=ON1X8Vlqc}F08Kctz%MKt4B?;!K>W_W@pw)Du zHQTyuANtz|hY#n^pJzrB4&Hd|lXzfqbdr^zxff`vrU&(xHcD?k+CboG^q=L;e-7(z z5f?vY>*VxdtSk^h=pv(K946;|b^k0}QOXg24IT|D8JY2ti6@=p-~trxe`&b*=B1X0G? z$YS+cTFdnd+#ZpHEW`64?ckA$AF!_20M%wLZ|YrJ9l&>bTzkF z^e)~j5>72HE_T0@+|T^H?sxhY3-jM~TZ*oh-vjXFJm`3Mo#?sSbvyEi3ab9(AmfqH z-E3%ZVng~*x>Wz36tnZ^w>332F`zN8U0j^IXx#VlNeLd>bqK3WXhmP9#dpsY9C_5H z2i{_gzEqNJ-AV_dWenZNUQ5YyrZ?25QuzOy19x^pERcz=Z1Yymzc-` z@3U&+b)Wj`d^IE_#DHhxJ}Zrb5d#yUb6bNB_pD)(KXiy5+-=>44NPcmyV~S54^8Ib z9Xu0bR0a@-CBfQ?51dtO`Y}x8(9el23|GUR8U|jDdY+=(tgy<<%PVG(cFB_7k50yG z!pK3Id8`XIJta>gKvc-f9o0yeTtL(fJ(7~pXXm+?An^OcZi%el&+ z&8l8G5atyOoV!XH^PHfHv8AfO-sr^AshS3mR9wLdlbzJu3x8n)6$~nz9{WBe)wmu z;*+>W2o^C5lAiM?%?f^fNqy(>LgN8zRuM{I5u|M;_NS>yd9Be zJ%9xL*>{zo_%cZ*hrBQ`H7)DCsxG>%ghzIO@7|@OhYkzP$ zZA0z&!yQ>b$V5!X5>-r(c7MVKbh&)_?UyecM8SfxP&x?Z<~s27j)84(Pz+G8RwtZD zy^F2d4ZEbY_{tKIvz086M7bfM3yX-*P&iTy;0l8G7P$9XusT8}OtJhF?UM_EdI$t|Bi%f+x)s>p5w~ zwY+jHzo>`T2|E`&eacIDgNwJlTurUyk-iO0m?{q3bVwuJL@_2E5}a;0(>&k_Q>4ty z#E1Ruaq}Kqr2?S#%gm2lt1f!~{(Wtp!+AAZ_CW)r$+_847cX2lKnXB zs#8IH7yvJNis7n8T;C~A4^$J?FFG(Dq1P-8oO0g9`=^ad##0i?ChO$0s;m)_g&}^@ z0%pUuFB*yc(+sw6-n>~*FOvaBrvgvH_IHm8N&q){vHhUU*LeQ)q((VVK0e5zFUAlZwz`@L#XuSMV{OtY(HKTqp2t@uMa|aUXN5Do9M+1k$E@RDRoV~Su-7~k^=9fN zfVO~6LU61hI{ql+3tuDoKj2tDzNAK0uM_L+?@w@ig8H3n05wRGQK7n}ysWG$*%A~4 z9SKI#)l)CeZQ8wi<)ItPcVzSg&%>uGhu8Nh-BcN%)EKXt>oy^z4F7&>`Hcm-g>l|yZL*8o&dALKfn9DoUlmX8AIGCwVXrxzFzk&;#e*!bPG z=G{AHaJCYx@+ZJ2^3%Yo?;qhm@nTC#1tb(d#6s^ciUJy0j z>w`kFgd$Wf(sP!P)S2n^K7V z%S&ABE9JWrPLk40$ZniQ)YX#s7Hr*ZPz96$jjmxQCGz>|By5kBem^WKDuahpy5l*u zJv=%EO`vNQO)+6WXvYqfa9T|uA}q9^hCxQn8K?U1Dy>--jjCo6&FAk#_swGG(*f!@ z-_efKko)niuprI0Q@#`Zfxad5YRk*>0l?u&BQ5m2KYspX#NKCzNx7*g2cjcwz_z)z zR45Tw<`}UqIC2CtK{>uP>;>3HOTGE_?R5BeaXvnI+e??&DTWmr_X&=KWz;+YV2-zL zFTCuWzJNN`^BE|SS?k5o)gW@g+;eS0ijj@vqXRfrX4RvkLI&G(6mSJnN*LMmV&z>$ z+*5GpnJURjpwf7Ijz=4OswI+#a{fMe|$UtUGmnBBEd*Zj! ziVA-=v*4g72|`m1UdJLNSML;33lpZno0w@^;`%OBr~=9ZML}WGojvgzl<8#Vg`nxN zx|k!(ds75pV2?yaXH%OmXRh5f) zI0K!M_r&^ch}y0#mD0*+Gf5e*K}MeHChLK;a#X8}pVD}w%OhwNQbKSIKqYD;+ngJW zaw3oLcuvze{rap32&ii1mjv^(6M`}UmbK4S)&}o9u@d>Cw3jd6LJP9(t!5+qcgQ&+ z;ph8?pyZMO=PBn&K zu6=$lZoGuP1k^49-*1$g6t)W&wtugQ41mQM4KHi@SL$n4-2=OZevN_^blH0bJmdp= zd;^pi6c|WRAkM2_9DR5d^8H)vb4iF21Y@O1;7rJL9DFXxfwgsrgdfGsa#QHC91zNCOk z?~N>xft+5?Xll4pgzIf1P}IlVn>!VHLX)oQE{0b+4SwT*D7$R zkXF{~>FFZSOuVu8DB>grpu+Yg28{L9_`jeI%wFfT^Fj!jtS`$$4wC4SMMp_z6Xm@w_SW+t7Nf5>rlt3(d@I7m-CI!EH--m`) zIy*a;mX>yapdavBltfdtoB3@9#l5>N&w*&JcX?!1u38mf5>NrC1)@`q?+nS#a~_V! z9q8&h-jR;ldIk@K|7Jx+g=KK@W=k+}@=g(@5x)3Ma&!TINuGrc-ZzAmLpB@t@86I5 z+kyQ?kRP--!wwJXspi#(R`>2%MITaCWg-p)@A>LVD6%nCzK^!7LkFRO1W<;Y2sv;> ztpsvGxIzp|9srfI3v{6ZG8>XNk0e_;`X;tNo)2s{eQ_%_HT8g6KN1KdfOi~mk@JC4 z&Oj^dv5RxsRS|FT6M&jjj}}|BWX8LEgdcqyvJxLWZ@`Q=&1W~lYT?kN+Um98%YN@3 zg)eEQQ);&M_s8=^9!eA2Ew5&DMyFxQR%K`MSny@P?yfC`7vX6GXGcv+#!0LM^2fQW z0Hj$By&tkLdH1o$BFnM!gF-@l5On0-vV{&<>%rrt5C6gjhq8adsnN7?+VN{FJ}2$) zoS7wUnWr0|#I;CpSKOvUH;55x_BbwXJIVi`9@9{uiKf72u-`Y_>HP8I7NRk?;$1+B+fU++#2vlphC$+!-Udom=jOcBpLOMi@6ZF-i zI(mDz@>^IpXVBx@8(mt#bp~3pj-;fPAZD4IYry$s4>dhEXZktmlw#e9GiT2DZaJgz zGd(S>_^I4AVdOQ*K7#T@ST5PP=<<9L68cHmS(m45sA4=$VdIkPYy8vGuvIm5d{`(V zH14*%=d!g`>3(TSK#-3QeZKSXUC0(oC`jHSX4R}#?L}PRs9|E}_u7UZ)!7e!lM?w< z`{qrQ2BQyraCkFoy>uOS#oY@G^d=-x_Tz#H_Gij{-RnDHgn%^tw$g{!|H^Gb)#}8$ zy30!U_d&u-VQMC>?sIRng!m-h(8#lE5CH@G0nl0j?zRNM8X}bgx-ErNezM@wuJQn$ z09fuj+lmgPC~w@j(N0`Y@Tm1=9cz569lSFU4z>D(q?&I0e1p?^(kdVN2d#CnBGu^aXro-MV!>KlLY@VI*}+dQ}EK&OFY+ z>y^yP0px{-Ot!;@t;5^346y&IJR%x2og?u12$IQm3m>04tdf?V{^rBu{RAlgYKVW9 zX^m8837x$ez}XFp#)M8VKw65}GKg^&ql-YrV4g?8gIp~a)%6!H%(B|{ANtEFY40uY zX=R%f$3p0ECOaVPLL+=Mgj#zoTW%weck3Op0|Kqw5|@q|3_)F&e)fT*76t6 zpOZLmR}E~TGWIX{v$CIj0i5FZQsaY&d{t%=aCtzD{#H_I+ut zlao`3`m(wB^sKBhlk{rzQKU1@`wwc%}(IZ)^|t1qeaUrRgyk&ZxS6@!PA_pykJu&{M@`AyLWY%*#ND%Q(a-_t#Vj#j^2 zC3TC+)S<7^hgH_5d52$rh76mXKtBxT%B&E68$?S;GSXy^f3*23n#zTsH?UH)%LxTV zDWSx59%*l_V`Ve%?PzFdkXO5+cN~rpxG^mf%T$l{Oz?v9HNK3-AhVkEV6jf!uRN-U zV_O6nGwc#BpXSJ3bMcwdoiqq(qvwA*sgXWAo@W}(PVnNVKlc~9WS;=Hv>|>pp9jf6 zkvjm244vHed!Y#ydJq9GAdGvGD+vaAQ@+zqh}&aq7Xd5Fu1AveHufe{>(@L7<@sZO z(0{)_+*60F$}^7*2-fC)fb38<71JpH{Orz4Gjnr^Z8t&JaKe{R;A|eq;aFym(G#xh zX%30-4%}PS!B8pLj~shpX0v($#b1hGl7>QW@n?_GXXrzvysdAgqjRRXg!E)j5)$F z>d=Q{5$t()DS5A5*xi&j9x_|&Fwdzvro$8g+@stdQtuVr^go2O4As+C5A{y7Ww(fk zX86iepE5Iw|42K_cr6;3>Ab>+DPURM7hn|*zj3XMXb(TbRizqcC#JFRt)zC$YKv(8 z6F&Td;L8~pZWggGTmivIQqEJZ^ZR#Q6GmFaT?mHZvvRy{`}H|_uE{!*kXlJOA=W4GAl8?^*o5URG?tW(Lmk&1dZbHQM zchCHFg9I24H=sBtB&j+gA87E}lS}sFxGHwnu56cYMIMR2=OLS|z{gfbJ)_Q4-P;Po z-aXa5hEVqjrP=3o2mMb^AkQ8Z4Q-`; zrwrUY&}(}ieynY7K9#g|oAvn%7Z}h0d=?%E8F2%{FA2PF+YT*UWc?9XQL*H)0|74$ z2MkcM<9GmO5%$y(O?UYQvBSph#}S`w%DKpczr1_O6J#XD94f1@m>3;J#5LrpKh*Oh zX!z~hH9+wkF9zrH7jVylw@?e+ixOw7fM)gKe%Lk!TF?1W z?!Vq3}ct#EN^jx>=F5fXwgH#{%6#78X6}J=OZ#0PP z*;A5kT0zunw1kbYx^H57rX|z-8svYHnnYSz7NQ!V&_ot&1yMEX&6+}P;b{1Q)sSK2 zz^I8zOJ-=`haAFo$I_85(u1vOrx>h9lAh%5 zKSoITLzcN^#DDDZJ~d+*MJFR=1%+bd5W!6q;qj7CEX+VY01xCXN|7WkI|()(J>F(^ z?pzmA`fs7+l5qzRn<%|33y6ktCMJ9~-(E4o3q-`pw>9Zxy1cR3UNeZ3x1q8}p@^%l zeMJBsVuRhV`k=$Ch7!>+G<26d7hkQv4v508pTj*k`kl}Rlpw28|4yrKuTB{+C)gN|jeWNP z1J;6Q(pYV|Jo{b*K)iv>Oi)}+N=zZi_S|ds>vz+Rh4?ZuI!eMXNTa}BphZlSq`P3> zLnmQ^q3Vsz_6$+frkCeeL5aNKgtI(7*?N(tXUj2s7zSk56_BK~>-)$BFxm-fDG>13 z%AoFg5f+pfhfr(Q-X8`4+LHj%gGk@V{5>7gUjT=!bh5N-P# zc92Av!}s@^2eNUJ{>shT4BDa3U)abz*HJ{+%oqEl7#^pjCy?_^ELGb+)cG5}zKo!y zLb`u6wYMJ8*WZYSR8C?o*$A3yol17Xxj*34&x%OV^^T4+WK0a@?GVCW%1-<`VAn-Q21(q@1rjdj ziV^V)Sf1q5Nh?5b#PQeX6?g$9JP8s@vua9{OA0iD>^&|z2O&$?YtbDb+yq&R3yleE z2nEoQd>}S!dppmG6azT~%#Unw)wIC5xX_j>LWno%B9Qm~4F-|G9Ug8j1U^`3eqgi$ zPx42hD~E|Y@DpE@jO;+RBeQ0Rqm9C{Aig9HGku`eLmR~Zw`K?YEu1l`|D+uQDBdF0 z(ciz@UQZh&dCVyIVb#r6V)YMd+Yaw%ApTB-B)R^N92@K>V%@GqQnj0h0InpkC z;zOQG-`tD1-Y`k`O2>jCVwbK)lWQ1q@H`EDK+)d)_uG#jcN>W$9K4w^2uP}HWyhX5 zj1Oe)ZdCn9+~Gy;SJY`{u}hj|_7Fd44fXv_Crm0%x<>sx5)|!z4kW=mvk(bA(#@Dr zUn~O7BVa@#{8_Kx5kdhQ&cfa=_@2~@{uE=i$cGs8bCzett`mApAF?cTJU+!UqF%7-Wy=*-<_wIG;*E8Z%9JoB0RGkfoFOTUh*1Og= zHoGx7wbRJ--93*XxGqfX?qhfA7J?6N5g6{n7*`asU#dA@>gv`YX}1~QAmAZ=9pi@X zZBkNFKKO>bLH~ZY1_d{_{95d@khua_>m+&Ebs4+iUHvgk6c8;SWc1Rn&uWL&)pak| zoI7`p35Qm@B_Mii3i}`mHDCkReiD7cUu3GdEtI_vy5!Pa$sU8 zL8a+Uc*MFN;VvsEF_4Wa0NiacPJ^eg8C2rNjT@l(SI~@?E??&j**D?#E`)##bj9xI z$c2T4ox!*cQAW_s|d=EB%5LICyWQ7f{@W(f|KOr06cHHGL2ZCcZA zNH3iz{4nE5O*Y7PWCS`WMc)a;;2PF58t-}pBW<@1UxINjwhB#dx`cG5{rGa1%&CGHCWU{n@Qmd=0l4p$C(p3|B;hZ(uD;z1!D@zfLS8w7<|crxmL}4NL&+r zt^^CX8WC31Q_>FGJ*K$Oe1dH(k)D8$+kpdHjvY5L(s_vThI zN!Q#=OpwJ+ zhvl1`78yOydCtn}Ha_$j;Ijj606D5v>K=W zEm(Qy@NfV~RW~%G+Am*}(S!iDd;muU!j)3DV3U(J1@OL^s6==_IEcL9l5n9A+9l+V zgmp*@1crQ9udLbjmggGA8J|H_CO!-;EiE~tc)pmpDaOHri~9z%la1nWpnxC$Vr$$jitOHBi>n~yWI;d z(+7ZohQe4hE5D^jKN{E>8=JcrM<-MSJa}ga9Xt8CND!9P3Jm@eD!e7lc9pjSy_+JD z|2-hWwJ%>N7w;7lOQ!a6@?i8sl6V4WM2>f zVaNlF2Bbtr-zda{;_fjem#PdK!?j31$W4SQNL8^)v6)N$A_nLT;Rl_VC}H<$Td<76 z;Ac@?Yq+@Xpjt$`&-|Y5yvd^~!O15;`zrOPmJ#X(3@a8Nid?3A7!WbT5@ zx4hAOrE`y@%>XILB6@N$KZ{3Qk4g05s{C(~5)w8Ho zWU@N^fE_JP92pfx5)I>J%aFu^^Aru%P3oDdg!VtrkFU;K0khqi*Zx_H|9tW9|ILP* zhg_omzVQF@uSki1?q&FACI9_5o9KUkm8`{oufsn(;=kA7|NlC8D1eg_{9fGH$cBD~ zVPi!08SxK{MH&hcTi0dWP2f69VxwQ?rl+@aDjV`dEiiHTjdGJkyfmj3hpSFhHztK0EUiT7&-mPkKe!yPW^ zx~3*lhLjg1ZH&##d;k)jVg4Uib8ukrQ&plNKo|ZC@&?yP3x>c=+C09U_}yeS7c-ae z$RMR%_!M2HEHi`{GzlURu?ckY5aa<+s7}ZaxL73`Nd|$ORZL7*V`F3MCXnDHk_Ngm z$#;=zj6lbRicllS<%B0f;-;Z+xdsChQ=JRB{R6mxnWzGU4?_{aD8mL+S7A|6LZFEN z1M%Q5xHCefWmvgVH~$;rb-?NuYM;^|yL1C7KoZpHJ}KjjpFw7O021|DqFC7C;aU{8 zwr+$#7Q@B#?~1L~j5q#kiFjm_3tLDy1l<@es*NG^D?F=F;5On(LNR@d>R_s8mLc?>y$(b^&dD5#&k9OlH^ds~k@jnNrbw4NGh-VB4W7^!zMP6kWQ zJktZ1X%xvHpI;b*U}0nPK?Jm$NJ&^ajMi1sRy=}z1d!hW%5lJbT9_mV;OZ}fc(L0y zGV(rzZC@2V(9(6{_FJs1tZ+-g3Tit9Jo6jM+PuV4z-T*^=M~^hx4ES@5G@P9;O`iC z4Cki*xpGx!{#m&Pga-hXcj5<96li1S#ddDo)p7*@5>1w@AVj!T1Q{tP(2>D32w9zY z%R762$f#rBjwlX~#LTZk7=0@5!$jhiXz8|)d&%F!1|V855Du;m`N^lEDo|~Q=I;k3 zj0|}Z8`}pGFP3~Ayu=dVvQY#K$&f!}x@o9;jpg9^g_470lnZPWBOl~`4Hh~QI`Dzj zPiUn)&;2%t3EQV$@-mP~L@e0ep|@bg00`GG%7yC;n1BwS$HsO;fhE&=fe z7gOFDr3j4dn@+$zS10by%v>&?sX12bdf#ul9CD9{Fs={{7dvd8Zfai$TO z4bFZj;k+B9cPX{~L<%Q%HE4Qd?300po2YtV7#dAR^76EVA$FIqpo%eE6s&t}PADFc z7oxJkee8llM?bU;U=^F2o+1G`V-uf%fK@PB>3I7NX5{1ujkxgaM0F)a2)8Ue!x;L{ zrQ6^I_KQSa7J9rRJVd8HvazlA(QZ<&yd`6Bwz+{ zYW4k0usjmG_EFKJp2o!Z5U`HYVREk;DgE0hrJ*NZMbhG$lq@YKVi)^LQnq-k0Tyi_ zo;ZNecfv$WZInh=##9;1It0{;d6RQW?_6SBwLF2d`nerZLPe*3WN~|P3Wqr>q+^OPa^CZVRXvl z@%K)Dp;E2?3pmbR{KpOU&_%uu*<^|mh6w;ai_Au5wN4B)uf=#J9R-!@d%V~vlzyaq z@DHNh>c%{%qu|6Gtvep?69N~*832dD)e7EqVIU!CsE;nLt|Y^UQ%Gj~;S=Guf=%k1 zV{YHPd2^kA@axwy7$qdZdO~&~o1iJWjC5S7E)vs0Y{>4^Bf%4E#MV2b-gW(k+7iOg zKQldDjQG5)dfgXj%}iWIkdw-K5gnDE+P0P9>dDnrZEYl_UIubkHjx*rAea;_{aVr0 z08udPw9*6(+!<3$$~dBaxH8}w(xJAFj-_Bit2b^;%t&pUQrFdGhl7QpLJy_l$B!S| zlck+t#r)r{JJ{YD$}yfqcLG*s)`y zr?haf5N6HSpabh%PQjf9Wgho+6ANHa5>Xa%K>BN(ye>p2=EV)#X|%M*c>>PO#iti z4C6peuf1Nkj5@Syp}3xWd2R*nT4O{>fLB5E7IL%LY1|l*vC3RWltTI;uFYxJWg#+{ zH=LO0If?Frg`+1aN~kAtCZ;tN4<>n_gOEN6)q+SgfA6P&Yk;dm zWFaqJnbPXV>yrY5DFm; z)~wkQVEj$za0Z!LIdO{viCP8*&E=|rGEgX(w_VR-#&d1p2k=bWz&>m%3`Sz6 z4w|BWh3qGcva9AI9aC3yc8bjUA)RCvA&rnkPqypaWO6M=xk9D{DPzCpx zLhn$WdyJZg9~+Q7FuImbj_~GO8)$(RQy&mdwfcko;XnM2y8?(OVNm%A8mw_g@s$mR zYXO#-=19Q=TuHXqU$tr#dE@4s8B+~dyzA_{^tKPad;k6@(4$|Sw07ay;jL;Z`f`=x z&e$%hbI>1)kd)q(vx+`M;sw-|9h~};ZMc-h+;I`ffa{RacW~)XI)6a7^MPf_1kA40 z5*ZpA8u;i@f~JulQdDYb85wIR22?sSiugNIbE<87br`)I3r=URbZf7txuqpDxq-wr zb?v{++T*ArDz?J-L%YFb07{)Uuz^?=HMhhY(iO__vi}&+ zMDr#M9_1CO`TH(^vVkr4+qB8T-O!%w`+pk67uULgIgGW?X^Ah7*%-pIV62(I7p;8% zJ~(l@4jVqbg``do+058aWOh^w?6mG&0vF(u6jz3Nh zXh(!3Ons7j+eq0)n<9XvxeuAtG(Zbf>ynC!WhfJeF#81Y{s#O4#AQje0KCT{@>(7q zXEY!iS$ur9ySTyz7j+#rpF`oJrN9?&tm27CO&3E*go!ke)1tz{vM>z@p~4+9ok;eO zJ238YieH7A?^_>HJ^M2@k}yw-f(-Qun1Lin;UZvyWFK)`F|);sJ0>yYM!Y;wz~J&= zTt)*3y-7@r6<@|QhXktfYSeJz0RpxnbxE$dBh%#==kRS!!8KPTm5qq*-?UV0bksoJ zai{le2o7`O8V3R05B8TX-N?z=k2$RC_M+N2} zV{xF9*KoRQvK)dbF1!x^^1RO^W3uST7Q>dHzM zT%WQ#$*@2-Y4?_(!`Crv)8m{IqI&o+%M^t5e*0hK&(RKPygT~)AEkQYCZV$Kf#G2x zun8V}wix^z9bF@>6Pmy9F31%CQB)Yc9<8pcp9&|$6DknB?>1;ned}4XLo6^Oi zxgiOk3M+h2lgGe6%xJI}1JP|L`E#O81vmWE+mPhnV?5Ixw~rWxJi#EZg|<&0utJwq zNWrYWrf$Oq44*fVG5wMG8HX5aOv97wi>eBde-an@NXFW+R<7CpzP_tLO8M@JlGxNj zUr{Wpj`vX&mDL7@hGqM(6}6i}Uo_qo%d;_yR7OM5ae2~Pbi*$pnwPU3AR z)J79YbcS~?Pw#71XcoJDm6suaN9J?flLoANZxKs~2WD0cVXx%1NHawXcf&c-uxiCs zSy`s>mXUM*7j%SR_`Z4j_7E9UWX}4NN3JvJ4V_1Y>2lN2vh3x5>=<-e2Jsd_RpT4BV}6U&ZnXdN+;p|X#ion0P-X(gU$oYzsD z7-ZG$ZD8R(@Z2FHB)U%l z9j+32fLW{U)*#fxe)siN&RbhVC$|`}0ct>95yt7GvJEx2S{N3&pE4K*l>n0x7(6m5 zy}Mc=nFI=pJ$gTbSkqKKq}2>+du@pp`Ye+?j(DqBK}}2OW?g-2Kto6k7BKyIFyY-dYy(KMJJrZ)bR12l%N~mu)m2|t2W;;d zu|t^oNOR`R;~83BHhA&cZm>pjkA_7o)B9DhGRb{aueSMB;|dw@GhE&%EL=>|61 zD+MtA-!6OP$c)Tmq}gbB&mVjRLLg^m1VGm%_K17}J-tWOlaVAB=KGFJrn$}2oI$FN zDMzaZ(f9vB-J8edoWFm(H#5dCW{f5Ku23P8on&{nSX!xMD}+!ANwV*Wy9JR*MTJI` zwX!7@6D5%)OG%O}C0cZzS2Oed&G&cCd7OXFALsO#-*4tib>H{r^Zs1R>w3Lj*Ls?p zYi^WHm@d{=gocG}Gn@2!%Y_SrfsuYdR`_cY5$p5Wl-swPmz=|`=rFL9OA+8}r@X2r z%~yMRer~_vzL>K`R6y2fjO=E$Q!p;FG{p)<&z?QaC#KJzH*X89b;T`r{T8oROdwk& z<&NF-t$oNgX8YUI;9C4j684X?P+K+<0*G4S*}d@f>+kNjX^E@r+0;1TPKHE-F0`6u%hJ*;vM@e8d^ffqIUvz{D1F9RrPsy%-tDA4| zkbI=-7nQH+W2-fO56-_82U|JzPdqpV;<80aM&I=th+HR_PmmRos3w0IE*4QTvR4>D zDa7e2c9@e4I%V2Q)geB@U?$q#%#L>{ZmB@cQcFDJITbP7hwF_;%!C^y5S_$`xM&Pt zuR#5IACEk77*&MKGXXq4AS#-}4IAYCDRke@)27DE_#S2Bg*wXL$wH&G;tKc`1T=TM zi>^wiMHzV~fKgCW9wD8Ob06n2gua}>9XL#ujNwEVi&2!~ z3YS_uB5J*eg*zWsO$BxpV3Ojqo__tni}X9^4-S1a_}r7kIXlFVi}CzX=Sm)jYYfeq zvW^2W0*^$E6!wRcy}ddmzW)~szj7S22MA#Rp2H6c?97(t-D=|hswGD)!Todi*TE+! z{Nm9;(_4N}BJZ~+YH+!~fUOZGfkf$FMQW{>4 zkFQguQ^%+x>2&EYFCoir+`Ku2NUZq2_pSxOWYC~Nx60ZRJA0R4B}xaDHE6NhjMT`= zPZrk2nu%&_5WCQ{*5`VamgII6Tn9#t*fIfdrCRi^>!RfanOVCynx9S5J%wiB{6V_4 zxEhFa&KIE5Xg%iX*YftD&J>oK*wK-=pW|3yAk{?CPYUp93(GU8+{QrI&VOcRWn$E3 z1xM*L)4f0a`h3H87|sp;f+A3H`1vc0YV|DB!lwZgFOuY)XLc0Je=m=&{sp~ z!{U}euRNMdFH;HH=i?^5rzAglV*7sD;RyuOqxh(+25CLG>cn&3@+?bBhn9^_)R z6{beBg!kLH!?4L06X-x(va&+A8S~hhd!*wAPjOO*ikeX0lfKjc&1*ef$8~|W zvlg|jX#a(y!-w3xV;;c8%TI7;8tQHC2Bf3eyQTWNNvnQS9EK*c;Bi{qo~Hu_f{IPU_FnE0bG&)C*Hs4>nmnf0{_D%L(cgzf{AIVL(Z-AK@rrz3=~mJN-hPnQ zEePJ(HD)us=xmGjjv+!lS&)6o8<3>yxtxpP%iNo4V#a> zihKeW9s^S49)u)3>ocL^xy(U)Bx5hDoGqPx=4ib03wRs8x9(bGI-4H1*ocQO9eg*t zp%haAM)#2hpG^I+BdmkIR~sv+3~sy~f$YaH>bYWY=1nK2KU4`;YnsK{4<0&n4$qgU zg~45XgkFk>S$nCM!JqdX#Gr8UTPMf+q_SV50NgE+ow+K{ow!?QNq`;OvK#pdD2AJdh?F0DNnX$>PxhIgV1 zhH!(yvWqL`i;BVNH6f7|;al~ljHf_2s%gEO0Q$I)(?r5vKxr*|)5`JehP?$G|bHS47CL#u=Ie)*YN< zy*oEPu!r&OjAg&()m-?SwgRHjw{qq8WPhrU}Vrxo=mf}v}Kxu+3 z*NY^JfH`8jvx9c>e3ujUzA1VU#z!Z@No1d-fi^kFbD#D}yFjr5uo z9Id3{@73iqE|bS5S9X3Jyy(-^S=g72wAIIW@kpha#c3160d)@lVPPNo*b3#(jPet5 zM*7=0?yT_ceJ4Z>n(N@8c3jv0 zo%8*?VXGH$wKJB{Lq{xK_40{ce1{zwz*jfadv)*LUEmz5=<7@ZUjSCXA`Kx}Nrw#q z+xF$hIh%<#C7ui6IN~Pn?--Bw5?U=307aR?=N-Zqt%HQ<1$NzQwKMqE#uKCUx)-{Hg8A(X7%meCN?k1vmSeH%lL`IjRK(1to3Q0@ah<6E{zL^|Wu+M{y-1 zj+GaQo;C0MTK8~r<~>} zU^NaL54lCPc*DMT;qs0fNE&r+J=(&>m`7^kDf6jacc4sip}$@ad29_q$Dp- z{bnoNV+J@+{xbM$-=O*y((d+7dzQStxkXlD)vC|0z0bdU{`~OS>0rP)o;@aJ*!Oa* z8=^POXh^+zjx51wdhFb=Wv_SUq>l-6OQ_|^3LNrj$2Mz9h0^I% z`0c?k3@uODYJSJQa#5I*L$g?@dBSy7xHpT{GC#hSGOvz8{2xfuJr;Ej!@!Deh@cm9 zdhydCvCdwL(4Tag7(LIyA?%aUTUCd1GZNRl{3Nz$XV>oy4i5f0){P=-pDJ?Gy_Z zTD5pUq3eH&xrjIn+z1ZK^isQTHN#V*cRI2AV7HRQadA2K*Vp^S-0~K1?V)SMh6k8( zQ6n&O94s6O=jqOqJbum_b~JRGrYPKbW8$R-pns#t6RzD`kJ z*c#wBxBw(?6HH2Nr?xDnp{MaTpZ`zkHfGbcTJd?sJ8=Y3(=t~mn4U=;Tc|gglSw-- z-BH(WEnvdd<=L^!NyI5>-(*BN9Yip(_T`T~3qAO>Eh??_ijUj}Ta?j^VsO?~xS1zr zPg)jk19f}i=`|z8WLnIti(8%(@e2dY8{(e$Yp;{*y`J=Z>zs(IY2OB~xW@bVfkOj? zImoY5hn+s4%8AEwB4eWp|G_U+uWTeqLMq}1aZGrkDM6#nJF zF!z|RRh10v*ufDv?-Jbb()Ukos-E_ztD?$nG3dSOKC?=h7%G;6C#|k5KG!h19}(}4wx>AuBReA?78QO+>j(Cvl8{)P=xjD=a61X@V?mba<+WbLapIPhA4?}m*_r-Ke zY&j1_O#RQ}sy}x~SbN#HHDyW^lpH{dbrb<&`3h#cfr3&jCqSO3+Sqi3ZXxV!IHP$g zEU?~cSWPo(&EGvq^qO%{>oiXqTqU!Iiy0@)9$M&;ijimoX5wG*C*MQkjIF^;;5qr9 zgs5Ld8O7w7!j~qrB^O{jIGMj^3rkbercy-TJ~+4;k8Q0QX2}vCv2NfEP#oa~a(?Qx zMhg6Wro7QPvSXF)wlVAR_&7|*0=@YrGaOlQgBg(wc+a~QHsz7(-c3g~(;R0IlrsoR zMrSd`FUJKM{*4gH#i;?$FU3rF+Y~S@->eiZCn4cSvD`kwuWqwsD8Ro=K7k`2CU_3T z{J+!0JA(3sX?ltufgQGtlLn<|Fn>u%F zCGJ>KsyJ^~cIw{uG?A`NiLWk3DHzpCCoGGlKo_oH%|R_&s|x^uSFGK?f7%`mBaqzM zYbQjnWd##yq7T!S2z3J&I(Dn9mzcpvK*A`WkvW2$`W&yGAc2hO~rsG_&V5UP1@+rcjO+q@Goe6meg-I^AY zHgcf&=9U&^H|A#qW1)?Bl^zPEflQ}33@;(WQ6!9NKBqfH3j4c$CLkbQ^@{_L~iGQ~| zVT|;jOMGMNj!YV8fssT_IPtUJ0cb~XOt#5{8_|wa4Z}N^%ww8Hel9CkYyk|{uB1CY z3?p2tlEr2}AWxH91OV{@xMBACT`>=hLfRO@y%Uo{3T164EEP7PB9;eiY{1$ney^Z2 zwW5I%!$_HGiMv{|PshCryY87F7UY}M&NaV4jXW;1APbQicyW&pXSc-o|A83w_&ql* z3y2xwq0ST+%|sF!e3AcN7{Csx+h$JM0+>B#8;mT&>EEUtrSz1?>!dB7+hjFUY#C~uaE4n7&2+o=Dvg;30-OfT62c`3!jD3Ti~Q$XIj@qMk7&$Z4C zlED3D)LH?Dqz53|6oXB2`z$Un2$nj10zT^AoUqeD>}xYHP_QTGlyk0}s|Hh3o5q4y zij#ETdxn`eHq!jRU^w&e!m{qD_BQ@4y+C@u33VeW@h7HJa_`A}V%Vp*Yj`3<-KB_4 zgso1!aszl%EX#3@qOj};A}i7#+1N)ABelIa+Tk2{4V$fcQ|}2F+68eEg)4lwk)TPk z1K?v?^?KC=^*#dzY^EO&;(4)F)h)OR z){ATc1kV)OqzGtG_5dra<*^bz3ECDJ5b~UBmBxnC1@IsVFHZQNjKsEsO}30oe*)n_ zT&9F}DI!q_n%H&YO5p?CE@t=8mUJ@Ykiw1g>F8C}gt})OQfe+XFp9yYRGY1aE3J0PON%b1wRy`}E3IFl1=zW>tcSex#?XC=_f*PA(i?Ha9^WD~!Ns_hL% zO^ouz)s4?$x{LLehS$mE)IUFUI@Yk6=Rr}8GsX=oB5}$lE5)t-KV9I`X{fbPJB@YJ zOYWm#_V{t*La3Hs`aNKztrcj&y!-YI>Ub z$#ke*-DrQkp*U+hy+S9?p#4)qt>uW^MxPG<%)s7^h!*qIoU@vQwoO1!0m}!4)<$=N zx`bOknQU5siAQlHdo?Az(Pgaj_qTA|Vz{XklvkFhh|pR9Jdrityg7;@4rv{R>tnej zvY*Ir`SE`~-Iz8n?r-_e|M<^;BpFBjBUj=*{^zUW$v+yY|J+IbW?DFjieCeVI1wa1%mkh`qJ3B%{q)J;4d7wu=YPSQDZ?& zbBxFKE^qa}zO%H+oDfURK6-i^H*Bc=Rb%r=#U}y~&;8ez*8d;1Qd72l{``3ZBtpCe zjfVR9Jfq@%UQ{IZ6MnZ=3K8kDdq0`xz{F%PW=D!zv2i&Ei{UC`4;dj*P#W+z4yk@4 z8SeKbS}@=UH~2_Dh$mtXPYaO17t85cWRVEvf;h8N;&eeGT{sA?&01PYmRQ*y9`v8y z@$VaX{~nL|o5W-+MFA2<$v`Ss1Xpq*#34#N%o$ z&>vNmy4BN@)W99`o}W{AOHJwH)#@a9R#H5el(?6AB(mDcA7Ih?w%}aEM87% zlr<^z^q3a?)a&HI*`Cs$f(wW>VJS2+<}z!K9GR8y%KDlq5knWiPYt_6l)>6OEW=_l^W|!tq5qh13j?P1?esA6!58?HU0u*bRdY+e4OT*QseOpHl1r+5GTwd74@YWjYXqA{a%YL1o*Y7TP*qE+f@dy5;7v_XHx<#l$ zdMJ$|se!~|TOoTPlRBgUOxIB`ob&U@WsU-C#igVgm|?<%30K5aa(M55Uwr->(*4+z zK7Q2bKK8=9VPB$Q%D z4)A0ha*bEa+M&R2q&9!(Lo!U*b&vY*<13#LBB_-#$F9E4r!Jv^mYRlAQ&tx8;BDgQ zpskh5*A9t1XGBD5w!u@R zDTHMcBc%`(`E{rdwQkAZ*VarQ&+}@)0I`k&{sl9K2-{B&IQRl5!|78l`rp~X+;-pb zvL?%LsMvR0(+$eg+Jw8Z)>Rzf$ehQ}ndE+)65Dz##Ei5NC5_B)+LDnGrXD9V@#%dG>!)@MMj~ zEtdpCNUTqX`yovyQ8+fUGcve8$l*o^49)dmr%NNnOxjVAPl2S!Ht8_6WHAPnSISLJ zM|jVryi>mXn;VeXuvMi$EAG;4)-98k`a&@HZ$ofRk$ho$u*W7FG)6i-aY>s(#m1eT zLrV`Fa}xwtk*BT)pSA^P8LaSe>t_G2=Zp@c+XBxUp-STmQFHNGJc6)HN?2LcD$Yg# zi~b9>4=6N#u8W;=V}(%5_o1c%)xAzfF$-c3lPEsh_2lI4lX`*<-Z(gTS>7`kO20Q- zij7p)y`dV@b8)AwdLMT)fn6+^*jZLhqf#N#iKarDKqBHy?xo9Pp$ijJ$3bS1om2?< zw<{MDYal?j5FIH<0TAlbj%DFeFiz}2N8Y|Zk!KL>CXZ)5PBF5bN7TLtm=RQR7YVMZRgE2cP6JD z-C!f8)Hd|rEEW7cz3u->w^OFv!*pQ61#E6-Dm7#vZ-0yOuT~3Z|Z?LY$}2*@H3(;nahG?nGfC76M6~J zzQpSNk5^s_d+0fj51AmvSMS(1tY~C?m;`*5oQfwCiZl5J*=2}a|Fe2Rohy`FBII99 ze;}v?HPTKAwMd`H+SdW2gd@Vg>aXuKli`X8Vxa$H578GY3tK>%Z>inkj2fjca1AOd zRCm3KCvSbzwP|gFBwLy099i80K<{S|PwX(ZCcEJysBrn7Xh!8$RWrk)NPh)mR(^I< zVA%TFio&p?Hs_z@%=~?mu+cI*rdr-!@a*znXf-zgzC~54xYd1uGRlsW&;YlThRG_+ z4yw32Cjk{2=EA!iP3{pqLM+{&Z35Sh7b^wq98Xvr@Jn(<;;6X7pzR|b+FFacK24|V zgd%7Fhl~W7mlwLUtwV_sh6$-A12xM~Wit0`r0BE!-PEv}{mt69?{$;sLc@QKXG4kJ zG}mo;cEkn@z1EHsVzexaA35GqPy9j1zVIrNksz?6^{kj@K)JhRZ9?T&JR4s?G9Ax> z6f-OdA?T|_oWy>R(k?q{jhDCT>wGb5ymzr8dHdX|4<|-Bb zQrS(8jHn@QkCinGy1QEzsf)7+tnC!9hMwpdS*K77sXTURWOv=&9Z^E7!?XD!#* zv0dh|MWok#!`yXYItrT)*`M0CR=iimTIj`e(WstB6%Swphk~P-n9-B?C{unw-adxl z==|aNNKkdzVpLUNsqd6%-Xj#3#!QXP9meTiD zsMitZ=o@S1lq4KI8ksO)*9hNd%{uMAJNj3XA;<448g@C{BYuLFfk7QLwO_{eoLL8q zMtyC@$loI}8gw#Q<=xv=7%;Hh?^4OZ)*Dj}d2Vm6 z83I)qjQAm=>AS#g;*e)PM}_EXg37V-MydL_r#h`QhodNw%A=GT;VnH-4)7^4JPb&# zD8Bre@4fl=Jxb#D7P~J?O-dR|cYH0 zAwAx1bKntx=vzEojI8gI5Zx+D9s4_G+_-;8caOG1>`6bI?s_Q2S(;Ow5&tI#-n8sa zgAG>?i#VSn3-B_oUsC!SCy96KSZpgmp2sw>g0PCPehTtP`PB|hZN z^yS8;4-Q4q-^{0kezZn?NfXD30PbJ9#_*M!^`;nMW`z( zULROuoRzN1ZCu{1N4J`TF>Tt&N4_$Q9wm$gNymX*RiPZzut-<$`cdePQRO@UC2#|_ zb*+5&e;)V;*pw~n!}KZ-|AV(Z&_(OEMuEBA!3iZzs?LhrZ-mDMj1Pi zW~?u-Jk9h&TeEQV)h+N!{b9@WRE9TwqS@>;<4?@MhC1OE2}{V=FD}HA5NUDI)iEh4 zsSsI`@iK3+#2waRs7jwKOW)t?3uTF+inJa0cD2h`O(Um|Qu!$}FbIgkuBWFDijeUQcLbytE?kJ< z(W{`Ipfv7v=2;pE%)U=6{&`8<=Q6P`iu>qY9={sr`E*tl^vo)gD86)6y*`3_TV4JQ5X^4mPFPR;{h)*`N%*pCu73Fg9kel4c}YU z!Q}g8@o0!-#*R8Z@Ad0&icLpM3R>?f|NPna>dd=@CAF3vG3TKUXvT*kzxC=a;@%&lLL*B9(0fQ?rf?fb!oFprJ`Hj5-fXxX0s1KFoJ6(xT% z5f;rImv%JoT2n$cHu`A;dC<=>S9+$IeQsrC^-sQpVS_?(D}MZtDMe&2l`7ISZ11gE z_-pb4h8d%)@7&3|5d{n|!7%7}K7|h>7Dq;!7eUzIH63ObLU@}ceOi@btzoFr|CZN` zMj-Wm3;g66)x0I7YuUt|Ksgt9@}Wdo-c)as5TmJ6_jQ|iu!k~|kQo`5mv=s|;mp?q zAR$<=bcw<2+b0*!koEl{hoglhx0#}^GY#s{kH&8L-!9}(M@23deiQaXdvHsWR3E3-)PM&im z1?brfM;|A*n9f1ZbN3q9<|l10m_B>eJ3PbX#6l{MgFW)UCp=sEdY0;xrig6iXr{zWj4zpRGJTkpPz! zkwko8_t7^i%SL{ zuawIAE$hr)AF=!`wO|^LWly#Nke)~Z$`%UxB)R+3IM`Y6j=>)Fo7Zv0+J}E#X!b%7u31(ZeIfIxu_Wa&I;!F-pnA#>SQ6 zk?5eQu0BRa@Ep(k6p)qOMtS1dx$4WKd9gH3l*ih<9m%+ahcfc_ynXN7bQUgLh&i5t zsuaY*{^F^Wp)+v!>viY1ALLQ&aF9n)dvJ=ZiVpYM9oC$nLX%^t;s9258l4u9ccbUg zd&eOb9nk-3^ypAk<3Bvk4L2SDiGq^Th@-H84HqP_D4x`asi$XKBV>cp5K2l;yLz>= z&FQBIGDOw~%wqx1h+caJ$80Y?(%=O;aK$~LVny5pmKm$k0|cX$h@^}D^!alp5#o$n z@lJ3`M_w!vYXDTTks!orNJA%$s6NXT?1KAwtkYvlI>m|oPsx$tb35p@DZANr2{`HO zRi9G2=VNLsHu%AwU-pNFdR5J}VyBnfE_Zi#M?$kiO7OkX`(~3O7=L}D#-IbrRNo-^ z0CL}59&jOA^q+s%KyX$4)hl-e2j8|u(!%KrAOGpxAM{8$ey4^ z+OxsL)UbSyU2F`+h&HQP?+n-g*ztf0Qu2U58=fcWe+xad@?`@x9Xnf~3>DJdH5G;3 z`=fiw^vH>E*dYqlbI(?3-+6F|@=qQe#9{Q3!$mpBl`&&o2}9_v+EL>xd?wUJSTj&j zU%%3NXZ+NhLPPC$aHI;MF|Xpd!~&L%9Y0_#=7Ry|v}n?_iwxRA|C=Fe@wGGLi?> zJYluFWX?M+iYDPTg>T%?_W)*2WR`15 zQ6(=O@GYo!k_KL`-g4QP&iJ!3&zq_UncVQcNW;`8U{lXV^S zwM`rWD>w#siNq%5!M>pxK%K0D9b1v-JE?iQb|=6n-|%;Bu=EHG4f^4I^{6?9vbhu% z5#|0|P2M*Xn0pk|77T}#8GwGVAWN+q{p4ACZ#;=24tdE@cqfjiEL5SKucD(q`nq=> zK`ogMynel4X#Tf@6(~b^R&+Gf)y3m=h}ZYacz>(4ZXH3zExC+}#ELPFi|w=`zBQXh zK|>iG|0R$jFp_D&rp=ode9B9SIei+{{eeLuU(ENucuO>o8~zE8g@TrT4Ah_vg4|X%F2Zt9(l(3 z5m!)Kc@hE5fW}1y5mfw;|2nMn?S?s#C9Xr+P@7z7A^dwz%g`%{SBn)%8t-<$eygW( zP6yD3M>3?DS6R_ob4J2tF%CiGHcmnW+?KPg&l}6n>AfQVw(m6~!InO( zkLT~Elwv1wPXLGyP^ak3?-4Gio$r=hElF@kOPvMGuI0G!fRw>*Uq~pAxCuc+f)jn> zQEG))=)F!)FJ}48oWGbdNN#lDp3C{OSr`@hxjZ0u*S2lUU8~L$5F2%T^i2_p)M*v* zFYEY}u7vL*qVSLv?+;=-Eb7%ll2!PU9S-Wfdr#!VPXVhcgW?+gcpeRs)x z%HPbrJD4pTc-@8)*%Hi6VqPXvouRQ{>U36q*mwxf5@XeU#R;;AlDs@Uew77a;DmFN zTw+h|wOgem|INJ=H&}g1-vM-+<^{fquKu=ezf5>tTpaY`6`9GgqNSGk1o@e=O^@<) zvZy0XP+aUK?!i}xB2zyrt#^_BaX$0aPai){;p&BFWjQTc!~vZP)8n>zJ+k*IX`ezv zBdzv39SAfNJ8oQdBlx%r_%Yd3Mt`B9q?d0)9Ve#TO6a?zF3!*jQ0f2jgdLtRyB1%p z@km^`PYqB-dJMU`f%<8nN`Htz9_J=AmG0BoI;_wH!84;{ECehB+Uwbd2NFG`gLENB?|T| zs@c+Ozv%QJTJZKw;-c|CK8aiY1j9%}S+tY_OIwxq?%fFh+yk$tpI>@}A!zXXVL%QQ zZ;twuKAIQ2Uu!4F)$z%BX9?{Z8X6ZJEQp^ZuA#~dW+WNKzN;C6<~Zs{*XLY(n3;ch zxr@H6>X2T&B%$(@*ac`91?>Ait2IC(elR$VOl3p)Ez3_Psk-U=FTlr&4938aq{=^M zY%dO)mU{Bj`}U8CbmleZ(-*y`%!&fTDvS#o*gHR9b75>tZA)V*fX_JOR=>W}W7mmc zIw|F=5)+WAt$O7>u$0_vuN_CL63O_$@z3sF8%~k`BX06|>!$U+aOu(kef5+D#`IE3 zv#VO#*{R$7%3VfiMthY!*gLKLijs$Z(mspxYfS&_cnVHqn>KSFcrm1LR3*fGuIiQ; z%^S|VTvF%xtfXMtaD$j9xeZ9l#>r#1H}BgXQc(DUDC!BBp=<4;T9sTzH{x&Kd=3Sc zK-{Q^CcyKYuD%tDaw(MnXP7b!j2}hQgFe5UvNo`f~ zkBjOq0ApFiLIs?2E3Vz{4K zlrgRatx17vDvJk7c}^)!xISs@l;ZWB*r6o;x?X$*-glBA@!&y&-jG*J&ZQI8%9s!I zpx1)b9uaBekpFFBf=}sp+qC0OyhsID4XlLDQm9-n_yYk{1p zzr8+CXea)g(jv$l3?!tVg~qkKavxe8(zx&YuhdOo9uB0E$$?+;J{DGgd(3thiTZ$4 zTL9<%wZv~N$FIi|poj6F(pf{P=rqDTv{|gWBSadhI7tJ?SY~dvsgtIrvlfAXPyu)` zv%|P;Rad;`X4LQfYf-i;q~yu5^3m7!WV@ZRcMEDZJM7_v!nbygS?K}Zg=;EKq&_Yx zSn=KI(~$M==GYe3_CaVbMmA$Hp?CiUGZsk{D?na6&)C#-THNO z&#uaXFEe!8qi62-RR39QV8quQqxELz*HgAME*M?c=fIjKxiL;2neHpbKW-Ggg6djt zn>9Hwo1o$f5=`0M&tknYhPwD!&D@wHT&bLkZc&bp2S}avnC`)keGW^Olr+)~vB`*E z=-F>r#E>tl>b<2UHRs&@qK*u$X+OEB$&9F#@c~0B%a_pox_tDs_I%xLw|e^#L1zyo zM{k<;uFQAQ)rFI6T)&Thv|;6vb6-o-8&!H<)E>q9!+Te2j9yn11^UJvYi2cj^}$`= zz3QC;5U?LK?^9(rwqt?dSu&EyC`~zGx|_@beO|3yyMwcW51yxg?>*gcl^rO>;>eBu zhXA^il>K`%yvtI$2j6yXH+|ka-S`p3@u_dJzK!qqZehv$h<$f$D(1hQ5vcYxapSpH z+1J#SvrdKeTm5T$n}g3(fueM6N4XTPk5Gpc;&~JqAk4WNnX!F)W$Qbetmiq{0#aT|7?8fuK&~Q2$WK^ z@3OFpG2pGUkGHBzFU}DGARJ)vt}f?k_lgdMV@_hPFP$>wy(v9nnJmYmBv*wpU!PA2 z7&vsrDImF{bgGe@^J#(E^Y!ND9I|!RzOcwS=I-6efPNrdw{miZ)L-$V(E-pHUN^6x zV4ig{7ss2CX;gM~FXo+RX}79V?kDQ)y4+;;kX_zi zb3fkMx-*)SHm{;mtZ1-v-M`>*4_p`2dQ3I|oKmUuj|yv^QA%c2d=K{>W4m`hwx}j3 z0tiW96*YvEuqXPI9;jjEIb+O0Sq1)3UR?GHIj;`!C%QC!t-if30!i(m(YN>B} zmT~9#`Pa}O4;|_V~NK& zrxgF&&;xXhf1il3F-LC=Rpu%jJTi-g10)@2Mva{|3H!EdkjnQw<-1uK5<->V*|GNH zUioUzqKrIN=sS!(>;s;Dq?LbgsU#talDAMAjG#&1Yi2>j%6s+7ch(ar6gxCZTN9{C zt}TbH&rTq%%DCf@=a*?9f2o<7X6gEsC&<*{=tjTp69@ zw4hX&0)rtET!GO`s$i$*~L> zv+z4qme=UoEe|_ksS8Y(`HiZBHX^jrc>m?HKb#{!_?>8H+k9F6#%ndx!!~|65L4Du zqcTM8^ANpZbsOzB4@^yO+Vp$*yaxbpy47d2zTJbh>sqjCe!`OjjkC4f+aFr}DW&Jh zk*OnUP&$!iyj?qj-?_UPJ$e9Ri?|O7rv2yZz1v_2Cn*y}Y;Pdy@ZH zbozlXBGeth2g8@-gi=2qrvo!o0q!+#-n`Uc&W-EGV0gZLbmo}ACNy=0G>e}~OU=pd zu^yQ--sYV4wA=EtTtFZSnXW`UqpwdHM zL=c%szdGrR+9NK~vKA{_sCS=@kFb(xwnzO=&`>LtIsMM?k2jjNxU_t6>qU$5Z8aP{ zEonQ2wng6CsYt^%(`<=$`jp-F1>FTcG9yLVSuu-Gi*C%}M-g(&P zRJ#UEl*JDwc6fB?aZXBgO|-MNQQxUUFU`wIw^e0#?!FM6OK?w(y4@4szdz?R+of{D znjpr~k+krtP$pD^v%FZg>Rb5WuzWUvr~y-onC@@)nG~6I@ZOq_Dsy;Bna-)YSu2Mz zIvPU;%K}Pnj#imgA4!E0c;(W9?AtjH_Q`KhJpcZrt*dV%%k7sur|cO&F!0aE?pA&$ zoYa+5=6$jEct&-iRDojpFZMt=h&VQA!`3t5JQ)I36Ni8(hGlw%h3ae_{_bG#$^nI) zI@`s>1T4#X?cVnH4(>lKw0Wea*;UC(q&b_b!9zM}I$n}u)!zk#qjIcV+zycY04e*Q@u}NX?YgQ}e$qN_Q%l{=zGJ>o((alU_dkImxz0$HL2K zA)Y>=p<|v<>P*XPBY?i2-&RIug|3w>Wtso^b*R&vqO7QSdj{$DV<@4_5J<4P!p+QW ze59jbzKwmjdfNFT{^(cP;tu!UIGNI|g?EG6>!MID`lY$QS^iB`=MaTnq9K z+Hg?2QmMMOTxZJasjn?!!6SU%&RbC)`nFe!&9Bdjd?)Eofzi5nRJtF7De3hzBUF|y z4!yvN+)AE3ot@pkrANspmnHW{FD|n)UHAG2`{H+dvN<`stL9ym_eWaOD+b(=F_L;A z*z3Uf^gCFf-*Ii%w5jUoU^~w;3#HN~x&zzHGZV_c?{{c$^J}Y3#nS>7+r0U6lS*B? zzSZE0(BQj)t-I&`r1Fp5`SEna>UqkF?1!-v47d88Sd^FdxXSNJ z){_-!*AAYzekNT%3PONVB@ON3V4!RzPno|u*R~fwa*}z9z|aD1A<77$eoskC(wI?I zBV7_ByO?d(RQf8Odi6q3+`bmshMJu_Up+f~_PVG+$*n37-KlOS3P#oBU5Vr^7d_;H50tZp#mP<;<#juF`H6b z1-ri-rR%f#srg}duM1g@d%Z9H(5$d;XBD8_p!t}$?%!pQ>A(~o(d-epvhs$ zlIRA_y6r2=p=Ax<4#Vn>VfnJWN_Pefr8k;QdIK~dvnnykf0|I8SX_Vfs8MHumx=O5 z9IXL1?^^eLI8@uUVAQ7f)}c4{^-!9QGC%%)(bp97t9Ro^TD4yEE@t>S@2F!>-d^sc zY*;yO+o-sjQ#UTL6imx;)1m_l4Vx~D%C;NOyWSmVqjI0Lo8w?G?uBnlNED5(6Pt(W zf5P>d`(zRj06B*P5pdh!SL$iZzk?PUU8AmNlVNOXC17ST!qrd`5K_*Z{HgwL?RqF{ zBXR_TITvT=6yK?Gqe9DWbz1-ldI3$MDwJGeMaIemR+8Z`O?rF%)l*>_UY5>C3Br4x zs_(Nd3tISEo4kGctd4^%S0W;+*J94<44wN(9$Js(@_O)QG|P_4A- zH^gdT@9c|%C*3HWpJO!9zm1WumTST4>47=J3x}9p(ss2nP3nCt)g=Gk`D23@cTa;d zdqm~~5Y%YBAM{Hf)|E<8>W!;cM>6Hkq(hYEuXUr%?WWHmp9_K$8v1q`gGdXiuzc8V zESfUol*mb4!wP_9&#dE0IiPSo!-F4v!Gl||^1M97wKIjxb3Fj`qhQ~qrKBij;K|bZ z2ho!%!so)4EWKL4W1hotpJ5xW?tbM}^{j|#hiBr_#5^2?lb+Q(fuO!)CnYu2W>X0RN*=?NT31 zd9im~)Q4Y6Zq0O>b8~2FAR~k#YB`x(z&6q0)NR_h@fKz%oX@-PYD-}=ya`(IEahW= zf#Ic60eq2`5C&1++qW`76xu=-%r9CTtA5WczCebL06SHMqSnfr=Aa{wX<@%s-s9%j z)t{vtcXSHbK1=kojPta?O`zI<{7eBE1h$wF>zH}-M7Ht72^Y&Fpy!@oqjo+p8|p>} z$KEmvI zeQe&U)o~~u#}3@`*M5;5llAnNwP!uM%O^gMU!Hc%Rr&d*%Hq`6eokzSctGhOaB=aY zGvndvnBnw%G2LmRRBU4x2!|@JLnV`lS4GWdSRF3R*1u{*`v|&sFjr zOA3!vcwI>Mh;nJ4wqf~;Up}u{^ZER&I~{dSl)p+Yh_|>HyC9~E&tku2u`NC=>enTD z^hKQqPjqE&b$O-pQFl7AK;7q{E8X>}ev55YTo>6Qz{2^e#t${Yd(w0}oHdN{hlU{=BlNlDcz9$1bc z@Tr1B9ur|?!6MiV@gbbBx{%SIb&UNs@(!iuCcRexgwoO!v)q*)h7naD?Nuy<7#R%+ zz$|#g9*=r**vJZ7uF?d$!QJHDNwRTo|C{`=$FK~Wj{2#RfH5Km7nnezl~Il|V^o|x zk`c!aWgxoL1D>mf<}`Qz3V{9E08MiMyp8_FYzQ+KfV-{;Po;1m%bD~;0x^mZVrp*g zhaf%_{F&w4o?=|bQ4){}-j2r8CH69LB(7S53ArK2X^;IN>=)KroKaUUi7yKPL|zh5 zS$oyxQ`~`!SH=-is`#zyEnDs#woAH=EAbQVr*7ef-G@cQFx?ow3;xar%o#pk{_U4< zlVP(lvFmLTG{k~uNZ3!ebxT-kp{@kKOE~25esJbD(8ISNk^$AvqOaHDFTXX7{j{I!5lMAz%`!`&0Wz2asI)Hp z@@seP_gg+q2rgvLT4ev}|-vaiob`eT=+OIfy%qz8S3 zb3F9=L$O2Plvs0{UEJI(2*+m`$QZL0^xjb!AK_wpoVS;TW+U=;pn<`PUUxJn!s^VWUiP`L!85*Ltjdj|UsR;PFy-3iJRyn{g}i)^TH^i9 zc=6&>S~G5rZ>clm-j|?Dv7QBsO@d0dZT|IbaFoZ&hvEdI3MDT{d^Nem$e9}A7}aPY z4eUKBT$G&ZY>ZsgH9UXlIh;pNH~zQGOLND`0;ujx2a;~wctbhU`pe(X+b8Yx0bTPR z%^hR6W!b3ScHseAeAs&DC2WWTt>0fU~g&8!stTY zkrg$Nh7t*pIn|3?OmEimVt#MjZw<2_VL+i4i4>}<3UcqMdBL8RfUN->W;>7K{$@46 z#8FtB;7!f;%Il?|@+q86+5~CU7-+>eDp`nNK~jn25Ay<`rx4Zz)ly$}=p^)jGxtv4 zVOjr+*Ou@!p7fv|-+4<8S6O0%+U++5@; z#!73IIWGSaSuqL8=H8lVEHDYypAOV32TN!?4EL2PdS#giG9g7mXHj?)4pFjey6i__ zzjhn8x(HFkF(&)^DZ{XafmKO^GSivOtXgqnvQzGb@lg+)?q=s!tJf5o85bX!^&;nE z`m;qdj=4rOef{&mY?6_NvR*@b+jTuJ^tw8z6 zMW!kgtSV4-))b<{{LK{8u2@$P8YT(3n18D^Yt~g6UWqlr%*}H?Z^*7+sw}U%n_aOW z>*BB0=bViqKdS2*r{7uIdXh=~lSDy^8`)(QUzR}}P8&VV0GKS9xGtKx`G;lqzQtyES zv+K7GNJ1buL4_!#-#g!fJNL!}+I%{ncUsdcVTJ9uIcGdf{`FSZ*55Sds1=h^I;g-uDRhEpTCO#OJ zFmTD!fF+yGTBI)i@u^<>OBc`Y+ts+;kdRp=tM?V}`#U0p2Tuwfe2mT~v~We@$$rZu z%d0XL@7ZtaV&gJHNtd<<;s6oBm3qk)ZhNwDs>nkTr!EN1BDyt{HZo!Og>L`4b5}Nu zA~=|xUph;ItPEa(7L8M^X-fx|feb}Lb3MY358pke7lTqcg$#qO$UB*5g9kZxdFm{T z1$7>r9#6k;vJdFoS~Ovz;^H}%sfI0gy-)eHy8FbSDMyb58<$^JclS_JT9|0-^|T%O z{n3H^Z@&J$ExKf-f6qL2tEPO@qeI=#9a^nhXovkx6rmY!wxJdt_&|Pn|MWm)wE81r z9J9&_@;J@g?)OD(x9@c4d|vRpJAn=h5+(UG=Q$+TWMR~vT>jI)KIU#SOZWDQm}ojq z!kVw)-rgs<8xa)74I4Jpja`AAr>Yb}qgLO(_g;P+Cb(26UP_&xej%djKmAOfq6+0b z9drEUx8EdzxYxZMK5COHPD{@!Kf*Eb>oJe-m8}yERo1Z|jNG0P*IQpdHMeq4P+ni7 zPk8hb12vR5rYsBmf)!7XVZ&^NuKD@1QC2`z4nC+~?5(R9i^wER+X-XJL^FN7Yq`}artaCkQv6UKlk!*PoLB>4e(_QFrr1q3<{n%<*R zCj+4*kvdJiy|-<F77+-z$ym@+BcCHDWpny zYe!!WfyV_qXf%b_>_}&BToxU(y7YubvGqc@BWReRo0q=W@+jA|>A!CMm3GAcWJ2RH zs_n=Ld%8$V0puA*wtz}1?bQDxqjG>zJ9$N#cLyh@lbUAXw_v0Y6r}Jf`*Uc?8;XGm zD{P6USrFIg>d`r`l8_*bDf;3!Aaj9(oI~WQQP`()ZTijo(~oN+B|8u0>y-(UMy#>& z_4+#PS^mPvgxt?smZt-reV=(*y)bQ(zG>M4i`#0ecA+b`k%LoOs;6NfK%eWMOs3Ez z8ub%%RsarjgyMm5aJc6~joQ9zmnHnid|Hg|hgOc_4QE5ExX(I_Upwj#yEBN&S=urV zw~(k9Y=zUO2%IME0UZ6msCt|A<{*X922+~^!Hp$9{t`z8DWP^Ed{>~i24#6_J7UJa zF1upzNCJ!a;Z_6Sip&A{;}`{${+{5VY`ea{5u|U5v?5O)U_G0<}&7w zi{b`h6?mLrE95sRFR8DEQYX)duxeK9gqz22JO zQy?A!`5MZn=LDQZQ8sq$*!-0V6N?`^R)6{yX>l}gV({=|TW?M1zv^OwW$%M0UyQcv zKPPW65u&|x=n?>nE5nDzY?@!@c=>uibw43CvqCqFf)?%acsk3RoZpYk);Cfu(@5&!dTzjXco?k~CB zE&7L*?SvDIQq{fvPchox7u${B?Kxn|-`Ct{*S2#@JN@UgK&46SIz_v&dbe)Se08I* z8!UhNYZ@0n{i@un+?X@3w>byw_`5Hv{jod`^##KXcrM-B*L8XI^5*4Tohr2wcV{=r zTk^m#2EWQ)K881yn4jEVCOQ9nHP$$zv0(b;m3`AZ zVJ=zxF>@qSM zwpnZMn~R2{i*dySGoFPAB0gaG$edG?|LdEdADPYa9VRWQF1av4q+dmKBIjQ=m9(^5 zITDt$AKm4JQU7%w)^1tmY+`R=9@}!yWG%Vn(J)%ed`&m;c&Uo}_8Y{A+7cY)bvE{k-`1g-1 zW-f0$HY7k@2q`chqObq!FFQ?QTr(c1m485-ZWS%<&<2KActKKbE@;e!=)9?H1?j1q zNCE&MAousw9id2b=k-LBg6squaQ?qv@59u=tH1ssA~fJ{i@&8=P`a|Ps05VLPhPf* zmW;l@lula49!)h2-#`(%PhuBIM890Xy2goHLN;c>aPFZLHNfXV|4Nf9haqh(0(`!Q z3OeI#&#EWgZL_9UNOQ@>iA&BMrO{*n#GT`g(xXHHivDT);-7cMzjZ}{Z>9=} zii#2hl{PW#?PtabQD7c&UITPyoPyzvLyBGNKL=AqnW-KCbM>ElyA|MTSYf$nW}#}$ zz1)_oQJ<%dYd9}(xM#Fq>rhUcppduKYAMQpy=tds9O(Y-y}$<@RcAv(e>}xRJ_2ND zySH~1+JsOmJ9mwzxxcV^$5Q6~>V+iF`+$gng=IzoJ@=;#CC=2G*3BNnWl8;X9c2h* zSH^)Q2@#Zj3&2BYPWoE?>kO}LL_`YIn!&w6P|)>yf`*~rWO}#15ETOhgNxM(c=rIO zO@{{CXW5OyOUP0D~dUr$gy&uF$Uah>pf1&7+NDBkg&Yx&pW~16-fw1!hNdBy@g-NZu<@$1%ohD z(L#jLS4IJ5xhz|ThH!UcqF2C6Wjg0K42H0fjX8wEw-(TyhIUapAZQc^^{o`YVAN($ zOPFQ)pLF^GEcPTU(+J8vkpOazWhTex6CIo|BrZN~*KNcM#4@e_^aL~TWXaho7V1gl z1p<#WSXWnyPhs&3Mo0l!xT}m#j(6qblkQnEi!-|l%9l9-jAAu~4x4d&22+r;UNzs1 zldWG?Q}H)+d;jE7bLnOgE1sgH#9w0K|H0Ooz~!8;@88V)X3Q8fW{iDz)R4+np_FwD zPubeETMI4rNF{RS(Aas}l$}IH5f!B{A&R0sTb4+LkZei+_nrA4J!pAA%{ z%bv_}?KOx)DHm`DPwOFh@6hr9&$;kDL>*-rHg>DlRc^#k?w<|eZsL-UcWUS?L}Gcl z?Vcf8G_TT~9$Jb5EN%mq2uuk$7ZSl)?}I=-5YY zFC~{~+L_1`tLG6*NQW%qKtvmtvy}Rb4>8u6bwyxDT};4jY;K>)oH*}JSA;mCIR%mr z41Wr`mgrR-DF8A&w_Zrv$QrZWU;XmkpPYuDA0~nQm2!J*IXA*Nlj$P2b!W3|)D<;Z^CUW2aPsCiogFMX(?03wxyA3Xe@dS8zA71I|n4 zF83LzJSaIiS)@7O1u~QDjRS)@K1>otI5`aIMqgxkTxUd|lU5<ffM6AYBW6KANa+Rz`91RFz;k`{Y=yxvxS^(5PsCN0r4O@v|+%SuGMBsXNn>{avVrQjI~!z-g!qF^sg$D(!ym(*rS$%k9Lth%a|hNC+pJ z57vZGK^hgRGkX&SydFk;B&V(j;X*P)ZoWz5SQ@XerGB*pSh0+Av82J%i*uA>4kMYQ z6P%}=ZtHBBYOQ6eNoNB;bSNBK2qp*mG{~4PttpwCFX$3GU)t6*MN&UrUOUvxOnsL( z*OldN&sE9Qvg}OqN?E+N$7ZZ%WAEi*OyMv64<4Mosd6uI*MM9Gq2F5foBsYgpu|Q1 zd{CNAki^hElZ|y5!vwBKvJ~?;IYqellToKh@BsiA$ucp?>&b!7(Kb-sS>R|9>AJaP zf3zLiyLa!@xlffsk72oIQ?#33?LX~o6KRKpvbi;`&*ErtzDVtzeqyN{l0<%W1eoUD ztafyn!-Q)fn}A!mA8FkoNK)-Do|vBc4eLc(E@+93!(S9us{m`73<=tuvD!a$Hn89sjIT2IKqbwi z;sBSg6!@&-3An)e-ao&V_->I{yW#hlYwR}O$zpzlQ-=yFd~VwT zuVl(-JH+sbyF4GYmn{nBbbQM47o%BLzozOhuKEZ*&@Z`805BG~qqX`|?x~y4pJQ&+ zx)wO)kQ^QoIXvf#F!+#Vph7#s#=3ngxIsql=nqXv4BoI*N9`+Y#|jynV0 zAaM=qypmSDY+kmRmpEzhZJf{E^#g_LPUq)Ts z@@;xAu7&__0imwuVzA8a8J7kh*kw&^^KV9l&i(y3?oyMHs|5wTnlcp@Kxo)V1vfoN z#I8A6G}PYtAR?-Hz>^6tpC*7;xSb6)HiK_BTX=T=Ux(M)indWkvGOMAb9DJXa7$B4 zC$zB1!s6LhwdnKl;M-|zNVfsZe*hF2Hhwopk)(T;BQ0TCgs#x=Pmn$n{dIANL?xFa zU>ktN9B#Y8vgdom|3D+6nbo2JYt@Bzt!#xji-TzPjRe|ul?>qqp4$I7*St zM!rv2PZTmTKv|8lD5Pb1Hs7)%fCqhbncJs`v_5v<=K(^>QyL4M zIr$=Ph=hQWZLAVk&=tjwXLl^XsJAmSOs_ntLg!M<5h&$$+6xadrx<2gEZi+N8n$ zShR;8b@=&pK#X$B$Xp7*3ItA*Oc*Z9bqW|ua=M(sno%dt%$R_(ueyrLOXL^2wIX}O zYIggIT@2rduAB)2TCh+;>Or`i(xd$5k8`iu>CV{_QOO~#voi|REH*wq^Qe*j<#9^k z!pE`s(*k_Wct49Qn+ZnMx#l79J=13DcF3_XxOsf`=CYIIY$2HsusI;Sv7|@#JN{cJ zKO(PiG?+S>!2*{8n&7aMGj6ws!Udpn&0Ny^a5Y1WFbQNIQd-dcP-KJ2nOzNB6m^d3 zz5AFqONYtfRVD+?ggsDMbK}aUCx>g3Up!$%RqBdUo-0o>g`Pv3eV4Vo>_9Ld5Uyo; z>Z>k#Wd$}@R!I@56iK(tO_G#4LYv|9()eNfiT}~81;+uWfU_0~4bAMRZ-SH@T;WgUNPpn1X-)BX!F`8+O@9x=%Xa;Y6_(-84pNSU zH?tyN%{bc3Jl%1}EryvymJL3&&wSQ^ip8)|%wz9zSj54+&`Mi4f23*pe>4^!d63NM ze?%0hi0&lI`h8QU2+Dv+9<`OCp;W7)VYFWUxr>#>URZ)cb(BO@bit?o&ige}mIpYG8|K>x> zUA)v0x<-8p9&I0imLc-DwJi}E2a{gx!mLStn-s$;^8DIY@z<)JKkwI>lGkD2!1e{- z_ixu+aGtVKWU`NXy(UC&Q=UL2yg~ah%mRO1s996ND>(oINk^>7Bno2|3FF9w)RPK;H9y9jw-E$p;1 zHr<9vsy;ObH|RgKHc^ChA}v@_{4wPAsPt0;Fu*@59UR>>ZWFK(A;XF*B4O5-9y$a` z>043e3=P>#kOu>O3?3$oGr)vF5@baN&d`>h@$^pZD%NA^Kq8Xhi^1 zQ=eNQ0cDnJil5I@^~;cB>B;9b?YevqEHU7Lv3#CxvL8crW3aCrw#C}F>Bg%@4W3>Z zIY5wfs^Hmo0@TQGeh>F31nQjT-=w=A-$@vaH$RMU+- zUF;6ZukHOqO@kv4UeHxY+$Jj6_?BoKF--V)Wf&Xq(&ft<=JMy$9c{Dz#v3d}{pN{i8*f!X-I!!q(FhWJ%Q@ ziyz-U*c@IG&WuOXd<=ItXTe()o_JW;`6Q4kr$P^&$(tkdWFVwMakJM7;;#NCX<|t4 zM@>wdXuOTxRGR(+B$=Frv(|1db0)~0PEJ-=PcAl_=in71CsJ=0ddXwqhh1KAbXLIj znr~iyfBaV$A$akxSD!CDGxSZy(gyv{ZE2*qFo3tY--M$lf@R4zl^{XtK5y^&wa=`F z)HKg*i|NfJrC^7g*%M(Y79pPC7WQu?@ov!XSr{o5s}Hdsdv~@6WRcL zG=;bF2!A2?gS$P&s@v81vu8^Wj072MA;O>3I4HcBm8}OgFDL8r zo4Mbyw6FEty<8dD;Z8ES!U2+C6D8Sbq_*X%F>5#K_cVb6g%{GITRlC`m^L_ForLU1 zDwPXf7eg)-#g>*@n^7~EA^a<+atYnbXdZz%m|6QwX*HsDwh%$Re2(-JvB?(5%jt7X zO|ne-MZRvS9J|yI{;L#EurXtxmkY}c&$lheOYx>X!c~T#1jX=#edfI>+TQ^q16v#b zM45M+Ik}u^*>d`WQj-$eAgPp4F5N`n7ZcXfBuY>1Hd&nQ@{A;`09px(CLM6g2(!_U zVQ=7qK!QETxE)BGeZ2p_7wrel@*)S(1MT^`Y10b0s%AJr(1G4>5fg&c6kh%m3V^Hy zeW2`%l$vi!M!k&;>VU*xFy2Xgh+rdRfz+uJ80UH&_71$3?Jn!jIYP>ghjA+;&)%ds z+zyV-WHpe~)*5ENv7v<$h`KI@q(LZ3;cCYYlaBd|!40fKUT;kF3&1~!^|J(Qu@+EX zpy{x%^qJsdS6(l!p|S3ajzG=|9+=erbU`OdF}-~GX<>E{2)4J>n~DNS4#t}Wmm?{~ z1*GIY>=3>t5d9&t8~B`gT!T9%ThjBy(`9SGs}yYznG5&7c&P-AQks0%AOU{qN4s9U zo!bAZ!?zpTqk?F%PbFFI3u(h#y==N1FbOw+l#B7iX zN=Zi-meX4cABxFW$mJQcRK7~x1z36vluuKst~u4j%#M`9Bw_x9%3)F9%yG~GlV4?< z2wlV-t9_~UoZrJCnS#VhlyTH5Q)TaM>x0CA|gYAgE`$LAvTu2k$=7J1POcu zy!#uZ%q-?`ur!$jNor=cruOO6r)WJx9-YRB4xmz(>0rWz1EUc-HHH$(XNE-%X(e9{ z6loK`rpFp^ui+FZhY4i`^mSbCj2f_%jBJw9LXId|ZB#=O1_i-6bD;<;O8I@MTJd7+ zFhzb!8sb!#K>8Xn<*T#%FVfZkH5pJ~F>CA2a(i&*8>c;wf4%?n+2`w*0X<2yLxR^z z`32H~%Hjz6_(;4}Xi}^f5bB6NCCd?#Q~{1`&#b6-N5hp2$Ri4aGz1H~@d?>gQOKf? zk-NZm%m=+)Lf>1MIUN1szokdHFPn;X*Qu1rOO4)@_!~ghmtzD5=VQEcBb9}6ElR_o}GPseBv;26)n1ep#(-z3u34w00+RsWY8aBy31)PAb4UG z^WMq@#}sb{%09*vjF&|SeNY+~9PJTu&j_-zL5_HTBrL`O49UGETPYy@6BA?7ZqWln zTd;(Gkq7<^f7j9VS-{`Y3F&44k6tI=9#=_gW_<>OJEf?XcYfgNXvxPe5kos(C&`?U zCm|+-88UxI`d$MpEnKt1)?RqP<~uh>w?mb|wviLJ3 zyqZ*d<`y!BQ=1-+NP52dVf?gZosK@*+%~`FMQZ4S>1Qi{=dkdt(XktS#P2G>MQ3gu zed1~|bQ2{Lc=^K)0yKSqjpI z`B9J5V-k3U&$M`r|K1W8Th0^bmg%35JbpTR>q^j}IEW%5h9iiZ(bUL!IumvgmCQl? zO3G0Ag4BC*uw89!mI%{Q>6|K7P69IpEwz( z+85Lu``~Si{$%Z&W%G_b+PoVY$6>nENL0E+)&&Q9nCP0TwaA?L?oC@Aj0{AM;#W@v z@cnR(pg|zoswQZ5QJ0(KdRX(E*Y#6ZpPylk0w6EKJneqLsEdi2g@Fa#9W%a6f`G6x zV}6D9u}7yad|Ex)mvIPTTMqNtrd#tw&9?FcXvLc6DSJ_|o z5Qm1$r^V*hl}-QiW?A&nN2gLlhc!9u5nYm09rSjkXP+1PCeQx6It9GGbmZ`}KEWN| zWF&ZY+NN!)Ie4%&&%1qm3+iMV4DI8YkVuP6KU|HW<56o7Syubm0?W20~Jr5QPAKVr1#s=&o=eS(9+vq6^Hm_ z(uu3y@;&-??m{`iwy29Y?NQ)*>AT3gNst5^QZ7n(+FfN$ItH_bpjh`-5aAccr_IBl zJFipFmeWtC)9ubX;cUL%p&`?w$)(`t-&75oHY5JGBW<*8!LP&ut*FZo+D(KflDEIO z=tb-b=6;S)N;1@b+0QB;xy;w?ik>NfSW;;5&6HzPtdM6r-=ZH8L-m*e{(vKTf2+X} zAK3TC2g(VW4X@_*Y@& ze$?}W=yZ1G!{;{?ka3#UgB=#T zyXVq*sTMSLNOOM@ztA&R-~xR9^Zd~Cic#wactawQN<{>)BpL@ep36D0#WF1{OuJKAj>=R9?}qZT!#^y~+bWKN0!0)i#Is zwQ2Y|-{t3rU+rmfzvypK*Wds4{eIP#HMuCG?3Gk5^g2cSj7c4ypU%m*YkqAs|d5O#nI75QvKkC;^l9t9nK% zZ&qT*n~AX&(7Q7RHa5~I5YDW~d1+Q&lh#6)F7u*IRPIKnX*|ZHOh?-5j|7aMRtZ68Ifb?_gD-*lLix-=--sOmqN}8M`hfGZ1G9miVMVL&1 z0hDxrMrS$y(xSKS=P$%h7P+y93_S+|kX1i9o+ZiRYx3$g7mllv4BNcEW~sU+w=`|O=Jck!TYn$jn0?sWMdk5tX`Z`0R&AF< z!w+e4wBJ8o_37Oj?FTkjzkmGddaGQikugHs2mYZ!btmNP&dX_YfrGHGOpj9uF z{O{-bzB*dHceIVJX$9G*7x)Q0CMR0R3gx|BUoTm_IOjj#{Zikx_VkRPI0|as0LJA_ zx(r@F*Jl~or?ikMi_GYrREwqwVZ-=gL{s30j5)gHpf)Eg2LJa{<%aen{j z1H}ngJco&(kZTe^q&3EjNpr7>=RK8>VfvV?R(ZO2e`c0x-#^YRC{BZJH(Lj#1A-?Nx^|m{3-LoCG#w!Om-wz3b*1VYWj-U3qR09f`BEu z4songWW(H&i3p)aQ`Q@G7yCl zYy4&U^pO%mglLLvXk6k`dVI>w8EZv)0W&Iqy7$>lT%eNSznM}y|KpptYyN1Z;ctRz zi|8?%HS_S^y{+L{9L;?dLL#O*yXxh$9_rb_6s_w*6O zE(Yzup5lo)Yh%HTtlx%1@>mify!|G)UUzv+er!3NwwYjs2siBl45O& zdJlmedEwE7%aDM9hyvwpxSLW7s$Zd$FNWrBC49$E6m!cIlp}UfgWW22&&+G zH+(l=zuekpBZ$+(SK2hdV#_5k?B+5o48-F5o~M-hLQFScMWsDZ2dm;fMSU z8RSjBDVxUxBWUaCA63$cWrpn!U#aVN9L}0}h08cKdOEX>Z;@o@3g%6jPyt_Ki`oyJ z22X;d5JCTA*c9{xV*4){wRFbJ%(g}V{kbCDR)z`7iY{nB*CzuXh0Y-i!O$5UxtQh} z@XpjVGkM(b)lOjFPlbS;XQ2uu7AKMa6OMq$lhtSrh;0cq6yd3Vbs1jYfN7z#+=x_C z2}^^VUW^dBgb@kpfWFnXi@#bu9MF?$N_uiYM8uQuad9wmg&ga=oH#F~#$weYrrVJE zgFJ;yUZ{9db_*|I-ksAC#7yio57F0f<04vAk=BoVqmKdm7D(TPgp(96Vua8}}FCPe|nnR{H^h2=_M;roC zDADJKZ~kCRv}@TGl{dP(Dt6Ake1;?Kp1 zY!HA8B*PGv5m87SpewlZ0-?hX{TD!CwVHj{%wml5h%JW`PAJT1WA$p>(d5fd?J z%w{w?+PY9Z8=@>-F3)0MMfrNlgI}}WDgN^JDuZ-aB>72wIEh^ z6#bUOg%8%K32}u#q8ET3EQ2WyQS?XJ91kBI-J;>g7(QwKjqx-aCF_MvFSD9-OWBK& z!f+s0yAE<&LQoMW6W%{zY$r4!lVG3;Gf75G2>X+?O$>|RV+!a5w^_WL8huWaHYU00 z<47P-4R*EtV88**)dVBS77r*4J@hP-{QdofcSC>P#!#hVzB`@DA|p07_dMT03{BD9 z+L$=pPZ1g@cSv~TG^f&!tEAe#eml=i>sRwt-OOX_Y+2y3adDRL5VV%8tr7mFny}h2 z*0`klTTKk_XiYXn9xd~1fP>&Vd7ZMqeo>Bs3u)1DdvRdhK&kTJ+Ta0Sz8?}{9l-cmW)T5 zcweQ!eE#@&VZNVWs$!rmy=d`;QTZ{ zM^nD$LtH+t>oK|VAxE12y4r=NceMKh0S4JE4SZ?vpC{gbPWj`%Cc#brqn8!9JmJwS z`^sA%jaFB$7>*-PGuK^14!`^AtXHXV3W#W3aB0ik$4h?vuR2Zc!tuRLf=0{5(+m%V zOetn2lHLQAn_^WWCva2c2~o9_FN60hB=$0Uu|uF}maZWSFdGMn@IiN$jT$(xjkocq zTf4>llb>$$7c*VF0zg4|@b!E!6%^^C|+uZcb`Ilkm36DTZ-~5#P zO%z=(tJ1X>eZ8XPStuhtElyT-X*JbzM+S8Ra4*UOm5komUVrz4y5^f z0#v&wf%v;_@pHs{`NJyDKl7#fIvUV4dFGu57iZLY>2%nr@3qZ@=EV54z}`-$JC6hi zAD2c}G%IQ7r%V6)ztxUgk3Pa4uyZNb(xBHb8U%>$jyL)`Eox`8s^M#w*>4>((}rCq zw)6+WGh-LTL-?3y(8oOd${*jVoNmh=Zz1ZxoB!aw+hg(!wK6EZc*=Q( zz9z8ZZY?=@a-uDvZQ~TaitXmB)lz0r-`# zkf733+`7V`;dk}j&|d4g?^jZuoavCSD~p9wqVq z>YL``4fc}B(GCA>-ka74T^s(W{%l0%uJXsQ{LmqlywZ1_ZJXa4p%~;ES7z9CLt+aDJ9=JF%~T*Z+*kgACXA9~ghlZL(Qa^H-l$(xYVIc6 zHeTrM;xq>J2{kUh4xt5mRlf3O^*>fAZG#-irU)Doogy+JLWnwsv#o3sP%`5ut`p^> zX;6xN>!U`gMwbo%fn*Xvq{O?+?H4RK3M2d3kbJnZ)B4T1dv;Z$Ul=}cl4(ZlmBi}l z>FGAqZ>h$%8Z~0A#^6|R;bgiSA>&-oh39e5fdNW&s8M|o&8Ch@p^LBCN`BuTZYAw!;cyRj1pO>i-bX|!f!Y7l` zr2GRZNHumGLRyAZ34e1kc*pM>+U*6u7CZ2uW(|<7=V#lib@ba{_d|aB_zO)tn|+9L zQ1QJi-lmh|o_X%NsnY%3{jVKTmW|L1 z);8O{YLuSWt@Edw{o?uOI-Z6-7w&B{JJvEch z*gbr`s*cQ6dMr%o3o3`v*`k21wG8cmEEt*@OqB#fC=pw-28h7f9#=?eUL9pH52O^v znR$t=m(+4_n5ztaYf|*n%9&>z#{~p>x3}>Q$f#K0JLdOE*On;E4)mUg!^E%SlehuB zt)luU)~(yHe9y6a_lAE;ageKypspoegfE7^6mWe`E>6B0no+sIOFz^ZP5f`P_I)CJ zp|YPDCPbwzAFm7}D;2Ed(t~fma189he8%us5@JSUW;N$vFjlAhQ_(7b4!{k=`2w7$e(;h0W%z z>}Y3b7c%fs&6(yF1gT384*8~8j65-8f_C=p)L~`M_Li;FA8K`M)k@9oAB+=k|FVDH z*o(jkf#%cIf48@Ma>_E%+woD|=H2^GuJ0Jw&ZlO?ov)vdeSD{<##;U7A8a;u${n++ zG%NH*=1%=1gQj|q4*oJLDqx~bb?3plF@OCwWq|eFFMc5iNo7a5A&hn6t{$dQge4Ot zqgfwX__V;vfFqYgAiY5Wa<{+C-2`Cl^cg}C2{9r!gx$}jc4$<6e zrw?^uhaE3&`zdGP$V=@n?0Pq9?ew;4Gym;)RM~%^{dXP{dcS`4uM5RPUksoB^`*+t zgymEAr%envvU6wt$7;V_K8cBu!7nVT2ki2SHr0Mwbjsnj_nAm_*ZA}cdD=0lUw6@W z9@I(CU&NtktfgndS=2`~?LyoU5tW6H)nV(c^A3FOa&#L6kW)4o$>R7LTsC9|jx@uR z#1L)r7WrAwOqsev=2MP6$3z{%M?Cf+0czkWkVq9oKml3cF7|?#CEh?ODM3Yv+BDE< ztmaUrfcf?W;TMApGX87S*VVu-QPh>r7a8$Ta6)qvg)>YnhFKhjp!*mG-pe;{kk+7V zs(dD34HLpbRxG8eW-egZ&#vd1G&!_Q@!JRsF#~`xHiy;Wk`ulWEP9-ekb605x2;%; z$v?Xp?^dB6iMdFmT|zWb`O2G;yL+b{T@n;$Wkj$DJeT6zd~*JP4!<~UNZk9@)jOcw zLK_~3@#Oee?dv5gzxrjyIKLNGbz9F}H;-5U;P`OIiBaY=N|s!3xbd$Cw{NaHb7J)5 zAHN?7p2d0yO+1Y2LVs zNxK*VNKH%fV@}3I3&WiuFETw%bPs@@aR|+H7Y~|y!$Nio4luk*OiExF2$@u7r&4Xt z=sjoJcVwM1>Wp@B80`o!Noh1oDlP{dT+v!Kn)5~6lgy=YiY({|(43rykmIBqB&}3_ zD-!nEx(P4Vdn<%kNIOIl6LcU#Xb{00pp$SvX?@8Qi`e7GbvAy7l2|M;)mwX=r2Oa3 z(T9!RvgIe_st$rAu}-vK8+Na>P6|}cz5YIS(W`)S>2O5)Szk^+b;vCvxtnG}*c->c zj_fxcXFA&C{nURSIDFX2Dt>O{Z(5qO=NARe?a|&%O}+QWi$gPtn?L{a&rR#Hx5rdx zpS3bLH0OmfH17Gt+QnLv>N})VVahOy>hgRoT19X#lwX%_aU ztXIZh&A~09L(R%3w>@`ke)uu%#hcolK7*Qn>=WI3BJ%mwT+?;dME>6=rzODW7<(0)Go(H%n?nlI22sTZOfE3tQ6mbesnwZa$#-1=A)iHd}iwM zVT2-l$3nl7UY2i09(-UHyW8%V%K*aoj$mB`;N6^oI)qGUo&@!v2Vr2 zikVZh2W13DnB}?HzPjMz>FJ&HzQgTedyj!9$FxY8bLfwiwR`*b9U3!w>ft_T=a~NV zc8Y7?6!ZU{^Ls9R|HBVU^ynzatXmK+rJC4DK~$a>{z5}TLk5Yd$*iisS{MAld|0)o z=)|x(%UWW{zDfj3S9dt{o0FeVER3@gFL6MzyZr0t+~B``YDx~EF*`UYK<^oZS%s1z zY#~qeltrfom(F;oeRt81F*B4mSF8Q3qj#tJ^9+^Fi^zLd$_wtgp%Qyaf@=lShX~iNLw9Q6S zDwXa$QU{;Uq@yh+R>u1T-k~-vsZE&r`+UTu8X9d|4*59c>vCiE9@DeS##Kr?hbSs@ zjty3PILSN3UKxT+FgpLtWMWg2%kG@AT(&gaqMUKHNb>g!ixWIr26GhsYFcab422>t zZFZ79)66^!$`M6Ow}}ooZ4aa(Qh2h^vZ;0NgQcM*G!}5Qx-1@YLtXsD}Sh&MLl zP|C0@Ui(W()Ls_L_X3|*%p5c*VcbGwisiUX+uV=-d)9DuW${$}3UcempdsP6Aa3L# zrV>wg^H}j}t8bMs_nV$d9-Rqwhm)_LGjT8HOkM4KX`yhG)IFd0#psZ#@3rrxqc>$%dWdGGN4378L*S_<_93YY%ZjtEkJ)U$(Ef>&gqugj>)MJxHn7$(99-o90vR@Cu6_fW3%Bt zg6dR2vyeyJu93)G>zOb(-`rl5p-nxL!}%CNMi_+)2fT~eI{z^#i8qS9b94i{B;TtW zHlXC&!`mW?k1YE^VRo^o*eTp;{8C->Uji0BYN1f}IQr$Nag7+hkO#@F80f0=05 zJx7<>Y%WQk8=hx>2nT7EzR9?pTpq8E`fQHR#9cjl^oa3%TCc&_xY zvzpIQTi5U9t;-sx%3h|J9q8j=v!wqsYGP*!d-CUaQsn_xlWlJaibT^y%vuG+5{m&U z=h9>W)fYgV6ark~{yxxCcS!#<*Eit>v4 zZXP!;oJ+Jc#pNwA> zW}Am*Ilk+%s$%za>*57#=UR15sm`}khDp9 zbiv>O7na4H9G`Q=%;$-V4B6%|7yl3hCDWtV=}0S&wjpMWr>A&6Y~9Sj{@|6cUyLWd zuhYzJKIrnj?)o`uRc6^?b}u?CIuo%l=+v6+euhye%*&@vy)LuzRluy$k6sK4KCWxiufJ9(8Tb+M z^ewD6@j97!Z0c2V1|-snT9xKzX0Nb7Hcp(RABD|cbV}5PT9}h%tmvZGJI_36`?l!C zySj=KePXm;PkeOu{j_Co)06FI>i>KG)_a|`PI(%)k5Jy*a<8>k!hOAmHr1w{0rUH$ zUi$c<^PO`qBd?F1cc3EUd7p*%4mhVj@BM4S76bm$X5NU_Rh9lW&7%M83?7Jh5IOWy0BuWmat&i`Q6-q8FX()%^fuv<}hX5|?QY5%BWOym|- z5&vU9Y?Dj6O^`w4fHuH?3Mnol`|!l`;VWlcoq2OX>Ez+@$H85T(nYk`gI-p%iZ@g+pV!psMu4scxZl$n=1?M zq|QFk^W}+@;`piav!)yhZ*ii}s}>=4aC{Y~R1PP9VZAU9U5Y5`7# zv=tB#AVbqN#XyI=LI#4{&4|-B6^VpGp$K{QK1IpS-U)Z(!g_P%XQVY>!Xwm*z6xq{&DQ|V*@<59|dGCJt#0bN#pSSU`Uiwt&#F7i5{Y7vDYU(wXlv*bXG znc`KZUeJj^N~cWbfHUyrIIKB&H}T-4U&s+?PQs=Kv%BWw9z4{e+pnpNV$Fe%ard&3OR})T66i@8Mipkul&YXK7tH?%uz_2*?gg)LSn}jT z*Lg5TU9CO`my@d;#&t39dUdFx?SlFlYE-KxkRYTAkSW@H>=Yfr?5WviNY zxhC3S>eI_vSy_4qp;pUxBq`pm82JHL6*~P0~DNmvX<(wKydi6PYlM8pn6%su!^j^SC&2 zsxID7`ZvrLmi!8G2BYRw2X8srbg#PA(EiS4P78R-o6y%Se*7^XkBI2ECg=1QKbiwd zQuLaN<5}qIL?zgzC^cnXadjuAnRcps5}j*>RlAUbh!dmQJ zTc{t^$~`!-tF(PZuxX5q+=IffFf|mJ6P00nX~!WK`!(kfu*GhvfLpD#xd8u00?#CR z*5b>B>dG0SS9HpW6S^Yv$KN{k+3BhYqGljxemk64=b zLdMEe7*ufkr!Ks8Aq9pAE!P{N)9QB5Z5@ZRZ#5YKpFi35=(JS`H6P=}_j;+(_V5Vk zZsovEVpq`UgL-wlXc-$HZBeT`gucG`XMiIOaB$t}52#VFmL{Y2_QMHNO@>Oa-t@%qIqmRQKsTHUONLZ=M z1mt=ZMFjmYsV zx^G%^k@-mk2$N^ePEGV7$}BMIsjlv@@YX}0;H+c+K4R7NbMeTt@@*}x7~)C<8$?zGk}hox?JCSCNe}4-0}V2UwWr z^j_ajLXm8uBv@+rJs2683+XEQWZH!k1gAWJTiI{tn>{Bt2TH{qT~?lCeH`w5Q9% z)iE(KXdKiOm=in2&m^NxC|;qda4CvC@UVYj&>^SLN5fRRXU{zdTN(W;Ww31Yv9`q! zI`XQc*?-C~x(zN+-fi-yBGkmbh52mXe}CL?#Z6;tVgIFlq_HTiTfqsOL&ZuD$E3Lp zBA|T1MU(z)HmztMZzy8#6SPE)UKh@k5w-vGF74C!U=z17u6*PEoE?!OjPJ2&Zl zzXXuR18_V=b|B;3NBDhS7JSuPTbidHJ>7aB5m;EPy4bY?3)`)kF6<1CJDvw2WQS4O zYYZM-7?k|Ip{YsYBp)g5f%>!`Pf~dttws=|28=fkcYZXu52v)ST&cb!yb||C35lp} zD;U(=7ez~}@5O=m{OC7|!5e*4-^jP4S$8epB@P;`TE~g9mB7R0wQ*Oa)J{9~Wz3;C zr;IkD0bes+`Y2zk(&iV88bI$;^JdF9$+l*$G&Wu*L!;Q(A>kU((US3dXpqp!wRCG* z;#`zL^OiHaP?lgL2N* z$<&M-Jp#udZYc}jK2NCznO?@EdjZk`49FB3!cj&e)?y;I?$qt}_tYdmnipn=!pwAbVa*OFC zFuiPSlqdRFA(4@QC2C$EYGl)H88a6S?gYiDq;*a5>C;-pn9P3u)62NmA*a5)57w31 z^ebvBrq(5v7s~&amIDN?xoaO%!sN4!+-y`-SWTkk35$`|ZKF3CLv~KsY3P~V_}b6z z2@5*XX{ok7-%Gt`Pmya|bxy(|R$X2F;WDG{pT7w|wXy^zkUm|1*$?T=q57FfZT6_QUqoaxNsmf zRwztK6$Okm(Gu5NSWFQ#?KFGz7$ve9qc^wq3j=`M>X_3+m9Ka564%SI=|YYt)*`Z3 zaDH&1ytcVW9<_a5q5|STi00Z1^mGb&jOXSPotCqA&2M%+L}KXD?xq=2;)A!!LZW8YL*#$p zz=1LjH)WV8w~>72C*wpA0efbaS1As29Lr-EJ9(sw&nwe<`zg5tz-Z+`cJq$uby0a5 zgTj|5S7s4}sRA>zv|}#aTQCRj0nrndxEn~?Q;y&=E6+Yhckz=m+Z@_4GPZPZ%l;@*^1Kitm-~p*d6>70}{paP;1ArMRBSK&3j>yY(0Uj%A8D2UFmV9c~_< zs|dLfvD4eFdH6$cp;z@iKto*0CTeJFKV3JIB?s2C(wV{n5AU^z^4w;R4f(*|)iV`E zLs$G-h1axT<}nO@tvz0h$PmSqC(e3aDRX;sbY~n^$z~a|}5kidp0F`KBSykP+F%&)m-WUO!yb zsb*MF)17z{IdD`JuK7oZims z-s-l|1`w!YF&@57yH{)Dbc!h%)ublA22e$1Oci*&!%T6^tgEeD8T3r$m}nAw9ECGx zPq;uN64^mBC}rqP1U;RK9x|Ce501q9#r8wMRqzj)o5+#kP6PcAA)M-WtCP1%E;zdN z*GHca^@A8f$*Zut}4HKcG$h6uJ_1>HF+4w zZ)z;lR&aWzU%E62QBOmBkGnto$4+aed#0)rI*s1mWyYg93Fy^@_N39DU`?4?(I{&A zemRqLiej=gEhZcVT1$=bt=JG^7uyZ4>(& z4isKw+H^;pfpv|Q%*cw8X@XSxcsm3$V}!`u8xZY(eUBhDQF%f_HiPsV5EK*+fJ`){ zlHst<&>b({=JWzAXc|3juCj24|0ucHy_OzHgHAq;zuD^NpZC(8l^RUucl`eQnvtVM z(W>e}FE6i=R@8p=AE$fU6$KpB*2&bd)|ywx!5MHzKeJ8d0RMhvrNJ+aE(b0r*ss%U{J@Ub5)pe-C8s^{W4<$ zOF}GUlqIF52W8r=dH6U}s(%Rq93@Ue$cOMvC~W;136d|>>RZwQO19v#2RsRx2f{Nv zbHCFc>opqQ72hxKbz71)na>~#zY%?);lDMp-taGz*NteFP26VLsj?@1_xHDA^}X7No;e&gVB6LW-JQuJXEoAWcl$$W8{hbgS#`a!A*zMS z1^n|Dn@T3>y>GIanGD?V@nJ)*+pxi6(V|%Kp0Qd96C;ITFZ6$qpB1Vm+D%G^*!>bG z@dNIRHDynrbEs17!Tb;pSiv&1MJ$}aKBWh^R8@xDtz&LZY;bDyYWh0|NcP36I`P4# zAu{V5NA6keXSZ8}3{9IB%JO%M{@4Bz`!5f3wQ`cPbw;Dh(o!2m981{m_$K8df|t+3-Q?a4g52h zp$|fl2Gy+~H4JtRjXyuvZnQSt5)Y3z_JRF$#+#!m15 z!~_ZeP~Km(H-hwue;DP1Oot76CjbLncM%LLl*BIRjY6;!wJb$l9^z1pH=Lbsak0Iv zqIXCm8M#!{rrR7jnY)L@P*4^%QnFSIr%ChbV#oP&K*kIfKiZ)5_ZOiG(nPUPk`5>1 zy1ICl{7k#lDYPTpIKM7lV`)N|VYG9CA`TJQV_KwQt1e?TNaiL}n8X{-W3yre!HBr$ zsV!m*mU2r6G#}N%@VRdzROiwvI!8}_LR3R4Urt^q8XNn{+S=OI{2fA`u}Vx~_EP6M zh7MtIF?Lc~^*ri0nYpZBbW*My=TM2pDrLxocpGP*1bYykZ}Ze^Q86izfMI=Z_Ln(l zJpNZC4JU`%sBG@H2cN+kD5#<^)Tt&(ov5TaQ|dOqt|QWC;3~mR$*FBS1kYUy43?KI z@5)``N0~$95!e!_D@sf|na(u_Y*-m4$_cz&8?P|^e$7)~PX0+WgawNs-q~k~%@FCB zx`IyR<)zU)Sz7N}3O*+Az(S4!bp^FU0jwOc#bSAC0v3cjm(vx@XUPSIhDLIxiFpz% z$2c8TR(U}L)`k^yNaO*DQM|&8NGk-gf^BqBPVORE@wpJ!=e5l14Z8SvUYSeLV-%!# z^n?L+z$RN^_8HU?HZS-ueU8{6pIJ0=BlQ;)S~sb1$)jNXFs8x_T{Ov`&7DCw|8@0w zwUzAOz7iFVLlzli(*4l3HE6ED7oeimt!#C(xCrs8L$B}hX#tCs@vsIE7;nB{W3%$e zzZiz}gC$S^;FZ)a3Nw1|GKmD`>Cl`)o<+g&n3&4e1*cE~Gq3F+){|_)W;T z-RcnA;IIu4zOt*{rjIr`_hh-+EV>3?N)t86s7s9Wlqpxpw-pobO$8_Sr@l7pSQ zeJx2`bZJ;vm}c~1&*F!RU#2UgT2SD*UKpUXcs!s;Wr?-wjF0>^1{y>m`WgWnE_YN5 zBQ=E#HFX^aUHZ$fzB<0ME570iIA158+!%Fv-){`2T9EPVkeaGE@Y2d3*S}oZ><`SV z6v~euswNgBfwLy3;`!O>Ys6%?o=6F{s1+x`mcWzu@q~ap>jpNx{cmD!JK@i(~n~|jXr>7 z(kp_viIRPP-)nxmHBj4RoFz(ZneJtQ2u|nfP9hyo@F5usz^qM&q}1#@l{Y>6E97`a zd{qm8BNLv(ooAJMKaDGRMIa&V5_^8qTl?^LGy+7!Pn^q*pKAx*alA5s%>WnOmAp|I zhV=#IjZ4tQd5T7k5u@!USVlJU2PThPS@%(?2ByTB)mvZR2A6=v8zU?tl+=FV*REX~ zxA>2L|N8{KnzGSF_Cd^v@%3#&%S$Sf?A4kr4kA$Wy!8%$DUZr20+!Lb+oHzKmcSrx zr!O^Cm!Z|9RxeLTDoS`2GM*T`u{Y>GY69zukQy8`smqGD)1|jd+o~vfA-mT_N^jbK zapLmHs8EUm>5owe?VXbBb@|@WkNZa6{6z+D?5(3wf_8F#e655k@D}DBcqggC+>`zz zu63001q6~^2bYL}JlUO8NNq+hU6Oxrg~O~Tg~z<0!igIj$eVmQu8R6XNjDR?7Qk0V zXs`D3yV<^3!-weGX)r--7v;X5B1JAHl4^QqrB_1?(HNPZ?$l)35y#_LQqS*wyOojm{l%K>^S zbd$&LGPL>q$Cc%4ra$Hc2vB3@$Av}}#GbP!Z^kWfafy^+4QwxZW#b3k4ny5}AUym= zqr?41e-Y^}R_fyj330jYT^6i*@@Jts? zV0~+nnRneTUmR8phTEG31%#bcUPw_WB^J%P#t?F^ew^+ykRavbdsp$$69P0-ZDKpa z(leC55Ovg==Iv)R{9vBVqO*B^uVd9rU3sQODf4*|jl&}>95HQ@f}FChC!G6r*?`$1 zwHA34&PDlh&;ilf{M;xT?+~Qs!jmuEW;#1GTqc_ErDuEgRZyM+SN~gWt7;peRTFSh zHz&oue4jVB$CT+bTrN#cerZyl+Rwf)K;`e7ecSqfW2^tz(A%d^GRv4S$bR$@Aw{j3 zeo_U@_5I$etL^OtYt>uB=P6fL$RKY3%D{1z(W|Go3q>s6y1SET^g8Iw+Az051qT_v zln5$H&gQ9;0N5PX+O{nY3HQu<*EX#vNrYJyrg_6>kMLVC zeuj~2EB9{;qJOIGtmPsol)zHIWExIfcPqqh*&WYX{nb^OJ(q7R1gjbfH&G7KH zGM+^U>r}vdCFXN>1*KiX{!A!-G+=>RDZ=-b$&8&Ff3Wk^9cy;}_oGZubW?T@!!Q@Ty zoe%A&o@n-H^|DGfceF$I_d~360VJf z_$@ftDyZ(SDw%Q)a)s-sL**w^6Y6d4BJcBm{w$`A(j_3p?FFOF++_UtB^$3kQ_&_a z*nh8lls6eGB}89Z*5fKA$iq^Dm}SN?532AHHpEw7EqPg@FnK)`If9xq9ex^^m$H$# z-Rq^c9ANe|1mRu$QPW$G6)@f9=W`g@)ebqi4wNqF1OMTM#ofev*8~XO!^{2 ziat3Y)yiNUE=;Dx+(k9OukChrqbMc;#e&A=@*P@={s4$>e9xsDli$t*#twR-YDKqF znmLrukKrvANU=eMpb7aDQEUJ>+DD+f*&ULoSQXjY3Ag?cdhhlFC+JB*B>t7as4`7H z2eghuk#EHDeB2@ve_2I*pep)u*J~O;9cS%ioc!TtI|DTdJ*CU?=Ogez?v->bbjP1*ZZrCH`EAwS%au-QE=B34 z*Hj`yle5*C`Ri;|me0CxMg`=?ny09_h~=QTp;13ca!!~1>EP6Ux|mXa`1UhzSAO~B z#v^NOSAKZmC-Nbxnb}_RfGo7PMQYEAQ0%01mslG)Ej@X@=2d; zE&^NHjul+zxI)oeoz#Hx%xIHaCu`GRYN; zk>7F0!>J&NmZ6Cft^B{o%`CkV0{2sV@EZdNo!YIb2J8U3kq&LA+Z>r$t!&eUK~jnhEeY49XR2%F!$#R({;8Q4rs66lqqrpS zJBLOuJ;jP(q0S*Au}yT3*m+N7Uk+b&lwJq*u^01L#8^Y7E;6D>T2f1&{7+`H{Ql>r z<#Ol`XeV&JfrJ(CpiuUl@)btCg{nDllUD5rYcl3Yk?s4%SJ4Io8B0fD0?r-Z=oagL zipdT>%kH25jus&%t!h>e==@T%x%_i6Mjd(xI!smXy4POz2N%KSF)I^kDrdVtMTBs)=7DXfqGxk_2Y zhcLVox*f&s3F;!Nscb1mDYTI4LUXZ5C06u-tZ}2W%F1ohsu!8L zv^}Nl78jO^+u#IJz1M>e;5yQ4vOcDg2Q*n<55SI4|4-_95 zP1WPF2~>P|x3{a0;VcMWUb^eNN*cBzIwWR>KApRgZ7SssMqX-)m(|sIlr19bm0QPS z5XV0{mD(F`Dt{a7m%4Yuh7DSk<8w~T@Ko7g19hlxXZaDJWEzxE?F4LxNTB)m*e+^y zb%juU>6-*Uy@l3FhW<(2A+tEiP!T98@IsgjPZ=d{TO&LVg3{(H_D$wsl12=5?IwwhX6SJtV9Bzr%>s%MI= zufum%LcgY#ep0?2j+E<0PS9B)H+dUj#fsWg#{2OfyEgI&d;GrP8pMq_Fl*_BMZbZR zSx(<6x}bmaKU7&YdzuV(ON{JyrhZT83>aXa1#uzzR(L&wu|SoeV2%+l2H)riD0k9n$GX=sIRlCQi7hWSmU~9y`7(Xknm>L%6eTY!(-#H~p{wQ?y+- zQ5wa0Z98{<>ea_8`c~_Llc)O{7~BAz^iz*gSyIElpwBGI5>CQj=>&-PT*&)hTDG*$ z!V1pHzVu~EMrlc)Q|q{o&t`=5Q~BN1#g^-)JX~uDyKn#oBu7yADRuEV5Zmw9Mlq`i zbXG0d4U}|(mYzvavAbLFwN=utqy0J>xF3r8w{&H4srj^K!dkAs?N;^`^&f4SGS$cW z%nBcda_FB)Q0W0K=q&o|I}8I<$e0Bt1m1N^!xl2Xx)TGDE>sI)a>A5{7$XM^J8(ti ztC#FbT{9~I_PFTtx2Or39Gw|llqamh!Ovz)B}UO?nP+qO>qgab6VLuw zw}BxkvRA7e;hi21mNq$Pu2Qso18(*QEWbJbOkh2h`PUqUDiGT3GCG6p4h-K5ElW&| zQg?R91%*~*bB4sh@YAXle=5e_swuG6YBSpVi$*!Xx(x(GG*I}U?Rc=nZ)JhVm{?gf zyf+)Y-UcYDreJ;XE~apG)db`>rD`L>+|JHhbWZo`hdV0B%2~@otGaXp*me0G=b!Gb zSTwkkpnlb4m{S*iAc%-jT|t@*O-WgPip>}a4l30F!7MPR%mRd5O{N#iNR@}jO5t!| z4HAF}Qn6}M((C??3A4jXarY`6JT%Gh0EicpAsxiLTSlu2H3+SmRBivSz4wlaI@`8| z?O=P_c)C>#pco28q7nrO#?}Hwg5;zGktAA(2!d#LtE~uXkwl^rR1lD)6d-6D$ev-wA`Z)D*4haRYhzh7l7ub(-d`kbZW`}_14ib0a6jIb{N#T+0a>?x|!r3U<3okr)2p; zZRXWQ=*-M!jJDRXJ;MB;u*Wa|9@xh@+sFHa*DwDFped7*Gq~?u|wtK`- zqD6%^2HCe`CX4whraBvy;Z^F~1Hi+MCjA$_qu!Uc7kZ%=^LgpI0ptZh3n69zD1{7C zl-omLWvLssaQLJdL1mY3;sL4Q~?c=h#KU3 ze+^w#WW=ibatr&WQ+*;(8J;s9wAUCVw-_r#O)Cgd`yzqDWd$%qb(bj8*=>pYP9GKmvcPodlCSGSGOMawb7PDZK7b!!(FTirZH5 z0&;aue}BI|rZgcpq-;8D_5St10?CWp;ncV{xZ_jOy%A=}Fh?L`b>x$VSiHwrOG>wx z+L%Ls0LvJMm`^IB1vM?t;7!`;u7a=r-EZTTYL=J?B)EYvvm%N)-s(Dp&ZJ33QBRp{ z*kbE$a4MPMaQbIEjbG)ATiH!J10M` zzkB&0NgzoR3DF?HR?hL3Fm8m->MoDJxrf4`FT5Ar@;t^>p8FZ*8)pNC>*PZs>nR z2r7JW?}d#CMMP>L=gE3JmYwJTUc&K zp%@@G8?(&#uU5>w{p3aX!-)%)Xhw&6uZgOc+xWW7TuylK>}^G7C8N?a2L>k2=5DKW zs#qW{sN4BaJju4iMez2OfUfN<&*5hmRTHN3AOCBynqBp^*CRG^e6$1U6ftSo^_10L z$cz`yNOpE&m#~u3uwcn{;0;Llbz%GfFaVziaSM20qry#$hSLT{g&{lHu_`8GAnUHa zEQ_p7DZcSLS+YkGUaXz<~`I^XZ* zXlxu6s%Z5O2|4-7@@zXdBB$c_$77FAE_D}`6;jY|4^#43TVHaxrunMY1>Vh4`v?b^ z=EBq1U-sM2$ltywKp(C`!lreo77}Of+zivRQ|CwRno_lAxbXJ;p+1ReV|&8dRG(_G)5co5`l=g)l5zqouEjW=(R6Qf z3&>3me}3r)`9JwpwA#@G%Z@&J^yv4#UufIocOPtwp^t>_Hae|Hi>S})xpSE^R5M%G zo!k?QsBepa$Bu?B+yK=InY^PzlpJ4L`d|PPh0-S;zQ%55Q@BB8L1Ju%?$sTe{OUuZ zP6xzi4vOd}#Ez91MTUs(U^6^ZJq(ZC|JSTdo^_!~Hg%@4S%Pn3f3+^ks+%+8HuLO^ zut=};c1UZQvLp|@+jzj&-toOd_>%s;`p@|~UDQVoJNob+Ke_9kD(^~0BK_`c%d@xU z3$3iIGSn>1n5c1qjo2{fsQvvVt^w2!R#Cwmck(%b-pdv>5^w$&OqsqvU24RuC+XHA za#`Hm>+e5*CZ4aU{qpNI`(ORdHS{rd^2<70Wp0f5=~s7mDk}c!rpat6ZE`)d_PlP4 z**$@w_FJu2OU%u-_$y_4ojn~O64IGbk>|UovZJ7T+SIBIV{S}-Rr8&0*YEh`r|>#C zkeY*PD{*#Kiygc|vi8jj-8NF?yYSG9?;sZ}NG}7lmqt^VVU!JZKFurxCm?qA%dy>L z4&aT(2oUUw(P}=_Elb1}Fqnx8jKXn2GRs}waw7L|I3Al84EMNp4!PU{QjyBACt3I{ z-6=O}(fS_~)fa}FCE8R3?|-mAB4>cDt;6p9=f$dSNxdbi>o+WZ895;^0e3g) zEgYB`h2H;7U?5SHk@!Vl@cZO1(EFyfSn_n#_qbi#u#0HMZOR!YM8fDq?0qoDd=D5( z8fHRb6i9I}Y;&F6)0IPN%V`iHc4r^5M`1Xc1Jxl~7;%l@a16#YK)M>}DS@Ss1DX6gA!x;YlseI4?VN24Yt z3VL-u?Qox-utT|Km1>f~b=6x=nw4w#TT25Mbc@QoII+`qQ^McTkW15~J!Fnoh+~2F z8PlJsT^^r?*QtZ10X>u{O$`k?k6wP>y^-g#hIa>_Idffi3#l0iZb8<f!3IFAkI>>m9BEr+>R8NRIvNG^ zQF=h^ejWW#3Oo>?yi#!1tVZ`Y)u~rmi4Ge)o+Db#txq6~*nb8u;mVDUluI0zat+=0S_bX4d*x`rYM8;%q@ zh=FzP?cz+H2MEN|`dRPIcn~JdFM!rSA4*jVOG~o;B3ujast0zC^k;bGqF48PUKZs1 z&|BKEN#qySV&+OA1;Z6%WBtP7vbrs$LL!S*au#j)F_9moq+qV%#-FcfZ%XPB`Jyk&TPg>#XA`09KD*pNi5J;Zfb%i%!8pyt5ptfHdp`=!c>gt*2D(!~dip znQOBnMG+Vvw2=uhsZA6p@$;<>WEO$(M@+Wap_j)+6Z!f0mNBWJ7CR%n{SdT~E~rA) zwU06En(kwl-~j^VNuGD&yyb?+ZwH?S^_Fy7$L&bJ$mf}CP0YWH!tq3=!$OqW-d zYt)5Cyv6l`Fk>qJe2L@^(RK?r9VQgAm)djxol#5ncmN;)=gL+L0ZJp2JyPC&RV%&_ zmmx0pv$-$Jt@QduU=lypwt45S(jA{ z@~@v1%oNc|*>U9LWIV&NL#C<{!QxE!8)N@^K6I$eAul1c?*#i*+3Pp&Om$N{ti!gK z4tEWl>6PL3JW!E$oxwSL=-DiVeV!>WVMQ~IJLJPO2bN~D8WLVeU8Tsgw~%hm7_k(5 zA6fbvf5qJVbJk?oG!TtIr%pYFY*^Of{=bL^z4iQH2;pG19$-V;6>bHjNNoQbsH8h4 zV3Ds8cMbYi(av8~(etg)$!`t?KpI+znVG&=HRc3!wIXCMEw z*d{^Site*46W0SZH9}0Eg`V1-KV>^;r3s4(cu1xPSnCUpLKDc2Mz5+(jVio*2(+TU zeI2yXYLa6mF&8NBNtI0fTVJCPZWeP-5RZWIF*Mp9x(C8A5efWkO(1D(SAZHY*48W? z`SyKT8|)%L=lBa@z#lVI zE;x<>)h4eJv~r3pS@L&Z4n$$3O(#u12`j;Orid^;o#P32egzP!g~mJrc*rRLho01W zj1@96Ph=&K9DpCnjZKHivl!F8?b;PUbr&4+?%%&Zd@sMJ*}?&@SB`WeWr2<+K69K& zdL|sp{Mn(`VWadoH{$gQ*Z$(%%+maPvNiK(Oq@MMFD9;X@)8sdgLZ>|Zsjg$-ob3Y zd_;$pcsbp@)cEv;ekd@~Wxbk}x6Cr4o&g}(moHy#+_xHY!@|To$>#|76GYA@rWKWs z!N?Q(6&&BnufZ2}l)7qr_wM~<7sMT+hE|tpFHk$Wm9#HZU|qV*OP4yyG6iVe*t+{} zuIW!9&(YpBHDf*dLahHY2kf;R5R{?0MSv!NPS@ax@3rm<(H%cFPVAq=MSpi+a{ znIF%kCfQ`aQ7g9_>h*FCh@QA;40wZ z8KuchPc#?!F(8K82C~6$5VS0&_jo)Q`07jjgAPWhC@Hj{`}g2&FY2tqj{bdZl85boLV48|mmGzy%;2Q@L%U=uSlGotR`pdsZ<0%;KnUb>qDOu)7Bzr*~c zslCNj0azz#e4(z=Lw1QC5wvZR4JDOeFz>ta3?oXpAKJ`-2NUx?h`#P#z_xLyohj52 zFAvn5I-!9klOQ=YEDIzeQyVY*^;gMHhg_aCsdgQvuFsn?FL!nUZuk$o6&dtXI}Ab7 z5(U#0s~SLsnfA}xS=&_xqCuh;2;?0)e7NO(i=U3U#{QNkK(h8J`grONdN!!#vNrF$$i^Y_w>v+Dl;ocd-cBQmzxlxHNGd}=+w!GXkdats~9Gq z!S#u`h2n>fev_dAT%wiahUkW)`CX*e04Es8piqHAUf;9< z7ZQ0@lHebD?INK3IAGckgFivyVN$>3kT{gfg{9E@^5jKO!lDbK_~2GZ4uJ8%3r7-X zB_3rMd6I1NdXiJj%GVEM8Spb zuNPBUiQxDMe3(DpjapJ*if|rU7qkh~m^3Mwib73?#orDo95kdEZBLILoq$FT1P#w` zC*5PBQ%2R&*W##9V3OZa_X#gS?qE>h%Lr-2Bt#o zS_^{UZPyu^%`&(-j>DfYOB?b_7v;^HHo=tXK@PmUiV~ImyRTgalbY3w2PA?_93b)F zi$7+Vc-q?9D!O$?19QXO+`V+1>SuzNrRBPW^#o}pwxDD0DpILx-*u=g?C9L&vj&z#NvmI?;FVYj*`7^`@SyFA^Xzz#WNy8 zBMOS#$4De3VA)T8h6sg*4}q(Z0uALMo&RqheEz9j8$>ta3!%zpkk(caqJ_X!*ta_9 zi$N9j1fzP?`)@&|h{}rG6`*Ch4!aYcN^T4c72~~}P1q#BF)C09P&YQs zd^wK|+ccC(mUQZBYT5|v=x=tGRlx&;i_166zvBMak1i>dNCKduiGAqwGM=d^0X?B8 z6q^uY8NP8VE(Lgkrp!Z?qQ8#I44&S*aS2{Fzow7&C3!K~C_UrjeW}qLMLECxw?a>z zmKLf4b;LF38z2SEd6h9wdiU#uTXk$GY=O6Z`;C9bW!VH-Brds1Joi`}o@=PfsaS#* zO^AtYOxD=6&L6Bl(eaR{dwl%mJ}Q-eJtY8Z%qGkR$}xKw;K0xCbo&VSXK{*B<;UBM zu?JJX(MRGFn|cZ}v8`#0*pc@yCy~o8_=WS(aZt~Aj)xny;sKtoz?CfN>T2x#a9n_B zyhE*K4D*yILVKUXvvVMM_PC#zrRbmh8Lp0m8l-+%WDR zafOVfqqbJ0xhReMV{b}`(A|)!hvb2^dh)jM!iJr}>-TeCkf8%9Z6{5dGv0vf&a?tD zKLSsIXcq*y{Cuk^Fjy4L$R}}qAa9}*jav5$R+E9Yjhlymi<9^Sdo|(NtBaH*pp4MR9Yw$^@O-#)#KUTz#w1X0 zB!r%`3_QlI-`=T#LUtq1!3xl2#O>z%3Mv~~PHrBy05K=H*pT57<#H7kYP1^Q2d1pl z{`yrSk80s5OwZz52!khTxWntP!^0CHn0^i6Kqc4MGCE{_Cfie&a8xDkdwMZ#l_hqnzDEoq{mV&t?7 z!_F;i?6K?E*e&jf=Zg;HlI7H_8lTJm!l_7F6LqdXZ*}8Hu-dU==@bmNzB^ERyAYp@n910NO3uKy7vR`C|Ws{w~@$~ zlw2h^%N|e1we6z&h=Mh659^`uufOfumo`ry?K_Xuo6uas+cDA3pIm?-8fk^l28D(u z7E9WPj4rlT>_8gdF?lmv2EJZ2;+zdd>M~o@AUEHPQ>9E``J4$~2Az?9vLZ2)Pvv1!fg)`a{ zb;%tFW){7Wl3asBNAN=;c5#OFKT@m47V}7R%dWy89Xg4|@5WubM?270gV|xDqwWJu zLCyjVt*zIf?s&>~uZd}d&vKbPPS$*G8Y0~)!y;Rdw1Fhp!!YH3P1(P z&4PB2+k>18Ghr+dfZOR?f-)7a=C&T#?XH4eAm4_WGiQo_vyo3vl~7F!NSK{Y-G)R| z(+~_KGG`pxqHsFe;m`GmrWqsP8{BjSLS}t&ccSs3LmAeG$BVm~aK5P39DPC($h|z# z!uYXSB)Ib_a_&NzZ3LKnTURYVF>RUz4idqu;6bsWD#4+LHavw^PK}KY>2DoEHFwr_ z8XZDi$J1|`x;P;HBYQh@`%a9S@1PB|oj7sgibabS={MlA|IcbM|Jb_Nzyt?) zey&`piBwEYT^&8((4f=VCKILcsoVsfgJsR$tZlKjnTczQ3n-1f7z<+pr*GP5SUaEo zGlOi;+KHX8y$B6X*;}=HP$jfMvWh&ECQf~mi>*C|pN|Kyl?fmJbd08>nu%y^7)UXg z)VD_s3>cfL*I(;=7KO(FENeU#?we1!h_|$F4vKfc6^1(Ia6MEZ;ehwBqP{uce3LTd z8aHe9Rv`Dbnj1Q9pybi^vpnlw*j@n(4I#^S&{Rc)n?WEzT~v5JaM`u98E-Ah;x#-1 zt6zdj#YaGarrOi=Bw%c0-~!h$jEcGYrgPjei@XE(5f=_VMjT#zfFtzvX*p`LQur9o zb)t$x)%9TVEogs>hG0@~e}iT4>XN%cNXy7616Pu>@(~1HMJQ_iV~Gp6@iUxZo^GwF zIEenv;YnlB>vqq@TL}u6jyoxV1_l!H_+wL_4m-Gzzi0# znRjA8rYm{rBP!AW6vWIMQsawMj@8_W3-0!nK(>RttQ|TF{{xG6#UuBi8OGFzC0Yu% zPa0;01Eag^>m`PbnI$y4nwpxif5xNDVmr1x(a^fa<9_PKDU!z(l*iAI1Y)b06*#Vk zVdttum9(I!$Z=5t-rix@N8?^*)`I^PJFc}%@oxdf2@aSJJ?^D=CbhqgfBF1H+l%x} zJjZoeN9IGrqluukH>*>Qu536fxIT9YqJ9b72z&<)8KMp&BrOwR$XU5gW*f;^vo>jKiCmox?FBr3QPpi=GqBxBNJ{NENa4~Uh=MJK@aokWJAb%+ z21@`9SS*waK%V1$eNLYyM>`l>@BJoTB>vFa+DbAAx?6Fw z{80W@C7IJce+>e+=n3@Wmk9!cbSi_GS+uH?t<^Vy>4oA-yc^Fw$dqXm)LzR;#pD16 z6A7tV&cnc&xWdWAbY!@vj0Tz0UgMzzhj;c4}HRj3IS)Ol@#2)PDv$Lwyv?;GZ*L8Avi3a%7^3 z5|DCUM#T>mcV4OHxD7aKKe?>qLW!QjVd}GzCD95EOtLiSY2w4@Z>Av=>7%J0rE9tX zevKS>apya5;dt{_fSpxn+j4OIv`jq*0O%p(wb?xu|%uk#O1kjO!nWsdRO@MoY_x~3NK3O z>p3~2$M1)KO|6iws7T_v>^D``b=CEZtJkf2+H;AdtUG^>yT~1#d81i$R*q-=1B?CF z-t~~KE7#0EA)Rl{n~-+&Qrx?qR<5R=KBg_wIy1fbFeC`9yA$U`YYOkky+&Q!m=ttN ztS{uV4b_gD)UG<{qN`hJk>~0BT>jRz?Lv-E=&_b6w1finT@E07U6r9tSS{?@t8s-% zfjtR@w`(gnaTSzY+JT=h2C?6g4X*dA21Hu<7(u6clTqNVn%iz?Q+>pb06H@c!3R~9B zvt>qcfIUdvto^ph6`&!Dg<7yVPjz(vwZe~|F!Hg{S=}*^@0X4@2nH47IEex?pz5pjH z|G3W)JhIpk?xMEkjqQO~kfnPjH#8?1!=XO4Turo+>8P%q@>>F&{UksJc!1OGPG}%0 z#MnT@nBnVFUxIY~4qe><6vGTA!23&^d{^*!yMe3oW5Dg;a9HI{hLHV}OthKO*Ns`2 zON)pWdU0KaPwTXH>7!h)F-8IT5w(OL-g~K!Bmwx>Aq)j{4YWg9rGt-h(Ex5x?O8v5 z`e%{fwWGcz3lDs=AEvKfgg(P9?BWo4N&4rj@&x=Ct>`~Rhl^=Mj6|fN=a*XQMjCae z1FeyV)UptK3X-k}d<|=&PZgYZv#qeniXq@awRm8iOZnBlV(GF*jDIXd*MpSjVDBvN z2b+9FXHDk3Ji*^~`E#(2+)6zKm;huXk=2n>i_CML{9Y41gI&iDw3Z8<6ku&R((M-! z4M4rmwE26rXIJ}R!P$Z`p#+%390M~kJ{{p%e4MROpA-BW3=}>Ez*ZvYiRP#qifmI$ zZ@>prt%k>W4?JS5(<}AVam=;F_$si%0)Vkv(FsY(2-XGLRRJNa?EL11hH$-Ws8zJjZ0;ItlcHQfSBPnttBG zdt^>Ouz$=w>U81 zmD+jd=){}mxV%#_ZpQ}ieXXJJ(p&qE{uZ3BiwuPqM}OeOrJI7$%kUy8_$ltQ(a6Oj zhEY`jbp6&}I){Eni5nhxlPNyR?blOTmAd*(XT64_e*2hQ9Cg~?cz$y;X4Y-NCFfkY z;z}c(e7UDP{~&Ko%>F7sU*LJzFbYGjRkP$)#MrbgYdh&PAil5}$j+`-E=yokV^w>o zORi1j(xnx6f8cr*@V=9=e>b+^a$KAU1P%|@TSD!6fu2`7*G%;DQMB#?c~FXN8^qvX zBAzpXrudBj2<5{EG2FUmYnWpCj;4>h(Eql+nOn?YAL{I^`)>gbne3^?v0-Da zkV{IYC4zEXF>5kPwptE*dO;#?xI|Nbv@xcc6sT$fQDY2BXs_&0tU zY-xxP<_zVc4YskYVlMvO6>Rk&(}PTQE!vB!fvZr2ivbVP3Q|82QM4GP(fJz0>7`x+ zMdmC#hghUkhAx%5sjp6p?kr0E7o)XNn7_7)sYnlnfdUw z@x#B5LG4EiitQ@}SYYc#nHBQ2`!?b0tMp~ILk#7GhTgPLrXnx^}d~Ke7$Jk`?lEmDo7S_yD7uG=P>$}BTWug zTTfZgh1gV}GSuORYEgul^7mxXXgr!=STn#H=p24j0z|QSv;?%SVYHj;qb}k|Pr0_a0DdRpUfsrqhMui?X7S zvJ~I|2jgoh0~tPoEIRFu93VqzB6U(K^^C1C=PDTImHL_i?2s2Arz52Z8#!ox?hOB+ zhwfg9silg+5V0)N2sOCf0rhvH*}IaaBIg^JMm*zQ939dixd%#EVOPS1*C9my4dMOC^Je?cL*TKDfh^a=4^KWPlS$Lvs`uTSCXJI2y8q!k+P9JcEUzOW7wi7eT%lHV zwGwN7hF@NSD3kAt=+bYQ;q`t4pwa+1JzN1xIPgI7$a$M^w`Hg%zJzg~zmXdRIhQId zCj}PvHiYe=?TJ|un-{q3po9r0N%>0HL8Cr>|H}K{O%gbVmy=BMK;Qb+Bc2_}dAl)o z&8*`BY)zIUGfLV&&HM19ym>aE=OM|!kyqg0#R6?3XqU0AXuqI0q^}z$4M-f4ygWxC z8yD{z1)-W-T=OVHw{XNz$2)7%C2d^P_WLIvU+=5Xe1V?N05^9u0J~5H$-=Q^pDw}} znnZ1~@9sp(F<9u$Y(}^$bgacDyRQ8ax`$2m%nz_qAf(%gdd zVRwjQAR~B{l zb&3uBT(M`z-yAH#u*5uv!Q)y98-O7_?84R6zNZMs#y8bd@=!QV9Lt=aXrd&sxm>Do6fY zi|x>gjGS$S7BM%BV`$@~WUfLEAi&CBJOJ9T98jPVP{je1*u0QQp$=gL%8lx_w;~m` z9#5u;svTOEY4Wuz=`wF?9JYNb@^aLq?(n}w)};!~FjSi8g^ko|4hLOeEQIS8bUT+}D8)rIyW|eqcQn`FN=!vTwTwyECfp-h=H$NLgbd10 zaOCx7%ID|2FH{gQ=F~@c>+tt4k~x@;7ytt^2jmb_9Q+vZiJ5H{xr%RbI89>}Nqezj zhO!I-tPwKCTXf%J{u2`t@;O=C@JPGC6{?0f-ZYe}-FL4zR6!c-%Z}-*UmUeol=IQ& zea6#=|2;p%Xr%3)MRG>1mmQ6%?#7XAhTN?fX>}7vmQUeiAD!KV{zNLT``) zhb-h9GL{z3sJ4qwP&3pn814?rHEBerdajcBkpXLzXgAmP=#Y?|L76^c3*WQ7*jtp& z^Kr^>JUs2ikvCpH@;thLo4Y##R}9$fTU#mSl#g(P=|+mVn!eml$XxZ8c%(t|R_-Yx;H zaRCBM-j)4whSvq)F)I9yX<9Ju2|+`*%<|BEe_P}VwFTRR7ciHH`9nIk8zH9{M(v?A z*ULa`H=?Zq>Lou0KKj9%Pc6I|=JH{^Z=SEA(SQU@pyZD=Oil8dIQGmwsj1O#y9PVW zu2KY5h;u=m=zL!_j_Mp6w>Rsq&Z6CQ%c*{)l5i;(QFX;tXxBIt)hO$((DOxL_6GWQ&FBvk_HQZa-$56U{N8hqq6 zz*SQjLHjTMdWsqpv9TUV9?Tn8<>A~^Xsi*ZdyFb<(q&Sni!;JzQ1C(-8HnSm=hTA? zr8(5Ov;c6BAO0j3p*UF1(Wq}`Cb7||jdsTb=!rb2+4zIbivE^YVqMDr3W%(lpZ*45 zg$F7L4%kF#3^zTqV2xZ$sIvxuwQRzbR0kiQO~Gl8#Z8{>0;dmXFOW86-$_A&6kqB) zkf2lk7K4(Eil&>jtfCIQVJkPu+))0Q2|@`S3>F-) zk-DuS1)d!AM^gn>ucU4A{WP5()ZF_GQIt}C(v0(tRdPid9>RMcB zEaY5`pn}SPAw)n5urdNbCt*3b_H97pl;L*BzB4l$|1^Q3xZCjCJZZ{UFRWZlnGJy@ zlyG3Oi~^vpRDd3V9mSAs`MC2cH9$3Cb9ekC>b$3=-Su z)rA$tNI z6O!vA)KeX|y}8S%whl72OPgNEEO;h;7#9Zr-i*a)_$$U`aSjNU6+Ey}4ce!a$kjBi zsUOfl#Ee7MHMTVzf ztovDeJB}8JJx%WEBHA=ku5W5h0HRQyZMrb03IRT5em_(jff?-XDo$tipv10Vk7pi! z`2W?ik@h!gyEUwGvaQTETjiJ(O7s{NA9foHU(%)cYN5G=l(~vj(huEc64pVk782$f z0pSDPY_|ori-L=?d}YpM&kW`U|Kty}O*6bLTFV~y4F{JH3#Y3Y@2Jt;=jw|XCX4(i z(ff;r(8NUQ%{WM+R0&3cz?tg@I#Q1VuMuwN~O-Lye zanaJK9_()s8fz{=?&*E2E$dHf?$N}%tfDpNd7tb20Bc27vHhy9rjf&^v)lhV?7_+R z7|73_o`0C*Z(4VXK>=J#ZFPerm5Lub>|;Nvwv;pk8Cbg(lTd}|L>Y;(im zmPWA)IM3n6-Jj@sz&yPO?5nJsU)VQIJ-Y<0c3f;4ND=`*khL2gek>};^z_4@UpYm$ z9Y$H$7}&nMQNtHyJL1a5r_!nUggH%^@Fd>I@rekZH;&PEO3I$>8CriyGvpH{GGT(u zzYvyPP)k%+CE8V@CZoO57SU1?62wbP;n7sl)&xEO7KFz3Y?V)MANkh4*T)EgL2D|0 zjt82<0+Itaws@~M$jQ*%XLK^)z)Zr%#N-KL5jps@MGK>}Xv0)w@nE;4|_5g#z>F?L*~7;_T*xxN`t)+4#MBg>-}#kE-Tc$L z`%KqfLvKN5eRR(8S@T7z@|v$23mws#>BH1Eh5XxB6M>?7jSMf&B;{( zpB$}t*c^bXrVV{zkxtcL^tZlWj!`c zxgr9Ma5D$&T;QLEyN=H;M|whVu^X~FS=-P1M5yw0#8($n{rr)y9%x(0|J>d3`1S0` z$E^pcDqs{G{q6@oBbusCA6NmP6G7AyeS5Ien+a#avkF0Vf6F}tjfYR}8S2Xg>+`wk zFAUZX&&hy_8>Y&5~wNAXb@t$fxT)`+}o2eFrY26$HE`XSc zN9urV#s|T8a}~~txdi&%YJp(70Y)ZzM2NW%Od_Fhwtxx<1E4-F)M8=}RwVx@mGtpJ zx||;XRvLg~Pel-~UIr8dK1CjQh{2KmtokOL%Q86e2P4V8vC*c$2tj`)=nfB zm1+D}?ntrSn$F`Fa@9L5;q?=ZoNo7>W}ZfIYTR)FouG64W;vk%oEYc@nB@VI^y?R* zS0f#4Yr+rey+yt+UJBs&m}POea@H)~>sg<7rI)LnrA=6ZCPMFQ*zXzHd>h(Hk!IwnYr-=3``cievUY0I0E1M-ZBJ+ zn*=E`rC5zPn|{w0cNH$}pxR<>Kall(zxS7JoP4sM=+% zDRz+tV#}C-Q`6ZcI362x`pbOy zUigAJ;$3hvl%gg>_^R+&nQ$eJiduAF!G~~3VUaOEedo`k{a$18xWl%y32uVu$%ztp zO1%}Jd}zP5T5bL3w{w&7!X#A|wjK&qPIkEZa8sHOx|QC00ashH||lH8iSgl&tyT&qMpJfyp`9fCA_rMHUpdGF<5ThjkTy62O$11zxe8^%jb=!?$_^hq$eaBrh-(2R{-J5Tycg? zhvLJzpx6Iy!X5c+KoUgrkM=~2Sz$_GH4bxo7DH_d?^$hb_Nj-W0EJm9Au23tgb4$$ zN>~*nQ}Gqggr|Lp7Cr$KC*P0pOYh!Fa16x?+#01QBuwjtwAO$6bW;K180!ICJ|4I| zL{6M$a#uiH=0Qj}t|{dmn^$~U{XN&|gRW(R;}t^XSU{Nype%w(uXeG87xTg&;*~Er zw_WFpV6zgDi#DTHoJ%xSXy@r_seQaNa5u~s^)b>^r_v`Bp7T?NLm9+mVCS^G+r{;Etrj zfa&-+BS==EFOjm!KX<(9HVzaWg%p@HSwEPqPyBA`-Jkf0k0>Cd4(CZStptP41Hc~2 zQ@ah_O2q3Cx&-Jr&B1ByQe9t;3N$*ztwCrLMr;b;UYy4U?XWJ=HKDP8DIs@#eK&xV zH6uow#&z=l!73hxcBRDZaAyEadShJLt8DQBh?^ozZ|1VC27AVcbx1_hS#M_zy?^0p z(L|n}g|;E4Hol2D6g2-VtN3sZ7-*h|2AEJ7=!;b(BmqVjO4abG7Ego`J?=0#B#HeV zLc(18yGl7$O&1b4+0BKkoO<4OugEKIkq+V}vTpMgH`ON!ILn@uj9?Yln&fyCuId>m zMsdDYq2lVMDB`8?XmJ-#^EMg7y=H?lAH;U`ul+7)9o`LN2X)U5(jpM?*aN;r2nf?I zU(m2X?0&9>4zwHo2F+mJZHWt?cyjo)s0=|g4pRV{iG8FhW9^Lkb2xUQL?zY$L z_f9Po?@=9oIfGc@R3JGmxno1Q&QG*9TZ85+b@CdjA$5`@W~+YRfQqCo^gub2$IPKG zvMH@XqiEsbj`PH`OcL0)R%1Kj{ThL47f{JV=pK?q+67pZT#{2H3#ygL zBXi|t5YIFFw7G^W3OkKfo|U}KDypKT6{ocqy;VHZ+3lKtP5Mp=bH%2>csGxM0ORK7 z?k4@c!ONSWSc?2bczv+Cwh7R+)3C#J*$+Zn5?wuqFRdcpXBCsgkfO_0)FI|hx!wh^ zV-+(QbpWnzy5FaCHefyR_vpLtt1LpFfSCh!S~AZ113>Nv&i5%JtUQ2rV8@seG(Z-> z@lOL@5zYa7G3AUWU^48x42xtaOCEs4mbjPo=)J#jA~NCk5W~!ZM78e~o&#kmc$ast zA2%46_0X-fiuo4Mq1%4!iYW;Z@UFeJxBh~^Wiz;qrX_iVaEsyRJ8N8zH37rb-%qj+ zA1``8px5HdRv;ns_O7+D)bVvw#Vz>&6$~lLyy0Lwu)fBlW`z8w`Q&RHQeU~|4n2Q` zOG#_1wg<#3d~0&pw4w{m-J1d(llMrNFKoJS3oDCvZ|*J^zGZ*cB-AQ*#3rx*3gc4XLDCXP=M$t6soyzV#Z&%!l~y_r|=alPK>Su_jis*wZD4e zDYxvK{yJF8XIdJ;Aw|f?9^M%{|7Z^}owl$e#toKoryDq&Ji+J0b_>0+Boz^w8F}9c z3L13*l6@>^QU(+=`0pi5P$ucu0XWZtG;6)!+!AxpDYx1@^)p!o>B~qjNAe9c&h5)V zc0j2*=k;Z8q6bJABVN6qKl9v6pjgb;-tyc zGqR0Q0*6Gh4i-=Y8w8(Vj1jcBMcjXPR{e-X?m&rC3SfST%C{m~9Fr-{Z>{wus;mCw zc3}*>q2Y`)$#YNvskGwh(R?t;wC7;@iSy{~sFq$HcA#84Og%;<=+h>< zxH3rcxfgr~t%4Zx;LbZn%4>~?M5ip@=dw7Qhaw}9O(qap$2JPAP5 z`;ugkE0N}hDgvVZK5uEmVLsP_X z<8o3=5jRLgil@4Wo`T5~oY7@C#Magl=4q^=8l{}!!o|+cZqJ2J+Am%eBpJ^ta4`(d zo0C+FrIpNO<=2{^6-kC~e)^srRKUT$MOMD{Rdj>y48lK6NVgCq)Z4 zXgP9S=*frRlPfu8zx|C@@j_9Sk9ZhC$`%L%^(d`DfDyk@LaIGt4k2$XMh9;&G$W0G z!AR=rj4QtxB!gMAvr}=Q5Z79E?0ko4LYg*G@f-&{3kods$%vt|g+0e!@cKAelZsqI zSv&RytAg_aoHGDmK-%N|KMZ2>dvN1RCO;&x8fpYeFrdvlZH?8v%v)WMdh+m%I!tUD z@u!rn&|@db0K$+1vc1GMNBY0Du$m`{+Hi`~ihz6R=Pu;gUp7fG)obqFBzh~mq@gHD z0PD#0VP)|B>O7Dl5@Zq-;O6nRP@L`-(UTwVmP=aRzqMSj^ew z-)05BdkpN2#D&3-6P2!RI1}@1H(eCOgd>#`q9Px?GbPAcr@O&>(i!$OdT06ZIgj*4 z2|_C+L0dHyvN0NAP1-UN55+9SW(37HDQa*;@`_QqG<9Ib3B@EoP!f#MKJ(Xm1?W_;8|%ti{|W(~YmmkmTr#jsJXCkVHnNAOwh1=(H``^yB8r##(!w`^?X zTf)2p+gCTm)k;T<_05G+%y_386m_IYB+!Fbiy$_U=G=Lk&$bO^hk?hF360c|;rEby zdR}sDuFvb~;!3{w99DI9o>|^Ab{dL)U`|6`GLCd2ziu2mAL{I>!uA*%@J`Tl#W-f>4wr8 zCA0a{6zC?f%SS$!-8`R_NZCuuiBOz58GPg~*-ILgyF8tk=LjmXLc-Ib{yKcgg_I_s zZ$qU^S1-j#vwTW>p?_>d4+{y@0s+U!Q))mPu+8x%gsOCXQn;h!rUOCO5aEpW$Z*%# zs1v?PfR2WO5L$Px1q|~0od~sOD`7rOrQiglsp33x!v=6fr6n=BBkrS7Rd`k{0{iZK zha1;31=9B>K#Mk*5PFx?J|twrh#(`;78755BY>A0q{>B1ibm$p`ZU>+fTKpbtpS*p z17@OaGNsWKtyX{s7Tdzg5q}s+%A89^^gF@f7n9OirA@Jmp>oX~Zj$Ul^TJYuNNYk@ zNIU^G9|z(oX-F7J+dwijyO>cm?zyCiDD-UetCt~Pt~!TP9rW9VQl(Wv+?W#{$aU-! z7q?uzkf(lWG4{2j&xues3kkw}CixiM_z_^bhK53u|E42{QbC$DqgpjSF+(t%RF4Ee zNdrnF4L-ir00^{j9B2g+x_w{s9GL^*(Ncium5^f(z{av1it!F`|47I!S-3j6m94th zwHpWVOxIw^dcH5BUX+P}o;@30^0H_tRIs6_x9AFh_x!QzP&n`aL{fU8R7)!CGNQyz;VUzVQ4L5S&jc`2@v_O@PwpPJu3EHW`OEN%$MGb#0G?tWOH zWad`fg#SVgb7Rl(SMs^9KTqa6YQ&As`O?R~r{n)-`n_Xp($ku$v-GutVu{ycZqnG8 JxPIsF{~tBAZ{+|0 literal 0 HcmV?d00001 diff --git a/runs/frontier-phase-factorial-v0/analyze_qwen30_prefill_fidelity.py b/runs/frontier-phase-factorial-v0/analyze_qwen30_prefill_fidelity.py new file mode 100644 index 0000000..a7260bf --- /dev/null +++ b/runs/frontier-phase-factorial-v0/analyze_qwen30_prefill_fidelity.py @@ -0,0 +1,462 @@ +#!/usr/bin/env python3 +"""Compare the frozen Frontier surface with conservative real capacities.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +import re +from collections import defaultdict +from datetime import datetime +from pathlib import Path +from typing import Any + + +RUN_PATTERN = re.compile(r"qwen30-prefill-real-tp(?P\d+)-mns(?P\d+)-") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--fleet-artifacts", type=Path, required=True) + parser.add_argument( + "--simulator-manifest", type=Path, action="append", required=True + ) + parser.add_argument("--output-root", type=Path, required=True) + return parser.parse_args() + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def load_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text()) + + +def sign(value: float) -> int: + return (value > 0) - (value < 0) + + +def kendall_tau_b(real: list[float], simulated: list[float]) -> dict[str, Any]: + if len(real) != len(simulated): + raise ValueError("ranking vectors have different lengths") + concordant = discordant = real_only_ties = simulated_only_ties = both_ties = 0 + for left in range(len(real)): + for right in range(left + 1, len(real)): + real_sign = sign(real[left] - real[right]) + sim_sign = sign(simulated[left] - simulated[right]) + if real_sign == 0 and sim_sign == 0: + both_ties += 1 + elif real_sign == 0: + real_only_ties += 1 + elif sim_sign == 0: + simulated_only_ties += 1 + elif real_sign == sim_sign: + concordant += 1 + else: + discordant += 1 + denominator = math.sqrt( + (concordant + discordant + real_only_ties) + * (concordant + discordant + simulated_only_ties) + ) + tau = (concordant - discordant) / denominator if denominator else None + return { + "kendall_tau_b": tau, + "concordant": concordant, + "discordant": discordant, + "real_only_ties": real_only_ties, + "simulator_only_ties": simulated_only_ties, + "both_ties": both_ties, + } + + +def result_files_by_anchor(root: Path) -> dict[tuple[str, str], Path]: + selected: dict[tuple[str, str], Path] = {} + digests: dict[tuple[str, str], str] = {} + for path in root.glob("artifacts/**/round*/results/r*.json"): + if path.name.startswith("warmup_"): + continue + round_name = path.parent.parent.name + key = (round_name, path.name) + digest = sha256(path) + if key in digests and digests[key] != digest: + raise RuntimeError( + f"conflicting duplicate real anchor {key} under {root}" + ) + digests[key] = digest + if key not in selected or len(path.parts) < len(selected[key].parts): + selected[key] = path + return selected + + +def find_real_runs(root: Path) -> dict[str, Path]: + candidates: dict[str, list[Path]] = defaultdict(list) + for path in root.iterdir(): + if not path.is_dir(): + continue + match = RUN_PATTERN.search(path.name) + if not match: + continue + name = f"tp{int(match.group('tp'))}_mns{int(match.group('mns'))}" + candidates[name].append(path) + selected: dict[str, Path] = {} + for name, paths in candidates.items(): + complete = [ + path + for path in paths + if (path / "remote_run" / "exit_code").is_file() + and (path / "remote_run" / "exit_code").read_text().strip() == "0" + ] + measured_counts = { + path: len(result_files_by_anchor(path)) + for path in complete + } + if not measured_counts: + raise RuntimeError(f"no successful run for {name}") + maximum = max(measured_counts.values()) + richest = [path for path, count in measured_counts.items() if count == maximum] + if len(richest) != 1: + raise RuntimeError(f"ambiguous richest successful run for {name}: {richest}") + selected[name] = richest[0] + if len(selected) != 12: + raise RuntimeError(f"expected 12 real configs, got {sorted(selected)}") + return selected + + +def campaign_resources(root: Path) -> dict[str, Any]: + runs = [] + gpu_hours = 0.0 + for path in sorted(root.iterdir()): + match = RUN_PATTERN.search(path.name) + exit_code = path / "remote_run" / "exit_code" + started_at = path / "remote_run" / "started_at" + finished_at = path / "remote_run" / "finished_at" + if ( + not path.is_dir() + or not match + or not exit_code.is_file() + or exit_code.read_text().strip() != "0" + or not started_at.is_file() + or not finished_at.is_file() + ): + continue + started = datetime.fromisoformat(started_at.read_text().strip()) + finished = datetime.fromisoformat(finished_at.read_text().strip()) + duration_seconds = (finished - started).total_seconds() + tp = int(match.group("tp")) + run_gpu_hours = duration_seconds * tp / 3600.0 + gpu_hours += run_gpu_hours + runs.append( + { + "run": path.name, + "tp": tp, + "duration_seconds": duration_seconds, + "gpu_hours": run_gpu_hours, + } + ) + return { + "successful_fleet_jobs": len(runs), + "gpu_hours": gpu_hours, + "runs": runs, + } + + +def parse_real_config(name: str, run_root: Path) -> dict[str, Any]: + result_files = sorted(result_files_by_anchor(run_root).values()) + if len(result_files) not in {10, 16}: + raise RuntimeError(f"expected 10 base or 16 refined anchors for {name}, got {len(result_files)}") + by_rate: dict[float, list[dict[str, Any]]] = defaultdict(list) + for path in result_files: + payload = load_json(path) + rate = float(payload["workload"]["offered_request_rate"]) + by_rate[rate].append( + { + "path": str(path.resolve()), + "sha256": sha256(path), + "summary": payload["summary"], + } + ) + tp = int(name.split("_")[0][2:]) + base_rates = [4.0, 8.0, 16.0, 32.0, 64.0] + refined_rates = sorted({*base_rates, *(tp * value for value in (5.0, 6.0, 7.0))}) + if tuple(sorted(by_rate)) not in {tuple(base_rates), tuple(refined_rates)}: + raise RuntimeError(f"unexpected rate grid for {name}: {sorted(by_rate)}") + anchors = [] + for rate, rounds in sorted(by_rate.items()): + if len(rounds) != 2: + raise RuntimeError(f"expected two rounds for {name}@{rate}, got {len(rounds)}") + round_feasible = [bool(row["summary"]["slo"]["feasible"]) for row in rounds] + anchors.append( + { + "rate": rate, + "rounds": rounds, + "conservative_feasible": all(round_feasible), + "round_feasible": round_feasible, + "round_ttft_p95_ms": [ + float(row["summary"]["ttft_p95_ms"]) for row in rounds + ], + } + ) + feasible = [row["rate"] for row in anchors if row["conservative_feasible"]] + capacity = max(feasible, default=0.0) + return { + "name": name, + "tp": tp, + "mns": int(name.split("_mns")[1]), + "anchors": anchors, + "capacity": capacity, + "capacity_per_gpu": capacity / tp, + "source_run": str(run_root.resolve()), + } + + +def parse_simulator(manifests: list[Path]) -> tuple[dict[str, Any], list[dict[str, str]]]: + configs: dict[str, Any] = {} + sources = [] + for path in manifests: + payload = load_json(path) + if payload["status"] not in {"complete", "partial_not_decision_bearing"}: + raise RuntimeError(f"simulator manifest has invalid status: {path}") + sources.append({"path": str(path.resolve()), "sha256": sha256(path)}) + result_by_name = { + result["config"]["name"]: result for result in payload["config_results"] + } + for capacity in payload["capacity"]: + name = capacity["config"]["name"] + result = result_by_name.get(name) + if result is None: + raise RuntimeError(f"missing simulator config result {name}") + entry = configs.setdefault( + name, + { + "name": name, + "tp": int(capacity["config"]["tp"]), + "mns": int(capacity["config"]["mns"]), + "anchor_by_rate": {}, + }, + ) + for load in result["loads"]: + rate = float(load["offered_request_rate"]) + if rate in entry["anchor_by_rate"]: + raise RuntimeError(f"duplicate simulator anchor {name}@{rate}") + entry["anchor_by_rate"][rate] = { + "rate": rate, + "feasible": bool(load["score"]["feasible"]), + "pass_rate": float(load["score"]["pass_rate"]), + "ttft_p95_ms": float(load["score"]["ttft_p95_ms"]), + } + if len(configs) != 12: + raise RuntimeError(f"expected 12 simulator configs, got {sorted(configs)}") + for entry in configs.values(): + entry["anchors"] = [ + entry["anchor_by_rate"][rate] for rate in sorted(entry["anchor_by_rate"]) + ] + del entry["anchor_by_rate"] + feasible = [row["rate"] for row in entry["anchors"] if row["feasible"]] + entry["capacity"] = max(feasible, default=0.0) + entry["capacity_per_gpu"] = entry["capacity"] / entry["tp"] + return configs, sources + + +def compare(real: dict[str, Any], simulated: dict[str, Any]) -> dict[str, Any]: + names = sorted(real, key=lambda name: (real[name]["tp"], real[name]["mns"])) + if set(names) != set(simulated): + raise RuntimeError("real and simulator config sets differ") + real_scores = [real[name]["capacity_per_gpu"] for name in names] + sim_scores = [simulated[name]["capacity_per_gpu"] for name in names] + real_best = max(real_scores) + sim_best = max(sim_scores) + real_top = [name for name in names if real[name]["capacity_per_gpu"] == real_best] + sim_top = [ + name for name in names if simulated[name]["capacity_per_gpu"] == sim_best + ] + worst_sim_choice = min(real[name]["capacity_per_gpu"] for name in sim_top) + best_sim_choice = max(real[name]["capacity_per_gpu"] for name in sim_top) + tau = kendall_tau_b(real_scores, sim_scores) + + pairwise = {"all": {"comparable": 0, "correct": 0}, "within_tp": {}} + for left in range(len(names)): + for right in range(left + 1, len(names)): + real_sign = sign(real_scores[left] - real_scores[right]) + sim_sign = sign(sim_scores[left] - sim_scores[right]) + if real_sign: + pairwise["all"]["comparable"] += 1 + pairwise["all"]["correct"] += int(real_sign == sim_sign) + if real[names[left]]["tp"] == real[names[right]]["tp"] and real_sign: + key = f"tp{real[names[left]]['tp']}" + bucket = pairwise["within_tp"].setdefault( + key, {"comparable": 0, "correct": 0} + ) + bucket["comparable"] += 1 + bucket["correct"] += int(real_sign == sim_sign) + for bucket in [pairwise["all"], *pairwise["within_tp"].values()]: + bucket["accuracy"] = ( + bucket["correct"] / bucket["comparable"] + if bucket["comparable"] + else None + ) + + confusion = {"real_pass_sim_pass": 0, "real_pass_sim_fail": 0, + "real_fail_sim_pass": 0, "real_fail_sim_fail": 0} + for name in names: + real_anchors = {row["rate"]: row for row in real[name]["anchors"]} + sim_anchors = {row["rate"]: row for row in simulated[name]["anchors"]} + if set(real_anchors) != set(sim_anchors): + raise RuntimeError(f"anchor grids differ for {name}") + for rate in real_anchors: + real_pass = real_anchors[rate]["conservative_feasible"] + sim_pass = sim_anchors[rate]["feasible"] + key = f"real_{'pass' if real_pass else 'fail'}_sim_{'pass' if sim_pass else 'fail'}" + confusion[key] += 1 + return { + "config_order": names, + "real_top_set": real_top, + "simulator_top_set": sim_top, + "top_set_exact_match": real_top == sim_top, + "top_set_overlap": sorted(set(real_top) & set(sim_top)), + "top1_regret_best": (real_best - best_sim_choice) / real_best, + "top1_regret_worst": (real_best - worst_sim_choice) / real_best, + "real_best_capacity_per_gpu": real_best, + "simulator_best_capacity_per_gpu": sim_best, + "kendall": tau, + "pairwise_non_tied": pairwise, + "anchor_confusion": confusion, + } + + +def write_csv(path: Path, rows: list[dict[str, Any]]) -> None: + with path.open("w", newline="") as handle: + writer = csv.DictWriter( + handle, fieldnames=list(rows[0]), lineterminator="\n" + ) + writer.writeheader() + writer.writerows(rows) + + +def plot(path: Path, rows: list[dict[str, Any]], metrics: dict[str, Any]) -> None: + import matplotlib.pyplot as plt + import numpy as np + + labels = [f"TP{row['tp']}\nMNS{row['mns']}" for row in rows] + x = np.arange(len(rows)) + width = 0.36 + figure, axes = plt.subplots( + 1, 2, figsize=(13.5, 5.0), gridspec_kw={"width_ratios": [3.3, 1.0]} + ) + axis = axes[0] + axis.bar(x - width / 2, [row["real"] for row in rows], width, label="Real vLLM") + axis.bar( + x + width / 2, + [row["simulator"] for row in rows], + width, + label="Frontier profile-only", + ) + axis.set_xticks(x, labels, fontsize=8) + axis.set_ylabel("Max tested SLO-feasible request rate / GPU") + axis.set_title("Qwen3-30B-A3B prefill-only: config ranking") + axis.grid(axis="y", alpha=0.25) + axis.legend(frameon=False, ncols=2) + for separator in (3.5, 7.5): + axis.axvline(separator, color="0.75", linewidth=0.8) + tau = metrics["kendall"]["kendall_tau_b"] + annotation = f"worst regret={metrics['top1_regret_worst'] * 100:.1f}%" + annotation += ( + f"\nKendall τ-b={tau:.3f}" + if tau is not None + else "\nKendall τ-b=undefined" + ) + axis.text( + 0.01, + 0.98, + annotation, + transform=axis.transAxes, + va="top", + fontsize=9, + bbox={"facecolor": "white", "edgecolor": "0.8", "alpha": 0.9}, + ) + + confusion = metrics["anchor_confusion"] + matrix = np.array( + [ + [confusion["real_pass_sim_pass"], confusion["real_pass_sim_fail"]], + [confusion["real_fail_sim_pass"], confusion["real_fail_sim_fail"]], + ] + ) + image = axes[1].imshow(matrix, cmap="Blues", vmin=0) + axes[1].set_xticks([0, 1], ["Sim pass", "Sim fail"]) + axes[1].set_yticks([0, 1], ["Real pass", "Real fail"]) + axes[1].set_title(f"{int(matrix.sum())} anchor decisions") + for row in range(2): + for column in range(2): + axes[1].text(column, row, int(matrix[row, column]), ha="center", va="center") + figure.colorbar(image, ax=axes[1], fraction=0.047, pad=0.04) + figure.suptitle("ISL=2048, OSL=1, TTFT≤1256 ms, 95% pass gate; two real rounds") + figure.tight_layout() + figure.savefig(path, dpi=180, bbox_inches="tight") + plt.close(figure) + + +def main() -> None: + args = parse_args() + real_runs = find_real_runs(args.fleet_artifacts.resolve()) + real = {name: parse_real_config(name, path) for name, path in real_runs.items()} + simulated, simulator_sources = parse_simulator( + [path.resolve() for path in args.simulator_manifest] + ) + metrics = compare(real, simulated) + rows = [ + { + "config": name, + "tp": real[name]["tp"], + "mns": real[name]["mns"], + "real": real[name]["capacity_per_gpu"], + "simulator": simulated[name]["capacity_per_gpu"], + } + for name in metrics["config_order"] + ] + resources = campaign_resources(args.fleet_artifacts.resolve()) + resources["fresh_server_anchors"] = sum( + len(config["anchors"]) * 2 for config in real.values() + ) + resources["measured_requests"] = resources["fresh_server_anchors"] * 64 + resources["warmup_requests"] = sum( + min(32, max(4, math.ceil(anchor["rate"] * 2.0))) * 2 + for config in real.values() + for anchor in config["anchors"] + ) + args.output_root.mkdir(parents=True, exist_ok=True) + payload = { + "schema": "qwen30-prefill-fidelity-comparison-v1", + "objective": "maximum_tested_slo_feasible_offered_request_rate_per_gpu", + "contract": { + "model": "Qwen3-30B-A3B", + "input_tokens": 2048, + "output_tokens": 1, + "prefix_caching": False, + "ttft_slo_ms": 1256.0, + "target_pass_rate": 0.95, + "real_anchor_merge": "both_fresh_server_rounds_must_pass", + }, + "metrics": metrics, + "real_campaign_resources": resources, + "real": real, + "simulator": simulated, + "simulator_sources": simulator_sources, + } + (args.output_root / "comparison.json").write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n" + ) + write_csv(args.output_root / "capacity.csv", rows) + plot(args.output_root / "qwen30-prefill-ranking.png", rows, metrics) + print(json.dumps(metrics, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-phase-factorial-v0/experiment-card.md b/runs/frontier-phase-factorial-v0/experiment-card.md index f291ab7..b2623c6 100644 --- a/runs/frontier-phase-factorial-v0/experiment-card.md +++ b/runs/frontier-phase-factorial-v0/experiment-card.md @@ -1,6 +1,6 @@ # 实验 EXP-SIMFID-PHASE-FACTORIAL：prefill-only 是否是 simulator ranking 的容易区间？ -> **状态：** 已批准，运行中（用户于 2026-07-17 明确要求先完成 235B mixed 与 30B prefill-only） +> **状态：** 已完成（2026-07-17） ## Claim 与决策 @@ -17,6 +17,7 @@ - **30B system context：** community vLLM 0.20.0+cu129，BF16 weights/activation/KV，TP∈{1,2,4}，MNS∈{8,16,32,64}，MBT=8192，chunked prefill on，prefix off；real 保留 runtime 默认 CUDA graph，Frontier profile-only 不做 E2E calibration。 - **30B workload：** fixed ISL=2048、OSL=1，64 个不同 token-chain prompts，uniform open-loop QPS；fresh server per `(config, rate, round)`，target-rate warmup 与 measured requests 分离。 - **30B SLO：** TTFT≤1256 ms，至少 61/64 requests 通过；primary score 为最大共同 tested feasible req/s / 实际 TP GPUs。 +- **Boundary refinement rule：** base grid `{4,8,16,32,64}` 先定位每个 TP 的 pass→fail 区间；若除以 TP 后的离散容量产生无法区分的 top tie，则在查看最终 ranking 前追加共同 per-GPU lattice `5/6/7 req/s/GPU`，即 TP1 测 `{5,6,7}`、TP2 测 `{10,12,14}`、TP4 测 `{20,24,28}`。refinement 不替换或删除 base anchors。 - **235B baseline：** 已冻结 `ISL=2048, OSL=128`、8 configs、68 个 fresh-server anchors；primary sensitivity TTFT≤1256 ms、TPOT≤150 ms。 - **Baselines：** real community vLLM；Frontier same-stack profile-only；historical Qwen30 mixed profile-only；historical frozen per-TP calibration 只作为 upper bound，不参与本 case 拟合。 - **Metrics：** top set、worst tie-break regret、Kendall τ-b、exact/non-tied pair direction、anchor confusion、absolute capacity、TTFT p50/p95、real trial variance与GPU-hour。 @@ -35,7 +36,7 @@ | Selective benchmarking | PASS for initial screen | 同时报已有 235B mixed success和内部 pairwise failure；后续 expansion 由预注册 verdict 触发 | | Simplified workload | NEEDS EVIDENCE | fixed-shape 只用于 phase isolation，不外推 trace-faithful mixed | | Calibration=evaluation | PASS | 新 case 不用 serving E2E 数据拟合 scale | -| Missing significance | NEEDS EVIDENCE until run | boundary anchors做独立 fresh-server repeat，保留 disagreement | +| Missing significance | PASS for ranking screen | 96 个 config-rate cells 均做两个独立 fresh-server rounds；两轮都 pass 才算 feasible | | Relative-only result | PASS by design | 同时报 req/s/GPU、TTFT distribution、rank/regret | ## 复现信息 @@ -43,12 +44,13 @@ - **Code：** AITuner branch `codex/fidelity-prefix-pilot-20260714`；Frontier upstream `d9cfeb6d8791fbf2f295dd9744c56a666171776e` + frozen known patches。 - **Environment：** 只使用 dash0 8×H20；Qwen30 venv `/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1`；model `/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B`。 - **产物路径：** local/remote `runs/frontier-phase-factorial-v0/`；raw GPU artifacts 由 fleet harvest，condensed JSON/CSV 进入结果目录。 -- **已知 deviation：** 235B 为 FP8/vLLM0.10.2/FlashInfer eager，30B 为 BF16/vLLM0.20/FA3/default CUDA graph；因此跨模型只检验 hypothesis consistency，causal phase claim 最终仍需 same-model phase pair。 +- **已知 deviation：** 235B 为 FP8/vLLM0.10.2/FlashInfer eager，30B 为 BF16/vLLM0.20/FA3/default CUDA graph；因此跨模型只检验 hypothesis consistency，causal phase claim 最终仍需 same-model phase pair。初始 continuous fleet monitor 未在 fresh-server 间隙保留 controller-level GPU reservation，产生重叠 launch；该 attempt 整体移动到 `invalid-overlap-*`，不进入统计。后续一次试图在“其余 4 卡”上并行 refinement 时，探针再次命中 TP4 fresh-server 空窗，把 4 个 TP1 jobs 放到了同一 GPU set。这四个 jobs 尚未产生 measured result；但当时 TP4/MNS64 已产生的单个 anchor 也按污染处理。两者都整体移到 `invalid-overlap-20260716T1750Z`，TP4/MNS64 从空目录重跑。此后的 barrier waves 不在运行中追加 job；但 Wave 3 的 harvest monitor 在未及时返回 launch 状态时已发射 MNS16/32，紧接的 monitor retry 又在它们的启动空窗发射 MNS64。三者都未产生 measured result，整体移到 `invalid-overlap-20260716T1836Z`。最终 TP4 waves 使用只含本波 jobs 的独立 queue state，不再依赖 pending-job 探针调度。 ## 结果 -- **观察事实：** 235B fixed-shape mixed 已完成：real/sim TP4 top set exact match，worst regret=0、τ-b=0.8944；但 20 个 real non-tie pairs 只保持 16 个，10/34 anchors false-infeasible，TP8 MNS×MBT interaction 被漏掉。30B prefill-only 待运行。 -- **异常：** 无。 -- **Interpretation 与剩余 alternatives：** 强版本“mixed 必然失败”已被 235B top-set result 削弱；仍可能存在 phase-dependent error magnitude，由 topology margin 掩盖。 -- **Claim update：** unchanged，等待 30B prefill-only。 -- **下一步：** freeze 30B simulator surface → guided real anchors → joint verdict；只有判定需要时扩展 decode-heavy 235B 或 trace-shaped 30B prefill。 +- **观察事实：** 235B fixed-shape mixed 中 real/sim 的四个 TP4 top configs 完全一致，worst regret=0、τ-b=0.8944；但 20 个 real non-tie pairs 只保持 16 个，10/34 anchors false-infeasible，TP8 MNS×MBT interaction 被漏掉。30B prefill-only 中真机 capacity/GPU 为 TP1=7、TP2=7、TP4=8，Frontier 为 TP1=8、TP2=8、TP4=6；real top set 是四个 TP4 configs，simulator top set 是全部八个 TP1/TP2 configs，无交集。worst regret=12.5%、τ-b=-1.0，32 个 real non-tie pairs 中 0 个同序。96 个 anchor labels 中有 8 个 false-feasible 和 8 个 false-infeasible。 +- **实验成本：** 接受 24 个 fleet jobs、192 个 fresh-server anchors、12,288 个 measured requests 和 4,512 个 warmups，消耗 12.0744 H20-GPU-hours。 +- **异常与排除：** fleet controller 在 fresh-server 空窗期没有保留 GPU reservation，产生了三批重叠 launch。污染 attempt 不进入 accepted artifact root，未产生 measured result 的重叠 jobs 也不被计数；同一波中已产生的 TP4/MNS64 单 anchor 同样按污染丢弃，从空目录重跑。最终 TP4 refinement 使用彼此独立的 queue states。analyzer 按 `(round, filename)` 去重相同 harvest copy，如果 hash 冲突则直接报错。 +- **Interpretation 与剩余 alternatives：** `H-phase` 的强形式（prefill-only 是 fidelity 充分条件）被否证；`H-margin` 与数据更一致。235B 的真机 TP4/TP8 最优 margin 为 2×，足以掩盖内部 residual；30B 的 8-vs-7 margin 被 TP-dependent saturation residual 穿过。但这是跨 stack comparison，不能把差异因果归结为 model size。 +- **Claim update：** “prefill-only 容易，decode/mixed 困难”的强假设被否证。新的可证伪命题是：config-ranking fidelity 取决于 scheduler-state-conditioned action residual 是否大于 real decision margin。 +- **下一步：** 不继续扩展跨模型 phase cases。在 Qwen30 prefill-only 上依次做 measured-collective injection、batch-composition-conditioned pure-prefill attention/step profile、TP1@8/TP2@16/TP4@32 的 scheduler batch/queue/per-step trace 对齐，最后再测 routing/graph。 diff --git a/runs/frontier-phase-factorial-v0/fleet-base-rerun.toml b/runs/frontier-phase-factorial-v0/fleet-base-rerun.toml new file mode 100644 index 0000000..5553f93 --- /dev/null +++ b/runs/frontier-phase-factorial-v0/fleet-base-rerun.toml @@ -0,0 +1,24 @@ +version = 1 + +[paths] +state_dir = "runs/frontier-phase-factorial-v0/fleet-state-base-rerun" +artifacts_dir = "runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive" + +[ssh] +connect_timeout_sec = 10 + +[scheduler] +gpu_free_memory_mb = 95000 +gpu_free_utilization_pct = 5 +prefer_pack = true + +[sync] +mode = "scp" +local_path = "runs/frontier-phase-factorial-v0/remote-sync-marker" + +[[hosts]] +name = "dash0" +ssh_alias = "dash0" +enabled = true +sync_remote_path = "/home/admin/cpfs/wjh/aituner/phase-factorial-sync-marker" +fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0" diff --git a/runs/frontier-phase-factorial-v0/fleet-exclusive.toml b/runs/frontier-phase-factorial-v0/fleet-exclusive.toml new file mode 100644 index 0000000..7de87b2 --- /dev/null +++ b/runs/frontier-phase-factorial-v0/fleet-exclusive.toml @@ -0,0 +1,24 @@ +version = 1 + +[paths] +state_dir = "runs/frontier-phase-factorial-v0/fleet-state-exclusive" +artifacts_dir = "runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive" + +[ssh] +connect_timeout_sec = 10 + +[scheduler] +gpu_free_memory_mb = 95000 +gpu_free_utilization_pct = 5 +prefer_pack = true + +[sync] +mode = "scp" +local_path = "runs/frontier-phase-factorial-v0/remote-sync-marker" + +[[hosts]] +name = "dash0" +ssh_alias = "dash0" +enabled = true +sync_remote_path = "/home/admin/cpfs/wjh/aituner/phase-factorial-sync-marker" +fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0" diff --git a/runs/frontier-phase-factorial-v0/fleet-refine-tp4-wave3.toml b/runs/frontier-phase-factorial-v0/fleet-refine-tp4-wave3.toml new file mode 100644 index 0000000..a7e6466 --- /dev/null +++ b/runs/frontier-phase-factorial-v0/fleet-refine-tp4-wave3.toml @@ -0,0 +1,24 @@ +version = 1 + +[paths] +state_dir = "runs/frontier-phase-factorial-v0/fleet-state-refine-tp4-wave3" +artifacts_dir = "runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive" + +[ssh] +connect_timeout_sec = 10 + +[scheduler] +gpu_free_memory_mb = 95000 +gpu_free_utilization_pct = 5 +prefer_pack = true + +[sync] +mode = "scp" +local_path = "runs/frontier-phase-factorial-v0/remote-sync-marker" + +[[hosts]] +name = "dash0" +ssh_alias = "dash0" +enabled = true +sync_remote_path = "/home/admin/cpfs/wjh/aituner/phase-factorial-sync-marker" +fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0" diff --git a/runs/frontier-phase-factorial-v0/fleet-refine-tp4-wave4.toml b/runs/frontier-phase-factorial-v0/fleet-refine-tp4-wave4.toml new file mode 100644 index 0000000..3cba17e --- /dev/null +++ b/runs/frontier-phase-factorial-v0/fleet-refine-tp4-wave4.toml @@ -0,0 +1,24 @@ +version = 1 + +[paths] +state_dir = "runs/frontier-phase-factorial-v0/fleet-state-refine-tp4-wave4" +artifacts_dir = "runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive" + +[ssh] +connect_timeout_sec = 10 + +[scheduler] +gpu_free_memory_mb = 95000 +gpu_free_utilization_pct = 5 +prefer_pack = true + +[sync] +mode = "scp" +local_path = "runs/frontier-phase-factorial-v0/remote-sync-marker" + +[[hosts]] +name = "dash0" +ssh_alias = "dash0" +enabled = true +sync_remote_path = "/home/admin/cpfs/wjh/aituner/phase-factorial-sync-marker" +fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0" diff --git a/runs/frontier-phase-factorial-v0/fleet-refine.toml b/runs/frontier-phase-factorial-v0/fleet-refine.toml new file mode 100644 index 0000000..786d090 --- /dev/null +++ b/runs/frontier-phase-factorial-v0/fleet-refine.toml @@ -0,0 +1,24 @@ +version = 1 + +[paths] +state_dir = "runs/frontier-phase-factorial-v0/fleet-state-refine" +artifacts_dir = "runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive" + +[ssh] +connect_timeout_sec = 10 + +[scheduler] +gpu_free_memory_mb = 95000 +gpu_free_utilization_pct = 5 +prefer_pack = true + +[sync] +mode = "scp" +local_path = "runs/frontier-phase-factorial-v0/remote-sync-marker" + +[[hosts]] +name = "dash0" +ssh_alias = "dash0" +enabled = true +sync_remote_path = "/home/admin/cpfs/wjh/aituner/phase-factorial-sync-marker" +fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0" diff --git a/runs/frontier-phase-factorial-v0/jobs_base_rerun.toml b/runs/frontier-phase-factorial-v0/jobs_base_rerun.toml new file mode 100644 index 0000000..2eb2f2d --- /dev/null +++ b/runs/frontier-phase-factorial-v0/jobs_base_rerun.toml @@ -0,0 +1,21 @@ +version = 1 + +[[jobs]] +name = "qwen30-prefill-real-tp4-mns64-20260717-v2b-exclusive" +gpus = 4 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp4-mns64-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "4" +MNS = "64" +RATES = "4 8 16 32 64" +SERVER_PORT = "8731" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp4-mns64-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" diff --git a/runs/frontier-phase-factorial-v0/jobs_full.toml b/runs/frontier-phase-factorial-v0/jobs_full.toml index c67f255..05d7621 100644 --- a/runs/frontier-phase-factorial-v0/jobs_full.toml +++ b/runs/frontier-phase-factorial-v0/jobs_full.toml @@ -1,7 +1,7 @@ version = 1 [[jobs]] -name = "qwen30-prefill-real-tp1-mns8-20260717-v1" +name = "qwen30-prefill-real-tp1-mns8-20260717-v2-exclusive" gpus = 1 gpu_model = "H20" hosts = ["dash0"] @@ -21,7 +21,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" [[jobs]] -name = "qwen30-prefill-real-tp1-mns16-20260717-v1" +name = "qwen30-prefill-real-tp1-mns16-20260717-v2-exclusive" gpus = 1 gpu_model = "H20" hosts = ["dash0"] @@ -41,7 +41,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" [[jobs]] -name = "qwen30-prefill-real-tp1-mns32-20260717-v1" +name = "qwen30-prefill-real-tp1-mns32-20260717-v2-exclusive" gpus = 1 gpu_model = "H20" hosts = ["dash0"] @@ -61,7 +61,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" [[jobs]] -name = "qwen30-prefill-real-tp1-mns64-20260717-v1" +name = "qwen30-prefill-real-tp1-mns64-20260717-v2-exclusive" gpus = 1 gpu_model = "H20" hosts = ["dash0"] @@ -81,7 +81,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" [[jobs]] -name = "qwen30-prefill-real-tp2-mns8-20260717-v1" +name = "qwen30-prefill-real-tp2-mns8-20260717-v2-exclusive" gpus = 2 gpu_model = "H20" hosts = ["dash0"] @@ -101,7 +101,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" [[jobs]] -name = "qwen30-prefill-real-tp2-mns16-20260717-v1" +name = "qwen30-prefill-real-tp2-mns16-20260717-v2-exclusive" gpus = 2 gpu_model = "H20" hosts = ["dash0"] @@ -121,7 +121,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" [[jobs]] -name = "qwen30-prefill-real-tp2-mns32-20260717-v1" +name = "qwen30-prefill-real-tp2-mns32-20260717-v2-exclusive" gpus = 2 gpu_model = "H20" hosts = ["dash0"] @@ -141,7 +141,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" [[jobs]] -name = "qwen30-prefill-real-tp2-mns64-20260717-v1" +name = "qwen30-prefill-real-tp2-mns64-20260717-v2-exclusive" gpus = 2 gpu_model = "H20" hosts = ["dash0"] @@ -161,7 +161,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" [[jobs]] -name = "qwen30-prefill-real-tp4-mns8-20260717-v1" +name = "qwen30-prefill-real-tp4-mns8-20260717-v2-exclusive" gpus = 4 gpu_model = "H20" hosts = ["dash0"] @@ -181,7 +181,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" [[jobs]] -name = "qwen30-prefill-real-tp4-mns16-20260717-v1" +name = "qwen30-prefill-real-tp4-mns16-20260717-v2-exclusive" gpus = 4 gpu_model = "H20" hosts = ["dash0"] @@ -201,7 +201,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" [[jobs]] -name = "qwen30-prefill-real-tp4-mns32-20260717-v1" +name = "qwen30-prefill-real-tp4-mns32-20260717-v2-exclusive" gpus = 4 gpu_model = "H20" hosts = ["dash0"] @@ -221,7 +221,7 @@ VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" [[jobs]] -name = "qwen30-prefill-real-tp4-mns64-20260717-v1" +name = "qwen30-prefill-real-tp4-mns64-20260717-v2-exclusive" gpus = 4 gpu_model = "H20" hosts = ["dash0"] diff --git a/runs/frontier-phase-factorial-v0/jobs_refine.toml b/runs/frontier-phase-factorial-v0/jobs_refine.toml new file mode 100644 index 0000000..55fd01d --- /dev/null +++ b/runs/frontier-phase-factorial-v0/jobs_refine.toml @@ -0,0 +1,240 @@ +version = 1 + +[[jobs]] +name = "qwen30-prefill-real-tp1-mns8-20260717-v3-refine" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp1-mns8-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "1" +MNS = "8" +RATES = "5 6 7" +SERVER_PORT = "8720" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp1-mns8-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" +[[jobs]] +name = "qwen30-prefill-real-tp1-mns16-20260717-v3-refine" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp1-mns16-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "1" +MNS = "16" +RATES = "5 6 7" +SERVER_PORT = "8721" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp1-mns16-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp1-mns32-20260717-v3-refine" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp1-mns32-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "1" +MNS = "32" +RATES = "5 6 7" +SERVER_PORT = "8722" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp1-mns32-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp1-mns64-20260717-v3-refine" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp1-mns64-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "1" +MNS = "64" +RATES = "5 6 7" +SERVER_PORT = "8723" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp1-mns64-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp2-mns8-20260717-v3-refine" +gpus = 2 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp2-mns8-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "2" +MNS = "8" +RATES = "10 12 14" +SERVER_PORT = "8724" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp2-mns8-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp2-mns16-20260717-v3-refine" +gpus = 2 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp2-mns16-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "2" +MNS = "16" +RATES = "10 12 14" +SERVER_PORT = "8725" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp2-mns16-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp2-mns32-20260717-v3-refine" +gpus = 2 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp2-mns32-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "2" +MNS = "32" +RATES = "10 12 14" +SERVER_PORT = "8726" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp2-mns32-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp2-mns64-20260717-v3-refine" +gpus = 2 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp2-mns64-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "2" +MNS = "64" +RATES = "10 12 14" +SERVER_PORT = "8727" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp2-mns64-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp4-mns8-20260717-v3-refine" +gpus = 4 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp4-mns8-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "4" +MNS = "8" +RATES = "20 24 28" +SERVER_PORT = "8728" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp4-mns8-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp4-mns16-20260717-v3-refine" +gpus = 4 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp4-mns16-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "4" +MNS = "16" +RATES = "20 24 28" +SERVER_PORT = "8729" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp4-mns16-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp4-mns32-20260717-v3-refine" +gpus = 4 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp4-mns32-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "4" +MNS = "32" +RATES = "20 24 28" +SERVER_PORT = "8730" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp4-mns32-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp4-mns64-20260717-v3-refine" +gpus = 4 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp4-mns64-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "4" +MNS = "64" +RATES = "20 24 28" +SERVER_PORT = "8731" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp4-mns64-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" diff --git a/runs/frontier-phase-factorial-v0/jobs_refine_tp4_wave3.toml b/runs/frontier-phase-factorial-v0/jobs_refine_tp4_wave3.toml new file mode 100644 index 0000000..b830690 --- /dev/null +++ b/runs/frontier-phase-factorial-v0/jobs_refine_tp4_wave3.toml @@ -0,0 +1,41 @@ +version = 1 + +[[jobs]] +name = "qwen30-prefill-real-tp4-mns16-20260717-v4-refine" +gpus = 4 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp4-mns16-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "4" +MNS = "16" +RATES = "20 24 28" +SERVER_PORT = "8729" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp4-mns16-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" + +[[jobs]] +name = "qwen30-prefill-real-tp4-mns32-20260717-v4-refine" +gpus = 4 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp4-mns32-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "4" +MNS = "32" +RATES = "20 24 28" +SERVER_PORT = "8730" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp4-mns32-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" diff --git a/runs/frontier-phase-factorial-v0/jobs_refine_tp4_wave4.toml b/runs/frontier-phase-factorial-v0/jobs_refine_tp4_wave4.toml new file mode 100644 index 0000000..b69872d --- /dev/null +++ b/runs/frontier-phase-factorial-v0/jobs_refine_tp4_wave4.toml @@ -0,0 +1,21 @@ +version = 1 + +[[jobs]] +name = "qwen30-prefill-real-tp4-mns64-20260717-v4-refine" +gpus = 4 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-phase-factorial-v0 && timeout --signal=TERM --kill-after=30s 7200 bash run_qwen30_prefill_real_config.sh" +artifacts = ["artifacts/real-tp4-mns64-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +TP = "4" +MNS = "64" +RATES = "20 24 28" +SERVER_PORT = "8731" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-phase-factorial-v0/artifacts/real-tp4-mns64-v1" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" diff --git a/runs/frontier-phase-factorial-v0/results/final/capacity.csv b/runs/frontier-phase-factorial-v0/results/final/capacity.csv new file mode 100644 index 0000000..de7a968 --- /dev/null +++ b/runs/frontier-phase-factorial-v0/results/final/capacity.csv @@ -0,0 +1,13 @@ +config,tp,mns,real,simulator +tp1_mns8,1,8,7.0,8.0 +tp1_mns16,1,16,7.0,8.0 +tp1_mns32,1,32,7.0,8.0 +tp1_mns64,1,64,7.0,8.0 +tp2_mns8,2,8,7.0,8.0 +tp2_mns16,2,16,7.0,8.0 +tp2_mns32,2,32,7.0,8.0 +tp2_mns64,2,64,7.0,8.0 +tp4_mns8,4,8,8.0,6.0 +tp4_mns16,4,16,8.0,6.0 +tp4_mns32,4,32,8.0,6.0 +tp4_mns64,4,64,8.0,6.0 diff --git a/runs/frontier-phase-factorial-v0/results/final/comparison.json b/runs/frontier-phase-factorial-v0/results/final/comparison.json new file mode 100644 index 0000000..db46764 --- /dev/null +++ b/runs/frontier-phase-factorial-v0/results/final/comparison.json @@ -0,0 +1,5860 @@ +{ + "contract": { + "input_tokens": 2048, + "model": "Qwen3-30B-A3B", + "output_tokens": 1, + "prefix_caching": false, + "real_anchor_merge": "both_fresh_server_rounds_must_pass", + "target_pass_rate": 0.95, + "ttft_slo_ms": 1256.0 + }, + "metrics": { + "anchor_confusion": { + "real_fail_sim_fail": 24, + "real_fail_sim_pass": 8, + "real_pass_sim_fail": 8, + "real_pass_sim_pass": 56 + }, + "config_order": [ + "tp1_mns8", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp2_mns8", + "tp2_mns16", + "tp2_mns32", + "tp2_mns64", + "tp4_mns8", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "kendall": { + "both_ties": 34, + "concordant": 0, + "discordant": 32, + "kendall_tau_b": -1.0, + "real_only_ties": 0, + "simulator_only_ties": 0 + }, + "pairwise_non_tied": { + "all": { + "accuracy": 0.0, + "comparable": 32, + "correct": 0 + }, + "within_tp": {} + }, + "real_best_capacity_per_gpu": 8.0, + "real_top_set": [ + "tp4_mns8", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "simulator_best_capacity_per_gpu": 8.0, + "simulator_top_set": [ + "tp1_mns8", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp2_mns8", + "tp2_mns16", + "tp2_mns32", + "tp2_mns64" + ], + "top1_regret_best": 0.125, + "top1_regret_worst": 0.125, + "top_set_exact_match": false, + "top_set_overlap": [] + }, + "objective": "maximum_tested_slo_feasible_offered_request_rate_per_gpu", + "real": { + "tp1_mns16": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 153.15452707000077, + 152.64549700077623 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r4p00.json", + "sha256": "d82e386ec3effdf57c148171311928298405da0e4f896584bf05db2485bb6bd4", + "summary": { + "admission_lag_max_ms": 0.11588609777390957, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 158.6234939750284, + "ttft_p50_ms": 151.05855697765946, + "ttft_p95_ms": 153.15452707000077 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r4p00.json", + "sha256": "1565be89131e3c8a9c352c6f29694ef2793734b0623a2db1cbea7e0b0e8a2bf6", + "summary": { + "admission_lag_max_ms": 0.11451996397227049, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 153.18561100866646, + "ttft_p50_ms": 151.18564700242132, + "ttft_p95_ms": 152.64549700077623 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 5.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 152.8308589477092, + 152.43642206769437 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r5p00.json", + "sha256": "e4447d94a326c541412b40157cfe283a6350680e3285b5d10fed6b8de5c0cd93", + "summary": { + "admission_lag_max_ms": 0.17109804321080446, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 154.35138496104628, + "ttft_p50_ms": 150.91424097772688, + "ttft_p95_ms": 152.8308589477092 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r5p00.json", + "sha256": "572c17e309c6c9f23c490979e7acb885475f142bc530ecfee88916b048259ea2", + "summary": { + "admission_lag_max_ms": 0.1321269664913416, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 157.11930498946458, + "ttft_p50_ms": 150.50161199178547, + "ttft_p95_ms": 152.43642206769437 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 6.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 153.56597595382482, + 152.16859406791627 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r6p00.json", + "sha256": "68c8b39154d46f1ceda17f47e8cbd84a8c8a9f088c5e82d8f0ac25a7180dc14c", + "summary": { + "admission_lag_max_ms": 0.11641765013337135, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 156.50876995641738, + "ttft_p50_ms": 150.41159105021507, + "ttft_p95_ms": 153.56597595382482 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r6p00.json", + "sha256": "a970a2cdf7034026a56af3ea4da59856fd588da35d66ef4f9e15481a892edcc9", + "summary": { + "admission_lag_max_ms": 0.11551892384886742, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 168.65782206878066, + "ttft_p50_ms": 150.03317291848361, + "ttft_p95_ms": 152.16859406791627 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 7.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 300.90478900820017, + 293.0445579113439 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r7p00.json", + "sha256": "1aa74c18adbff53b36ae8a6f07b5b0865f6b68fb3e7862a70b36f67c18f0bba0", + "summary": { + "admission_lag_max_ms": 0.10505830869078636, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 304.15459000505507, + "ttft_p50_ms": 263.44375393819064, + "ttft_p95_ms": 300.90478900820017 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r7p00.json", + "sha256": "e14bc9dba35d453862839eeed77262a1ed9b8394404ca90c60ec8cb26a1aa73b", + "summary": { + "admission_lag_max_ms": 0.10963028762489557, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 295.1545650139451, + "ttft_p50_ms": 243.97043196950108, + "ttft_p95_ms": 293.0445579113439 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 8.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1323.4397460473701, + 1315.5151740647852 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r8p00.json", + "sha256": "9aeee7453ffe5098d203e02b84f5009ed347c478b736e89297bd3695dd81d477", + "summary": { + "admission_lag_max_ms": 0.1251589274033904, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.90625, + "passed": 58, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1477.699184906669, + "ttft_p50_ms": 776.2576789828017, + "ttft_p95_ms": 1323.4397460473701 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r8p00.json", + "sha256": "7630f4901cf4ec4ea955738b119a1c32aba53f9ca1edb8819556fc60ae1fda09", + "summary": { + "admission_lag_max_ms": 0.11316593736410141, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.921875, + "passed": 59, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1404.566578916274, + "ttft_p50_ms": 761.7035870207474, + "ttft_p95_ms": 1315.5151740647852 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4892.253074911423, + 4893.758205929771 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r16p00.json", + "sha256": "d3a2a85cf06a4f080edc9a3809f05493ac55b0aa1d890806ffbc551c8eca14f8", + "summary": { + "admission_lag_max_ms": 0.39830803871154785, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4971.215195022523, + "ttft_p50_ms": 2742.7029330283403, + "ttft_p95_ms": 4892.253074911423 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r16p00.json", + "sha256": "67110284c38a9a60228f00b1d2b0de9477d6dec660f1c91eb4f39e1b79e272a5", + "summary": { + "admission_lag_max_ms": 0.11306500528007746, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4972.818040987477, + "ttft_p50_ms": 2742.512951954268, + "ttft_p95_ms": 4893.758205929771 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 6650.756464921869, + 6751.845097984187 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r32p00.json", + "sha256": "affc62b27382400497bb82e9da31ed9891b60155318fc4f250dca0704ccc5e02", + "summary": { + "admission_lag_max_ms": 0.11010200250893831, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6920.621327939443, + "ttft_p50_ms": 3650.839229929261, + "ttft_p95_ms": 6650.756464921869 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r32p00.json", + "sha256": "df0c8d703b023692814be41f47da4db4ce1b7eb0c3423a742c10884b6447583f", + "summary": { + "admission_lag_max_ms": 0.1288279891014099, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.15625, + "passed": 10, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6897.979240980931, + "ttft_p50_ms": 3730.34580796957, + "ttft_p95_ms": 6751.845097984187 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 7732.2855240199715, + 7744.056030060165 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r64p00.json", + "sha256": "fbfad78129c843c85905c3f94f278f468a0a019c0cdf0b62140d0fea8f0aaffc", + "summary": { + "admission_lag_max_ms": 0.11090096086263657, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7836.245508980937, + "ttft_p50_ms": 4328.587644966319, + "ttft_p95_ms": 7732.2855240199715 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r64p00.json", + "sha256": "0f005c642327708e1ceb058f71f806b4d46ea666f45695605552e8f234521608", + "summary": { + "admission_lag_max_ms": 0.10862003546208143, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7847.871358972043, + "ttft_p50_ms": 4332.9262170009315, + "ttft_p95_ms": 7744.056030060165 + } + } + ] + } + ], + "capacity": 7.0, + "capacity_per_gpu": 7.0, + "mns": 16, + "name": "tp1_mns16", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z", + "tp": 1 + }, + "tp1_mns32": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 153.44141807872802, + 152.2580359596759 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r4p00.json", + "sha256": "211fa276e4a99a061126882085b9850549b453520ac3414dd2d332c89e52b431", + "summary": { + "admission_lag_max_ms": 0.09491504170000553, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 169.56262104213238, + "ttft_p50_ms": 150.8563900133595, + "ttft_p95_ms": 153.44141807872802 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r4p00.json", + "sha256": "17428902a5b20e9084f20792963f0620a1af06cdc1afb76ee99fd15dd905b89e", + "summary": { + "admission_lag_max_ms": 0.12033001985400915, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 153.99700601119548, + "ttft_p50_ms": 150.5822929320857, + "ttft_p95_ms": 152.2580359596759 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 5.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 152.05012692604214, + 152.00478909537196 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r5p00.json", + "sha256": "b0eb19ff7ab8a26e34e91f213a983e58689c63071285e27b3719bda0a765ff32", + "summary": { + "admission_lag_max_ms": 0.11265103239566088, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 153.48999004345387, + "ttft_p50_ms": 150.21017810795456, + "ttft_p95_ms": 152.05012692604214 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r5p00.json", + "sha256": "3b857dff42dd16ebee95356ceacd6b58ff7bd0a9b91f3d6208d489b84985697f", + "summary": { + "admission_lag_max_ms": 0.11149293277412653, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 153.33813393954188, + "ttft_p50_ms": 150.45341395307332, + "ttft_p95_ms": 152.00478909537196 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 6.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 152.74227899499238, + 152.82704099081457 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r6p00.json", + "sha256": "7615fa664a99f02f9f6d4f1fb0f2d6cba11842772c086634b939bc08cd0f2f4d", + "summary": { + "admission_lag_max_ms": 0.1096969936043024, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 154.2669020127505, + "ttft_p50_ms": 150.6725640501827, + "ttft_p95_ms": 152.74227899499238 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r6p00.json", + "sha256": "345e3cd3f8cd4d7c7a5e9343f937dd9b1b5928500133830f3046b743da6df64e", + "summary": { + "admission_lag_max_ms": 0.12192933354526758, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 153.12304301187396, + "ttft_p50_ms": 150.68741200957447, + "ttft_p95_ms": 152.82704099081457 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 7.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 302.5522669777274, + 294.5978350471705 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r7p00.json", + "sha256": "3f54a3e3f2279a5325a173ff714d600fb1a2cdfcfba1e8e5c980f7b21b99be59", + "summary": { + "admission_lag_max_ms": 0.11122168507426977, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 304.24335307907313, + "ttft_p50_ms": 262.4604320153594, + "ttft_p95_ms": 302.5522669777274 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r7p00.json", + "sha256": "0832169cdf1bef49c5006464c6cbcd50f4567e0ab90bec98d722f64f7b476763", + "summary": { + "admission_lag_max_ms": 0.1059951027855277, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 297.1431679325178, + "ttft_p50_ms": 249.51713997870684, + "ttft_p95_ms": 294.5978350471705 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 8.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1321.9368209829554, + 1318.7565059633926 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r8p00.json", + "sha256": "a0f5900ac9d2c47dcc7d7ac08d3f3d23e1d6f529fdd0200989afa819dd3df055", + "summary": { + "admission_lag_max_ms": 0.17414195463061333, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.90625, + "passed": 58, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1475.8616799954325, + "ttft_p50_ms": 775.7647309917957, + "ttft_p95_ms": 1321.9368209829554 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r8p00.json", + "sha256": "c0fc3c85095ddacc66849e6f58be0851abab6d4022ee4eb7120c8cb8ea9e5d2c", + "summary": { + "admission_lag_max_ms": 2.749277977272868, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.921875, + "passed": 59, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1407.0930189918727, + "ttft_p50_ms": 763.8335369993001, + "ttft_p95_ms": 1318.7565059633926 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4893.3289990527555, + 4893.17379205022 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r16p00.json", + "sha256": "4e9ccaf5384f622b1da7c58d3de803c86099dd9deae91e363aabccc14aeabc05", + "summary": { + "admission_lag_max_ms": 0.11893699411302805, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4972.85822511185, + "ttft_p50_ms": 2742.8449960425496, + "ttft_p95_ms": 4893.3289990527555 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r16p00.json", + "sha256": "c207e415186aed1148e16085145bcdfd189314e7b92f74ea72fb2e4972ef4eae", + "summary": { + "admission_lag_max_ms": 0.09541702456772327, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4971.2590809213, + "ttft_p50_ms": 2740.232800017111, + "ttft_p95_ms": 4893.17379205022 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 6653.632597066462, + 6647.155530983582 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r32p00.json", + "sha256": "780f563a2739d892761905031f82ca8bd0ac4f4eb485c454390dcb2bcdfc1b81", + "summary": { + "admission_lag_max_ms": 0.105265062302351, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6922.717701992951, + "ttft_p50_ms": 3652.452490059659, + "ttft_p95_ms": 6653.632597066462 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r32p00.json", + "sha256": "2a5eb8fb37f03e759de149d24f9b23c812a26afe72ffaaffa030688904634e43", + "summary": { + "admission_lag_max_ms": 0.10375294368714094, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6917.111590038985, + "ttft_p50_ms": 3649.0042679943144, + "ttft_p95_ms": 6647.155530983582 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 7747.191094094887, + 7839.257381972857 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r64p00.json", + "sha256": "b94a39a91842ab39d545c473fc4f872ec79eb1888943c3e03ab03c18961a9105", + "summary": { + "admission_lag_max_ms": 0.6591080455109477, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7851.370444986969, + "ttft_p50_ms": 4335.578468977474, + "ttft_p95_ms": 7747.191094094887 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r64p00.json", + "sha256": "69ab47305c4bb14f3b4ed5bc0b18a37b8a5e7c1cdbee9232e6d2b9616ed10f84", + "summary": { + "admission_lag_max_ms": 0.10753795504570007, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.125, + "passed": 8, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7885.349316056818, + "ttft_p50_ms": 4009.1016669757664, + "ttft_p95_ms": 7839.257381972857 + } + } + ] + } + ], + "capacity": 7.0, + "capacity_per_gpu": 7.0, + "mns": 32, + "name": "tp1_mns32", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z", + "tp": 1 + }, + "tp1_mns64": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 156.76817402709275, + 152.9719429090619 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r4p00.json", + "sha256": "df798dd7a1cde3cfaf698286f44bbac2b0730a2c806f125268bdf7226e2bb47c", + "summary": { + "admission_lag_max_ms": 0.1147780567407608, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 162.9874319769442, + "ttft_p50_ms": 152.67050196416676, + "ttft_p95_ms": 156.76817402709275 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r4p00.json", + "sha256": "df80057224d69ce2a79f937aa45c2093d66d131765f3c7bfcc2289a9aaff3d6a", + "summary": { + "admission_lag_max_ms": 0.2131829969584942, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 153.917470946908, + "ttft_p50_ms": 150.96506499685347, + "ttft_p95_ms": 152.9719429090619 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 5.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 153.4973110537976, + 155.23677493911237 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r5p00.json", + "sha256": "e614ffaf879859407fc6420847f0140a81bda9a94fcc438fb6782c9019bf5378", + "summary": { + "admission_lag_max_ms": 0.1283780438825488, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 155.08639602921903, + "ttft_p50_ms": 152.1085740532726, + "ttft_p95_ms": 153.4973110537976 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r5p00.json", + "sha256": "13e825eb5ced3d0cb2b5dd68d9243fdf1cee8779b90e9aad77be04053be22be2", + "summary": { + "admission_lag_max_ms": 0.1683390000835061, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 155.6691420264542, + "ttft_p50_ms": 152.42570801638067, + "ttft_p95_ms": 155.23677493911237 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 6.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 152.53963600844145, + 152.85827894695103 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r6p00.json", + "sha256": "ac530f98f57ec452538ebe2311291d0a7c83bbd5554e82f53e78a2311802e411", + "summary": { + "admission_lag_max_ms": 0.13490929268300533, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 157.18225098680705, + "ttft_p50_ms": 150.29372205026448, + "ttft_p95_ms": 152.53963600844145 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r6p00.json", + "sha256": "2f1445ee0890b0972252d10180ef6f00cbad45ee73bd174f21b370cdf950b654", + "summary": { + "admission_lag_max_ms": 0.16514293383806944, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 158.88926689513028, + "ttft_p50_ms": 150.2534201135859, + "ttft_p95_ms": 152.85827894695103 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 7.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 346.96662798523903, + 295.1050230767578 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r7p00.json", + "sha256": "c4765794d442daf7eeb64a2f587792701ebe7659c7702a3bbbfc8900801fff0e", + "summary": { + "admission_lag_max_ms": 0.16206549480557442, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 354.881492909044, + "ttft_p50_ms": 310.15722593292594, + "ttft_p95_ms": 346.96662798523903 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r7p00.json", + "sha256": "30addb463839e23a7356a5cf6d4060ec1eebfa2b9564fa9500be5bcb7aecb537", + "summary": { + "admission_lag_max_ms": 0.10446517262607813, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 296.6241450048983, + "ttft_p50_ms": 246.06937903445214, + "ttft_p95_ms": 295.1050230767578 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 8.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1376.107804942876, + 1321.1636180058122 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r8p00.json", + "sha256": "89e2975f9c7319e477789ea77015541d5235efa61d90fb8bd5b0420a58f7f186", + "summary": { + "admission_lag_max_ms": 0.1416490413248539, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.890625, + "passed": 57, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1501.330335973762, + "ttft_p50_ms": 802.2130889585242, + "ttft_p95_ms": 1376.107804942876 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r8p00.json", + "sha256": "9ca6935f1320033c3ec4b72a35c5d9909ccd7b0fff2e4b2c92c6a26f52887df1", + "summary": { + "admission_lag_max_ms": 0.12422900181263685, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.921875, + "passed": 59, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1410.047525074333, + "ttft_p50_ms": 766.8001820566133, + "ttft_p95_ms": 1321.1636180058122 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4950.155022088438, + 4892.750842031091 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r16p00.json", + "sha256": "4fce2749435bf2e8c77999e11be0a788cd85f459a647760f3db34f1dbc0aee85", + "summary": { + "admission_lag_max_ms": 0.10520406067371368, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.171875, + "passed": 11, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 5031.207721913233, + "ttft_p50_ms": 2775.8131299633533, + "ttft_p95_ms": 4950.155022088438 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r16p00.json", + "sha256": "33ded0192d8986dbcf0a7f5ab6329a440994243d4e528d6c1b278bb6b92720fd", + "summary": { + "admission_lag_max_ms": 0.14005706179887056, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4971.664474112913, + "ttft_p50_ms": 2743.070787983015, + "ttft_p95_ms": 4892.750842031091 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 6750.775643973611, + 6754.7571890754625 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r32p00.json", + "sha256": "44030c4d3374cecebe67b886b428ba8014c87d941c00a178d6049bc57c809f46", + "summary": { + "admission_lag_max_ms": 0.7358170114457607, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.15625, + "passed": 10, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6899.517056066543, + "ttft_p50_ms": 3728.6347220651805, + "ttft_p95_ms": 6750.775643973611 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r32p00.json", + "sha256": "f397bad36552c6df5bf5bcee702cfa968ddf3e20b20e9976554ddfbe838ef3b2", + "summary": { + "admission_lag_max_ms": 0.7476879982277751, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.15625, + "passed": 10, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6901.606322033331, + "ttft_p50_ms": 3729.3128239689395, + "ttft_p95_ms": 6754.7571890754625 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 7800.616211956367, + 7748.703246936202 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r64p00.json", + "sha256": "61eadc2af60386d6972d69fe7b083874e705864162073d54d1bde2bc0e6de7e7", + "summary": { + "admission_lag_max_ms": 0.11387700214982033, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7905.77321103774, + "ttft_p50_ms": 4366.223473916762, + "ttft_p95_ms": 7800.616211956367 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r64p00.json", + "sha256": "db02e9a814154a9259cbe3fefdf15700923458f7e1208596e861b69b6a6ddd41", + "summary": { + "admission_lag_max_ms": 0.14900392852723598, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7852.748095989227, + "ttft_p50_ms": 4337.09563605953, + "ttft_p95_ms": 7748.703246936202 + } + } + ] + } + ], + "capacity": 7.0, + "capacity_per_gpu": 7.0, + "mns": 64, + "name": "tp1_mns64", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z", + "tp": 1 + }, + "tp1_mns8": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 154.07429495826364, + 153.62840006127954 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r4p00.json", + "sha256": "23d5ab491b70c02f51c2f06dec0407edc0627819a33b9bcef1349d974a4e20fe", + "summary": { + "admission_lag_max_ms": 0.14362391084432602, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 171.46877304185182, + "ttft_p50_ms": 151.62539994344115, + "ttft_p95_ms": 154.07429495826364 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r4p00.json", + "sha256": "563aa230e0f5e420cf453dfd9b829d7df493919192d638cbe76f85eb77b6d2e7", + "summary": { + "admission_lag_max_ms": 0.11323392391204834, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 155.62100906390697, + "ttft_p50_ms": 151.69239102397114, + "ttft_p95_ms": 153.62840006127954 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 5.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 154.20477103907615, + 153.81942794192582 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r5p00.json", + "sha256": "6f2a04a5c11eed5826e2a6e1ce6e6f8d98504b31d0fec30d9440df2cdb62e56d", + "summary": { + "admission_lag_max_ms": 0.10794296395033598, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 155.37187200970948, + "ttft_p50_ms": 152.59731002151966, + "ttft_p95_ms": 154.20477103907615 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r5p00.json", + "sha256": "27a0bfa65a66e5cc77e10a5cf39a846f0fd1dd266c10926dae9db7a80c581d6e", + "summary": { + "admission_lag_max_ms": 0.1204309519380331, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 155.40366806089878, + "ttft_p50_ms": 152.0398510619998, + "ttft_p95_ms": 153.81942794192582 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 6.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 153.00951700191945, + 153.5524509381503 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r6p00.json", + "sha256": "4562fae05d510cd50f2c9d7479916d056c8ac4d04851d2388b98f8410192c23a", + "summary": { + "admission_lag_max_ms": 0.10347133502364159, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 156.3970319693908, + "ttft_p50_ms": 150.414744974114, + "ttft_p95_ms": 153.00951700191945 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r6p00.json", + "sha256": "bd13283e2807150396b278f46ff630ae9820895e0c054a9efcfd8fc213947d47", + "summary": { + "admission_lag_max_ms": 0.13600767124444246, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 159.63539096992463, + "ttft_p50_ms": 151.83425601571798, + "ttft_p95_ms": 153.5524509381503 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 7.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 336.8800369789824, + 329.65395506471395 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r7p00.json", + "sha256": "fd5afe52238d6ddb72bdf7f7b142fddbeb3f62e160e4a1b3a639b308717964e6", + "summary": { + "admission_lag_max_ms": 0.14346837997436523, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 341.1952300230041, + "ttft_p50_ms": 294.81438896618783, + "ttft_p95_ms": 336.8800369789824 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r7p00.json", + "sha256": "b0fdc1ef617313261c490010adeb2f4ee93bbb053f1db10734ed5a63fd37fa76", + "summary": { + "admission_lag_max_ms": 0.11047185398638248, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 334.06115300022066, + "ttft_p50_ms": 281.1476809438318, + "ttft_p95_ms": 329.65395506471395 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 8.0, + "round_feasible": [ + false, + true + ], + "round_ttft_p95_ms": [ + 1289.2397560644895, + 1254.356418037787 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r8p00.json", + "sha256": "8877ed44b90f421afd22ec110fe6337581c67cd973b44378bcb7528fc8624de3", + "summary": { + "admission_lag_max_ms": 0.09899691212922335, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.9375, + "passed": 60, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1454.3529660440981, + "ttft_p50_ms": 813.0663179326802, + "ttft_p95_ms": 1289.2397560644895 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r8p00.json", + "sha256": "7ab24f5ab33bf166589af7e2f5c2a274ceef6a64f0029a96ec1879b42332123d", + "summary": { + "admission_lag_max_ms": 0.11694896966218948, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 0.953125, + "passed": 61, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1347.0055809011683, + "ttft_p50_ms": 744.0344559727237, + "ttft_p95_ms": 1254.356418037787 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4903.538785991259, + 4900.28300601989 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r16p00.json", + "sha256": "363d9b43164b5451bc3dd668caa0ce141d15114087cf444e7fd79d12b7563fd6", + "summary": { + "admission_lag_max_ms": 0.11420203372836113, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4982.48829995282, + "ttft_p50_ms": 2749.9175920384005, + "ttft_p95_ms": 4903.538785991259 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r16p00.json", + "sha256": "8aec70dfd6b69f4903bccc02c5d5474165ed07d9dd010c860ff9b447429da950", + "summary": { + "admission_lag_max_ms": 0.19414699636399746, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4979.007771937177, + "ttft_p50_ms": 2749.115635990165, + "ttft_p95_ms": 4900.28300601989 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 6760.187636013143, + 6534.733187989332 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r32p00.json", + "sha256": "7684ac0d2d6f851218b88fb37db3a5b54d877a6f1ead3bb604a6cb4d9a61cf3f", + "summary": { + "admission_lag_max_ms": 0.43534801807254553, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.15625, + "passed": 10, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6907.940509961918, + "ttft_p50_ms": 3735.540736000985, + "ttft_p95_ms": 6760.187636013143 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r32p00.json", + "sha256": "947974a9ae04e170b8c1aad32ddbda8c1000f3902a9b572c6338a7d0d4f23146", + "summary": { + "admission_lag_max_ms": 1.221106038428843, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6798.48151106853, + "ttft_p50_ms": 3595.78589303419, + "ttft_p95_ms": 6534.733187989332 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 7754.704045015387, + 7756.428897031583 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r64p00.json", + "sha256": "abfc2a158087f0ad75b560dd420604a9f81359ffdf39f0ce8e2ea25b451e6904", + "summary": { + "admission_lag_max_ms": 0.13654399663209915, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7857.916672946885, + "ttft_p50_ms": 4341.527819051407, + "ttft_p95_ms": 7754.704045015387 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r64p00.json", + "sha256": "192c0c850a52d2b64780bc6321cbfd10faddcc0ff10741ddad283b7d789b9def", + "summary": { + "admission_lag_max_ms": 0.11000398080796003, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7859.458995983005, + "ttft_p50_ms": 4341.894763987511, + "ttft_p95_ms": 7756.428897031583 + } + } + ] + } + ], + "capacity": 7.0, + "capacity_per_gpu": 7.0, + "mns": 8, + "name": "tp1_mns8", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z", + "tp": 1 + }, + "tp2_mns16": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 99.1211449727416, + 101.08128399588168 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r4p00.json", + "sha256": "f149882da390114f2a2a140a3bd2ca6f15b6f7761591ea7b49b5a961c9d18df5", + "summary": { + "admission_lag_max_ms": 0.10085199028253555, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 101.28417506348342, + "ttft_p50_ms": 95.90165293775499, + "ttft_p95_ms": 99.1211449727416 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r4p00.json", + "sha256": "154dfaf2729001a8ef5b7430290f4ca906df1d4758e867d5b350489410f36d5c", + "summary": { + "admission_lag_max_ms": 0.11863000690937042, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 105.76263803523034, + "ttft_p50_ms": 96.27402503974736, + "ttft_p95_ms": 101.08128399588168 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 97.1850419882685, + 97.03529300168157 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r8p00.json", + "sha256": "9338d5cb994b0c12a6420f030ae2a9335a04986b4a81e31f4781ac838aeef2d4", + "summary": { + "admission_lag_max_ms": 0.12352492194622755, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 102.26434096693993, + "ttft_p50_ms": 95.07436107378453, + "ttft_p95_ms": 97.1850419882685 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r8p00.json", + "sha256": "a35b67970948976482ee835e595ee5631fa921607276d5ea7928460514f0d06b", + "summary": { + "admission_lag_max_ms": 0.11170096695423126, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 101.72453604172915, + "ttft_p50_ms": 94.03157397173345, + "ttft_p95_ms": 97.03529300168157 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 10.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 96.10817302018404, + 94.37024802900851 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r10p00.json", + "sha256": "f67e0f4f6b41861fcbcbee04b4efbf0dee46e273239ec94ef22d650ba6afa22a", + "summary": { + "admission_lag_max_ms": 0.10012194979935884, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 102.40214702207595, + "ttft_p50_ms": 94.17163301259279, + "ttft_p95_ms": 96.10817302018404 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r10p00.json", + "sha256": "5a64b52f0298e9ecda2fddb8230cc27ab4b9c1db01575f68b5331816b324d1d7", + "summary": { + "admission_lag_max_ms": 0.10465399827808142, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 101.07159405015409, + "ttft_p50_ms": 93.24954205658287, + "ttft_p95_ms": 94.37024802900851 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 12.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 447.0475659472868, + 454.8899739747867 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r12p00.json", + "sha256": "c6e27b9078281a54feeb1fab09616385a70de819791ce14f46b001bf540a80ab", + "summary": { + "admission_lag_max_ms": 0.12051803059875965, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 455.56937297806144, + "ttft_p50_ms": 280.09015799034387, + "ttft_p95_ms": 447.0475659472868 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r12p00.json", + "sha256": "861f9c4186c33e9ee6365d19c53e98c10615c5ee529a84e69f84dc292f53ac4f", + "summary": { + "admission_lag_max_ms": 0.11322635691612959, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 457.4526828946546, + "ttft_p50_ms": 290.1579710887745, + "ttft_p95_ms": 454.8899739747867 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 14.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 1000.5082850111648, + 1004.6314439969137 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r14p00.json", + "sha256": "2dcaf1087edd1f12fbfe0618a6ab636b22484756a5729919bc0e51de153f8f4a", + "summary": { + "admission_lag_max_ms": 0.1881921198219061, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1089.2393708927557, + "ttft_p50_ms": 611.2175739835948, + "ttft_p95_ms": 1000.5082850111648 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r14p00.json", + "sha256": "16937b31f72298c7a50ec3c7b9c1b9c798f305d83469b3d3c058efc4d53135da", + "summary": { + "admission_lag_max_ms": 0.10659568943083286, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1096.6680150013417, + "ttft_p50_ms": 617.3943240428343, + "ttft_p95_ms": 1004.6314439969137 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1458.4544260287657, + 1471.3564389385283 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r16p00.json", + "sha256": "5576a7fd06977f51f1c43bb73c3c474cc21a93cd288b422ec3d09db671b3afe7", + "summary": { + "admission_lag_max_ms": 0.135264010168612, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.765625, + "passed": 49, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1583.2410319708288, + "ttft_p50_ms": 904.5663280412555, + "ttft_p95_ms": 1458.4544260287657 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r16p00.json", + "sha256": "2ccf16426046c412829442f72932268d5b27ce051a4312d135d32b240b48ec7c", + "summary": { + "admission_lag_max_ms": 0.11741300113499165, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.78125, + "passed": 50, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1537.9131329245865, + "ttft_p50_ms": 896.1611289996654, + "ttft_p95_ms": 1471.3564389385283 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 3348.027400090359, + 3348.575382027775 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r32p00.json", + "sha256": "d258760d15f1fdf2adefca72f7a3505c0b839036799dcb074450f7f526f76718", + "summary": { + "admission_lag_max_ms": 0.18610397819429636, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3442.3487790627405, + "ttft_p50_ms": 1801.6119640087709, + "ttft_p95_ms": 3348.027400090359 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r32p00.json", + "sha256": "edf5cccc1400639a94b5430897a04f20d2214bb1ffee7f4bfa62514150179a6a", + "summary": { + "admission_lag_max_ms": 0.1144439447671175, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3443.0311269825324, + "ttft_p50_ms": 1801.435066969134, + "ttft_p95_ms": 3348.575382027775 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4286.911493982188, + 4287.898648995906 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r64p00.json", + "sha256": "2a6303efb826637c3c6448d6f766f7de4fc4d7b63db163a48f72c20df68394f4", + "summary": { + "admission_lag_max_ms": 0.10997895151376724, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4340.311317006126, + "ttft_p50_ms": 2396.4726959820837, + "ttft_p95_ms": 4286.911493982188 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r64p00.json", + "sha256": "6fafe3bf206583b8689be1b2b57d6b92cbd1ba54f7fb230ca05f6c68b3ab9757", + "summary": { + "admission_lag_max_ms": 0.12669397983700037, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4340.538130956702, + "ttft_p50_ms": 2394.112486974336, + "ttft_p95_ms": 4287.898648995906 + } + } + ] + } + ], + "capacity": 14.0, + "capacity_per_gpu": 7.0, + "mns": 16, + "name": "tp2_mns16", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z", + "tp": 2 + }, + "tp2_mns32": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 97.2925309324637, + 96.13083000294864 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r4p00.json", + "sha256": "812f8a108a9390fb759b46e5bd41e3ddad66bc144d47bf4024d834e42b47c47e", + "summary": { + "admission_lag_max_ms": 0.11386198457330465, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 100.76773096807301, + "ttft_p50_ms": 95.72782705072314, + "ttft_p95_ms": 97.2925309324637 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r4p00.json", + "sha256": "70f922eb7651d46d81d7d394b8453c5b778f921069cf48fb2ae2677afa94a842", + "summary": { + "admission_lag_max_ms": 0.30313804745674133, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 129.06646099872887, + "ttft_p50_ms": 94.53632694203407, + "ttft_p95_ms": 96.13083000294864 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 95.42673197574914, + 98.94769196398556 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r8p00.json", + "sha256": "24097d38b0ffb45f6cdbd93f0aad04dce3b7da41ff45e948eb3a794902b6547f", + "summary": { + "admission_lag_max_ms": 0.11571904178708792, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 100.04934400785714, + "ttft_p50_ms": 94.34181300457567, + "ttft_p95_ms": 95.42673197574914 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r8p00.json", + "sha256": "7251f076d34ba9484b08fa3fc4b898d2ba8d1199cb1eaeaaffc19930f8c889b9", + "summary": { + "admission_lag_max_ms": 0.1177559606730938, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 100.93720292206854, + "ttft_p50_ms": 94.77824496570975, + "ttft_p95_ms": 98.94769196398556 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 10.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 96.28185909241438, + 95.9081610199064 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r10p00.json", + "sha256": "85796457715a25c15fc909f6def32776fbd49956eb7890df1c4abcef8c5afe78", + "summary": { + "admission_lag_max_ms": 0.14786410611122847, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 102.46025596279651, + "ttft_p50_ms": 93.54631893802434, + "ttft_p95_ms": 96.28185909241438 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r10p00.json", + "sha256": "0bf4d4d006bf0b5bd8b989ceeeb4bacb0e3ee1b8ea66f30b3deb120a14871cf0", + "summary": { + "admission_lag_max_ms": 0.11162098962813616, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 100.54603894241154, + "ttft_p50_ms": 93.77408598084003, + "ttft_p95_ms": 95.9081610199064 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 12.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 439.0145930228755, + 429.9719400005415 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r12p00.json", + "sha256": "68de6e36bd7c5624cab107fe5cb21bb817711691e545439a958c926a27e3fe29", + "summary": { + "admission_lag_max_ms": 0.10475562885403633, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 445.90794993564487, + "ttft_p50_ms": 270.3119309153408, + "ttft_p95_ms": 439.0145930228755 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r12p00.json", + "sha256": "d8d3a44a07b46f1673817ece2948fd97c30f41bd451d93a2fa02edf074536c63", + "summary": { + "admission_lag_max_ms": 0.4323029424995184, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 434.6071509644389, + "ttft_p50_ms": 265.2625450864434, + "ttft_p95_ms": 429.9719400005415 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 14.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 1014.2896320903674, + 999.8802930349484 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r14p00.json", + "sha256": "10196c2f9a9b69c2e47f64d06bf5ee6863af4cd2b612301622f0e304949a942a", + "summary": { + "admission_lag_max_ms": 0.09005970787256956, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1105.1907669752836, + "ttft_p50_ms": 621.6445650206879, + "ttft_p95_ms": 1014.2896320903674 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r14p00.json", + "sha256": "c10cab85588a3563bed36fc4480462635d34d240938beaa0e5034f13186578a6", + "summary": { + "admission_lag_max_ms": 0.11327012907713652, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1090.6395060010254, + "ttft_p50_ms": 623.5586369875818, + "ttft_p95_ms": 999.8802930349484 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1488.3801550604403, + 1484.6198010491207 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r16p00.json", + "sha256": "ef5046c49c32178855bed6a0488ea21e8da22f786ddaabc503bea2d1025bb2f3", + "summary": { + "admission_lag_max_ms": 0.7892940193414688, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.765625, + "passed": 49, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1613.616122980602, + "ttft_p50_ms": 905.2404230460525, + "ttft_p95_ms": 1488.3801550604403 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r16p00.json", + "sha256": "de02c1d49414cffe7e20023abf4d502a4efc6d2e01fcb3e49e928081aaef77b6", + "summary": { + "admission_lag_max_ms": 0.10548799764364958, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.765625, + "passed": 49, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1550.9842830942944, + "ttft_p50_ms": 904.9503420246765, + "ttft_p95_ms": 1484.6198010491207 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 3391.811708919704, + 3362.9186518955976 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r32p00.json", + "sha256": "876c5444d17c4ce81eca907ae17d5cf7062ffa67fa266b491580e52f712614ec", + "summary": { + "admission_lag_max_ms": 0.12557301670312881, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3486.074637970887, + "ttft_p50_ms": 1840.204154024832, + "ttft_p95_ms": 3391.811708919704 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r32p00.json", + "sha256": "02b4cd93ce1f9816e822022fb608ba8d4c89265f673f6d27c8fa3d4f1523ffe0", + "summary": { + "admission_lag_max_ms": 0.11014204937964678, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3456.965696066618, + "ttft_p50_ms": 1812.343467026949, + "ttft_p95_ms": 3362.9186518955976 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4306.351027917117, + 4302.602957002819 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r64p00.json", + "sha256": "13ba282f179b2d6ce13a0aa2fd04eb29f82e579b7995e3406cb0afd69a2d5ee2", + "summary": { + "admission_lag_max_ms": 0.3526840591803193, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4359.505696920678, + "ttft_p50_ms": 2411.1130589153618, + "ttft_p95_ms": 4306.351027917117 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r64p00.json", + "sha256": "767622d62f0757ef2d8c7651ea470bfa2e62edad0ed4e75f66f84a0b722e35da", + "summary": { + "admission_lag_max_ms": 0.10626309085637331, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4355.409953044727, + "ttft_p50_ms": 2409.5974090741947, + "ttft_p95_ms": 4302.602957002819 + } + } + ] + } + ], + "capacity": 14.0, + "capacity_per_gpu": 7.0, + "mns": 32, + "name": "tp2_mns32", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z", + "tp": 2 + }, + "tp2_mns64": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 96.54812794178724, + 95.08875198662281 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r4p00.json", + "sha256": "6e6a45d0c1010094c3395969815fd8cf9c35b919db202d772fb514744f981ebc", + "summary": { + "admission_lag_max_ms": 0.11306197848170996, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 99.27749610505998, + "ttft_p50_ms": 93.01548008807003, + "ttft_p95_ms": 96.54812794178724 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r4p00.json", + "sha256": "58aa71d694704cfea15c37cb29420dbac1dc505045c29a731e8be904846cca5f", + "summary": { + "admission_lag_max_ms": 0.11853000614792109, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 98.74943597242236, + "ttft_p50_ms": 93.57558900956064, + "ttft_p95_ms": 95.08875198662281 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 94.24941800534725, + 96.21043398510665 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r8p00.json", + "sha256": "311350af236cb0306aa635f05f8999b66bf5080bad7cc1f6703fffe0be31aee6", + "summary": { + "admission_lag_max_ms": 0.19771000370383263, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 100.52211803849787, + "ttft_p50_ms": 92.4453770276159, + "ttft_p95_ms": 94.24941800534725 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r8p00.json", + "sha256": "60fe989a36a526e76f6f5c98e217b9feaf081bc680b16b9d53650bf085047c06", + "summary": { + "admission_lag_max_ms": 0.3933720290660858, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 102.97230398282409, + "ttft_p50_ms": 93.26327103190124, + "ttft_p95_ms": 96.21043398510665 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 10.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 97.98661107197404, + 92.9473100695759 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r10p00.json", + "sha256": "d0a04444a0e41bf76f71ac6ea3815bfbe76f7a51f8ddf669976bc3a4896dd0c1", + "summary": { + "admission_lag_max_ms": 0.10270101483911276, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 103.49742788821459, + "ttft_p50_ms": 94.55623896792531, + "ttft_p95_ms": 97.98661107197404 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r10p00.json", + "sha256": "f330f26f45f6039c48150a0b6b146ca3dbb9ea8b2dff086a9efd5938ab981eb7", + "summary": { + "admission_lag_max_ms": 0.11239596642553806, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 97.4133670097217, + "ttft_p50_ms": 91.2688790122047, + "ttft_p95_ms": 92.9473100695759 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 12.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 260.68170997314155, + 319.04888805001974 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r12p00.json", + "sha256": "fed5eacf18018402b0c2675b8c5595f92693a9bddec30f354d989098f9fe4965", + "summary": { + "admission_lag_max_ms": 0.11784560047090054, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 345.19017208367586, + "ttft_p50_ms": 211.41935500781983, + "ttft_p95_ms": 260.68170997314155 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r12p00.json", + "sha256": "f2be27cbeff3e5c1a95050418f2dc3c712a4648cada8cdb3d36382cfffc9593a", + "summary": { + "admission_lag_max_ms": 0.12930459342896938, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 326.86293800361454, + "ttft_p50_ms": 229.8808820778504, + "ttft_p95_ms": 319.04888805001974 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 14.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 895.7287209341303, + 912.1733580250293 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r14p00.json", + "sha256": "c2e157808e0308cc443f3abdf5114898cef66c435688d24321f80bef2d966208", + "summary": { + "admission_lag_max_ms": 0.14902697876095772, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 962.3544370988384, + "ttft_p50_ms": 576.2168710352853, + "ttft_p95_ms": 895.7287209341303 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r14p00.json", + "sha256": "749ef7d31bf8782d42719b75ee1a2e4a443f097510af721d02cf53df191bc66e", + "summary": { + "admission_lag_max_ms": 0.42896345257759094, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 986.0872740391642, + "ttft_p50_ms": 569.7114520007744, + "ttft_p95_ms": 912.1733580250293 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1364.6171110449359, + 1366.6857269126922 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r16p00.json", + "sha256": "7545ea9210348019de88ede5d2cba7527913534274deee034fa6811c173a6e72", + "summary": { + "admission_lag_max_ms": 0.12612901628017426, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.859375, + "passed": 55, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1435.8308439841494, + "ttft_p50_ms": 848.9949750946835, + "ttft_p95_ms": 1364.6171110449359 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r16p00.json", + "sha256": "d75e27137f6a7811126d8c957bb5a081944fc1bb7643852cdd785111773f6010", + "summary": { + "admission_lag_max_ms": 0.1258660340681672, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.859375, + "passed": 55, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1436.2916270038113, + "ttft_p50_ms": 846.2129919789732, + "ttft_p95_ms": 1366.6857269126922 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 3243.9196659252048, + 3214.524411014281 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r32p00.json", + "sha256": "366ebddc167338ed096f373de5667740721422c13d2db1f887ed09186afafd19", + "summary": { + "admission_lag_max_ms": 0.09669095743447542, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3337.068362045102, + "ttft_p50_ms": 1751.3202880509198, + "ttft_p95_ms": 3243.9196659252048 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r32p00.json", + "sha256": "01b8f5081e36a64d5776285c8c5f423b53b1ad684c4b8f95a8caa5d997c6bf09", + "summary": { + "admission_lag_max_ms": 0.8310690755024552, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.328125, + "passed": 21, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3277.297410997562, + "ttft_p50_ms": 1790.5721209244803, + "ttft_p95_ms": 3214.524411014281 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4319.725521025248, + 4367.031275993213 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r64p00.json", + "sha256": "92202709fd6bbd558ff6f1986df213848254110c4274447a7bd5879153290d46", + "summary": { + "admission_lag_max_ms": 0.10392803233116865, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4360.145599930547, + "ttft_p50_ms": 2403.5812759539112, + "ttft_p95_ms": 4319.725521025248 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r64p00.json", + "sha256": "9660d15d31bc70c5550ed29477f563313d4d81baf0e561126aebd3b7d097cfa3", + "summary": { + "admission_lag_max_ms": 0.17276196740567684, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.25, + "passed": 16, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4414.442001027055, + "ttft_p50_ms": 2260.2788450894877, + "ttft_p95_ms": 4367.031275993213 + } + } + ] + } + ], + "capacity": 14.0, + "capacity_per_gpu": 7.0, + "mns": 64, + "name": "tp2_mns64", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z", + "tp": 2 + }, + "tp2_mns8": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 97.17482794076204, + 98.03147299680859 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r4p00.json", + "sha256": "0e315f6d9e0087ad553e874b0c16c2e6de79eb6eca22eb118dc5deca52e6efe3", + "summary": { + "admission_lag_max_ms": 0.11708191595971584, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 101.92840604577214, + "ttft_p50_ms": 95.13642196543515, + "ttft_p95_ms": 97.17482794076204 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r4p00.json", + "sha256": "baa394a24d2a3db40a3a7ae6dae0c5ce49ff05499a1e17eaa0bca4b479ae33bf", + "summary": { + "admission_lag_max_ms": 0.10946101974695921, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 100.52333807107061, + "ttft_p50_ms": 93.19468098692596, + "ttft_p95_ms": 98.03147299680859 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 97.67980605829507, + 93.80627400241792 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r8p00.json", + "sha256": "303df27c6d4604d5af65f9056acacb7c4373fb6eb7edb6e5d42689eeaa384aaa", + "summary": { + "admission_lag_max_ms": 0.11111306957900524, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 113.2329449756071, + "ttft_p50_ms": 93.05417106952518, + "ttft_p95_ms": 97.67980605829507 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r8p00.json", + "sha256": "cdbb1bca17b24b67e18ef3dd4997494d6bd2a887b3b5d1264fbf411e688c53bc", + "summary": { + "admission_lag_max_ms": 0.12463307939469814, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 101.12245299387723, + "ttft_p50_ms": 92.26030297577381, + "ttft_p95_ms": 93.80627400241792 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 10.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 94.45723297540098, + 93.57301203999668 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r10p00.json", + "sha256": "fe17ef2417319307d651bc10b01a618234d61bab797ad87a7332cab20d78ecb2", + "summary": { + "admission_lag_max_ms": 0.3267280990257859, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 99.28593400400132, + "ttft_p50_ms": 92.22474799025804, + "ttft_p95_ms": 94.45723297540098 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r10p00.json", + "sha256": "550f693708a34624a82d77afd4da0ab4bc01e09f42242a3a79e4532897681a51", + "summary": { + "admission_lag_max_ms": 0.11238199658691883, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 97.77756198309362, + "ttft_p50_ms": 92.07128698471934, + "ttft_p95_ms": 93.57301203999668 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 12.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 320.49866404850036, + 336.9630330707878 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r12p00.json", + "sha256": "edab8169453151de732923c076b8f7dca4082ee47ccd2597e65e7ba68516bc2e", + "summary": { + "admission_lag_max_ms": 0.09914662223309278, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 327.930542989634, + "ttft_p50_ms": 231.4880370395258, + "ttft_p95_ms": 320.49866404850036 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r12p00.json", + "sha256": "a6dde4363d362ca10254409e3dc734d0f0e640e63156426f3f4fd7115db49e16", + "summary": { + "admission_lag_max_ms": 0.17016706988215446, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 428.6657409975305, + "ttft_p50_ms": 237.09688894450665, + "ttft_p95_ms": 336.9630330707878 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 14.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 911.7195709841326, + 910.7652489328757 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r14p00.json", + "sha256": "7eba81ee5c85c457a056f9aef28abd94d375fdaa116929cfe0a0c282a0738bd2", + "summary": { + "admission_lag_max_ms": 0.13163150288164616, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 999.2953530745581, + "ttft_p50_ms": 580.023527960293, + "ttft_p95_ms": 911.7195709841326 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r14p00.json", + "sha256": "6a113d3d876ff01a9c1101e8b75efa2fc71f96683d6bb430e26d5976426b5d23", + "summary": { + "admission_lag_max_ms": 0.11634768452495337, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 988.8870939612389, + "ttft_p50_ms": 578.9327350212261, + "ttft_p95_ms": 910.7652489328757 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1389.8832038976252, + 1365.0342189939693 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r16p00.json", + "sha256": "d677251ee1a941c52469052a239c2e13be0a837285069aa8182a16e196445e48", + "summary": { + "admission_lag_max_ms": 0.11070503387600183, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.828125, + "passed": 53, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1460.5093500576913, + "ttft_p50_ms": 864.3530959961936, + "ttft_p95_ms": 1389.8832038976252 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r16p00.json", + "sha256": "f5177f53aa0f87a4e7291f470978d86ad8bc46bd1a8362b79f48cd128646dc10", + "summary": { + "admission_lag_max_ms": 0.3672370221465826, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.828125, + "passed": 53, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1491.050804965198, + "ttft_p50_ms": 841.7225609300658, + "ttft_p95_ms": 1365.0342189939693 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 3258.010295103304, + 3258.664960041642 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r32p00.json", + "sha256": "02a6ac2b9c2b9ab97854d6fcc13b64bbb36a079e4afdae84231e4cb187fac4ef", + "summary": { + "admission_lag_max_ms": 0.1296499976888299, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3352.278529899195, + "ttft_p50_ms": 1755.0864539807662, + "ttft_p95_ms": 3258.010295103304 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r32p00.json", + "sha256": "9ca2f44fc7e3cb3d38ada548af858903b31bbee5cd50f1723c29255ebd4224e9", + "summary": { + "admission_lag_max_ms": 1.1343059595674276, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3352.7101600775495, + "ttft_p50_ms": 1755.3453800501302, + "ttft_p95_ms": 3258.664960041642 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4190.255103982054, + 4189.1050330596045 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r64p00.json", + "sha256": "53908d3a254e19691839831b4337014b09591ced9189c2a9b29e0d92e80fcb56", + "summary": { + "admission_lag_max_ms": 0.11591101065278053, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4242.029857938178, + "ttft_p50_ms": 2340.5529640149325, + "ttft_p95_ms": 4190.255103982054 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r64p00.json", + "sha256": "1c8c85c873aa96c01dc0200f848474a53c1173ef72f3828e8c23aeae70e92b11", + "summary": { + "admission_lag_max_ms": 0.284243025816977, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4240.770195960067, + "ttft_p50_ms": 2339.3583690049127, + "ttft_p95_ms": 4189.1050330596045 + } + } + ] + } + ], + "capacity": 14.0, + "capacity_per_gpu": 7.0, + "mns": 8, + "name": "tp2_mns8", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z", + "tp": 2 + }, + "tp4_mns16": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 74.66826005838811, + 66.5705680148676 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r4p00.json", + "sha256": "fb354fc8a097caa21844d4ea5515b98b03459f400816e5e5590a6cda81cf4815", + "summary": { + "admission_lag_max_ms": 0.1035250024870038, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 93.38432096410543, + "ttft_p50_ms": 65.75745099689811, + "ttft_p95_ms": 74.66826005838811 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r4p00.json", + "sha256": "ac8e8412f79a8a32ebf085445a94aa548dd2d646fce0b9f56b4378081dcc8960", + "summary": { + "admission_lag_max_ms": 0.12108997907489538, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 79.89856600761414, + "ttft_p50_ms": 62.386377016082406, + "ttft_p95_ms": 66.5705680148676 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 70.44799195136875, + 72.29184894822538 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r8p00.json", + "sha256": "05ebbd68bbf24e9c8b25684474411c7db122b92701cc1617f9d786cf19592751", + "summary": { + "admission_lag_max_ms": 0.1825150102376938, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 96.92095802165568, + "ttft_p50_ms": 60.598687967285514, + "ttft_p95_ms": 70.44799195136875 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r8p00.json", + "sha256": "d24f7daaf41073cc8aa030b74c06203783684c3e556770366883c37b826406f2", + "summary": { + "admission_lag_max_ms": 0.10628998279571533, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 75.13270596973598, + "ttft_p50_ms": 60.161596979014575, + "ttft_p95_ms": 72.29184894822538 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 16.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 66.02151494007558, + 107.58775402791798 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r16p00.json", + "sha256": "a29b2128983af630e1da8f970cbda802849036bc522eb506821bf2fe1a34172f", + "summary": { + "admission_lag_max_ms": 0.10468997061252594, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 72.93136999942362, + "ttft_p50_ms": 59.5597909996286, + "ttft_p95_ms": 66.02151494007558 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r16p00.json", + "sha256": "275ae35651cbe2200d16c9b06a68568efda527564065255a10dee6fe93346010", + "summary": { + "admission_lag_max_ms": 0.3144120564684272, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 119.78317995090038, + "ttft_p50_ms": 59.12985489703715, + "ttft_p95_ms": 107.58775402791798 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 20.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 193.8455159543082, + 187.52167106140405 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r20p00.json", + "sha256": "4cb485f30b81c004bab379e6522d94b811c0bdc0be6d8e42c9d8ae842d9600dc", + "summary": { + "admission_lag_max_ms": 0.10407401714473963, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 197.7908629924059, + "ttft_p50_ms": 142.73560303263366, + "ttft_p95_ms": 193.8455159543082 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r20p00.json", + "sha256": "63aa9dc03a08b781c5d6e8805f2b61698c75e71673608ecd38bd1132c62e6987", + "summary": { + "admission_lag_max_ms": 0.09386602323502302, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 190.29632199089974, + "ttft_p50_ms": 136.1723990412429, + "ttft_p95_ms": 187.52167106140405 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 24.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 568.2198900030926, + 573.4539540717378 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r24p00.json", + "sha256": "5e4d5b44850090d07fe1d3abc44eddea3bfdf2b682f59b12a45a866621e38f98", + "summary": { + "admission_lag_max_ms": 0.7511250441893935, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 613.2891150191426, + "ttft_p50_ms": 375.27791305910796, + "ttft_p95_ms": 568.2198900030926 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r24p00.json", + "sha256": "239e7789aed2e9b2bcf1d127215a6b122071eb856be2db64ccc184fda153aa48", + "summary": { + "admission_lag_max_ms": 0.11180061846971512, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 614.9205249967054, + "ttft_p50_ms": 375.5133739905432, + "ttft_p95_ms": 573.4539540717378 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 28.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 860.0242879474536, + 886.1482660286129 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r28p00.json", + "sha256": "8b249b52779e5b3cd9028aaee30ee73ef34b0b0a64dcd75b9aad57266c29fbc2", + "summary": { + "admission_lag_max_ms": 0.11425174307078123, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 930.8854410191998, + "ttft_p50_ms": 534.7980660153553, + "ttft_p95_ms": 860.0242879474536 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r28p00.json", + "sha256": "c9b28c8f3187542a343ecf1067783569ec2abfa64631d6827a94ba6876aa7b63", + "summary": { + "admission_lag_max_ms": 0.312068616040051, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 922.4315020255744, + "ttft_p50_ms": 549.7413279954344, + "ttft_p95_ms": 886.1482660286129 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 32.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 1147.1082780044526, + 1137.236590962857 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r32p00.json", + "sha256": "d27ff9f940a6cd7fba339d9a176de39cf4b5bd164dcc500f2af36bcb07cdef30", + "summary": { + "admission_lag_max_ms": 0.16146607231348753, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1201.626160996966, + "ttft_p50_ms": 696.4708579471335, + "ttft_p95_ms": 1147.1082780044526 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r32p00.json", + "sha256": "093044e87fffdaa222e5bad3807cbf4900aa64891d5b4ba27f187c7a411252d7", + "summary": { + "admission_lag_max_ms": 0.10430300608277321, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1201.045956928283, + "ttft_p50_ms": 684.3201500596479, + "ttft_p95_ms": 1137.236590962857 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 2064.763988018967, + 2039.6780440350994 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r64p00.json", + "sha256": "fad7bdd80d5ca603790a4e5d19dadd4f494540b106ce583e3e8c1c05d2374a58", + "summary": { + "admission_lag_max_ms": 6.394341005943716, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.53125, + "passed": 34, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2144.80376499705, + "ttft_p50_ms": 1158.5425049997866, + "ttft_p95_ms": 2064.763988018967 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r64p00.json", + "sha256": "24fdb945706dbbdd29916e0f08e0eff807334db212fded94c729da2e1b46efd3", + "summary": { + "admission_lag_max_ms": 0.10294502135366201, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.546875, + "passed": 35, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2125.4275270039216, + "ttft_p50_ms": 1131.1213460285217, + "ttft_p95_ms": 2039.6780440350994 + } + } + ] + } + ], + "capacity": 32.0, + "capacity_per_gpu": 8.0, + "mns": 16, + "name": "tp4_mns16", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z", + "tp": 4 + }, + "tp4_mns32": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 67.80304794665426, + 64.59337309934199 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r4p00.json", + "sha256": "2e82a30f02c2702668e59661930431cf82323746a111d42bf38b7fdc0d083b3d", + "summary": { + "admission_lag_max_ms": 0.1332589890807867, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 72.74050498381257, + "ttft_p50_ms": 64.10286005120724, + "ttft_p95_ms": 67.80304794665426 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r4p00.json", + "sha256": "8294196c7dc52765a6820f5e1d8439c480490daed9505263f707810f73c95783", + "summary": { + "admission_lag_max_ms": 0.1283179735764861, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 70.6796320155263, + "ttft_p50_ms": 60.962425894103944, + "ttft_p95_ms": 64.59337309934199 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 70.46496798284352, + 68.22343391831964 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r8p00.json", + "sha256": "38bbfeab47e5eb6296fcccf20f04ef25dc24db658be92300c492734bbfe333cf", + "summary": { + "admission_lag_max_ms": 0.10555307380855083, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 87.7126749837771, + "ttft_p50_ms": 62.65173899009824, + "ttft_p95_ms": 70.46496798284352 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r8p00.json", + "sha256": "716f90177d6f203717c80a8e9b28697e53141f204af470493c299bc73a1ea3ee", + "summary": { + "admission_lag_max_ms": 0.24518708232790232, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 80.40577603969723, + "ttft_p50_ms": 60.29119901359081, + "ttft_p95_ms": 68.22343391831964 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 16.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 89.53388908412308, + 76.93439698778093 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r16p00.json", + "sha256": "063016efe822c31c2425e7d963eda4c054f3f7c0341df0a3cd73ab86a942e873", + "summary": { + "admission_lag_max_ms": 0.2996840048581362, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 97.41822991054505, + "ttft_p50_ms": 60.97789097111672, + "ttft_p95_ms": 89.53388908412308 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r16p00.json", + "sha256": "eecc6bdeff90a8bf195dfc5387c6bc2f4db20ed7c9aeb9b4ebe050efccc6051a", + "summary": { + "admission_lag_max_ms": 0.36267400719225407, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 79.42048599943519, + "ttft_p50_ms": 60.29245804529637, + "ttft_p95_ms": 76.93439698778093 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 20.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 206.2494510319084, + 201.13301498349756 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r20p00.json", + "sha256": "22a62729358fea7c38ca166a1a49f1235e98c7f07ecfe4948322110cda4b7178", + "summary": { + "admission_lag_max_ms": 0.11367700062692165, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 255.14698098413646, + "ttft_p50_ms": 155.46583198010921, + "ttft_p95_ms": 206.2494510319084 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r20p00.json", + "sha256": "be103a759100bd2ab6dd31ee4ce41b6f0da40d232a60ebe137e3d933194a487c", + "summary": { + "admission_lag_max_ms": 0.17853605095297098, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 209.22932599205524, + "ttft_p50_ms": 155.12292610947043, + "ttft_p95_ms": 201.13301498349756 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 24.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 580.5047769099474, + 577.3971510352567 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r24p00.json", + "sha256": "890a733a97f399deb4c5efcca661d9a442ed76080cd5d8b6a2c09368aef701b7", + "summary": { + "admission_lag_max_ms": 0.1084543764591217, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 621.5670789824799, + "ttft_p50_ms": 374.6346529806033, + "ttft_p95_ms": 580.5047769099474 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r24p00.json", + "sha256": "87e8543652b1743b1b9b8c3de0fc680401b0bf9a33d61fa6132120c944b2a6eb", + "summary": { + "admission_lag_max_ms": 0.11801638174802065, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 623.2662099646404, + "ttft_p50_ms": 382.11427396163344, + "ttft_p95_ms": 577.3971510352567 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 28.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 873.0683700414374, + 889.4930009264499 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r28p00.json", + "sha256": "f877781730a5437b4cbdb76948915f1c871574b19a6df27076d21b65bb18e678", + "summary": { + "admission_lag_max_ms": 0.6807943573221564, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 943.9258739585057, + "ttft_p50_ms": 542.9852310335264, + "ttft_p95_ms": 873.0683700414374 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r28p00.json", + "sha256": "82151dc539b35302489c17976cfe89b8f4c64944298954607fad8f5c9cb9e7d8", + "summary": { + "admission_lag_max_ms": 6.750226486474276, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 936.5667289821431, + "ttft_p50_ms": 546.0105559322983, + "ttft_p95_ms": 889.4930009264499 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 32.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 1135.3467140579596, + 1146.743991994299 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r32p00.json", + "sha256": "082cc4c47b2eec6e49f0edd14cff5a51b880bd998d98da8b9830bbf7615483ef", + "summary": { + "admission_lag_max_ms": 0.10443700011819601, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1199.8225760180503, + "ttft_p50_ms": 680.4794340860099, + "ttft_p95_ms": 1135.3467140579596 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r32p00.json", + "sha256": "517d3cbc1104b5ea1bdb0e381c033cf978181a93449b6498d30b52cd3dd3e770", + "summary": { + "admission_lag_max_ms": 0.4415180301293731, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1210.0602959981188, + "ttft_p50_ms": 693.0603379150853, + "ttft_p95_ms": 1146.743991994299 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 2084.9693010095507, + 2041.9401369290426 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r64p00.json", + "sha256": "7e11771ed598826fb3c5594c7e05d0989e01030be2d8b36fa21813aff462496d", + "summary": { + "admission_lag_max_ms": 0.10768393985927105, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.5625, + "passed": 36, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2132.128676981665, + "ttft_p50_ms": 1113.0280290963128, + "ttft_p95_ms": 2084.9693010095507 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r64p00.json", + "sha256": "cf550423832a92cba942b7e1dc14a42000e8c9507403de8cd49aa56af9db6706", + "summary": { + "admission_lag_max_ms": 0.11262902989983559, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.546875, + "passed": 35, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2122.8132860269397, + "ttft_p50_ms": 1131.2310450011864, + "ttft_p95_ms": 2041.9401369290426 + } + } + ] + } + ], + "capacity": 32.0, + "capacity_per_gpu": 8.0, + "mns": 32, + "name": "tp4_mns32", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z", + "tp": 4 + }, + "tp4_mns64": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 66.89354998525232, + 83.23234610725194 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r4p00.json", + "sha256": "266cba4088f8a9483ceb41fdc02d04a2181cb7ffb6b52372b6b0687e62f2863e", + "summary": { + "admission_lag_max_ms": 0.10568101424723864, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 72.09669100120664, + "ttft_p50_ms": 62.014322029426694, + "ttft_p95_ms": 66.89354998525232 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r4p00.json", + "sha256": "26f3926c40504b96a37cb7b9dae1af5040c085c1a730e20feaf1d3e9ddbf8e54", + "summary": { + "admission_lag_max_ms": 0.108166947029531, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 96.42452490516007, + "ttft_p50_ms": 61.261708033271134, + "ttft_p95_ms": 83.23234610725194 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 67.26283801253885, + 67.34374002553523 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r8p00.json", + "sha256": "e84e6ba2c8822e012aceea7cc6f376ea1b6265fe4a3bb68b7badfc36b8813545", + "summary": { + "admission_lag_max_ms": 0.15694997273385525, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 86.54619695153087, + "ttft_p50_ms": 59.63437003083527, + "ttft_p95_ms": 67.26283801253885 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r8p00.json", + "sha256": "dae50073c9f79919181e8dcf274ca8929ba41c512f64363e40187de68839b87e", + "summary": { + "admission_lag_max_ms": 0.09962497279047966, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 69.00161597877741, + "ttft_p50_ms": 60.2562150452286, + "ttft_p95_ms": 67.34374002553523 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 16.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 71.37343694921583, + 64.9534990079701 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r16p00.json", + "sha256": "776c9956f65acf7c9b6c8df517a4f8c4d462289893a19e625a2a0ad7dcbf5506", + "summary": { + "admission_lag_max_ms": 0.1066719414666295, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 82.25136005785316, + "ttft_p50_ms": 58.52357891853899, + "ttft_p95_ms": 71.37343694921583 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r16p00.json", + "sha256": "90057204a5f6343ee4ee9cdd08323d87d83660d5bf8f02480882eb479d2e587e", + "summary": { + "admission_lag_max_ms": 0.10102998930960894, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 65.3582609957084, + "ttft_p50_ms": 58.98014397826046, + "ttft_p95_ms": 64.9534990079701 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 20.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 192.98715889453888, + 258.1990590551868 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r20p00.json", + "sha256": "6ce029388b206d0663acd41f4da1d67a6562389e1c84cf1826ce77c05d176d9a", + "summary": { + "admission_lag_max_ms": 0.12691703159362078, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 198.67739104665816, + "ttft_p50_ms": 142.75032398290932, + "ttft_p95_ms": 192.98715889453888 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r20p00.json", + "sha256": "0dbfb3f012574988ddebfb0af1532c735ac62d6d88863d24cd820aed50a92a59", + "summary": { + "admission_lag_max_ms": 0.3961039474233985, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 259.55891504418105, + "ttft_p50_ms": 162.6282810466364, + "ttft_p95_ms": 258.1990590551868 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 24.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 583.3089610096067, + 571.3486529421061 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r24p00.json", + "sha256": "0774ba2f285d81447c63343b32551d6fc33369b54010629381efa9a37df23a9c", + "summary": { + "admission_lag_max_ms": 0.09822729043662548, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 628.714332007803, + "ttft_p50_ms": 366.9840869260952, + "ttft_p95_ms": 583.3089610096067 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r24p00.json", + "sha256": "1986a05c366ebe73af0f89d2cd0e4efd3e577c800cf1e80997cd26ea0aaaec09", + "summary": { + "admission_lag_max_ms": 0.11414836626499891, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 617.9165309295058, + "ttft_p50_ms": 376.65714998729527, + "ttft_p95_ms": 571.3486529421061 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 28.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 869.708820944652, + 866.4831441128626 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r28p00.json", + "sha256": "ab987729fc833a59817eabaa18c320b0d0a2ecd402fd02bdb534b6c773c029d8", + "summary": { + "admission_lag_max_ms": 0.578614417463541, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 941.3277570856735, + "ttft_p50_ms": 540.4732850147411, + "ttft_p95_ms": 869.708820944652 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r28p00.json", + "sha256": "38f9c82c9ee30927cc0ba857e5e3fce2e79b8a24de03efaec77ce27258633a7e", + "summary": { + "admission_lag_max_ms": 0.12532551772892475, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 938.2033230504021, + "ttft_p50_ms": 541.139594046399, + "ttft_p95_ms": 866.4831441128626 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 32.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 1132.6635719742626, + 1144.196804962121 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r32p00.json", + "sha256": "358f5b8e1b72bff5752c076f2966b966d98f6f5767ce52d5a255b7f68f061e54", + "summary": { + "admission_lag_max_ms": 0.32093306072056293, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1197.7118930080906, + "ttft_p50_ms": 678.4568330040202, + "ttft_p95_ms": 1132.6635719742626 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r32p00.json", + "sha256": "3f6dbb760582fdd7470422d8edf6b62949d158ad0d437f04015441a0a4509003", + "summary": { + "admission_lag_max_ms": 1.1304179206490517, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1193.1529619032517, + "ttft_p50_ms": 692.5535890040919, + "ttft_p95_ms": 1144.196804962121 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 2042.27216495201, + 2043.943466967903 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r64p00.json", + "sha256": "a24eae7db6eb4cd105ed903917988e8fe6eef5d4cc2549d4e05e9a03dfafb107", + "summary": { + "admission_lag_max_ms": 0.134881935082376, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.546875, + "passed": 35, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2132.23939598538, + "ttft_p50_ms": 1137.3268909519538, + "ttft_p95_ms": 2042.27216495201 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r64p00.json", + "sha256": "e85efaada0b64bb9dd686f59282171ccb256a1d8483c6f037589a68e3e1a104a", + "summary": { + "admission_lag_max_ms": 0.11892011389136314, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.546875, + "passed": 35, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2125.3299129894003, + "ttft_p50_ms": 1134.724210947752, + "ttft_p95_ms": 2043.943466967903 + } + } + ] + } + ], + "capacity": 32.0, + "capacity_per_gpu": 8.0, + "mns": 64, + "name": "tp4_mns64", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z", + "tp": 4 + }, + "tp4_mns8": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 82.33969006687403, + 69.57727600820363 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r4p00.json", + "sha256": "e69b89ad2e07e6560ebd870773532b33e4415e9ca1515e585cbe465616678569", + "summary": { + "admission_lag_max_ms": 0.14186999760568142, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 86.54833806212991, + "ttft_p50_ms": 62.749656033702195, + "ttft_p95_ms": 82.33969006687403 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r4p00.json", + "sha256": "2c76dcf112a828a6720e079d29bb54a6a65d37d26abaaafdc6d5580751dbad65", + "summary": { + "admission_lag_max_ms": 0.11478399392217398, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 81.3983449479565, + "ttft_p50_ms": 62.220132909715176, + "ttft_p95_ms": 69.57727600820363 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 63.71171900536865, + 68.14328604377806 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r8p00.json", + "sha256": "0e5d687dfcdb4b6a0d1d060ceefe8a74faab94a5b241580d45b53c452f893b23", + "summary": { + "admission_lag_max_ms": 0.09818794205784798, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 69.63772000744939, + "ttft_p50_ms": 60.89008797425777, + "ttft_p95_ms": 63.71171900536865 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r8p00.json", + "sha256": "1bf016b603473d46fda4bc6d4daeaee5e1091ff4cda1cab7224e0c5edea512f8", + "summary": { + "admission_lag_max_ms": 0.11399004142731428, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 76.76690397784114, + "ttft_p50_ms": 60.9520060243085, + "ttft_p95_ms": 68.14328604377806 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 16.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 133.86176899075508, + 61.95738597307354 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r16p00.json", + "sha256": "197808b1b5fa8082e603a255f358826b5f8f48707223c7fd63abfe029674b817", + "summary": { + "admission_lag_max_ms": 0.10530801955610514, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 191.68931001331657, + "ttft_p50_ms": 61.92394997924566, + "ttft_p95_ms": 133.86176899075508 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r16p00.json", + "sha256": "53e809470daf9efcdabb254e85e8cc3b45d0ab8593c6c2f3aa3396e60d1656da", + "summary": { + "admission_lag_max_ms": 0.1288399798795581, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 70.40882098954171, + "ttft_p50_ms": 59.417470009066164, + "ttft_p95_ms": 61.95738597307354 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 20.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 201.29013701807708, + 200.0734859611839 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r20p00.json", + "sha256": "05b977824fb301517cdbe2d077b360b6053d57fee1c20acad3537321827d027a", + "summary": { + "admission_lag_max_ms": 0.3848400665447116, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 204.5107699232176, + "ttft_p50_ms": 150.0862929970026, + "ttft_p95_ms": 201.29013701807708 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r20p00.json", + "sha256": "baf422b7a3c95b0d3d376a31b0a30e075f69216c794ebc2ef51afafe1ec55229", + "summary": { + "admission_lag_max_ms": 0.24754798505455256, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 204.8192109214142, + "ttft_p50_ms": 148.99871195666492, + "ttft_p95_ms": 200.0734859611839 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 24.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 564.2021730309352, + 560.4996709153056 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r24p00.json", + "sha256": "029b2a9695322d536539c3dd7665725f933b21a8bb48fa94c0448c9f7f8bf0eb", + "summary": { + "admission_lag_max_ms": 0.14956865925341845, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 610.4218009859324, + "ttft_p50_ms": 368.3909500250593, + "ttft_p95_ms": 564.2021730309352 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r24p00.json", + "sha256": "94b46583af3a7a4bbb9f7256c4c4a5ce804b3092459abe3984ea0ce67f591f3c", + "summary": { + "admission_lag_max_ms": 0.11110503692179918, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 606.6705230623484, + "ttft_p50_ms": 367.50321893487126, + "ttft_p95_ms": 560.4996709153056 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 28.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 870.7952990662307, + 894.6108610834926 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r28p00.json", + "sha256": "07bfe263107bb3d2acaa356257343f90045e4295c799178bc0a0b5f1c1994284", + "summary": { + "admission_lag_max_ms": 0.6798935355618596, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 942.792383953929, + "ttft_p50_ms": 543.340316042304, + "ttft_p95_ms": 870.7952990662307 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r28p00.json", + "sha256": "861dd55076ef40a78c34824345c5279dff86262b604db120e9e38efa27b5be55", + "summary": { + "admission_lag_max_ms": 0.09912776295095682, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 930.2741150604561, + "ttft_p50_ms": 549.9696529004723, + "ttft_p95_ms": 894.6108610834926 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 32.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 1136.603599996306, + 1134.0074760373682 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r32p00.json", + "sha256": "332b805322b571087b018ee459dd91c443dfe0b2bb539b5bdfb4de902bba6da2", + "summary": { + "admission_lag_max_ms": 0.19067595712840557, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1201.8779760692269, + "ttft_p50_ms": 683.3873559953645, + "ttft_p95_ms": 1136.603599996306 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r32p00.json", + "sha256": "996a12a5ed962218dfbe6f30be688119030f10f6539647b6e4a9f25f032b22cb", + "summary": { + "admission_lag_max_ms": 0.11958007235080004, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1198.4891929896548, + "ttft_p50_ms": 682.7383439522237, + "ttft_p95_ms": 1134.0074760373682 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 2040.8174369949847, + 2082.3236129945144 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r64p00.json", + "sha256": "c5f17f625071e905b5f1bf8045cd233b2fea0fe6ca00044935dee9c663d1526b", + "summary": { + "admission_lag_max_ms": 0.7995429914444685, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.546875, + "passed": 35, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2131.160691031255, + "ttft_p50_ms": 1134.8516750149429, + "ttft_p95_ms": 2040.8174369949847 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r64p00.json", + "sha256": "775db5b941347d9fd5d6df32d6ee037f8168bd47b9eae9a7bdb3dbd8c40aa990", + "summary": { + "admission_lag_max_ms": 0.17620902508497238, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.5625, + "passed": 36, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2129.518053960055, + "ttft_p50_ms": 1112.3740070033818, + "ttft_p95_ms": 2082.3236129945144 + } + } + ] + } + ], + "capacity": 32.0, + "capacity_per_gpu": 8.0, + "mns": 8, + "name": "tp4_mns8", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z", + "tp": 4 + } + }, + "real_campaign_resources": { + "fresh_server_anchors": 192, + "gpu_hours": 12.074444444444445, + "measured_requests": 12288, + "runs": [ + { + "duration_seconds": 908.0, + "gpu_hours": 0.25222222222222224, + "run": "qwen30-prefill-real-tp1-mns16-20260717-v2-exclusive-20260716T164730291172Z", + "tp": 1 + }, + { + "duration_seconds": 540.0, + "gpu_hours": 0.15, + "run": "qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z", + "tp": 1 + }, + { + "duration_seconds": 937.0, + "gpu_hours": 0.2602777777777778, + "run": "qwen30-prefill-real-tp1-mns32-20260717-v2-exclusive-20260716T164731376588Z", + "tp": 1 + }, + { + "duration_seconds": 558.0, + "gpu_hours": 0.155, + "run": "qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z", + "tp": 1 + }, + { + "duration_seconds": 972.0, + "gpu_hours": 0.27, + "run": "qwen30-prefill-real-tp1-mns64-20260717-v2-exclusive-20260716T164732482666Z", + "tp": 1 + }, + { + "duration_seconds": 572.0, + "gpu_hours": 0.15888888888888889, + "run": "qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z", + "tp": 1 + }, + { + "duration_seconds": 886.0, + "gpu_hours": 0.2461111111111111, + "run": "qwen30-prefill-real-tp1-mns8-20260717-v2-exclusive-20260716T164729267199Z", + "tp": 1 + }, + { + "duration_seconds": 525.0, + "gpu_hours": 0.14583333333333334, + "run": "qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z", + "tp": 1 + }, + { + "duration_seconds": 983.0, + "gpu_hours": 0.5461111111111111, + "run": "qwen30-prefill-real-tp2-mns16-20260717-v2-exclusive-20260716T164734745039Z", + "tp": 2 + }, + { + "duration_seconds": 569.0, + "gpu_hours": 0.3161111111111111, + "run": "qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z", + "tp": 2 + }, + { + "duration_seconds": 1033.0, + "gpu_hours": 0.5738888888888889, + "run": "qwen30-prefill-real-tp2-mns32-20260717-v2-exclusive-20260716T170909930153Z", + "tp": 2 + }, + { + "duration_seconds": 584.0, + "gpu_hours": 0.3244444444444444, + "run": "qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z", + "tp": 2 + }, + { + "duration_seconds": 1063.0, + "gpu_hours": 0.5905555555555555, + "run": "qwen30-prefill-real-tp2-mns64-20260717-v2-exclusive-20260716T170910954319Z", + "tp": 2 + }, + { + "duration_seconds": 597.0, + "gpu_hours": 0.33166666666666667, + "run": "qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z", + "tp": 2 + }, + { + "duration_seconds": 957.0, + "gpu_hours": 0.5316666666666666, + "run": "qwen30-prefill-real-tp2-mns8-20260717-v2-exclusive-20260716T164733581881Z", + "tp": 2 + }, + { + "duration_seconds": 563.0, + "gpu_hours": 0.31277777777777777, + "run": "qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z", + "tp": 2 + }, + { + "duration_seconds": 1008.0, + "gpu_hours": 1.12, + "run": "qwen30-prefill-real-tp4-mns16-20260717-v2-exclusive-20260716T172922998830Z", + "tp": 4 + }, + { + "duration_seconds": 559.0, + "gpu_hours": 0.6211111111111111, + "run": "qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z", + "tp": 4 + }, + { + "duration_seconds": 1024.0, + "gpu_hours": 1.1377777777777778, + "run": "qwen30-prefill-real-tp4-mns32-20260717-v2-exclusive-20260716T172923955862Z", + "tp": 4 + }, + { + "duration_seconds": 556.0, + "gpu_hours": 0.6177777777777778, + "run": "qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z", + "tp": 4 + }, + { + "duration_seconds": 959.0, + "gpu_hours": 1.0655555555555556, + "run": "qwen30-prefill-real-tp4-mns64-20260717-v2b-exclusive-20260716T175248149074Z", + "tp": 4 + }, + { + "duration_seconds": 567.0, + "gpu_hours": 0.63, + "run": "qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z", + "tp": 4 + }, + { + "duration_seconds": 995.0, + "gpu_hours": 1.1055555555555556, + "run": "qwen30-prefill-real-tp4-mns8-20260717-v2-exclusive-20260716T170911977980Z", + "tp": 4 + }, + { + "duration_seconds": 550.0, + "gpu_hours": 0.6111111111111112, + "run": "qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z", + "tp": 4 + } + ], + "successful_fleet_jobs": 24, + "warmup_requests": 4512 + }, + "schema": "qwen30-prefill-fidelity-comparison-v1", + "simulator": { + "tp1_mns16": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 171.58529929215405 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 5.0, + "ttft_p95_ms": 171.58529929215405 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 6.0, + "ttft_p95_ms": 333.9001759332376 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 7.0, + "ttft_p95_ms": 427.64609771465524 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 516.4107605377559 + }, + { + "feasible": false, + "pass_rate": 0.3125, + "rate": 16.0, + "ttft_p95_ms": 3193.8404181099845 + }, + { + "feasible": false, + "pass_rate": 0.203125, + "rate": 32.0, + "ttft_p95_ms": 4962.713638565687 + }, + { + "feasible": false, + "pass_rate": 0.140625, + "rate": 64.0, + "ttft_p95_ms": 5853.338638565687 + } + ], + "capacity": 8.0, + "capacity_per_gpu": 8.0, + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "tp1_mns32": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 171.58529929215405 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 5.0, + "ttft_p95_ms": 171.58529929215405 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 6.0, + "ttft_p95_ms": 333.9001759332376 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 7.0, + "ttft_p95_ms": 427.64609771465524 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 516.4107605377559 + }, + { + "feasible": false, + "pass_rate": 0.3125, + "rate": 16.0, + "ttft_p95_ms": 3193.8404181099845 + }, + { + "feasible": false, + "pass_rate": 0.203125, + "rate": 32.0, + "ttft_p95_ms": 4962.713638565687 + }, + { + "feasible": false, + "pass_rate": 0.140625, + "rate": 64.0, + "ttft_p95_ms": 5853.338638565687 + } + ], + "capacity": 8.0, + "capacity_per_gpu": 8.0, + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "tp1_mns64": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 171.58529929215405 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 5.0, + "ttft_p95_ms": 171.58529929215405 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 6.0, + "ttft_p95_ms": 333.9001759332376 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 7.0, + "ttft_p95_ms": 427.64609771465524 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 516.4107605377559 + }, + { + "feasible": false, + "pass_rate": 0.3125, + "rate": 16.0, + "ttft_p95_ms": 3193.8404181099845 + }, + { + "feasible": false, + "pass_rate": 0.203125, + "rate": 32.0, + "ttft_p95_ms": 4962.713638565687 + }, + { + "feasible": false, + "pass_rate": 0.140625, + "rate": 64.0, + "ttft_p95_ms": 5853.338638565687 + } + ], + "capacity": 8.0, + "capacity_per_gpu": 8.0, + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "tp1_mns8": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 171.58529929215405 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 5.0, + "ttft_p95_ms": 171.58529929215405 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 6.0, + "ttft_p95_ms": 333.9001759332376 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 7.0, + "ttft_p95_ms": 427.64609771465524 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 516.4107605377559 + }, + { + "feasible": false, + "pass_rate": 0.3125, + "rate": 16.0, + "ttft_p95_ms": 3193.8404181099845 + }, + { + "feasible": false, + "pass_rate": 0.203125, + "rate": 32.0, + "ttft_p95_ms": 4962.713638565687 + }, + { + "feasible": false, + "pass_rate": 0.140625, + "rate": 64.0, + "ttft_p95_ms": 5853.338638565687 + } + ], + "capacity": 8.0, + "capacity_per_gpu": 8.0, + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "tp2_mns16": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 122.24062760264154 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 122.24062760264154 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 10.0, + "ttft_p95_ms": 297.1650738307572 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 12.0, + "ttft_p95_ms": 384.6764910356253 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 14.0, + "ttft_p95_ms": 386.4490667329015 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 962.8278877316862 + }, + { + "feasible": false, + "pass_rate": 0.390625, + "rate": 32.0, + "ttft_p95_ms": 2758.1797930156335 + }, + { + "feasible": false, + "pass_rate": 0.265625, + "rate": 64.0, + "ttft_p95_ms": 3622.636520081023 + } + ], + "capacity": 16.0, + "capacity_per_gpu": 8.0, + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "tp2_mns32": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 122.24062760264154 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 122.24062760264154 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 10.0, + "ttft_p95_ms": 297.1650738307572 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 12.0, + "ttft_p95_ms": 384.6764910356253 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 14.0, + "ttft_p95_ms": 386.4490667329015 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 962.8278877316862 + }, + { + "feasible": false, + "pass_rate": 0.390625, + "rate": 32.0, + "ttft_p95_ms": 2758.1797930156335 + }, + { + "feasible": false, + "pass_rate": 0.265625, + "rate": 64.0, + "ttft_p95_ms": 3622.636520081023 + } + ], + "capacity": 16.0, + "capacity_per_gpu": 8.0, + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "tp2_mns64": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 122.24062760264154 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 122.24062760264154 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 10.0, + "ttft_p95_ms": 297.1650738307572 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 12.0, + "ttft_p95_ms": 384.6764910356253 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 14.0, + "ttft_p95_ms": 386.4490667329015 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 962.8278877316862 + }, + { + "feasible": false, + "pass_rate": 0.390625, + "rate": 32.0, + "ttft_p95_ms": 2758.1797930156335 + }, + { + "feasible": false, + "pass_rate": 0.265625, + "rate": 64.0, + "ttft_p95_ms": 3622.636520081023 + } + ], + "capacity": 16.0, + "capacity_per_gpu": 8.0, + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "tp2_mns8": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 122.24062760264154 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 122.24062760264154 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 10.0, + "ttft_p95_ms": 297.1650738307572 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 12.0, + "ttft_p95_ms": 384.6764910356253 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 14.0, + "ttft_p95_ms": 386.4490667329015 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 962.8278877316862 + }, + { + "feasible": false, + "pass_rate": 0.390625, + "rate": 32.0, + "ttft_p95_ms": 2758.1797930156335 + }, + { + "feasible": false, + "pass_rate": 0.265625, + "rate": 64.0, + "ttft_p95_ms": 3622.636520081023 + } + ], + "capacity": 16.0, + "capacity_per_gpu": 8.0, + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "tp4_mns16": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 93.12333977685495 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 93.12333977685495 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 309.403713596319 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 20.0, + "ttft_p95_ms": 718.1022232545545 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 24.0, + "ttft_p95_ms": 1155.549457433053 + }, + { + "feasible": false, + "pass_rate": 0.765625, + "rate": 28.0, + "ttft_p95_ms": 1515.6009336189102 + }, + { + "feasible": false, + "pass_rate": 0.625, + "rate": 32.0, + "ttft_p95_ms": 1787.92236218991 + }, + { + "feasible": false, + "pass_rate": 0.390625, + "rate": 64.0, + "ttft_p95_ms": 2661.056949107159 + } + ], + "capacity": 24.0, + "capacity_per_gpu": 6.0, + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "tp4_mns32": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 93.12333977685495 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 93.12333977685495 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 309.403713596319 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 20.0, + "ttft_p95_ms": 718.1022232545545 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 24.0, + "ttft_p95_ms": 1155.549457433053 + }, + { + "feasible": false, + "pass_rate": 0.765625, + "rate": 28.0, + "ttft_p95_ms": 1515.6009336189102 + }, + { + "feasible": false, + "pass_rate": 0.625, + "rate": 32.0, + "ttft_p95_ms": 1787.92236218991 + }, + { + "feasible": false, + "pass_rate": 0.390625, + "rate": 64.0, + "ttft_p95_ms": 2661.056949107159 + } + ], + "capacity": 24.0, + "capacity_per_gpu": 6.0, + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "tp4_mns64": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 93.12333977685495 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 93.12333977685495 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 309.403713596319 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 20.0, + "ttft_p95_ms": 718.1022232545545 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 24.0, + "ttft_p95_ms": 1155.549457433053 + }, + { + "feasible": false, + "pass_rate": 0.765625, + "rate": 28.0, + "ttft_p95_ms": 1515.6009336189102 + }, + { + "feasible": false, + "pass_rate": 0.625, + "rate": 32.0, + "ttft_p95_ms": 1787.92236218991 + }, + { + "feasible": false, + "pass_rate": 0.390625, + "rate": 64.0, + "ttft_p95_ms": 2661.056949107159 + } + ], + "capacity": 24.0, + "capacity_per_gpu": 6.0, + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "tp4_mns8": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 93.12333977685495 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 93.12333977685495 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 309.403713596319 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 20.0, + "ttft_p95_ms": 718.1022232545545 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 24.0, + "ttft_p95_ms": 1155.549457433053 + }, + { + "feasible": false, + "pass_rate": 0.765625, + "rate": 28.0, + "ttft_p95_ms": 1515.6009336189102 + }, + { + "feasible": false, + "pass_rate": 0.625, + "rate": 32.0, + "ttft_p95_ms": 1787.92236218991 + }, + { + "feasible": false, + "pass_rate": 0.390625, + "rate": 64.0, + "ttft_p95_ms": 2661.056949107159 + } + ], + "capacity": 24.0, + "capacity_per_gpu": 6.0, + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + } + }, + "simulator_sources": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/simulator-tp1/frontier_surface_frozen.json", + "sha256": "2e3c4711bf9a5046daed3517c58f23241880467ee8f8e70191dc379413ad027c" + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/simulator-tp2/frontier_surface_frozen.json", + "sha256": "faffabd900cc38c17a349f13840257078543af9d5da39c363a5d77a87d2eef93" + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/simulator-tp4/frontier_surface_frozen.json", + "sha256": "e54e08cb263c376daf92f418a90ad79ae1afaa2edbed69271513218ab104e93c" + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/simulator-refine-tp1/frontier_surface_frozen.json", + "sha256": "b4530266c764e2bb1b420a68052d70e24f63eb9c30f3b3e8f8c9e4865a25f280" + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/simulator-refine-tp2/frontier_surface_frozen.json", + "sha256": "cd06f00b5c6f514896842e9f4482bd6270fedb0bf003823f1b974d136acdeffd" + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/simulator-refine-tp4/frontier_surface_frozen.json", + "sha256": "6ec105c50563e669ef145aa2b9a431e695a6308abb6d5db951b8a1f9d4a9e2f4" + } + ] +} diff --git a/runs/frontier-phase-factorial-v0/simulator-refine-tp1/frontier_surface_frozen.json b/runs/frontier-phase-factorial-v0/simulator-refine-tp1/frontier_surface_frozen.json new file mode 100644 index 0000000..48c05b5 --- /dev/null +++ b/runs/frontier-phase-factorial-v0/simulator-refine-tp1/frontier_surface_frozen.json @@ -0,0 +1,461 @@ +{ + "capacity": [ + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 7.0, + "maximum_tested_feasible_request_rate_per_gpu": 7.0, + "upper_censored": true + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 7.0, + "maximum_tested_feasible_request_rate_per_gpu": 7.0, + "upper_censored": true + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 7.0, + "maximum_tested_feasible_request_rate_per_gpu": 7.0, + "upper_censored": true + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 7.0, + "maximum_tested_feasible_request_rate_per_gpu": 7.0, + "upper_censored": true + } + ], + "config_results": [ + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "loads": [ + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 10.505136728286743, + "offered_request_rate": 5.0, + "offered_request_rate_per_gpu": 5.0, + "request_count": 64, + "request_metrics_sha256": "914a1261d25b03b812dfa30880c3b0a0d7b697a8c13a9f3a26a7bb6eaecd0734", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 5.011124187030024, + "ttft_max_ms": 171.58529929215405, + "ttft_p50_ms": 171.58529929215405, + "ttft_p95_ms": 171.58529929215405 + }, + "status": "completed", + "trace_sha256": "98e4ac5df766b1cb5bfb7469f2d76368808c2e085e22eb3553cb8627592cacd7" + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 10.398303508758545, + "offered_request_rate": 6.0, + "offered_request_rate_per_gpu": 6.0, + "request_count": 64, + "request_metrics_sha256": "d8f0fe2fbf2ffb42b7e5033f611fc22f3293264859264b5f8d67f13f3a5b66c9", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 5.915042512446176, + "ttft_max_ms": 432.43322440414465, + "ttft_p50_ms": 282.618152646152, + "ttft_p95_ms": 333.9001759332376 + }, + "status": "completed", + "trace_sha256": "e0f9adf28d29542fcf834d482c990781724426f3d5cdcc043af53888393d1f64" + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 10.264978170394897, + "offered_request_rate": 7.0, + "offered_request_rate_per_gpu": 7.0, + "request_count": 64, + "request_metrics_sha256": "7ba9181767b8cbd741db3794bc7729920d7741d63b795ade4aa1fe28a89f3a63", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 6.89299458621687, + "ttft_max_ms": 432.97081304433016, + "ttft_p50_ms": 297.34516049744997, + "ttft_p95_ms": 427.64609771465524 + }, + "status": "completed", + "trace_sha256": "3734627d83aa731c9f09a4c4fea7d994f110bcbe937e157906354a4eda9cbf48" + } + ] + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "loads": [ + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 10.103909015655518, + "offered_request_rate": 5.0, + "offered_request_rate_per_gpu": 5.0, + "request_count": 64, + "request_metrics_sha256": "914a1261d25b03b812dfa30880c3b0a0d7b697a8c13a9f3a26a7bb6eaecd0734", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 5.011124187030024, + "ttft_max_ms": 171.58529929215405, + "ttft_p50_ms": 171.58529929215405, + "ttft_p95_ms": 171.58529929215405 + }, + "status": "completed", + "trace_sha256": "98e4ac5df766b1cb5bfb7469f2d76368808c2e085e22eb3553cb8627592cacd7" + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 10.05237078666687, + "offered_request_rate": 6.0, + "offered_request_rate_per_gpu": 6.0, + "request_count": 64, + "request_metrics_sha256": "d8f0fe2fbf2ffb42b7e5033f611fc22f3293264859264b5f8d67f13f3a5b66c9", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 5.915042512446176, + "ttft_max_ms": 432.43322440414465, + "ttft_p50_ms": 282.618152646152, + "ttft_p95_ms": 333.9001759332376 + }, + "status": "completed", + "trace_sha256": "e0f9adf28d29542fcf834d482c990781724426f3d5cdcc043af53888393d1f64" + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 10.048887729644775, + "offered_request_rate": 7.0, + "offered_request_rate_per_gpu": 7.0, + "request_count": 64, + "request_metrics_sha256": "7ba9181767b8cbd741db3794bc7729920d7741d63b795ade4aa1fe28a89f3a63", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 6.89299458621687, + "ttft_max_ms": 432.97081304433016, + "ttft_p50_ms": 297.34516049744997, + "ttft_p95_ms": 427.64609771465524 + }, + "status": "completed", + "trace_sha256": "3734627d83aa731c9f09a4c4fea7d994f110bcbe937e157906354a4eda9cbf48" + } + ] + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "loads": [ + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 10.058743476867676, + "offered_request_rate": 5.0, + "offered_request_rate_per_gpu": 5.0, + "request_count": 64, + "request_metrics_sha256": "914a1261d25b03b812dfa30880c3b0a0d7b697a8c13a9f3a26a7bb6eaecd0734", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 5.011124187030024, + "ttft_max_ms": 171.58529929215405, + "ttft_p50_ms": 171.58529929215405, + "ttft_p95_ms": 171.58529929215405 + }, + "status": "completed", + "trace_sha256": "98e4ac5df766b1cb5bfb7469f2d76368808c2e085e22eb3553cb8627592cacd7" + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 10.106712818145752, + "offered_request_rate": 6.0, + "offered_request_rate_per_gpu": 6.0, + "request_count": 64, + "request_metrics_sha256": "d8f0fe2fbf2ffb42b7e5033f611fc22f3293264859264b5f8d67f13f3a5b66c9", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 5.915042512446176, + "ttft_max_ms": 432.43322440414465, + "ttft_p50_ms": 282.618152646152, + "ttft_p95_ms": 333.9001759332376 + }, + "status": "completed", + "trace_sha256": "e0f9adf28d29542fcf834d482c990781724426f3d5cdcc043af53888393d1f64" + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 10.1020348072052, + "offered_request_rate": 7.0, + "offered_request_rate_per_gpu": 7.0, + "request_count": 64, + "request_metrics_sha256": "7ba9181767b8cbd741db3794bc7729920d7741d63b795ade4aa1fe28a89f3a63", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 6.89299458621687, + "ttft_max_ms": 432.97081304433016, + "ttft_p50_ms": 297.34516049744997, + "ttft_p95_ms": 427.64609771465524 + }, + "status": "completed", + "trace_sha256": "3734627d83aa731c9f09a4c4fea7d994f110bcbe937e157906354a4eda9cbf48" + } + ] + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "loads": [ + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 9.897645235061646, + "offered_request_rate": 5.0, + "offered_request_rate_per_gpu": 5.0, + "request_count": 64, + "request_metrics_sha256": "914a1261d25b03b812dfa30880c3b0a0d7b697a8c13a9f3a26a7bb6eaecd0734", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 5.011124187030024, + "ttft_max_ms": 171.58529929215405, + "ttft_p50_ms": 171.58529929215405, + "ttft_p95_ms": 171.58529929215405 + }, + "status": "completed", + "trace_sha256": "98e4ac5df766b1cb5bfb7469f2d76368808c2e085e22eb3553cb8627592cacd7" + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 9.957297325134277, + "offered_request_rate": 6.0, + "offered_request_rate_per_gpu": 6.0, + "request_count": 64, + "request_metrics_sha256": "d8f0fe2fbf2ffb42b7e5033f611fc22f3293264859264b5f8d67f13f3a5b66c9", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 5.915042512446176, + "ttft_max_ms": 432.43322440414465, + "ttft_p50_ms": 282.618152646152, + "ttft_p95_ms": 333.9001759332376 + }, + "status": "completed", + "trace_sha256": "e0f9adf28d29542fcf834d482c990781724426f3d5cdcc043af53888393d1f64" + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 9.799486875534058, + "offered_request_rate": 7.0, + "offered_request_rate_per_gpu": 7.0, + "request_count": 64, + "request_metrics_sha256": "7ba9181767b8cbd741db3794bc7729920d7741d63b795ade4aa1fe28a89f3a63", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 6.89299458621687, + "ttft_max_ms": 432.97081304433016, + "ttft_p50_ms": 297.34516049744997, + "ttft_p95_ms": 427.64609771465524 + }, + "status": "completed", + "trace_sha256": "3734627d83aa731c9f09a4c4fea7d994f110bcbe937e157906354a4eda9cbf48" + } + ] + } + ], + "contract": { + "arrival": "open_loop_uniform", + "input_tokens": 2048, + "output_tokens": 1, + "prefix_caching": false, + "rates": [ + 5.0, + 6.0, + 7.0 + ], + "requests_per_anchor": 64, + "target_pass_rate": 0.95, + "ttft_slo_ms": 1256.0 + }, + "frontier": { + "git_head": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "git_status_short": " M frontier/config/config.py\n M frontier/entities/request.py\n M frontier/events/cluster_schedule_event.py\n M frontier/execution_time_predictor/sklearn_execution_time_predictor.py\n M frontier/metrics/constants.py\n M frontier/metrics/metrics_store.py\n M frontier/profiling/common/layers/rotary_embedding.py\n M frontier/profiling/moe/moe_impl.py\n M frontier/profiling/moe/moe_vllm_kernel.py\n M frontier/scheduler/cluster_scheduler/__init__.py\n M frontier/scheduler/cluster_scheduler/base_cluster_scheduler.py\n M frontier/scheduler/cluster_scheduler/cluster_scheduler_registry.py\n M frontier/scheduler/cluster_scheduler/sticky_lor_cluster_scheduler.py\n M frontier/scheduler/replica_scheduler/base_replica_scheduler.py\n M frontier/scheduler/replica_scheduler/vllm_v1_engine_replica_scheduler.py\n M frontier/scheduler/replica_stage_scheduler/replica_stage_schduler.py\n M frontier/simulator.py\n M frontier/types/cluster_scheduler_type.py\n?? data/profiling/compute/h20/\n?? frontier/scheduler/cluster_scheduler/prefix_lor_cluster_scheduler.py\n?? runs/\n?? tests/unit/test_attn_prefill_prediction_fallback.py\n", + "source": "/tmp/replayserve-frontier-rs1b" + }, + "profiles": { + "coverage": { + "attention": { + "1": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + }, + "2": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + }, + "4": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + } + }, + "manifest": { + "attention_tp_coverage": [ + 1, + 2, + 4 + ], + "environment_contract": { + "dtype": "bfloat16", + "frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "hardware": "NVIDIA H20", + "model": "Qwen3-30B-A3B", + "tensor_parallel_sizes": [ + 1, + 2, + 4 + ], + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + }, + "inputs": { + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931" + }, + "outputs": { + "allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3", + "attention.csv": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d", + "attention_true_mixed_fused.csv": "43ef4be90bddc9aeac6dbbe339feec24162cd1f2129a08fbd959e6ee4eaf5f60", + "linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "moe.csv": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0" + }, + "profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-fused-mixed-total-conserving", + "projection_contract": { + "allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity", + "attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target; true mixed rows use a total-conserving compatibility projection", + "attention_true_mixed": "The directly measured fused total is preserved in diagnostics. Frontier's two targets are projected by the same-TP pure prefill/decode reference ratio, with projected prefill + decode exactly equal to the fused total; the split is a schema compatibility attribution, not an observation", + "linear": "Frontier profiler using vLLM 0.20 CUDA operators", + "moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero" + }, + "row_counts": { + "allreduce": 24, + "attention_frontier_compatible": 132, + "attention_true_mixed_fused_diagnostic": 30, + "linear": 36, + "moe": 72 + }, + "schema_version": "frontier_qwen30_vllm020_frozen_profile.v2" + } + }, + "root": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2", + "sha256": { + "attention": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d", + "linear": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "manifest": "af40545e75aff55c6333cd2d5379ccf042a5a0b7d7fc7df4f745ce256cb290eb", + "moe": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0" + } + }, + "schema": "frontier-qwen30-prefill-surface-v1", + "status": "partial_not_decision_bearing" +} diff --git a/runs/frontier-phase-factorial-v0/simulator-refine-tp2/frontier_surface_frozen.json b/runs/frontier-phase-factorial-v0/simulator-refine-tp2/frontier_surface_frozen.json new file mode 100644 index 0000000..da9191c --- /dev/null +++ b/runs/frontier-phase-factorial-v0/simulator-refine-tp2/frontier_surface_frozen.json @@ -0,0 +1,461 @@ +{ + "capacity": [ + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 14.0, + "maximum_tested_feasible_request_rate_per_gpu": 7.0, + "upper_censored": true + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 14.0, + "maximum_tested_feasible_request_rate_per_gpu": 7.0, + "upper_censored": true + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 14.0, + "maximum_tested_feasible_request_rate_per_gpu": 7.0, + "upper_censored": true + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 14.0, + "maximum_tested_feasible_request_rate_per_gpu": 7.0, + "upper_censored": true + } + ], + "config_results": [ + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "loads": [ + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 10.598209381103516, + "offered_request_rate": 10.0, + "offered_request_rate_per_gpu": 5.0, + "request_count": 64, + "request_metrics_sha256": "3597984a1acaffc6b3e8b1baa3ff41bb94ee1618f8780fcc6cd8d3bdb3d7201b", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 9.814433634929404, + "ttft_max_ms": 300.1784228926625, + "ttft_p50_ms": 211.39433537475938, + "ttft_p95_ms": 297.1650738307572 + }, + "status": "completed", + "trace_sha256": "92e3001d8467e3f97a243bea8323440ecf254a8ca211e52226fe886ebc9404be" + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 10.660333633422852, + "offered_request_rate": 12.0, + "offered_request_rate_per_gpu": 6.0, + "request_count": 64, + "request_metrics_sha256": "06641f2758bd3f801fb7b9c934670bc766372ad7353f5e14aea2988dfbec462a", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 11.55742285377119, + "ttft_max_ms": 389.93612032123883, + "ttft_p50_ms": 296.08352841701094, + "ttft_p95_ms": 384.6764910356253 + }, + "status": "completed", + "trace_sha256": "cbc40c05fb9e400fa5edea6dd52c4122a957418fe523d102f1e72a784321e681" + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 10.409451007843018, + "offered_request_rate": 14.0, + "offered_request_rate_per_gpu": 7.0, + "request_count": 64, + "request_metrics_sha256": "94cf0f3c510015ba50495eb38fd6fb452dd23d16ce0c788dc926229ecd557812", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 13.374732701347012, + "ttft_max_ms": 388.3013907870674, + "ttft_p50_ms": 309.4635231444016, + "ttft_p95_ms": 386.4490667329015 + }, + "status": "completed", + "trace_sha256": "3e43801bb724218ceaf0d52f17cd4d0fb3863e70f95c4d8d5a43ce7f85c75c01" + } + ] + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "loads": [ + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 10.360321521759033, + "offered_request_rate": 10.0, + "offered_request_rate_per_gpu": 5.0, + "request_count": 64, + "request_metrics_sha256": "3597984a1acaffc6b3e8b1baa3ff41bb94ee1618f8780fcc6cd8d3bdb3d7201b", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 9.814433634929404, + "ttft_max_ms": 300.1784228926625, + "ttft_p50_ms": 211.39433537475938, + "ttft_p95_ms": 297.1650738307572 + }, + "status": "completed", + "trace_sha256": "92e3001d8467e3f97a243bea8323440ecf254a8ca211e52226fe886ebc9404be" + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 10.402549028396606, + "offered_request_rate": 12.0, + "offered_request_rate_per_gpu": 6.0, + "request_count": 64, + "request_metrics_sha256": "06641f2758bd3f801fb7b9c934670bc766372ad7353f5e14aea2988dfbec462a", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 11.55742285377119, + "ttft_max_ms": 389.93612032123883, + "ttft_p50_ms": 296.08352841701094, + "ttft_p95_ms": 384.6764910356253 + }, + "status": "completed", + "trace_sha256": "cbc40c05fb9e400fa5edea6dd52c4122a957418fe523d102f1e72a784321e681" + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 10.40966272354126, + "offered_request_rate": 14.0, + "offered_request_rate_per_gpu": 7.0, + "request_count": 64, + "request_metrics_sha256": "94cf0f3c510015ba50495eb38fd6fb452dd23d16ce0c788dc926229ecd557812", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 13.374732701347012, + "ttft_max_ms": 388.3013907870674, + "ttft_p50_ms": 309.4635231444016, + "ttft_p95_ms": 386.4490667329015 + }, + "status": "completed", + "trace_sha256": "3e43801bb724218ceaf0d52f17cd4d0fb3863e70f95c4d8d5a43ce7f85c75c01" + } + ] + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "loads": [ + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 10.155991315841675, + "offered_request_rate": 10.0, + "offered_request_rate_per_gpu": 5.0, + "request_count": 64, + "request_metrics_sha256": "3597984a1acaffc6b3e8b1baa3ff41bb94ee1618f8780fcc6cd8d3bdb3d7201b", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 9.814433634929404, + "ttft_max_ms": 300.1784228926625, + "ttft_p50_ms": 211.39433537475938, + "ttft_p95_ms": 297.1650738307572 + }, + "status": "completed", + "trace_sha256": "92e3001d8467e3f97a243bea8323440ecf254a8ca211e52226fe886ebc9404be" + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 10.103453874588013, + "offered_request_rate": 12.0, + "offered_request_rate_per_gpu": 6.0, + "request_count": 64, + "request_metrics_sha256": "06641f2758bd3f801fb7b9c934670bc766372ad7353f5e14aea2988dfbec462a", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 11.55742285377119, + "ttft_max_ms": 389.93612032123883, + "ttft_p50_ms": 296.08352841701094, + "ttft_p95_ms": 384.6764910356253 + }, + "status": "completed", + "trace_sha256": "cbc40c05fb9e400fa5edea6dd52c4122a957418fe523d102f1e72a784321e681" + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 10.255317687988281, + "offered_request_rate": 14.0, + "offered_request_rate_per_gpu": 7.0, + "request_count": 64, + "request_metrics_sha256": "94cf0f3c510015ba50495eb38fd6fb452dd23d16ce0c788dc926229ecd557812", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 13.374732701347012, + "ttft_max_ms": 388.3013907870674, + "ttft_p50_ms": 309.4635231444016, + "ttft_p95_ms": 386.4490667329015 + }, + "status": "completed", + "trace_sha256": "3e43801bb724218ceaf0d52f17cd4d0fb3863e70f95c4d8d5a43ce7f85c75c01" + } + ] + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "loads": [ + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 10.106109380722046, + "offered_request_rate": 10.0, + "offered_request_rate_per_gpu": 5.0, + "request_count": 64, + "request_metrics_sha256": "3597984a1acaffc6b3e8b1baa3ff41bb94ee1618f8780fcc6cd8d3bdb3d7201b", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 9.814433634929404, + "ttft_max_ms": 300.1784228926625, + "ttft_p50_ms": 211.39433537475938, + "ttft_p95_ms": 297.1650738307572 + }, + "status": "completed", + "trace_sha256": "92e3001d8467e3f97a243bea8323440ecf254a8ca211e52226fe886ebc9404be" + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 10.112579107284546, + "offered_request_rate": 12.0, + "offered_request_rate_per_gpu": 6.0, + "request_count": 64, + "request_metrics_sha256": "06641f2758bd3f801fb7b9c934670bc766372ad7353f5e14aea2988dfbec462a", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 11.55742285377119, + "ttft_max_ms": 389.93612032123883, + "ttft_p50_ms": 296.08352841701094, + "ttft_p95_ms": 384.6764910356253 + }, + "status": "completed", + "trace_sha256": "cbc40c05fb9e400fa5edea6dd52c4122a957418fe523d102f1e72a784321e681" + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 9.847446918487549, + "offered_request_rate": 14.0, + "offered_request_rate_per_gpu": 7.0, + "request_count": 64, + "request_metrics_sha256": "94cf0f3c510015ba50495eb38fd6fb452dd23d16ce0c788dc926229ecd557812", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 13.374732701347012, + "ttft_max_ms": 388.3013907870674, + "ttft_p50_ms": 309.4635231444016, + "ttft_p95_ms": 386.4490667329015 + }, + "status": "completed", + "trace_sha256": "3e43801bb724218ceaf0d52f17cd4d0fb3863e70f95c4d8d5a43ce7f85c75c01" + } + ] + } + ], + "contract": { + "arrival": "open_loop_uniform", + "input_tokens": 2048, + "output_tokens": 1, + "prefix_caching": false, + "rates": [ + 10.0, + 12.0, + 14.0 + ], + "requests_per_anchor": 64, + "target_pass_rate": 0.95, + "ttft_slo_ms": 1256.0 + }, + "frontier": { + "git_head": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "git_status_short": " M frontier/config/config.py\n M frontier/entities/request.py\n M frontier/events/cluster_schedule_event.py\n M frontier/execution_time_predictor/sklearn_execution_time_predictor.py\n M frontier/metrics/constants.py\n M frontier/metrics/metrics_store.py\n M frontier/profiling/common/layers/rotary_embedding.py\n M frontier/profiling/moe/moe_impl.py\n M frontier/profiling/moe/moe_vllm_kernel.py\n M frontier/scheduler/cluster_scheduler/__init__.py\n M frontier/scheduler/cluster_scheduler/base_cluster_scheduler.py\n M frontier/scheduler/cluster_scheduler/cluster_scheduler_registry.py\n M frontier/scheduler/cluster_scheduler/sticky_lor_cluster_scheduler.py\n M frontier/scheduler/replica_scheduler/base_replica_scheduler.py\n M frontier/scheduler/replica_scheduler/vllm_v1_engine_replica_scheduler.py\n M frontier/scheduler/replica_stage_scheduler/replica_stage_schduler.py\n M frontier/simulator.py\n M frontier/types/cluster_scheduler_type.py\n?? data/profiling/compute/h20/\n?? frontier/scheduler/cluster_scheduler/prefix_lor_cluster_scheduler.py\n?? runs/\n?? tests/unit/test_attn_prefill_prediction_fallback.py\n", + "source": "/tmp/replayserve-frontier-rs1b" + }, + "profiles": { + "coverage": { + "attention": { + "1": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + }, + "2": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + }, + "4": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + } + }, + "manifest": { + "attention_tp_coverage": [ + 1, + 2, + 4 + ], + "environment_contract": { + "dtype": "bfloat16", + "frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "hardware": "NVIDIA H20", + "model": "Qwen3-30B-A3B", + "tensor_parallel_sizes": [ + 1, + 2, + 4 + ], + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + }, + "inputs": { + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931" + }, + "outputs": { + "allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3", + "attention.csv": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d", + "attention_true_mixed_fused.csv": "43ef4be90bddc9aeac6dbbe339feec24162cd1f2129a08fbd959e6ee4eaf5f60", + "linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "moe.csv": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0" + }, + "profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-fused-mixed-total-conserving", + "projection_contract": { + "allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity", + "attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target; true mixed rows use a total-conserving compatibility projection", + "attention_true_mixed": "The directly measured fused total is preserved in diagnostics. Frontier's two targets are projected by the same-TP pure prefill/decode reference ratio, with projected prefill + decode exactly equal to the fused total; the split is a schema compatibility attribution, not an observation", + "linear": "Frontier profiler using vLLM 0.20 CUDA operators", + "moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero" + }, + "row_counts": { + "allreduce": 24, + "attention_frontier_compatible": 132, + "attention_true_mixed_fused_diagnostic": 30, + "linear": 36, + "moe": 72 + }, + "schema_version": "frontier_qwen30_vllm020_frozen_profile.v2" + } + }, + "root": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2", + "sha256": { + "attention": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d", + "linear": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "manifest": "af40545e75aff55c6333cd2d5379ccf042a5a0b7d7fc7df4f745ce256cb290eb", + "moe": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0" + } + }, + "schema": "frontier-qwen30-prefill-surface-v1", + "status": "partial_not_decision_bearing" +} diff --git a/runs/frontier-phase-factorial-v0/simulator-refine-tp4/frontier_surface_frozen.json b/runs/frontier-phase-factorial-v0/simulator-refine-tp4/frontier_surface_frozen.json new file mode 100644 index 0000000..0190387 --- /dev/null +++ b/runs/frontier-phase-factorial-v0/simulator-refine-tp4/frontier_surface_frozen.json @@ -0,0 +1,461 @@ +{ + "capacity": [ + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 24.0, + "maximum_tested_feasible_request_rate_per_gpu": 6.0, + "upper_censored": false + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 24.0, + "maximum_tested_feasible_request_rate_per_gpu": 6.0, + "upper_censored": false + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 24.0, + "maximum_tested_feasible_request_rate_per_gpu": 6.0, + "upper_censored": false + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 24.0, + "maximum_tested_feasible_request_rate_per_gpu": 6.0, + "upper_censored": false + } + ], + "config_results": [ + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "loads": [ + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 10.649231672286987, + "offered_request_rate": 20.0, + "offered_request_rate_per_gpu": 5.0, + "request_count": 64, + "request_metrics_sha256": "2bf5e46241bf3462e21f707f715b4a9fe2c932b19da35b19b4a875f88605eee0", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 16.74355310200617, + "ttft_max_ms": 779.2433711652614, + "ttft_p50_ms": 476.39050138849325, + "ttft_p95_ms": 718.1022232545545 + }, + "status": "completed", + "trace_sha256": "da52e3a35a89135dd82d40415cf2dd43738eceb4183f322fcd38d96294a845d4" + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 10.245610237121582, + "offered_request_rate": 24.0, + "offered_request_rate_per_gpu": 6.0, + "request_count": 64, + "request_metrics_sha256": "79986860bea420cd6e42385521fc8377dbc237a15010a4daf014bc9a94f71aa7", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 16.93744325871021, + "ttft_max_ms": 1219.4533647214066, + "ttft_p50_ms": 708.2221064125775, + "ttft_p95_ms": 1155.549457433053 + }, + "status": "completed", + "trace_sha256": "abae88091c5b7bb5a4b7f3f6fa553e132b10e7943c0dbe18dea203ac85c289fa" + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 9.953459978103638, + "offered_request_rate": 28.0, + "offered_request_rate_per_gpu": 7.0, + "request_count": 64, + "request_metrics_sha256": "7f33dcb51a6e2a47acbc2f4ee968746f86575c0cbf62482ae08cfaf58216d459", + "score": { + "feasible": false, + "pass_rate": 0.765625, + "passed": 49, + "throughput_requests_per_second": 16.898601280793688, + "ttft_max_ms": 1587.02950504691, + "ttft_p50_ms": 885.3220562610811, + "ttft_p95_ms": 1515.6009336189102 + }, + "status": "completed", + "trace_sha256": "603b9e57429538d162af2ab5fd99c15611246ab92ab067b09acb4da3963d40e1" + } + ] + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "loads": [ + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 10.059623956680298, + "offered_request_rate": 20.0, + "offered_request_rate_per_gpu": 5.0, + "request_count": 64, + "request_metrics_sha256": "2bf5e46241bf3462e21f707f715b4a9fe2c932b19da35b19b4a875f88605eee0", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 16.74355310200617, + "ttft_max_ms": 779.2433711652614, + "ttft_p50_ms": 476.39050138849325, + "ttft_p95_ms": 718.1022232545545 + }, + "status": "completed", + "trace_sha256": "da52e3a35a89135dd82d40415cf2dd43738eceb4183f322fcd38d96294a845d4" + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 10.000702381134033, + "offered_request_rate": 24.0, + "offered_request_rate_per_gpu": 6.0, + "request_count": 64, + "request_metrics_sha256": "79986860bea420cd6e42385521fc8377dbc237a15010a4daf014bc9a94f71aa7", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 16.93744325871021, + "ttft_max_ms": 1219.4533647214066, + "ttft_p50_ms": 708.2221064125775, + "ttft_p95_ms": 1155.549457433053 + }, + "status": "completed", + "trace_sha256": "abae88091c5b7bb5a4b7f3f6fa553e132b10e7943c0dbe18dea203ac85c289fa" + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 9.852763652801514, + "offered_request_rate": 28.0, + "offered_request_rate_per_gpu": 7.0, + "request_count": 64, + "request_metrics_sha256": "7f33dcb51a6e2a47acbc2f4ee968746f86575c0cbf62482ae08cfaf58216d459", + "score": { + "feasible": false, + "pass_rate": 0.765625, + "passed": 49, + "throughput_requests_per_second": 16.898601280793688, + "ttft_max_ms": 1587.02950504691, + "ttft_p50_ms": 885.3220562610811, + "ttft_p95_ms": 1515.6009336189102 + }, + "status": "completed", + "trace_sha256": "603b9e57429538d162af2ab5fd99c15611246ab92ab067b09acb4da3963d40e1" + } + ] + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "loads": [ + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.849465370178223, + "offered_request_rate": 20.0, + "offered_request_rate_per_gpu": 5.0, + "request_count": 64, + "request_metrics_sha256": "2bf5e46241bf3462e21f707f715b4a9fe2c932b19da35b19b4a875f88605eee0", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 16.74355310200617, + "ttft_max_ms": 779.2433711652614, + "ttft_p50_ms": 476.39050138849325, + "ttft_p95_ms": 718.1022232545545 + }, + "status": "completed", + "trace_sha256": "da52e3a35a89135dd82d40415cf2dd43738eceb4183f322fcd38d96294a845d4" + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.80930495262146, + "offered_request_rate": 24.0, + "offered_request_rate_per_gpu": 6.0, + "request_count": 64, + "request_metrics_sha256": "79986860bea420cd6e42385521fc8377dbc237a15010a4daf014bc9a94f71aa7", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 16.93744325871021, + "ttft_max_ms": 1219.4533647214066, + "ttft_p50_ms": 708.2221064125775, + "ttft_p95_ms": 1155.549457433053 + }, + "status": "completed", + "trace_sha256": "abae88091c5b7bb5a4b7f3f6fa553e132b10e7943c0dbe18dea203ac85c289fa" + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.798201560974121, + "offered_request_rate": 28.0, + "offered_request_rate_per_gpu": 7.0, + "request_count": 64, + "request_metrics_sha256": "7f33dcb51a6e2a47acbc2f4ee968746f86575c0cbf62482ae08cfaf58216d459", + "score": { + "feasible": false, + "pass_rate": 0.765625, + "passed": 49, + "throughput_requests_per_second": 16.898601280793688, + "ttft_max_ms": 1587.02950504691, + "ttft_p50_ms": 885.3220562610811, + "ttft_p95_ms": 1515.6009336189102 + }, + "status": "completed", + "trace_sha256": "603b9e57429538d162af2ab5fd99c15611246ab92ab067b09acb4da3963d40e1" + } + ] + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "loads": [ + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.800058603286743, + "offered_request_rate": 20.0, + "offered_request_rate_per_gpu": 5.0, + "request_count": 64, + "request_metrics_sha256": "2bf5e46241bf3462e21f707f715b4a9fe2c932b19da35b19b4a875f88605eee0", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 16.74355310200617, + "ttft_max_ms": 779.2433711652614, + "ttft_p50_ms": 476.39050138849325, + "ttft_p95_ms": 718.1022232545545 + }, + "status": "completed", + "trace_sha256": "da52e3a35a89135dd82d40415cf2dd43738eceb4183f322fcd38d96294a845d4" + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.804483413696289, + "offered_request_rate": 24.0, + "offered_request_rate_per_gpu": 6.0, + "request_count": 64, + "request_metrics_sha256": "79986860bea420cd6e42385521fc8377dbc237a15010a4daf014bc9a94f71aa7", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 16.93744325871021, + "ttft_max_ms": 1219.4533647214066, + "ttft_p50_ms": 708.2221064125775, + "ttft_p95_ms": 1155.549457433053 + }, + "status": "completed", + "trace_sha256": "abae88091c5b7bb5a4b7f3f6fa553e132b10e7943c0dbe18dea203ac85c289fa" + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.80262541770935, + "offered_request_rate": 28.0, + "offered_request_rate_per_gpu": 7.0, + "request_count": 64, + "request_metrics_sha256": "7f33dcb51a6e2a47acbc2f4ee968746f86575c0cbf62482ae08cfaf58216d459", + "score": { + "feasible": false, + "pass_rate": 0.765625, + "passed": 49, + "throughput_requests_per_second": 16.898601280793688, + "ttft_max_ms": 1587.02950504691, + "ttft_p50_ms": 885.3220562610811, + "ttft_p95_ms": 1515.6009336189102 + }, + "status": "completed", + "trace_sha256": "603b9e57429538d162af2ab5fd99c15611246ab92ab067b09acb4da3963d40e1" + } + ] + } + ], + "contract": { + "arrival": "open_loop_uniform", + "input_tokens": 2048, + "output_tokens": 1, + "prefix_caching": false, + "rates": [ + 20.0, + 24.0, + 28.0 + ], + "requests_per_anchor": 64, + "target_pass_rate": 0.95, + "ttft_slo_ms": 1256.0 + }, + "frontier": { + "git_head": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "git_status_short": " M frontier/config/config.py\n M frontier/entities/request.py\n M frontier/events/cluster_schedule_event.py\n M frontier/execution_time_predictor/sklearn_execution_time_predictor.py\n M frontier/metrics/constants.py\n M frontier/metrics/metrics_store.py\n M frontier/profiling/common/layers/rotary_embedding.py\n M frontier/profiling/moe/moe_impl.py\n M frontier/profiling/moe/moe_vllm_kernel.py\n M frontier/scheduler/cluster_scheduler/__init__.py\n M frontier/scheduler/cluster_scheduler/base_cluster_scheduler.py\n M frontier/scheduler/cluster_scheduler/cluster_scheduler_registry.py\n M frontier/scheduler/cluster_scheduler/sticky_lor_cluster_scheduler.py\n M frontier/scheduler/replica_scheduler/base_replica_scheduler.py\n M frontier/scheduler/replica_scheduler/vllm_v1_engine_replica_scheduler.py\n M frontier/scheduler/replica_stage_scheduler/replica_stage_schduler.py\n M frontier/simulator.py\n M frontier/types/cluster_scheduler_type.py\n?? data/profiling/compute/h20/\n?? frontier/scheduler/cluster_scheduler/prefix_lor_cluster_scheduler.py\n?? runs/\n?? tests/unit/test_attn_prefill_prediction_fallback.py\n", + "source": "/tmp/replayserve-frontier-rs1b" + }, + "profiles": { + "coverage": { + "attention": { + "1": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + }, + "2": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + }, + "4": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + } + }, + "manifest": { + "attention_tp_coverage": [ + 1, + 2, + 4 + ], + "environment_contract": { + "dtype": "bfloat16", + "frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "hardware": "NVIDIA H20", + "model": "Qwen3-30B-A3B", + "tensor_parallel_sizes": [ + 1, + 2, + 4 + ], + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + }, + "inputs": { + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931" + }, + "outputs": { + "allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3", + "attention.csv": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d", + "attention_true_mixed_fused.csv": "43ef4be90bddc9aeac6dbbe339feec24162cd1f2129a08fbd959e6ee4eaf5f60", + "linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "moe.csv": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0" + }, + "profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-fused-mixed-total-conserving", + "projection_contract": { + "allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity", + "attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target; true mixed rows use a total-conserving compatibility projection", + "attention_true_mixed": "The directly measured fused total is preserved in diagnostics. Frontier's two targets are projected by the same-TP pure prefill/decode reference ratio, with projected prefill + decode exactly equal to the fused total; the split is a schema compatibility attribution, not an observation", + "linear": "Frontier profiler using vLLM 0.20 CUDA operators", + "moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero" + }, + "row_counts": { + "allreduce": 24, + "attention_frontier_compatible": 132, + "attention_true_mixed_fused_diagnostic": 30, + "linear": 36, + "moe": 72 + }, + "schema_version": "frontier_qwen30_vllm020_frozen_profile.v2" + } + }, + "root": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2", + "sha256": { + "attention": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d", + "linear": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "manifest": "af40545e75aff55c6333cd2d5379ccf042a5a0b7d7fc7df4f745ce256cb290eb", + "moe": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0" + } + }, + "schema": "frontier-qwen30-prefill-surface-v1", + "status": "partial_not_decision_bearing" +} diff --git a/runs/frontier-phase-factorial-v0/test_phase_factorial.py b/runs/frontier-phase-factorial-v0/test_phase_factorial.py index 6e30aa5..4172926 100644 --- a/runs/frontier-phase-factorial-v0/test_phase_factorial.py +++ b/runs/frontier-phase-factorial-v0/test_phase_factorial.py @@ -37,3 +37,9 @@ def test_grid_and_trace(tmp_path: Path) -> None: assert len(lines) == 4 assert lines[1].split(",")[:3] == ["0.000000000000", "2048", "1"] assert lines[3].split(",")[:3] == ["0.500000000000", "2048", "1"] + + +def test_kendall_tau_b() -> None: + analysis = load("analyze_qwen30_prefill_fidelity.py") + assert analysis.kendall_tau_b([1, 2, 3], [1, 2, 3])["kendall_tau_b"] == 1 + assert analysis.kendall_tau_b([1, 2, 3], [3, 2, 1])["kendall_tau_b"] == -1 diff --git a/simulator-fidelity.md b/simulator-fidelity.md index 728ce14..b646d5b 100644 --- a/simulator-fidelity.md +++ b/simulator-fidelity.md @@ -1,6 +1,6 @@ # Frontier simulator fidelity：以 config ranking 为目标的阶段性评测 -更新日期：2026-07-16。 +更新日期：2026-07-17。 统一实验平台：所有新增与重跑实验只使用 `dash0` 的 8×NVIDIA H20。Qwen30B 的真实 P1 artifacts 也来自 `dash0`；早期文档中的主机 provenance 标注错误，本版已按实验主机和远端 artifact 路径更正。 @@ -14,11 +14,12 @@ - 将 operator profile 更新为与真实 serving 一致的 community vLLM `0.20.0`、BF16、H20、TP1/2/4 栈后，新的 profile-only Frontier **没有恢复排序**：92 个真实 anchors 在 simulator 中全部 SLO-infeasible，12 个 config 因而全部并列，最坏 tie-break regret 为 `60.91%`。这否证了“旧 profile 版本不一致是主要原因”这一简单解释，并把问题收敛到 execution context、operator composition 与 mixed-state schema。 - 在 Qwen3-235B-A22B-FP8 prefill-only 上，补齐 FP8/MoE profile 与 serving semantics、但不做端到端 action calibration 后，Frontier 的最优集合与真机完全一致，Spearman ρ 为 `0.9487`，20/20 个可比较非 tie config pair 同序，选择 regret 为 `0`。 - 在 Qwen3-235B-A22B-FP8 fixed-shape mixed 上，Frontier 同样给出了与真机完全相同的 TP4 top set，Kendall τ-b=`0.8944`、worst tie-break regret=`0`。但这次成功掩盖了一个明确的机制错误：Frontier 认为同一 TP family 内四个 MNS/MBT config 完全等价，真机 TP8 capacity 却形成对角为 0.30、非对角为 0.20 req/s/GPU 的 checkerboard interaction。34 个 config-load labels 中有 10 个 false-infeasible；20 个 real non-tie pairs 中只保持 16 个方向。 -- 现有结果表明：**在已对齐的受控 case 中，绝对 capacity 有 gap、甚至 action differential 错误，都不必然妨碍 Frontier 找到当前 surface 的最优 config。** 统一 dash0 campaign 已完整覆盖 Qwen235 FP8 prefill-only 与 fixed-shape mixed，但仍不能外推到 trace-faithful mixed 或 decode-only，也不能忽略达到这种 fidelity 所需的真机 profile、KV capacity、runtime-semantic patch 或 calibration 成本。 +- 新增的 Qwen3-30B-A3B BF16 prefill-only 对照在没有 decode、prefix reuse 和 true-mixed attention 的情况下仍然失败：真机 top set 是四个 TP4 config，Frontier top set 却是全部 TP1/TP2 config，top set 无交集，worst regret=`12.5%`、Kendall τ-b=`-1.0`，32 个 real non-tie pairs 全部反向。这直接否证了“prefill-only 是 simulator ranking 充分容易条件”。 +- 现有结果更支持一个 **margin-aware fidelity** 解释：action-differential residual 可以很大，但只有当它足以穿过 real decision margin 时才会导致选错。Qwen235 mixed 中真机 TP4 相对最好 TP8 的 margin 是 `2×`，足以掩盖同 TP family 内的机制错误；Qwen30 prefill-only 中 TP scaling 差分更细，Frontier 的 residual 直接反转了 topology ordering。模型大小或 execution phase 单独都不足以解释现有结果。 因此，现阶段最准确的 research statement 不是“simulator 排序一定错误”，而是： -> Simulator 的 usefulness 取决于 compatibility envelope。问题是需要多少、什么类型的真机信息，才能使其 config ranking 可被信任；以及当 workload、runtime 或 execution topology 变化时，如何低成本判断该 envelope 是否仍成立。 +> Simulator 的 usefulness 取决于 action-differential residual 是否小于真实 decision margin。问题不是简单区分 prefill 与 mixed，而是定位哪些 scheduler-state-conditioned execution residual 会随 TP、batch composition 和负载被放大，并用最少的真机信息判断 ranking 是否足够可信。 ## 评测口径 @@ -89,6 +90,22 @@ Frontier 原生 profile schema 可以按 `num_tensor_parallel_workers` 选择 TP 该 case 是有意设计的 prefill mechanism-isolation workload：output 固定为 1，prefix cache 关闭，并使用固定 64-request cohort。它不保持原 trace 的 output-length distribution 和 prefix reuse，因此不能被称为 trace-faithful workload，也不能代表 mixed serving。 +### Case D：Qwen3-30B-A3B BF16 prefill-only + +| 项目 | 设置 | +|---|---| +| 真机 | `dash0`，8×NVIDIA H20；每个 job 按 TP 独占 GPU | +| model/runtime | `Qwen/Qwen3-30B-A3B`；community vLLM `0.20.0+cu129`（source `88d34c6409e9fb3c7b8ca0c04756f061d2099eb1`），BF16 weights/activation/KV，FA3，runtime 默认 CUDA graph | +| workload | 64 个不同 token-chain prompts；ISL=2,048、OSL=1、uniform open-loop QPS、prefix cache off | +| config surface | `TP∈{1,2,4} × MNS∈{8,16,32,64}`，MBT=8,192，chunked prefill on，共 12 cells | +| simulator | Frontier commit `d9cfeb6d8791fbf2f295dd9744c56a666171776e` + 当前 compatibility patches + vLLM 0.20 same-stack `profile-v2`；无 E2E calibration、decode graph=`none`、analytical collective | +| load lattice | base system rates `{4,8,16,32,64}` req/s；事先声明的 common refinement 为 `{5,6,7}` req/s/GPU | +| real anchor | 每个 `(config,rate,round)` fresh server；两轮独立运行且第二轮反转顺序；两轮都 pass 才算 feasible | +| SLO | 至少 61/64 requests 满足 TTFT≤1,256 ms | +| primary objective | 最大被测试可行 offered req/s / 实际 TP GPUs | + +这个 case 是对“prefill-only 是 simulator 的容易区间”的直接可证伪测试。它刻意去掉 decode、true-mixed batch、prefix reuse 和 KV-residency feedback，但保留 TP 改变下的 local shard、collective、batching 与 scheduler queue。Frontier surface 在查看真机 ranking 前冻结，没有使用本 surface 的 serving latency 拟合 scale。 + ## Workload fidelity 修正 `thinking_w20260327_1000` 的 600 秒原始窗口包含 15,479 requests，字段包括 exact prompt、arrival、`input_length`、`output_length`、session parent/turn 和 block-size=64 的 `hash_ids`。只读审计得到： @@ -126,10 +143,11 @@ trace 的 source hash block size 为 64，而 community vLLM 0.10.2 的 CUDA KV | Qwen30 mixed / old profile-only | 12 | TP2,MNS32 | TP4,MNS32/64 | miss | τ-b=0.000；exact sign=37.88% | 25.63% | 排序错误 | | Qwen30 mixed / vLLM 0.20 profile-only | 12 | TP2,MNS32 | 全部 12 configs | 不可辨识 | τ-b=0.000；exact sign=7.58% | 60.91% | 同栈 raw profile 仍不足 | | Qwen30 mixed / frozen per-TP calibration | 12 | TP2,MNS32 | TP2,MNS32/64 | hit，非 exact | τ-b=0.9668；exact sign=93.94% | 0.76% | dash0 | +| Qwen30 BF16 prefill-only / vLLM 0.20 profile-only | 12 | 全部 TP4 configs | 全部 TP1/TP2 configs | miss，无交集 | τ-b=-1.0000；real non-tie=0/32 | 12.50% | topology order 反转 | | Qwen235 FP8 prefill / best-effort | 8 | TP4,MBT16K,MNS64/128 | 与 real 完全相同 | exact match | ρ=0.9487；non-tied 20/20 | 0 | 足以选最优 config | | Qwen235 FP8 fixed-shape mixed / frozen full profile | 8 | 四个 TP4 configs | 与 real 完全相同 | exact match | τ-b=0.8944；exact sign=24/28；real non-tie=16/20 | 0 | 选对 topology；漏掉 TP8 MNS×MBT interaction | -最重要的区别是：Qwen30 的 high-fidelity 结果依赖 per-TP E2E calibration；Qwen235 的 high-fidelity 结果不依赖本 case 的 E2E calibration，但依赖同 runtime/hardware 的 operator profile、真实 KV capacity 和多处 compatibility fixes。二者都不能表述为“拿 stock Frontier 零成本预测即可”。 +最重要的区别是：Qwen30 的 high-fidelity mixed 结果依赖 per-TP E2E calibration，而新 prefill-only profile-only 实验证明“去掉 decode/mixed 状态”也不足以恢复 ranking。Qwen235 的 high-fidelity 结果不依赖本 case 的 E2E calibration，但依赖同 runtime/hardware 的 operator profile、真实 KV capacity 和多处 compatibility fixes。两个 model 的 stack 和精度不同，所以不能把差异直接归因为 model size；二者也都不能表述为“拿 stock Frontier 零成本预测即可”。 ## Case A baseline 结果：Qwen30 mixed serving @@ -311,6 +329,53 @@ MNS 128 fail pass 原预注册的 TPOT 40 ms 因最低负载已不可行而失去 capacity-ranking 可辨识性；本文已把 150 ms 明确记录为 post-pilot protocol amendment，并保留 40 ms 全部失败的 sensitivity 结果，没有把 SLO change 隐藏成预注册结论。 +## Case D 结果：Qwen30 BF16 prefill-only + +![Qwen3-30B BF16 prefill-only simulator-vs-real config ranking](docs/assets/simulator-fidelity/qwen30-prefill-ranking.png) + +结果不是“绝对 capacity 有偏差但 rank 可用”，而是完整的 topology ordering reversal： + +| TP | MNS | Real | Frontier profile-only | 差异 | +|---:|---|---:|---:|---:| +| 1 | 8/16/32/64 | 7.0 | **8.0** | +1.0 | +| 2 | 8/16/32/64 | 7.0 | **8.0** | +1.0 | +| 4 | 8/16/32/64 | **8.0** | 6.0 | -2.0 | + +单位为最大被测试 SLO-feasible req/s/GPU。四个 TP4 configs 是真机 top set；Frontier 却把全部八个 TP1/TP2 configs 判为 top set，两者无交集。部署任一 simulator top config 都只能在真机上得到 7 req/s/GPU，因此 optimistic 与 worst tie-break regret 都为 `1-7/8=12.5%`。Kendall τ-b=`-1.0`；32 个 real non-tie pairs 中 0 个同序。96 个 config-load decisions 中 80 个一致，但同时有 8 个 false-feasible 和 8 个 false-infeasible，它们刚好跨过了不同 TP family 的 capacity boundary。 + +### 错误从哪里开始放大 + +代表性 `MNS=8` anchors 表明，低负载单次 execution time 并没有数倍偏差，错误主要在接近饱和时被 queue 放大： + +| Config / system rate | Real TTFT p95（两轮） | Frontier TTFT p95 | Real / sim SLO | +|---|---:|---:|---| +| TP1 @ 4 req/s | 154.1 / 153.6 ms | 171.6 ms | pass / pass | +| TP1 @ 8 req/s | 1289.2 / 1254.4 ms | 516.4 ms | fail / pass | +| TP2 @ 8 req/s | 97.7 / 93.8 ms | 122.2 ms | pass / pass | +| TP2 @ 16 req/s | 1389.9 / 1365.0 ms | 962.8 ms | fail / pass | +| TP4 @ 4 req/s | 82.3 / 69.6 ms | 93.1 ms | pass / pass | +| TP4 @ 32 req/s | 1136.6 / 1134.0 ms | 1787.9 ms | pass / fail | + +这排除了“只要给所有 kernel 乘一个 global scale 就能修好”的解释：相同 stack 在低负载下接近，但在不同 TP 的饱和边界上向相反方向偏移。更符合数据的抽象是：小的 per-step composition residual 改变 service rate，再通过 scheduler queue 的非线性反馈放大成 TTFT 和 capacity 边界错误。 + +当前证据可以排除一些原因，但还不能唯一定位根因： + +1. **Decode/true-mixed schema 不是必要条件。** 这个 workload 没有 decode、FULL decode CUDA graph、fused true-mixed attention 或 initial-KV state，排序仍然失败。这些机制可能在 mixed/decode 中进一步增大误差，但不是本次失败的前提。 +2. **Communication 可能解释部分 TP scaling，但不能单独解释全部误差。** `profile-v2/allreduce.json` 已有 24 个 TP2/TP4 实测 rows，但 base Frontier 为了保持历史对照使用 analytical 600-Gbps/1-µs collective，没有注入这些数据。下一步必须单独做 measured-collective ablation；但 TP1 无 collective 仍在 8 req/s 出现 773 ms 的 p95 低估，所以 collective 不会是唯一根因。 +3. **Profile 覆盖了 token count，却未必覆盖 scheduler 实际产生的 batch composition。** linear/MoE rows 覆盖到 8,192 tokens，但 exact-2,048 pure-prefill attention 在每个 TP 下只有 batch=1 的直接样本；实际 MBT=8,192 可以产生多个 2,048-token sequence 的组合。这是一个需要补测的 coverage hypothesis，不是已证明根因。 +4. **Routing、step composition 和 scheduler batch state 仍然是联合候选。** pure-prefill 只排除了 phase mixing，没有排除 MoE routing、TP-specific local shape、launch/fusion 和持续负载下 batch/queue trajectory 的耦合。 + +### 为什么 235B 能选对，30B 却选错 + +现有数据不支持“235B 大所以 simulator 更容易”这种模型大小因果。两个 case 的 precision、vLLM 版本、attention backend、graph mode、EP 和 profile closure 都不同。能被数据直接支持的区别是 **decision margin**： + +- 235B mixed 真机中 TP4 最优 capacity 为 0.60 req/s/GPU，最好 TP8 为 0.30，有 `2×` margin；Frontier 虽然漏掉 TP8 的 MNS×MBT interaction，仍预测 TP4/TP8 为 0.40/0.15，所以 residual 没有穿过全局拓扑边界。 +- 30B prefill-only 真机 TP4 相对 TP1/TP2 只有 8 vs. 7 req/s/GPU 的 margin。Frontier 预测为 6 vs. 8，TP-dependent residual 大于真实 margin，因而把最优 topology 完整反转。 + +因此，“prefill-only 容易，decode/mixed 困难”的强假设已被否证。phase 仍可能改变 residual 的幅度，但它不是 fidelity 的充分条件。按预注册 decision rule，此时不应继续无区分度地扩展更多 phase cases，而应在 Qwen30 同一 model/workload 上按单变量顺序做：`measured collective injection` → `batch-composition-conditioned pure-prefill attention/step profile` → 在 TP1@8、TP2@16、TP4@32 对齐 real/sim scheduler batch、queue 与 per-step critical path → 最后再测 routing/graph。 + +本 case 接受的 ground truth 共 24 个 fleet jobs、192 个 fresh-server anchors、12,288 个 measured requests 和 4,512 个 warmup requests，消耗 12.07 H20-GPU-hours。运行中曾因 fleet controller 在 fresh-server 空窗误判 GPU 为空闲而产生重叠 launch；这些 attempts 未被合并到 accepted artifact root，已整体隔离并用独立 queue state 重跑。最终 comparison SHA256 为 `c9a9cac9f60c7be804d1cb9466c455f8fe9e3a8dc60b9cec3329bde6a8c19334`。 + ## 尚不能纳入 simulator ranking 的 case ### 原 internal-runtime Qwen235 prefill-only @@ -328,11 +393,12 @@ MNS 128 fail pass 当前证据的结论是： 1. **绝对 gap 不是否决 simulator 的理由。** Qwen235 中 11%--50% 的 capacity 低估仍可保持 zero-regret config selection。 -2. **但 rank 成功不能证明模型机制正确。** Qwen235 prefill 的 TP8 MBT differential 错误，fixed-shape mixed 又漏掉 TP8 的 MNS×MBT 非加性交互；Qwen30 还有 28/92 anchor SLO labels 错误。只是当前 decision margin 足以容忍这些 residual。 -3. **alignment 是结果的一部分。** Qwen30 需要 per-TP E2E calibration 才能恢复排序；Qwen235 不需要本 case E2E calibration，但需要同栈 FP8 profiles、真机 KV capacity 和 simulator patches。论文必须报告这部分真机成本，不能把它隐藏在“offline profiles”中。 -4. **目前不能建立“simulator 全局选优普遍失败”的 premise。** 受控 prefill、fixed-shape mixed 和历史 mixed surface 上，经过 alignment 的 Frontier 已经能选到最优或 0.76% 内的近最优 config。更有证据的 research premise 是：aggregate selection success 会掩盖 action interaction 与 state-transition model 的错误；若要声称全局选优失败，仍必须在 dash0 的 trace-faithful mixed、decode-only、prefix cache 或跨 topology case 上得到稳定反例。 +2. **rank 成功不能证明模型机制正确，rank 失败则表明 residual 穿过了 decision margin。** Qwen235 prefill 的 TP8 MBT differential 错误，fixed-shape mixed 又漏掉 TP8 的 MNS×MBT 非加性交互，但大 topology margin 保住了 top set；Qwen30 prefill-only 的较小 margin 被 TP-dependent saturation residual 穿过，导致 τ-b=-1 和 12.5% regret。 +3. **prefill-only 不是 fidelity 的充分容易条件。** Qwen30 在没有 decode、prefix reuse、initial KV 和 true-mixed batch 时仍选错 topology。phase 可以改变 residual，但不能单独作为 compatibility-envelope 边界。 +4. **alignment 是结果的一部分。** Qwen30 mixed 需要 per-TP E2E calibration 才能恢复排序，同栈 operator profile 或去掉 mixed phase 都不足；Qwen235 不需要本 case E2E calibration，但需要同栈 FP8 profiles、真机 KV capacity 和 simulator patches。论文必须报告这部分真机成本，不能把它隐藏在“offline profiles”中。 +5. **现在有了可复现的全局选优反例，但尚不足以声称 simulator 普遍失败。** Qwen30 BF16 prefill-only 是一个无 top-set overlap 的稳定反例；Qwen235 两个 case 又证明 Frontier 在某些 envelope 内足以选优。更有研究价值的 premise 是：**什么 state-conditioned action residual 决定 ranking 是否能跨过 real margin，以及如何在不做全 surface 真机 sweep 的情况下检测这个风险。** -后续每个 case 建议使用同一 gate：worst selected-config regret ≤5%、tie-aware rank correlation ≥0.8、足够数量的 informative pairs、ground-truth bracket 不足以反转最优决策，并单独报告达到该结果所需的 real-GPU profiling/calibration cost。 +后续每个 case 建议使用同一 gate：worst selected-config regret ≤5%、tie-aware rank correlation ≥0.8、足够数量的 informative pairs、ground-truth bracket 不足以反转最优决策，并单独报告达到该结果所需的 real-GPU profiling/calibration cost。当前下一 gate 是在 Qwen30 prefill-only 上做同模型单变量 context ablation，而不是继续扩展跨模型 phase 矩阵。 ## 数据与复现 @@ -340,6 +406,10 @@ MNS 128 fail pass - 画图脚本：[plot_simulator_fidelity.py](scripts/plot_simulator_fidelity.py) - Qwen30 audit：[report.md](runs/frontier-multicase-sufficiency-v0/results/qwen30-baseline/report.md) - Qwen30 aligned metrics：[metrics.json](runs/frontier-slo-alignment-v0/results/metrics.json) +- Qwen30 prefill-only experiment card：[experiment-card.md](runs/frontier-phase-factorial-v0/experiment-card.md) +- Qwen30 prefill-only comparison：[comparison.json](runs/frontier-phase-factorial-v0/results/final/comparison.json) +- Qwen30 prefill-only capacity table：[capacity.csv](runs/frontier-phase-factorial-v0/results/final/capacity.csv) +- Qwen30 prefill-only analyzer：[analyze_qwen30_prefill_fidelity.py](runs/frontier-phase-factorial-v0/analyze_qwen30_prefill_fidelity.py) - Qwen235 fixed-cohort comparison：[v2_refined_comparison.json](runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/v2_refined_comparison.json) - Qwen235 full report：[report.md](runs/frontier-multicase-sufficiency-v0/best_effort/fixed_cohort_evidence/report.md) - Fixed-shape mixed comparison：[comparison.json](runs/frontier-multicase-sufficiency-v1/results/t0-final/comparison.json) @@ -357,4 +427,17 @@ MNS 128 fail pass ```bash python3 scripts/plot_simulator_fidelity.py + +python3 runs/frontier-phase-factorial-v0/analyze_qwen30_prefill_fidelity.py \ + --fleet-artifacts runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive \ + --simulator-manifest runs/frontier-phase-factorial-v0/simulator-tp1/frontier_surface_frozen.json \ + --simulator-manifest runs/frontier-phase-factorial-v0/simulator-tp2/frontier_surface_frozen.json \ + --simulator-manifest runs/frontier-phase-factorial-v0/simulator-tp4/frontier_surface_frozen.json \ + --simulator-manifest runs/frontier-phase-factorial-v0/simulator-refine-tp1/frontier_surface_frozen.json \ + --simulator-manifest runs/frontier-phase-factorial-v0/simulator-refine-tp2/frontier_surface_frozen.json \ + --simulator-manifest runs/frontier-phase-factorial-v0/simulator-refine-tp4/frontier_surface_frozen.json \ + --output-root runs/frontier-phase-factorial-v0/results/final + +cp runs/frontier-phase-factorial-v0/results/final/qwen30-prefill-ranking.png \ + docs/assets/simulator-fidelity/qwen30-prefill-ranking.png ``` From 95f4af3d996b75b715a73eb864d80361d72652df Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 09:44:49 +0800 Subject: [PATCH 096/148] Add Frontier fidelity envelope campaign --- runs/frontier-fidelity-envelope-v1/.gitignore | 8 ++ .../experiment-card.md | 76 ++++++++++++ runs/frontier-fidelity-envelope-v1/fleet.toml | 24 ++++ .../jobs_attention_composition.toml | 43 +++++++ .../materialize_frontier_allreduce.py | 115 ++++++++++++++++++ .../mock-fidelity-envelope.png | Bin 0 -> 133230 bytes ...vidur-large-payload-model-prediction.patch | 84 +++++++++++++ .../plot_mock_envelope.py | 75 ++++++++++++ .../profiles/measured-allreduce.csv | 25 ++++ .../profiles/measured-allreduce.manifest.json | 15 +++ .../remote-sync-marker/README.md | 2 + .../run_flashattn_composition.sh | 65 ++++++++++ .../smoke-report.md | 15 +++ .../test_fidelity_envelope.py | 69 +++++++++++ .../run_frontier_qwen30_prefill_surface.py | 54 ++++++++ .../test_phase_factorial.py | 17 +++ .../freeze_frontier_profiles.py | 2 +- 17 files changed, 688 insertions(+), 1 deletion(-) create mode 100644 runs/frontier-fidelity-envelope-v1/.gitignore create mode 100644 runs/frontier-fidelity-envelope-v1/experiment-card.md create mode 100644 runs/frontier-fidelity-envelope-v1/fleet.toml create mode 100644 runs/frontier-fidelity-envelope-v1/jobs_attention_composition.toml create mode 100644 runs/frontier-fidelity-envelope-v1/materialize_frontier_allreduce.py create mode 100644 runs/frontier-fidelity-envelope-v1/mock-fidelity-envelope.png create mode 100644 runs/frontier-fidelity-envelope-v1/patches/0001-vidur-large-payload-model-prediction.patch create mode 100644 runs/frontier-fidelity-envelope-v1/plot_mock_envelope.py create mode 100644 runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv create mode 100644 runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.manifest.json create mode 100644 runs/frontier-fidelity-envelope-v1/remote-sync-marker/README.md create mode 100644 runs/frontier-fidelity-envelope-v1/run_flashattn_composition.sh create mode 100644 runs/frontier-fidelity-envelope-v1/smoke-report.md create mode 100644 runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py diff --git a/runs/frontier-fidelity-envelope-v1/.gitignore b/runs/frontier-fidelity-envelope-v1/.gitignore new file mode 100644 index 0000000..008f888 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/.gitignore @@ -0,0 +1,8 @@ +__pycache__/ +a1-native-smoke/ +a2-measured-fix-smoke/ +simulator-a1/ +simulator-a2/ +simulator-a3/ +fleet-state/ +fleet-artifacts/ diff --git a/runs/frontier-fidelity-envelope-v1/experiment-card.md b/runs/frontier-fidelity-envelope-v1/experiment-card.md new file mode 100644 index 0000000..c1563db --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/experiment-card.md @@ -0,0 +1,76 @@ +# EXP-SIMFID-ENVELOPE-V1：Frontier best-effort fidelity envelope + +> **状态：** 已批准，准备执行（2026-07-17）。用户要求先把 simulator 现有能力跑到最好，并同时覆盖 fixed input/output 与真实 trace replay。 + +## Claim 与可证伪假设 + +- **研究问题：** 在不使用被评测 config/workload 的 serving E2E calibration 时，Frontier 的 measured operator/collective profiles 与 scheduler state abstraction，是否足以找到真机上的低-regret config？ +- **H-CC：** Qwen30 prefill-only 的 TP 排序错误主要来自默认 analytical all-reduce；注入同机、同 TP 的 measured collective 后，已知 `2048/1` surface 的 regret 降至不超过 5%，Kendall tau-b 升至至少 0.8。 +- **H-BATCH：** 若 H-CC 不足，错误主要来自 pure-prefill attention 只有 batch=1 profile，而 Frontier 在多请求 batch 上使用没有 coverage 的 `attn_prefill_mixed` 外推；增加 MBT 可达的真实 batch composition 后可恢复排序。 +- **H-STATE：** 若 measured collective 与 batch-composition profile 都不能恢复排序，则缺失量位于 isolated operators 之外的 scheduler-state-conditioned step composition；继续增加静态 kernel rows不是有效修复。 +- **成功门槛：** worst selected-config regret `<=5%`、tie-aware Kendall tau-b `>=0.8`、真机 capacity bracket 不足以反转 top decision，并且没有使用同一 surface 的 E2E scalar calibration。 + +## Simulator ablation（先用已有 ground truth，零新增 GPU 成本） + +| variant | compute profile | collective | 目的 | +|---|---|---|---| +| A0 | vLLM 0.20 frozen profile-v2 | Frontier analytical | 已冻结 baseline | +| A1 | 同 A0 | Frontier 原生 Vidur + measured TP2/TP4 CSV | 检查原生 profile consumption；大 payload fallback 保留并计数 | +| A2 | 同 A0 | measured Vidur，cache miss 直接调用已训练 estimator | 最小 correctness fix；消除 `>100k elements` 静默 analytical fallback | +| A3 | 增加 pure-prefill batch-composition rows | 同 A2 | 检验 batch-composition coverage 是否是剩余误差来源 | + +A1/A2/A3 都重新运行完整 `TP∈{1,2,4} × MNS∈{8,16,32,64}` surface；TP1 无 all-reduce。所有 simulator variants 在查看新增真机 case 前冻结。A2 是单独标注、带单测的 compatibility patch，不与 Frontier upstream 原生能力混写。 + +## Workload matrix + +| ID | workload | arrival / prefix | phase role | +|---|---|---|---| +| F0 | fixed `ISL=2048, OSL=1` | uniform QPS；distinct prefixes | 已有 real ground truth，选择 A0--A3 | +| F1 | fixed `ISL=512, OSL=1` | uniform QPS；distinct prefixes | short-prefill、多请求 batch composition | +| F2 | fixed `ISL=2048, OSL=128` | uniform QPS；distinct prefixes | true prefill+decode mixed serving | +| T1 | `thinking_w20260327_1000` eligible trace | 原 timestamp/order；exact prompt/output/session/hash；prefix on | production joint distribution 与 cache/scheduler feedback | + +T1 只排除已经审计的 72 个超 40,960 context rows 和 6 个 zero-output rows，eligible universe 为 `15,401/15,479`。负载轴只使用 trace 已有、同 session 共享的 `sampling_u`；入选 request 的 arrival、input/output、prompt、hash 和相对次序不变。真实 runtime 设置 `min_tokens=max_tokens=output_length` 且 `ignore_eos=true`，逐请求核对 usage。 + +## 固定系统与 config surface + +| 项目 | 冻结设置 | +|---|---| +| machine | 仅 `dash0`，8×NVIDIA H20 | +| model/runtime | `/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B`；community vLLM 0.20.0+cu129；BF16 weight/activation/KV | +| simulator | Frontier `d9cfeb6d8791fbf2f295dd9744c56a666171776e` + manifest 中列出的现有 compatibility patches；A2 patch 独立 hash | +| configs | `TP∈{1,2,4} × MNS∈{8,16,32,64}`；DP=PP=EP=1；MBT=8192；block=16 | +| runtime | chunked prefill on；fixed cases prefix off；T1 prefix on；fresh server per `(config, load, round)` | +| score | `capacity(c)=max tested offered req/s with joint SLO pass rate >=0.95`；primary `capacity/actual TP GPUs` | +| SLO | TTFT `<=1000ms + 1000×ISL/8000`；mixed case同时要求 TPOT `<=150ms`；另报告 50/100/180ms sensitivity，不用 sensitivity 改写 primary | + +F0 沿用已经冻结的 rate lattice与两个 fresh-server rounds。F1/F2 先由冻结 simulator 给出 boundary，再加入共同 per-GPU guard anchors，避免只测 simulator 预测附近而漏掉真实最优。每个 boundary anchor 两个 fresh-server rounds，二者都 pass 才算 feasible。 + +T1 保持原 600 秒 arrival window。先在 simulator 上冻结 `sampling_u` bracket；真机只运行 topological guard set `{TP1,TP2,TP4} × {MNS8,MNS32,MNS64}`，若 top set 或 bracket 仍可能被未测 MNS 反转再补 MNS16。每个入选 source-row vector在 real/sim 两侧必须有相同 digest；至少两个 session-hash folds，若本轮只完成一个 window则明确标为 single-window evidence。 + +## 诊断与停止规则 + +1. A1 必须报告每次 collective prediction 的 measured-model hit 与 analytical fallback 次数；不能只看最终 rank。 +2. A2 对超过 100k elements 的 payload 必须由单测证明走 estimator；A2 若不改变任何 TP2/TP4 step，立即停止并检查 CLI/config 注入，不进入 GPU。 +3. A3 profile 只覆盖 MBT=8192 可达的 pure-prefill composition：F0 为 `1/2/4 × q2048`，F1 为 `1/2/4/8/16 × q512`，TP1/2/4 分别实测；不做无边界的 profile sweep。 +4. 若 A3 在 F0 仍不能达到 fidelity gate，先采集 `TP1@8, TP2@16, TP4@32` 的 per-step batch/queue/component residual；禁止用 per-TP E2E scale把答案拟合正确。 +5. 只有 best-effort simulator 在 F0 通过或形成可解释、可定位的失败后，才运行 F1/F2/T1 真机；任一 case 的结论不外推到其它 workload。 + +## 预期成本与产物 + +- simulator A1--A3：CPU only，约 1--3 小时总 CPU wall，0 GPU-hour。 +- attention composition profile：3 张 H20 并行，预计 5--10 分钟，`<0.5 H20-GPU-hour`。 +- F1/F2 real boundary：预计合计 12--24 H20-GPU-hours，smoke 后再锁定。 +- T1 real boundary：600 秒原 arrival window使单 anchor较贵；预计 30--60 H20-GPU-hours，必须在 simulator bracket 和一配置 smoke 后重新 echo 精确预算。 +- 产物：variant/profile manifests、full surfaces、anchor-level request metrics、rank/regret/confusion tables、profile-consumption counters，以及 fixed-vs-trace fidelity summary figure。 + +## Benchmark design audit + +| 风险 | 处理 | +|---|---| +| selective benchmarking | 预先冻结 F0/F1/F2/T1，不因结果删除失败 case | +| calibration=evaluation | 禁止使用同一 surface 的 serving E2E scalar；microprofile GPU 成本单独报告 | +| trace filtering | 只做 context/zero-output correctness exclusion和 session-coherent thinning，不按长度筛选 | +| simulator-guided real sampling | 使用共同 guard anchors；未闭合 bracket 不能宣布 top match | +| absolute-vs-rank metric | 同时报绝对 capacity/latency、rank、regret、tau-b、pair direction和 SLO confusion | +| hidden fallback | A1/A2 强制计数 measured-model hit/fallback，并写入 frozen manifest | diff --git a/runs/frontier-fidelity-envelope-v1/fleet.toml b/runs/frontier-fidelity-envelope-v1/fleet.toml new file mode 100644 index 0000000..612e4f7 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/fleet.toml @@ -0,0 +1,24 @@ +version = 1 + +[paths] +state_dir = "runs/frontier-fidelity-envelope-v1/fleet-state" +artifacts_dir = "runs/frontier-fidelity-envelope-v1/fleet-artifacts" + +[ssh] +connect_timeout_sec = 10 + +[scheduler] +gpu_free_memory_mb = 1024 +gpu_free_utilization_pct = 10 +prefer_pack = true + +[sync] +mode = "scp" +local_path = "runs/frontier-fidelity-envelope-v1/remote-sync-marker" + +[[hosts]] +name = "dash0" +ssh_alias = "dash0" +enabled = true +sync_remote_path = "/home/admin/cpfs/wjh/aituner/fidelity-envelope-sync-marker" +fleet_root = "/home/admin/cpfs/wjh/aituner/gpu-fleet-fidelity-envelope-v1" diff --git a/runs/frontier-fidelity-envelope-v1/jobs_attention_composition.toml b/runs/frontier-fidelity-envelope-v1/jobs_attention_composition.toml new file mode 100644 index 0000000..7e2aa3b --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/jobs_attention_composition.toml @@ -0,0 +1,43 @@ +version = 1 + +[[jobs]] +name = "qwen30-attention-composition-tp1-20260717-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-fidelity-envelope-v1 && timeout --signal=TERM --kill-after=30s 900 bash run_flashattn_composition.sh" +artifacts = ["artifacts/attention-composition-tp1-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +TP = "1" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-fidelity-envelope-v1/artifacts/attention-composition-tp1-v1" + +[[jobs]] +name = "qwen30-attention-composition-tp2-20260717-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-fidelity-envelope-v1 && timeout --signal=TERM --kill-after=30s 900 bash run_flashattn_composition.sh" +artifacts = ["artifacts/attention-composition-tp2-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +TP = "2" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-fidelity-envelope-v1/artifacts/attention-composition-tp2-v1" + +[[jobs]] +name = "qwen30-attention-composition-tp4-20260717-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-fidelity-envelope-v1 && timeout --signal=TERM --kill-after=30s 900 bash run_flashattn_composition.sh" +artifacts = ["artifacts/attention-composition-tp4-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +TP = "4" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-fidelity-envelope-v1/artifacts/attention-composition-tp4-v1" diff --git a/runs/frontier-fidelity-envelope-v1/materialize_frontier_allreduce.py b/runs/frontier-fidelity-envelope-v1/materialize_frontier_allreduce.py new file mode 100644 index 0000000..bb57bc0 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/materialize_frontier_allreduce.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Convert frozen vLLM collective measurements to Frontier Vidur CC CSV.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +from pathlib import Path + + +FIELDS = ( + "time_stats.all_reduce.min", + "time_stats.all_reduce.max", + "time_stats.all_reduce.mean", + "time_stats.all_reduce.median", + "time_stats.all_reduce.std", + "rank", + "num_workers", + "size", + "collective", + "devices_per_node", + "max_devices_per_node", +) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def convert(input_path: Path, output_path: Path) -> dict[str, object]: + payload = json.loads(input_path.read_text()) + if payload.get("schema_version") != "qwen30_vllm020_allreduce_frozen.v1": + raise ValueError(f"unexpected input schema: {payload.get('schema_version')!r}") + + rows = [] + seen = set() + for source in payload["rows"]: + tp = int(source["tensor_parallel_size"]) + tokens = int(source["num_tokens"]) + key = (tp, tokens) + if key in seen: + raise ValueError(f"duplicate collective row: {key}") + seen.add(key) + if tp not in (2, 4): + raise ValueError(f"unsupported TP: {tp}") + expected_bytes = tokens * int(source["hidden_dim"]) * 2 + if int(source["payload_bytes"]) != expected_bytes: + raise ValueError(f"payload mismatch for {key}") + + # Frontier Vidur consumes only the median target. The raw profiler kept + # per-rank distributions but not aligned per-repeat critical-path + # samples, so do not invent critical-path min/mean/max/std. Repeating + # the observed critical-path median in the unused fields keeps the CSV + # schema explicit without changing the trained target. + median = float(source["critical_path_median_ms"]) + rows.append( + { + "time_stats.all_reduce.min": median, + "time_stats.all_reduce.max": median, + "time_stats.all_reduce.mean": median, + "time_stats.all_reduce.median": median, + "time_stats.all_reduce.std": 0.0, + "rank": 0, + "num_workers": tp, + "size": expected_bytes, + "collective": "all_reduce", + "devices_per_node": tp, + "max_devices_per_node": 8, + } + ) + + expected = {(tp, tokens) for tp in (2, 4) for tokens in (1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192)} + if seen != expected: + raise ValueError(f"collective coverage mismatch: missing={expected - seen}, extra={seen - expected}") + + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", newline="") as output: + writer = csv.DictWriter(output, fieldnames=FIELDS, lineterminator="\n") + writer.writeheader() + writer.writerows(sorted(rows, key=lambda row: (row["num_workers"], row["size"]))) + + return { + "schema": "frontier-vidur-allreduce-materialization-v1", + "source": str(input_path.resolve()), + "source_sha256": sha256(input_path), + "output": str(output_path.resolve()), + "output_sha256": sha256(output_path), + "rows": len(rows), + "tp_coverage": [2, 4], + "target": "time_stats.all_reduce.median", + "unused_stat_policy": "repeat critical_path_median; std=0", + "payload_contract": "size=num_tokens*hidden_dim*2_bytes", + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + args = parser.parse_args() + manifest = convert(args.input, args.output) + args.manifest.parent.mkdir(parents=True, exist_ok=True) + args.manifest.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + print(json.dumps(manifest, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-fidelity-envelope-v1/mock-fidelity-envelope.png b/runs/frontier-fidelity-envelope-v1/mock-fidelity-envelope.png new file mode 100644 index 0000000000000000000000000000000000000000..0e63a4e1036265f16fd579d28755cbbfd2fe7bb4 GIT binary patch literal 133230 zcmeFZc{tbo`YryE21A8PBtK4<*i+nqUVBk-D5padE0!Gre?WO z=8DbbA(DaX3Rn30DqZ{`ec9EkFk{^rnb{w41Dy_S;*Px^2fyy`lQ_mj<4fWC?=N0q zr@%tW(f|H>Y-qh!=D+_+vou4P!t`HXr)kS7|Gz)EYwcEQ#{c>*rU;?`yFP4UOW5(9 z_>Y~dtKd+=Ld|_i_uRb2veMGAhczP|WfdL?iHSMdQV9u_p7#Ujs@vKOB|^XSy>D+f zUI<=TTns7{GZwWyiCvnD$^5(IPj|g&3|mr(5|Zt5+jA;5#>z`Y;powy{v2L1^~oxy zi#^;}+1N^(o7r+Kzxk%83%dMi_WStp`?zv?dU~jjkB>)$p|-mEhPCU~?Fc`f zasK`76@ABcH4HYr6n3Ao)>f;#Sc4~SHNF{eepF9!SM872?iHT%ZH2BChU~mMCA@J4 zPTSY=4XwawsExeXl4~>56)KHiA1`8f<;vk)>!#a%3yBE{LKby9*~J}pHe}DtTrRJw z>MB`YD!Z|6t6HMsI)39~_H9R8zKSjH)6YFyd27q%g_+D!kv+S2ch)8zYkYZ;p?G=8 zeeHG$*73fYV?EUo<3$4MB@1KpUh{trQG6af+OdL~y7rk)x<}t0W_I>l+1a8Lnb7T$ zbTJ$e1|>^#e3UnzKCLVBXAt~R%J}r>@NlTc6ICk8)|HIm>_#6_b~YEf3Ldm=rQtjx z5|or=Ho7^0ZdHvj?uDj7Bdbq9K!9lSlPCVSZmliyoZq}=J-bYMv8PtD^yLxG&d$#6 zUah{bPc?qE6&~8KVFQJ-d-v{N%{hE}uWW-w%&LDb&QIsLjqfIV6)9-kIn-A8t1gND zo`7NCi95TWemug>O}%lCYI*bltMcLMNWoCmhld}XPE^du%#>A9qQj>7;aPX^^RL~z zcP|YMjZa#dZSfU*8ebp$ct?84(uRhM!#`e|AAu_6FqK|zrW!a*bV=X zjObsUAC`9d)6C2v?X{9Z{=uptO<{Jd$1!Pfak^dDb;#gjoHU27u5Ofso5=I$&xNcT zg_34ZMT?%Wra625{B?z(%_f75nRMLj3lsIH0zyNphD(-1q?Z>o%|`>Cuh>14%lW$;%2W1-vk zDwjC+sXjM)%_id_@6%NkdakqlCa?eDjI69u-&OQD##d9do~0VD(e3Z=XO*~|(a8Ab z`}fU{9zD{tqQT-JCNbL;%`XWh1K+s>aqfA+(@eM{wwNBxsi4Pu@S4kA}3fBWISP$<0G&ja>f{2Xsmc7v_& z5p6#s)xm=YXD!b&Zrir*=Ft^5k8*|RZ`rmjc&|qCo2I5MIGT7%xo6rhGA?{#JAdIq z>c>;jk%fgvD4aWXP|gK-Yb0P_`k>@N}-VF6)*pZ-4yQZe(PXXVEmcc?;QI?a6>&w>FxXXwD_B5xwvU&dwV zoN9gT{z~z~b1dl24Yap{gDZBNakw%yxW#L(e~+l;R~l_?Z6Q%n@^oKbySvWJ&-cyF z)SS}mFLwC-^{Ih{+46;&*cUEaj03T^0RzrI-%thjT6|4m!l$iWyLM#ikAA+z`}glj z@T?5w2<`p!pbVK~Zf*{jP3BK?j<9X3m_dox>o+~I$Xm^h$Gqw`Pk2yMria_BT-!bU zaa)3_B6D&M#5{ahn`U@K`N6?;*s*#KGv++^X#ub0Mc0X6wbUUYAv{{Cn~*(Drx}Pl z{r*aeFs!Jg6j&`$T3YJxr+#{c}0cE(!#8HTfu?B!9kLz z^!4?#ZQIy_SOhEm893;&y?iCBB}GF^d-|}kY`R%=b&cm$P8w|QrVk@k9NsAgm516r zxRbXXu)JI691+WX?%X+H$6n>HFZ5Y@dwYcr9Xi%k8Jua?A-8kS9;+hTRr$%h%^U*8 z#ka6?k1W5X3JD9p&slUdT0Zcp9`R@W`t{g-#Ry(4q01v@N4LoG;ymfUvQ;Q_`J;w) zz?pu8OFGm4BrYr}Dy4Rc=hl2p;<3;Kophs=Gnrguxsc8$hTDycI5h87VX-TQ+1c40 z{xpaW$E}imbASKEkq?3;@rH#iowIf2*DY?PH0-`N;k ziZ73LN<<7}<2s9O+_)j<;Zf8&fqNBHZ)0QQ_x0;niJtUZ4<0;-x@2yig3k=Qdv_-; zYEb(TadFyE4(adJV-Bf$IjIzxkmm!H&=Nh>1B|@pOLL|YC<5CLt2gaoc`Xe8gNo&`F)j^*FmrkGXr9zkN2j!uJt zo90t>1>4xDDAq^VUds=I0|R`IlalrtPF00+_^sb|nCD^pyLZPMi`-rcS$*e^Xpz;( znomgRk~(le3n4+X(CEsQE2Y&De9Qt}$eO-KxtJ(4xGviI$$&>evsc{Q_9~B0w|mo9 zR#x(U(0pZU7||!v=un}UcGv?c{1I&q_|q-StlSv?Lij@hYdrm z`H7-&U%sg2dtSjUq6;mHX;>>QE&cZ6N9CR@97DdCBd!AN13ZQy)>{@97Bpi^zP1HD ze)5DKH$uCxZS`7KPLH{Xy$uV|*5kf0W%O5!N5*8tsYHF%!yV0iI8y~O6ddN1WuE@#B z>ALswqK?`2)Fp@Q#yG4E=EXo~w8QNsItA;aTjMO{rX)BvY~Z@ZA#G&hc0PR%07Gf{ z&5f%70U`#z6uCKe;ECxX=lkpTJOXxNvk5Z$aIYuYxJ@cLGCMWb%I$?gD_?M2QLmVPtNi9y4|^V)iQz*|xsH$JC~r=FaaA4-Yr3kyns>>v&~)Xj_^= zKG%|WV}^+v?vSZ@+H+tM6 z{}9!ocs*m!$p}`eQbav$o_|mfy_2)E{p^_8bK{cj0M!a5UW;R6j@@IO74&P_M2`>t zY09zE$?UV%*4~WsV7EBu%r5Q4j_r#6uF3Q@2_)Rd_E_un=&u&6|f%7Ss>i2C{XP-f=lO6%)^v~zax@zG-ed5nuC2rNRJ zzPnFo4Km?a;)sfYaJIk>#b`!r&OL)$0LNL`X_~<shdNAe_^BX+VF8&#lgWLYTL?+g~{_;Jj~C}FZyFBYyEbK0LTSp;82N* z!Ez4|JBP)@+``{6^H|$q7PgHm{(R-X56|p2)l0mqiDFMaw+1695j}cuE%+7gi|`F+lNakOd5iE;9kuP$;sDgo2Ld34q_N zUdMJFi#*G#4s6Zg)&YX!V zb;2oQ;nhjwr!Zo@uQxU}8dx|?rT%U&DXIR&fZg6)4|*{z2bhy43i##w>eZ_#TkKGN z9TK8kj4c>yD{_ZUG(tVSR;p$=pRpGg^}^EpaOVU>hrNccomgenW!H|9vtT8mZw?2esfk<*4t~>E0v`dgh+2w6wGg;6n1+1p^J~aiwNGX3Of4 z2L&Z3l~(l6vioG1l&wzc$A^}W-%LtM3cG##HlK=}T}G*6w5F3&HWAIJ-VK?h^N*c4 zt+@Kj_gTf&!52itADS!n!?%G)I^I=s~tUz z2vlxV5515PX+J@z4Z(LM_$og4^aJCp*9J)+JjQ!f(wquihd0-MeXgrvVgvwa8V!|( z*P!OtFZ2A^=IVsR?jD20y*6zuz*HWKlMSZP>BVzO3sQW1Cz0_J#r^<@>ga86`jcj@ z-+k-o^uEamJ~UF7zc&v4J%~@5{?2&uA&e{ z#v#dlyf>K8s?q4Fv7QeG7IfoF^(`&>9mU+90xLKnG^loreSPTYc-V6;J1i_L-f0dz zvYSh1uYOHwz*>roowKvnmhiLKR#QDB-wx<3`E>{qwB-8L_jnY4c=wJmofaqQ^kuKx zfj4h5;V6|Dw6kcPr|ea!F~0Pph$&I#fJNQWo|i^ge#k2?{sq?lcqh8};nvX|)wAC* zF9HUBdPUgc2F+YKseSuY*4{aMWx{;aT`QRCdzvAi>Hvho;%c6GkzE>eZwsq3_h_ zECs&Prh8wkW?i2gT>WLN`DbS*7bV({R%iFcb3Soib0;jC9~`HUFD)3dS8=bXu?@O7M{4@C0yMrE4n0cw?N5ZZ zQkz>M_n1lni@rYq_-zOTQbajlBB7P5Vpj7a&`5$PU!4;iUNf>5Zb)8&#b?8h@08Qj z)Jzj~YbyDgv}kO#b_7W$v#tb8NvR&WL#b6DtD&KYj+T~bya^1cR8*mfRYt#WYk!b7 z^zMjtwt-3y4%)2bbkCzGI2di!T<1+FDkK!DcedrD3$u<978gH8?dW~beuE)PFn3~d z(hu;LN2MiIi%<9upsJ(FrN)d$VvA6>l*aSzI(#ql6T;T>`bI7rwfT6l4CBfj;^Mk# zR6WpgSILHz43ADsXy%7j53yWZy-q!P2)NEvp6%SZE(1aT95u;y&2<=i<{{jctn$D?MezB24ch@r9-DyZs3@L;MUGBRno)}S z(+uK~e4bLHlapFe$wF+@5Ij<>d2YwV@UN$>2kDKN_5<5~q5o<#g$8k6&cPvTW)bYD z+)On_OxU<*$zUrr7)5{u@<(c}HDMROyF;9n@aJPHi@c9{d1CBhb}|b~$izS@cTOg% zf}hXm8uA#Lo$qs>pL)?M+O*tv$lF=UYk|irR?MCY#FoxXoWi(qBlnH#Un?1@DDlsp zJrgl-!WMnWJa>%p22`{nBcfUv)r=k|iO^29-uhgthV?j?YbZG#508bgKNab= z1$F^Ep-=$IweCk27fY>P&rU+1rN37UP{`~xdLGdGY|x?SS35U*xRyofa7)~KVbzXHVvfgP&kiGHW19b3V@xex6-e3%Wykm6I|Jdohn z_gQJ&0x3MI0hzS>Mi?)DbAezO)bUc_B_P$6obzmsph^o1<25KrZry6#ns?fkW~G+q z!v}<{FMS6l2J8uFb+R$kaI}O@_q!oI{q*dCZ@5L|Ds*In!@c?O!)Rd{cr~K8>-TS7 zuh02{EBsuS#UU#um4)5EfBz8*6yWct)Ya8Bdz1EExNspw|CNAGf@H(=pB9}~+dxP+ zn}mghDOvv0>dwYA3+=P~z@Y5a)D$9z)&zPD2~u84@jm$MFAj* zt9HUA(-vQZmfYEg*!|4VMNZClW)`KajeLm^MnWbgoG3DgKVP-a7eKxmh~bEhjV0AL zN<2{5%xqsAQQkD4pAl|%<8_>$vdO#}g0-qFsX>;_o|-?Sr`Oq85oC$`t{Wa59f-5` z=)$LH3TMNain20!v+aPH$YvFIrEdbK^l?2hq0C+P^Xot`x;#Dfur89iCQe#sW-;Y- zq7WhrvYZK|n|z~;HW19#^{=l?`UVFFw@yHQBKTTninUI>YR~7V^8V|igwL(^>GZy8 zu))pEZ3m+TD(17J>)?-GjRh$ zL#<-=^$ZNhz}kEUE8TzR-kF$NnNnhFDc3YF*v{lI}e&JAM==0A$^tNS6;R3Laz$@EC=a2I^wXw3g{{*+26lPu%N zP}>oJsHcXxhp{9ybabV~p7TU3Ak2#f5KV0fJg+jy}|xpB`Ql*4CwKI4q;0^+NB!DGQ@w zkEfy!3q&N{r<6UO+V_6klCLD|?4Ui^>)e#}dIpkR{HTMYfWFJ(wr$(KeSUU=P3y4h z(5)TvH_qa10&ssJMhl@rL+&cz!%^IcFE8{DXUUAp2uf$5| zclGodEZVQnk5Y`fkMd56L}Ep2}K!TJxqR zKfE`0v%U`n7)-d-UX1iutk@qkW)K%Sem|9mA+UR4oAwL6RSXOa&^ApW%QT{_4T434 z$h?Ar1njo?x72Se(<=;hbXb-b#u?_O{sf^`Zrzwcwe_I24=Q!2wEcu^gG-%i!%q&} z&WgX*kGq$C{P;miac~+iF*1VahJ}YyVec=yyIaMCS`YW8q@;L6D7X~IM5y8LvTUUW zg_W@xPF(o%?VEy(39KQM`{MXn{wg*ZFaJZhI*__uG_h#M+YC*oMYvWJ9=}6JoGt1#XkI& z3J9H~*kj&0pvGil!>thbdpf*LI_URwZ8`R91ma@g@+G9R*4Eaumo71&bR=AEr>wjd z6+CK+k~|6k)N`Vl*-!MIi#0nxP1o4uZun|N=!^F{Y8|#&WkGFjhl~%^c5NV z_SZk@0-#sO;)mKD?Efy@%+@vpG0zllmS^9!8j0HPu=DSL`U6|Ieu9m%ikuIa$$Ey8 z5)m`8+t5sQOp7SZnW}wT(%OWeDJ&%P29AeJNEOfZa_DMnYZ+Hky|Qg9Mc&D9`J)H1 z!W0Z+2Cs6~);2c1XKJ8<0Wp;lw(V0q(3Sa@cq-6Te-v(R3O!hX~RZMJK5cobESgm&8aTwRC*N6C?J8Qfu={;sNmANn!>gTqsdspy5S65dp z3OmAIy&xBntBc=Dotv90BqFk5Yq)Zp)HYn1&FSLVA3Qyeb682nznx01z`X zqug?Ga*P``_`;TioUxtM;c?@}U$=IWk0YAN%ZoLYkL3NsVq-&~uk3)X^7if9nVv{v zwI{0E2%Se+T*ICv68uWa@t+d=toBp&UD<$;cxy)gHI%F}hrV4NZb6&(#V00S1MQc% zGQp$6preAb8xW_TW4Y_l3y4Fed4u>6xo2zZa87U-ke2+Q>2dAawF;W;8-$de%KCi) zkct92VspSKbZIPfvjpFm@ePbCxfor#bO|U4IlITTX~WH<)_J!%vLc4v+FWLLg&+Sp zFhFl?Y>Zz_10}p7-?5LpHDW3^ih@wq%7F0f`|FYookG?p zm2u(go?heOys7QSHr5Hv18g>yc$d^=zC5svBll#aK#D=WNUW4+Aeco^FqO+t>t01! z!Uh^&enpLC;raWy55?ziKRlEkIObO#c>x81&JSHsV9(ViC{VJvn~(dOTQq0yg`Up* zSsHs}KM&bj+vF%s{I4n3y;>#txiyDYuU;K>*qJw3^3t~7U~b4) z)Byf`*AJwK!hC+KYjJTgQ4cGp)o}EXb#or`e4amLqnGcH1g8axtEAL`!mZ9Qtv zUTV$T5mI))!6+PB$vt{Uo)&8Ljrf65V1BCe{v991wUewcKjO-UA*vAXinFp~=gy<-?Wkl70L#Fh zM7}@EIBPF#H0BqNoMi5E%a70YOshQz_2^w-P|!7inw7FX;C}_K!|N%RXGV4u&-QK& z9(S7=c~m;uZuYd`p93>;<)Mkc>fwp-y_tK=tgZQx zgd_UKdQczRjtC4}njob~vJl)%`=`)1U(TPX`A|5u`K>;n3_yTPgvI%-qNtLKwtvWi z>k9%ng#vORSZA+{GG}>>+m>PRCs;&z6qsaOPMvms?e|#Ff}S87oZJhe%UJ495sG?S zQim$!ptw|=n}gv(p?Ray-2CImk4fg4{rm(7xw9`Uj4{GOLqmhvR?y~-WHC~ZW-{Vm zO64qSDuIG4zQ54-M;f`=k_IzF#>`%-yw}Za}dv}3L2b>xE<^2 z7^3^I-jAL|Q*}*u*}i8j#wRyG=0PcqM#d#^d%v4}vy7_C6;)sr1z5T$e};^!-x*z` zAA7*kGn;JFl6wYMer9=reL8x36=YsLtCchqTwJD_j#c?C%xjv){hXq>wZ1;I5`BHo zLP>vmRhD81`%PFVs3=~GlR6~Rpdhu1LI~_FO2TKHboXe_z@NY385`QEVC zhMNTp&cqjunZg5N%FqiPyyu8PE_4A_`%Z+L#>{j6b=f8+CZ=C+mG1mAN+;v7m**tg zJ*~FG13-L_6yqe^xTZt|>2RPs`;5I85469I8T|97HeQyR@&+HEwcQmUkx~9P%kHMk zI1rr_(KaOTpM8B#?2AFJHrA{`HoF%@_bAm>jFRaqftsR@n3XO-)0x*-4)_cPS}Uw8 zek^!~sLdi9ZlsRix@%3-k+9yvxD`Io-CpZyXx<=pmk(%>x^uGf7Qb# zuM&7NF}+&zP#!JR?OBnRDK&?Ff4=kEyx?KObKS&mMvqb2W2(9?1*Mzi8z;dV5SLek zc}gdx%95wASbPdEh{Nf`$NxS|>QKlkp?O1*oLCsG;s8Joz@fZ3U4^<~ZC=$BOoI`s zLBYYyTj|L{H{^9R7bM@?S>e3=RMjZWy>46$*b&0~-Q+u-Q?oIIp1>7G$qHq$D7#;p z@Vh9zKgSwMm7cg)w&xP>uLHqVVZ|95Q~dS8(i+s)DjKDonSw>|ZV8!m#($sZ-Al^3}~6zrs{Y99jYb z0<*tucGxg-{;60V?K*~s+SUF#EJ=PP%vWV?ZIZ@rDPtgkLE&r8S5#L1JuAEFAD_4b z`A$FHEErCP!H6RGpa}Znb-!Nhy}TV-au|IrthBsZs2^>locHH~<*%^uja)eJ7O^M* z+I={n!WO%cLgIBRBm@LLxdz5n8RRKC+3ww^HlzCumx`gJGe?>Hx1kWhQZ zpnC3}x)%*W@t{rfV-Zg*Xz%BT`acyJfL>UP)#Ah*SA*+oH8~2RK#d>MX8m&xMYqn2 zKxBS`MsQ$`pk}o>(kG@d5Pz#*lndttB5!%M_U86)j&)Ki3-8o(F` z{wQ)YJsK^0ZFR@HaBkjuc)wNx@Gw_x7F5mZ(?!kI-h7WP)TN4e*=ywkt-uHABSzeU z4I(N~tAE0P(V6oE=YaT$8oN-EYCe;7p4Mm3oeh#=u#o^*3Nw5zu4%*4UJC(0!a?EL zh_m;fMdqQryGy~o%{%o@70N}&Sc#X7{Vzv9o&#R4j)xe%Mn#5~6*PiTgb}#-RflK?D_C9tg4_16w zDNY5cs}m&%u*@GOEo&Q_@TjN&@MKeLQfFftfDkdIcVE%u9cwW5-T^}dR0?@)IZlDk zD*Daw0NLh^89S`4tx;<8B}Q#zO&WT7K0Ae%SFaMteG^p_q8Q9zYl+lB+y5Eo%B>T_O7l~NRwX@k1@d7#{rA(QTPDtx+>QZ;l(tn7VBhE_xQvrqgX5&){h=) zGst=;EdFnZp#rHW0q_CBrGlv`7hHVTVT$AedP912cr-F@1)~GNGn;Cx#7#J65D$nF zU}tZC^~H;QkkCiwCap5pbfKgOup9oY^c$S2#QRcGA`SD-7t{*Ya!9d5kGUQavYyx* z0V+9(*=KID3{DWLcmBTLG>;#rA^Umc$PuT(MkdsCd>%gB2Kr7+S%d8*#)FHt2$is% zc2}3-^NBwRb=X?sQk|K31J~AC-DW6O@V!#~H7KLZM4X}sX7KL4MN|ctXCS%|>}i*m zmn4aGXKBIhy1zdmObEfB#tV9~Q2>RVqyx#dvDouS$acv2WBs;3Fy&5^ZURC=6mdGh&4N(2CMIm?jg^QugeHRhbYd%4 z2^8{z_OR~1u$K7oq1_UX2=}p&qo@l)kqbqJD1~weDdieU(9J9fS6Y$Bmo4sn30t=CCdOd0n?RKV3x%AkfNRxIAfXtD{2aCkd-Zc zC%bxj{I=}BKoqMp+2%~Z2EIw*7w+B{GVvAbR)x1o2+9_8pVE&X*MjKy;tW95yWh8X zl1W}(9(5TiupkAvVOrbTgk1*Dh+5RG#2z6>y}>eZ(yV~6w!^4oQ|M-Y&JQBiNV@%z|@LRtJU~scL{1<*CCdj#WBHN%D@X#3z^?(sC5f$uU{G(j=^w`loJHF z(Ft`FYPfk;9z$OLzj+=a9>0dlqpTNX?`0viibt)7XITRu?a*6CGv zNutDDZpTg1(@fPoVJlAH4t#;WcrGFw5`9JJT;xOT}-2J2NcmcN6O!{*@fujbC5^B1RDi=0smb zg0vs&J|W??KfBpIBsIvYcX_qw03}!HT|zk=W|~qk zjL|X%>3CdM2eD9`WfnHJREPVJ zZ{=`6Dki|;Hl8lT@?022}h(tkdb2BqEz>pi^K)P!igwsrCZ*Q+O zekU2mviiKW>#0oqPkvI6>gXm~pkMU23d|}d1{4j;Sji^(o zof9`#(s=)`UoeNcH zGcoPJa$06vgLZYGVwFD@CaEeq3B8zDjoJ3G_>hz4H|sGoA`8!*=HWQ3Gb1A-=f~&+ zC%ew+GCR+kqM-XF*G4~LKm?BX4tRbYs*-na-lw*fP`r|P0l!K2ebrh z%^rbSL?f|X=FYe!@$q+kXl*qRv-{~&S*0Uea|jE<_DLU%umeZS^3{`62gFW2izS$Q zN}=U_O>;FjH!q&*M_8gqe*9)`_~OM2KX`mRUOb75i;D`4i!;7jkA5n895!|RUZ^6p zq;OR5e8J#@=;<%>zBf1S39JVKdFmvN8zBer9wp1|8nv(E2S?$2+zSh!tc5tZ>vgz1 zQVkB-7-)=VNgsb&xYW1nl*;ffhaWAx8ega5q91QV_fDNiw?QQQv-AJ~O7$QM8bmnP z_NLYvbL5DKh}cB{N{{EYLF3N@T5u;-&QG9v91*L7?E82u|4UPoZa`pQU3Ko)iG?2G zdA@Q24k09QVj5CxiTR7^?V~6JpbUx~Ino6H0|!nOl*`&Y`*U|;wFI{&Nz3wU%2n(g zvHFn`B|t>XlLb!?54+_h&y}lInIQU+3TgG-z0|Pb5L0JuEfbtom2uunH(`@G3vQpH z_DJsP_xE4EFe3CZm7PWb`4N&2I%j+p$4|Oz{qFC7gzU45H%J79M*kxQUT6~e+0k(op0b~) z|D&vXtyBUfaUUdSQ*1+u#*_7^&{o7Aug*5FT>-12Jjw$oP*Knv5fvRxcolkS%I|pf%2*T*{IcsGV1htDaZ4kj&QL=$JoMHOrH^^fI6~w7?y6V4s_I6`SOWC(? zT0=)vO8=bPxdA;)q)dnRcrkNC`}r9rluXM&6yS0+dNH|OR8+Ja2Q?FnllWIjgN%rX z9NMmg?K_X+Y1ZM}>%~fZV=s|3OIkMZ3g+LRZvlLcPfuq6yg-GUiJS+bUqa7i0M7h4 zedQH42xYh`^x{whMZtOc1{F~lso#>9$GfpKJF`(B{)HfncR{bjj!CLP@Mnc5v+4m3 zFv062NrTaMHnvu%U~>tQKYDbfho|T197`?I41|6m;07WEV_(7i)gMNme|BkEa?4mx?m!o#>_%T1Ja5@96}+Q0wW)2DlpEO_oI zG%@O`|8E7L8o-n+P+nnS83O+;y7+e}`7mzZ9)d#}G<}`mBe*Fto%+=cialmuhQ@P; z9~Xk1lO${~ysHl$Z2edJ6Lb=?T*MR%UQhZDp-JY?Li@9W{|+4|1Scs#&OoCH9ZGIE z)Py9k0!$LGH*soSNE zOHCwvx}+HluRH=*-cJ;cpwPP&4{f~(0n|I_DC4HJy~jxrFpe2T#p~Cvzd^~9^g+Jg zfB3K-dtn{KdNk4Ly^nMlXn7?F^DR^_cp8e?Rx&;n!Mh48PXOY4!A+gf;EMd_5xGQ zC#1V^tSa$DkTKS4GwVeWY(l=4Tr=*GKY#5%1e-K^jq~z?Wa;p0NGZd-K6xD)de@Od zyLD?9wo1%hBOnGTLySnU0I~Cw7>V&ZK*ZJ`VeRnAH^Yqr? zQU89`_WAy8!G&#*6S~HZ9Xm+pT686r7yKN+!1@v2DF22TM|4;Cjw6DIcS1t>vFeeiBRH!v;5N)j8~W9-T?K#Q()!4f5pH4vwh)HA|rC~ zn*7s1{U6_%a^2({$(H}|m9XL8ukt_N_y0cZf7T-4|E<}7EXM!w=dAh(D!%eu;WAv| zoc#RHkKfzE!4YcgHA_l(*YDrog3dC|@Z;zu*_#Z-h~M1j zeaH@h)9S?4U7u2TY8rg1Z+v=M@V3K7<|O15dyY6scNbiGx~9k}c7rp41p-ApnM?i4 z89z!!`k7s+0*)6Vl@i^8;&YViEgq}m=TEino{U$os-VtqQsqOtFVGJu(L4S6LPhFd zxw!(J8@${gR9*us;qf@F+JtVUia%v*xPgkGZGz>7Fdn~ptQDLNA<*t0LC-}=R1x_;=(%-t0h*8r!9`l? z6rejD-svd&{!M%4$nSHyEUJ$s;TF*J^;no;f~MZNyh_#w@&}4W4W|5BwxAn?6OF)u zi-{B~fu63##c@FHINho+|NK*&v~kg_1^6@cU^O(T;~CLt#&W4cinNT2Ise}DxA{Mz zrXJE`x%++&j=1Ib)d*(^w_(IW+du~*ZJz7!cFew50fBE2t)z`dBuP^W@-C4;Y=0Ep zgyhPCyj}tHVPa~k{@m=&B=~f|Iw}+jZy;-uHaZFnBBx^%xOp`_1bDX09NBE-$v;qf z^X~u$r7+ZhI7S*HCF$nLXx7KyS)_SAF5!xx@?)SpNt&xUqK2 z3Ry;C34%(%V$pQ>W>653jVYvjaf_+QUkaAHEt2mQTqmSpLy!rAaBkCk%jK5!5q@4)vpqtB)H3x>GeU)+;Q!>wlbLxkBf+9RnHbC zA;N+Qlm~2s$aE&tbQKJ#WI_c>SwOy$4IWU@qR`4r#!=uG!~RY5ID*@uA12x9iof=l z-GEDW9Yh5Y5>zM@Vpt;NPcrNP<^o^*OP+`15MWq=or}v2oC`wZE?)G9ZWE6UBBdFl zqRWp)58gfOKJ`5B6lpj&iH2r}hY09jv0}xhyOrS*qxRHgrIKepP|H=^Q>+_oE(6dTjqC+<~e- zf`ZC=#DyPC#R)D;u;oDrT|@2x;-iU;%{tt`GiT1kqke?`X!bPYk~Dia6ct7QOcP1X zLRgro<}|dix4493uVYim5EPK*U+pE*SXacj6@(XJDA96hDFok5Wt&jzB zkUSOz=~3X-3o;2Qr;Kw`cazd-PbIt;#)AYSl@RqGN+}uRfUfxX+Lt)^^MB{$NICq`+Cvp;ArATiUGXi0%Ughs>j_z)9^&o>1-)Up6&4W^ju{ncy%cg4d`qP3M%fl;W za`EDx@2=pXrYLM#)|{$1?VWnFlvl?qBz<8iBwGE|^2wUS=j8HbWQPro9*n!DB(_fD;#3*SnHNt zl|OduVE!{GmBP|QJeqHCHXzfHsY|dD3PD8Aao*j0H*SucW>bU`9ODv$ZB}r@{2Uqz zfw|(Tpvb_cvk?WYWC#5ey&I|0O+AJA`8V77#!>x;9=l#?d_T$y=)Qhl zdl*AGAhZ87n?fY|{(V~1$~B_>LqkKWJ$pyHsydxeP0qK-8hHNu5)8~}Dv~>XJYWW$ z16_CTo!~_ysIoL@eNI0!nRTXU2IEcU2X3d9+Ap6Q8KWe4LVQj{3)fgUh?(=@CLQg|t`vQrn(f(iI z;9QANndFAHbbS;Oznd?LI>Sgox^8S{+uGWgKQE)u`nIA%p{EF^hBPVOy@!Sg%^~(Y z%$Bk@)|q$!Q?)GPO4;KlPi|Da#rnu8ABrX!wG|hY=>z9WD6scXf&Qe6+Yc+1Yz|3f^#uQe)OHeYMEAG{YhT2~9nI&HL&} z@^*>KoDj2*|0!s0Y2k}8-ddwAKA?e?j>lt5Maa*kkaU@QC$S5Ehh8jt;j(`?P?9`7 z75(^=Tl*qlF3-M`FvM1Pj^Yr=5hr1?Of9lo&Hy~g8`=^x`8jL5lm`!)6S6N39sfk8ju=RNF}J5* zUZ_zhMMZxZ} z$oE}#b2EVqhuJ&6U=<{6cJe37L6UKh6hqfLOu6pWbYiZdkaP_)1Yk7Q##rGAjXk7(v$P3jd{0Ac(EwJQ7_Bq~2Zk?Wx9|Q?u*;{|u6_F@iKj z25*ETZU76Al0NEQJo>r(=+IzdVWtV>GV}vpLBo>cL`2 zrygcopq*}oPMU!%N)u3A*W+pY+!m2wVPH<3StdV%P=ks&oyh+EOwbZx&mgzcY~UTt zB!K5@06}lSC_?%lfJC?t9^6b?eCsi|hZWh1IdL;F0>Bg@k4re({eB}MQkX|kj%_B} z22)1$WB`>9DasJHU3x15aNy~kq{m_7#%mC=yI@P8(~n&O8X;MRih@J{L7LbzVZpl! zx6TeU?Xd_L$Y9>ljgkNffJvgY!_`h^Kau7NNy*uk$1!ev`u8T>@rK!!-*_7)P^8!a zp8$o_9uTX^kC15#9UaHR7tk8+V#qO4<=jbbnnW!qsVM)%hk`9$VbrNGdIYcdMg2^pF|2FG2FA@wdy zb3?sk1%-fbV%7kUA!A~Q5&ekEpbQ)hUQ4qVN#h%?A@2t#Z~{^f0lI@lQzTl{!WXtR z+_0n8i&E&$hDE{@Z-}W*^29I#Jb)+siSjy8R7nd5Di-IEm^+7`e?)B{pNw8DU#Pj`El(LO!ejEfqAwr@*=gODkB^8lu-o3a!ZypjKd zGDY*>EACf!snOD`Aq@~@R>s#KEGLF1Oynv7neo$Bm9Z zXnOC9ag~^`O?~0#rh}m%hSTcZEmY0|R#3d8Mq^#15D{Q3nPr#4A z7Qu>)Q6!EXRIp0Fe%*u_RB!Q&#HCG^58!k2u)G;IdmXlZLqpYv53pf35XyuZO+Ls> z5Gmx?tFu!`X97ST=?y@py^69l>B)uNZ9j;8cvzSZ*ah(sVWbxyDfW?@KHij4Myc~*`qoY=g{8OHG^9TMsoUiqv`qjf=0Nl8m7T&qk(qFb6(mTES!yZha~@=0%PKJTew& zh%^q5I8pzO((XC^FzPb^7@1ZVHH6nd-;i&e2X>j049NS)$Iq{9QV{cko{aE9ai?49 z6{>2&DyXA>>)qADCX7)NB=Q?PdxfjYx34hy@$6HpKj~=)ux1Y``hmo?8@G-l#Q7r4 z*KLkeNBCuSe}W963uJqv)C`j+WyugZ3Jsjvo8Y5!NZNb+JVm9!@H>b1!j1-B&HJz# z1ish>Gg16OV>6hVcu3Pg{{-?0@dB9Mhu+DYj7^DYT=5OAM>JV#M)#o##jMy8CMiWl zgF>=aG%@J(BuEjraeiI*VE+$jwgwhtl%oAyf^fg z=IbenI&>-ajG{uz3n)t~C>XtgZkPP(OExz9D2!lZ_xTu_cs9at!ED=J{E$xt8ZtVW zIg|C_u`eB~Ub=xaff+$z)syw##ZMZ3Fl$N?#e6^RgaE}V|gLl5xPnGLb;olPP(%+D<)sPxhMNo8%sdrUC| z#lX11$D(bJ3MxEsRZ|ZdBTUxa8kAJ|U);TURL}eO_x&*+8IO4$3uQ>1%=4@gDn&_1 zp%Theh|EM(N@NJlrBVu!j2Q}1X*4F1gi0agdTgBU_qVQVU2ENI-GAI`-PSq-s?U3P zz4mMG=kvLrdn#(Hu}hYmT8NF(#t-99(wE2r4XkgOM3i(aJ`@@6|HMuj6rdX}{72&oOpL^;_R#gN1R>_-%Fm|{r)lnD)`1ss~l|&2*s*v@CQ>9 z^j9rke!D``nYuTB*C*CmuNa3=n}#1Lb6;%$6s9;wzxmAxmEt5JH!x9h!M3wYT`aG^P_@%ccB!b95E;-D~Izbg}QVqlSI*I&n@E{OZu3hI6PmuA^?=+AG5CO>-k z*N)ZWD!*_S=aOS31vt2xj7BY`pY!}WqsflXKmPQ~aK)?Rny|^&*HqBDm0E72Hhj2Q zp9S})8`4^D;qT6EyUx#CVXVnnsdx7!Elqdg_7Ln{(+Ps7V*8LiHuaeaIJ>RMf|y%{ z;s~?)N9?Nm1{J40(XgRBGN#>XYWUjof{vUwGrDzrXw3Bu?JFqT_L(c|*N6WVv+Yj%qH1^@Zx{wY&gPlq2)O=Rq zW*z-^H37&d$Y^5au^gl@uRbnzU0j}{R@mS$2fXUNVji!)?|bUe{8$8NudST}l)^Pe zgZH!;c)ZotcXcnv6i3;dpPVzXkLysig7t}2D7w90<2K$W$d)$C$fDe(4Mj>7|0;B7 zlRJ$oj=yO;?r=qLZa{=X%goXW6|M*nY_J{R!#0D=?`Z{9L3hfmqM8VP0Lt(68hy=g z1Df(Vt8;EvC6}r0xvB8YXkw!?bjb;}^R>OsxfjkB+|{idj0)NB%lnjT*T%v|c^PMx zZ1iZn^Udy77|U$mP*zr!+_KpP(heZm8VX*wNy8V$UMN)s)gMJhre&MikRsT^7kSJA z62>~FdtoKtW=1qnh>$Gz6Qw88q1bEIXy%r$SW(NpV~-ESJ)3gfsZ%GC9*P^RLb6|M zxINbI7~Zd;__2-~Isktlvq=-cH|v1q2By~;({x;ywT_NEyB_gsJ9zo=Hmm2fyLo;v z*~bZT9)$}=dXnMb#bjx@^P4pf`vDT)v3>WGI`|G+)>uJ-U=i7W6L|Aa0Xe2tIy?+ga~ zA@N(kxmLHe*DlQfxM^={-~lKn7M<9AzpRDGKF}J6dRqYa8j8vf&!WnPK0`K$lx^$_ zEshdY+2gz!`90{JY`kbu;c)LWU8MD8#r~DqzDGm` zO+#Ug+liMw8n=~p-w7#Kt|&Ma42GD^o(};g*BXV4*&DkKv%CTLaNK|YIh!xn)}D=ti%W9vI?ijV zPov;t<>t=-REuKs$))>4H}pD^Nz+37{5CL}QCfZp!;6K^!nbhx&`Z*E@xW3@Dhh6+ z*E6Tb+{$J(MLA?&5!b92>s=RLte!Dj$sjU6BSt9X+h+(c?{()Z3})EObCK&lDR z*C4%jpfi*fE-Vl(nOTh=Q&e2+PlLY5CZ$|EcC26Z>olM5I0|f{)XP0ynJowqF1XrQY<9d z7fCA;QKC?Xu(yRk|AM0sh_*2tjljg)zN!xWT=*&>c|9famIG(mp3CYz`UF>brO=Ika(L)+~FF(fsrLz8sirbjyCP9>o25;NFRVBUPv8N~Vv zBcIJW>oHp2XP>C?cAboNA4T2EdzD5JY3qcmTm3G36KGG$B}>&MV#jc6rg0DjnS*-h z3bo`#H%_j`xCMFIW7{-^QPq-9AX)gE7kL{5w>MuKa3DAL zYl6>qz^642>lTef?)bc@C{)urUss z%;?~i5D~YKl(by)P-)xQ%0h7D1*ChTrsgI~k<5!t_LT5D>y+$n+x~b?>#;mcX`-a1 zXtIuuL2iv=s?osr76{QpY-Jx` z-ZdmZ)kfr{jcNB@Tf3beIS@TDcv`dH@%rmfEc;yOAy(sHN2XDpQXW(rk=*)0j+lQ64QdhFCSO3OJjIwr>8>$-wh zCOxAnY@D1k@ANMEuoS9v;y%NPlP1-p-$(p)30NKn4g|%{-!iaxT3l$CS8AtT7V8qw zw*do46)i9>0tC?@^UCXr+J%(*Vf>0k5gFq)G*-GWs=;P4n5++Lub&;S`?uax!^K;G zexMhGh07i@wfWo3F^3CxNhzW~g=2Of68Pi+87IG^;XJ8+aLGaE3Hmhx$bp2-5?EA> zeay^`*7z)9I?xunK!(SrJTyj;><>X z4KB1XGA!gq`>*1y!%5L@&Yrlh^E^E~x^Afc_+m?D*`t9uG1h70#uYfzPPz{$uR4Du;UvUGjHrYz2M)Q2ZGNoOg3;gwRWHu;W0W?Br&j zZ%KQ;`mq(;za6+#sySf5y){k&8Ac1`9P~3PX%Is@fW)47r$FQv4;Y0($KRBOBe30c zX7hI@Ul6DZ9NUFdv=%FG|KTk9&k4~Y7U6ruF$_U)x8I&Bv}$R9o@^m0osXNu*RUlu z3dVigv3>hB7J|SA>ze8Y>vpeAa!>=YHSKiX1T9(cQ<~ms^p}#9gYe0Y39QdvjmDW|V}`f~aoS5{;e%u|MHA~adPRsxW62a*hY=Q5}BT20-F&duae1nD+v6?Lh3w`ILiN z)&}!iP3sQY*^xXadQobpB`6(1pqkxU?h{92cyw16{7IY7NfxE#|qkdo|!s!(W}=W*Th~^vL{}^#Ho;bmViwLMP<( zdXwTU)&`Ag;G!-c9oVFO*w}%|dtaHiaGPp5+cP}3wWW4SU`Wdg1 z_Y=P0K*hn=?Q_$&*wLVH@dqw6HJNDc#*{R|*u=48Ytq+UpZbYuT4dz4Za2;& z!L&2=HH;TFteW^Uiok?nZu8)xzMAh1y0mn+?ytp6#Z4XMQ@Frz=!c)4sb9aa4Q$X?TcOjlEA3l(97p{Ox5o} zZ^o%{AxxR!`U17SCoU;SIf>wZ6cVVHw4vg!+d|T6`p3~yCC*AVG+2M^-i4i$BQO)B zFzYe5Jb7Y^UQ4CTaqc@AR-`n0wjVAJqbpO+(N5AEcOlDtuZ%m0BP)txPB@aBD_w@& z{8%`)@}z3VVx5*o)|TU%w0x93Y1>smiVl85d4RO?ASn6W#xY%77WzKW4EWYxA<@(- ze1Xxo<($e%smV7lWcHweB|7k8k8iqQNZBvPK569Z@86yNl~U@v?&!gU{Dn%zgXt+m z!ZYvt;j|sWKCS=!WQ|NbA{G1HdHn?mlyuR!Woq;+(%qoo-Hv8lOPkOWNNXJW{tQ(A z9_XeBC=#}R@0cF#OGfYSI?XbD@we^gPTx1lHN6E)Yd2x9#ZR{O9e2dqTt%Fwm-5Sm=cJAW7$+nkF z=l|@N=KPu!Zqxn*OC2f$FN{U1?YkWhsA{%XeU(!2teCTBe*rs~C1FU>N!%o;$rYAs zG%0&+s+G)Y%DQ%&gapJNyQS&tKlI8#Son98w%9bLZC8vgjl7N^r$02LqG|iuUa{-K zPKt%n++I1YtJpjEm>kF*Wqq)mB%-^fX3Tllez(9wI^;hmbyvETzTfzQ?kxx79vsM^ zJ=+YX>IuC|TyRj&!c^q2MelvxVon@^8k!M=A8Qhss`peeYW`x7PcO3Hgxz*I?&F=D zvOE2=aL?dW3M}b^v+wX0mio?uyaPEJ-goYS*ljD&0rTE& z_=1j`biYZ5bXZ+41 z;_bm2E<_<4vmUDoF99@#9f_3KP| zPV(f9YsNG!7#i5if%p}Db+mow!%1Tf-|{)Tagv_(+CFIIq^JL@`Bd~R0xaUKT0?Me6l&)<^VDTX>e|I4yz$A;@?yZeXS*>0Ai8}b1b zKok@B5RR5el&@u%VCGSx*U!!=;x;;n86zj>B7vl6Qg{tI_HbZM$X{V=ixJ3+>_w?C z-Rsh&OJ7U|YOT?~r>@dIe-u#Bk{ZYU`Lq$3^G00_5{KsdT2B~PdvZ$8C#j;jE!$Mi z)5_mnj7TA5iJe)~+>wjomgE?$AB`g0eem$@-9&DB4xq3%NoMz{>7Nd2>*!nqv6yin zl((5*NE)Fw;wG``uuo8c`jRskdoXFNz^Pg^EN})R1Pt@ zurf5wnw=??vipyVYu;w4PIHnu8b=B_p6e`aD)9b#y5N{BF_h@k2+^WwUX!+ug;7LI zJt5J@77^PWJR-43olL~xCC@Wnn!qU`txV}fQ*doC)ZER=#lfH z-NO4vOx~pIAK6@S1!}nb@j+RMNn;+JiI07ub^JnfC4B#=Z}ungtCT60E34Q5%h9na za*oywxEwrV%&`V7JVx9+(8uItS>QWZ))x0aUmpA4^;f5E-B#zzkC6JB#ikv(waVUZ z%gG??IgfzxlGY6JYp<6zpV^x5#1r*fXa{osEnr{-riv?1j~QK!6WGJni+M}=DSPmT zS=^!7YRX`Xoj<6yU#~4fcG39P7{-%j3yt$IKS=T!KtD!m#b(1?2jix-anT%+^?mcN zVUK9NmYwi1YK}~h!`ypae2=`Q{^##(Ur~kwL7TFk-MZBjq&qi3c&D@DkN)qTNs9+H zCkx46v}%O@3eUuOPG8GZXyB;T<=czVm9K|hww%2&^`OSM`6^e?DhyZ~uZ0a{sns(G zX!O?sS|!Al<;Zx}dwuzdMU2_{2RRp3gd%RC!+XRj+){Lbuzb?yo~rXWT@lz+~&;h zvEK^g)QM-}*ME(1oRTc%_aC#R&_0I9Y((3yxB|XbO6FV`HltbX8a%r0fH+ZCO~df3 z8LYALIjR$h3~F}A$=;6?*Evh+iLSJMmy10g(ye`?b#)BzqN;i8icg(}4o2Z<-c48kJzM;9_J}Lk@ey`;Z#qAAudN>EwauJGSrtjwAt$U?`{+E7Lq0r@B zUb;5)ATO~IkbpuVJ}d9ulwThIXXiPsUF*6^W+xfUy1b5=Tyf*ZF3pLl11%N5_FKx( zjnxTltg=ZEPW`iAZL;$_9~Nql{6g^Kne#`y;(mbMo~3r%9N8^qRffP9j*SyRq#l0_ zKM%~&XXX~s=CxRMB5`zby{A1+rXE-lwZVF)v!37}D7R$VY$-w`?2J0|n2FMn53)F=OPjkQPTg-w?%{V%HT+N&F4SOBYBmLISTx~c&>ob zOES|GD|)d}7vmdXKWTA*o|b+$U7_r}XvtKW;(SoKQ>c z;Vn(=SG6^E&Ehd`Z8U0L{!!VUaSl5XV;$!JlU@*+CX32Z`iCS@pgn!M%nlrJb9U_F zO~9q|f$b-2YhOhKDi1^ypd2O>F>9B(arCx42RJ^6r{SitiDa!Z3Jh~eBnCampSQ{f z&Aq$|uH-tBn%==HGFT-IbgI!^3y;H7~J*%5PvsiLmYZ zgjl=0Ry>gw>1~;D$$jaRcj`Y+LA2aLFFm>*C6a+_e8Y?U{F;m~qJO_FS!?UwP5*ca z4SiO8Qq}I*8L@9;aE6QoPNEraI?Yzh=xWAyk7a*JmCvaW?{kn;85gE$?3R1FaO9y)%ciO&(otWC4q zqMDxvp(Jx`Nk_$7iVqWGnGjJ=G6xnNnrI8T-SH9`vCNMjPLJkRGR|%kQc|a4ING0B zmJw#uKsqDA{+Ug*y0k_)R%t)_tlK___W?z8oikAQCA`|(BUTa13T`GzsX@JZ_3#uJ zH8(|?Oaz4`qZe@?k3=SLJpgZ{xokA%+uAJhXJAg=Et*wAWKThwzrUKy4s7D%Kd z80IKIG95;=wl2=mnJuY1y54*qsRX?T%oSlocSn|hjKzjZh{hf^+P*~E?ZBhrheLnw zm350INwk6DkVRs14Fa5pDB5qp(H(q_j2H8}*FV@5(PAA)Qh||$M&WzM;qp$7{_8FE zmnpDDx!)4Iq}CY%l+AQ6h2X(p6xCl|HeoKB1yRc4hanQN!FP{f!Ha;`L* zxSDtzM%&wCDkETfPahIKBedJ=3UQ8l&B%)ug(e8@9>4c3{%<#8qHlA5Tm2KqNu0^8m#g$NZ|2>c0T1`ZlRFq*?3o`~D`E9!oM>f){ zMf;-5&Ai@-UY9c#$k*?kAHp@f1>MY?I3q$HFG3vOXX@wDQvF$bk&a=aV^kEg=$>z& zLmnB~)G1gPcLf;Kx2j5zz5*2F0kKP%pV|~PW60`ZlSx^pVmxSc@zO>PKC^P2eW}zK z1)<*=nW5%m#HxG&6s%N;1TSmxPjuveW6GyCXc502=ey<{TgjLR-0Jp6o z+KE>Qh&-<0Midm=5D=lAU#$Tne*U$&>c_OzYq{}Xy*I_r3ksBpDAS8#e)rypxgq@w zrzRFE(O}Yb;6URepD9%~bEy7qvjRKt>NlzKOUF?g)<+ikcCZEG7o}ZVS!ixGruSo| zsU?=G-Z*{^rA){?E$3Dq)vVR}U`8`}WoIHTd~3hT@p5-V^N**2Md3Mc3B*C-z4u<+Rajxzv`?@G@Q7<@Hv^5k~9HbcudI#~>qpDgyNxppTaY<11cd^N3H4@Q!(T!ZHvV7jPT?Z(Z)(mmXiWNPJ` zHw7jA-?It^->l$t85OQ!Ec$}davBel4)}~-eft$!B8lCxNZ%w! z*%e?AmKUN~>0M#~$>JrDnNT+qc}oqfF8!Y-OvkQ;>yF!1D<`6})h&Is5L9pZ#BP}& z$rh8LduXfu@fbXaTdt5jt;4jBl9Y+P)s!Yg4IMU&b^B9hSooI9F{RsdGE$&|?iIOPFyewN}`U zzud`aud|IgfW^2{=-?hbdN3+|JK}saM#~{o5uA-H?tAOe!^|NmJV>g8D`=Pc!(efg z)gjO!TN{H~sPxgyxnd>=cCan2_#jQU_OvVsW;==Q&3OwK z*2V8nOqe9krh~Y~q)E4Gst2RzcKrTDqq_RrvR~=5ni26sOj zR+n|RnQ*dQ>`R;Ns!ef3_H1nzm}SU5GReYm#YUH1FQK zr;nkCv>#(nd=-knKNfa1`$n-G#aXT<=JwDEGmz0IOk!fsa3Pn(ZU5Zrsl=$XR-9vx z6P%h;o{a%QVLdm(>tiegc2H1|h1MDA%A%#T=!`Fm1M!gtCJq|O!7wlQZSBOBSK3DM zPuu$qgQAfnmZiKI9zY1m&yn$Djc9JO&I zU4A>Up=;%6;_KP5+qg$(jP=tqOO4uG_U}QLzPw&(4-#({5=f9zaZ&kRmZB~kJ84*? z<}dFSwgM`=xKMQF z?lqjc{O$V9tpyX&7F|n8(QUY~1zu&m{uawNVFNQgt#p}840!M@pd(XpCh@kVF$(Ty zZin46LVx&AW}40WLOX*9-f{C zkCVEdRPWGT+_q}as@QT};XvEWJ&kaHdUJ+^tw;2tdpsT2crD-xekHR9l<@gbh`I9V zt}fd}9?|9E=TU#_kNNGMAJchqz&Jz1710=ejUVroNA6mRq_&oV4q**wON)NL9XmW6 ze$TfV5XpK;GV@ix&fc)+98rwsL*x@@G7@uU6n#ZjgolPO{jQysOgvq$h@BqRvVKuh+muGyVh5OPD9RN?RF|4_tE5BlI6Y)eubjsw61Y0q#S>v%Yq_#WbSzD~>`unT? z^Y6*)+O+-OjJ^K9v6{M+-%oL8|77K)IkkU3oT|<=pWC8WHlJui<}G z8vZt<`yXHS|K}gpvKtU1Ejx4rw$`a;PQYlegg@RFoKdwDR9Lu6H?TVN--q<+YkKeZ zxR{VtTG0O@m9;eGv0cRq506j*#@4T2FZ#o6^ zKSx!*AC`FY+mDxDzA1aCcxGR;$OxKYU6LJMspfhkOfAsZTGcwvCOjw1qs@ffxi=P{ ztn6fzu}9BijD^*aXPv*kTHkTVuB{zH9Y=KDSHAf1=S60pj}}Z#jgR_P_4xe!vn6cp zD$3S4PPRJr>J?)Roe7ll36$+t@ny!2(nCFo^YJ0UYnk_#xmz)`Aw%im|L|b$2}Np2 z^_n4u!?uneveAA-(k9fJGK8oMJu529bJGCx3{0=y{DL+?J9^w_clG00z zatLz<;1j6*G=~fuHoJq^RRSRCEpJ?}-i{Xg+TSUB{W=K9B`NQHetxrMdGNkybd-fu zaw;o$C?pmqH?dY{$cpL@L6%ftICMq=AAj%JyoMsdch5wB%trcdx!T45VF)ma3GBeR z?6V}Mxzw2?Ccywsw=qi~iazvI(UmLdJLkd-z#qpulU#m!f|&YNAMQP~-x+6?!IY0b zm;Jdhh4ox+45Z0`!n}atLnJ++qc_#`3;?MCX48Shdq_vjr-zd|TjLelVUgQ7@` z%`Gw(+g9w37}%bf@CpurRN6U(*1-X*ww919|gIar zLz9_XCFE{XAL&d*mN$2~xf9)-`*!Lv!uKMD+LQBY=~Rf3h#vbuldYrL4e z%_Noc9_hRm0jM?!MLKJYho-XSYE$)xHxS(c_{`W;vrYkJFS;8e@;1OkE!)=2cG_}P zoV>MN=d>RBzT#8W1Hcb`GHRpaqiYNqJh&In*(NPu7ySu8AK$9;{Ob?wC_OF1hC4vC z{h;llRbC~DhIu!a7KfETzCOI{Ey)mj=CY^OhyBHw^wyiO483>n-W{{>o|=sA*uGuI z6FmHsoQIg$d+}E^W_!*vFk{F$O~GNbWK*kZqlW?YKEMlU5oIUt%x>+2In~vu{A13Y zn@Jk;C_uY8qqg$$zetVE>+bNbFUj{&KghUeo1%`o`v{&AR}P@cdPumJQ_)08u}PNpj0JZ5DeUtnugNtIza} zY{&$6xAC9Xl;j$TfI_(lOg;JVpbsg3H8B2&@~SW5UkY}Es!v5Sub8K&M`&DF5@UI% z8!`r3H6O&5@I9UZ6ffGIw4EQPk_n?!rG$R8;#MrZ<8g#bJojxFEAH_6ZYWr1C{J~5 zam2Ykm0y{MVRUkQefhxvH5rsKv2-`DdY7$_z0W=wF<38P-YA!dTBgqFf`~gJ*NO^5h6Pcb_=tj@_GMmf5_GFEdfna3SI~3j*Y1B~$tmB#5|Kj@* zQ6#J3untPNEKnuC!a{`B@x!l|Jv_OMLgo<-JOFEx*+dUP4`YFG>eR0NhLdBU+bU#T zMyF=!)9tQLRGxuP!qE}yjM=V+{MLMMW^mGzf^(nNbm_|dWwH1kVb3pb67cb#AK&q$ z)&;XqBiO%G@&0#3nd?~(ZTsAW9&f$5I8p~&C}-NAT^M2cIXcB-T&K%@T+fXtyV+>Y zl4o6WUJmhg(v9yqb>;aMYfy7KlRlx%XJ!dJ+Gr7U4aqI_7LnLHn(4+)O zc{}#U{%Ip0vNc6ZR#B3d73&NNV?)q$z41+%OM*o0QMKlZC%dDF5;Rlh6yXr#)@g(( zSb^`TjfoWOCqUza9Z&rUJ}wy^K|b{z5A;lVPJ2tT#bKUIC>j_CdwY-l`}Zrg=yjYE z?W1?E6YZT!Ok?Ki4_Z2_LrR;LKi0o|60`n#kJk@hwiq^*5H8KTQ0dbd!5|V)XU#~7_AYUYP({;X{3avk&d$bCpW+Y4w6Uqtz0+s@6n#P$yMCx>DNn5O^1uu%hmRf z9mZptN~xxyTie|t>i_}qG^mTmDUBM#N0F8g(^JTM$YM0@KC^HhF4pSX`C}iXDM8hM zVy-i5yfPu=v{u(v>vt&3@{Z?1&g>~GIu3ET5nBR42sn6{E_>?(e|1SR?gs_)5Q-y? ziOT7#2MszW2fP=@4h|kVvO5WD`6X){99+-cOBrKnnqLYFyzRXz3fsT(9B{aq*Vp`r+E#q zH70kueX6R~pO^alQDv@iT?mXvT&jvTLEXrs+k`%6FEZ(XJS~7kL5+Aw!|(%g7c!A< z_(c*!P44u2xW}tvrF8*m*;_{E$~laX{y2b~Z^W$mXd8eg=9jEM3I`V3r-zp`kIR~Xm=Z6o5sn|P zEcr>~ZGQB#w!Q80o}X^Kfdf)bI`DpBsNzbMuu}3$0Rj!#oqBnk?omttx=kL#(W&YE z^PDob#AABIw`qY9$FT@<>=VlG4=X)ONYM2Jn2LA&mTz&%;gzfP?8g-PHT7(W3H!;FB@qau<~Ldxi5Wtf6)e?eA!5xaJ@-9 zv`1ttav;YB?0;#uZ&)HwvFY$c-~nCN5s4f`hv$^{Rat~X4&P^2dUl53l^EYOT!<&mhZlZa+F~EBB2V9F=+1 z$@$Wy18breSXgAhsL#2)VGfx9;mtH*Al8Lx3|7fFIiW#IRfF+V6RudCk%bNbz>v^L zC1Ob4F#s4rIn}Oz8_$e>Dmq>ANO#_iym|Lf-}@b%2fd5GxqJ1!dG-mn>oxh*Wn1#_ zo&mfHUuF+c`y2=;|N3Ojvm&0CX^iDAM3E6JUm6)~f`W&02A^WR%qCMyr_^ZHu^7<9 z(_-2AS-!&}HAj>FL9HrllSYTlj5>y%zv3yqRh-x)58jLtkpr7YrlzcXV@U!eg7$%p z=K0O);eU8;`~psvjfSTR27VlV#Bi9)qRFAp_MN}nlDv1%poJXNakK>WoppLN&4tm~ zqgXxw#~d;jKaqb+%ibMV7Q;GEi{3ScKT+qeJ;@C(IDTXFknP*f=hWZVujcHr<@waV z??mQn=tnP6D5`Q<4J_1GFJ5$Av}jRIcJ^X|m}D?H6>`_6v5P&Bv4wJiz76)<;HiA6 zB6GIHw44i-yXG`_x%bVZ)ssB$ z!sG`lfJ@;kCJf4mbVvjR+q?w@Lp7@zOVueu8>yY$3;IhN8 z-6#*daje53AC(Qw2M<|t#!B~LX_|K*4~qpV%8`3Mp~~^gsYGF(hWu#&V}w}8hGSD^fo*^+^I}T37T&*{<)Rn-gQgB&k-HJY3b%+z7tA{rFTek0*B&1dFR-9edqWl z>~W+}(M}`nxSRQ+?#HqxiO#d>%5_)9F<~54*XitjCoH9noDy;>2fu8hPw)mN1D-D0 zbum!}XVveNlAidpdL50p^5)dc&Y#XnB#*0!LB_$;6(Shh7o) zf4vV(-l!`_t$1H3md=YoIf5?Rf+-sFe-Q&ayBm<$vZ<9z%gS_0n*M$*E?Merg~|)k zQE(m^h_-gbb`mfQexiuys)$wz*xF_F-c0n@c=*G9P!9u#5d~R~_#hSSI!Y}je_B6r z!h}#--pPYA7Wp~%(5680Y4PpShL{gTsDkIuKi>T?4H)_pclVZa;%(l!sGLNe-4Aw+ zr-k3$J?(qH*WLiup6(VhnDFPR3|?^fUdfv`y|hV{7bK)*kQG1RFp(49`OAwPyKXiJ z2wD_e91!VszSFm=s<)(iYBM8`Q2#85z6Zl4=?I_(!GY`me8igq>fJ z)O5|9J4Aql1r94#>~Hb2UH2sOGXdGfdXE!Xn6?nVL&;@4Q&TyzW=N?LHsj8cb(o-F z^D@u%dYYq2jHN!KH8L1)FY$h^bA>k=o&{?iM!qkqLhy3%=rHY|+L6KghV|QYC2c)t zO%^|R6+jYY%a231T4gQdg|v#gMX`6L%@!5=!o0N4*F4W#sWh`W{A=UGg~-7hGTEb> zBme}YNJJQ6NXlFJoU2i-o4;XFb@7*tPQPE__tx0~s}<_$uw(-uu(i`j=QuDg7j_B< zEeLAz6#=4OvllOhFsd+Z#?vZ^aY1F(sTcmH>F1doo(o?chi&sFa}`T(etBA6(Dv~P z9*-xY8ngzd{$va<2jmU1$>>EYakKa(AxE`RJEJECoVMJ3tHL`+hMWob>HTiQ(Cus+ z8v?m0G;5N$&`vdYRa>Fn<9(GT6Q}MS8_hh4lsv#6apNRSxl#O+e9t%X@-)}k z+mmv>Zfr0`JO~WF$5fN1J>p>8Z*Lh*6UVFf@B5KFq`Q6ZnRMgE!8zgO1$X?332&$p zOAmA+V0z0T@~+J(`Y3-T$%K8;@A8CfIdo%~OUx_7dgDlL*P4H#n!#;b!?wQ+tVeA~ zP_~9-Y}bF@XthaOy-w5qsCTgc7&1yjmWx-{adjoJpg$^|9PSbPOrs8i`(LdFN`D~E zLKwdsTzd0XTQW%e0mFeG-g1}`40jjg=bxN5@@v5%`*>Qqd9qS7N%RGt=*1I}h>s>B zjD>H@(I3>)P$iU>QaB{BH;f=lSEVEzN>czL>kz(amRXPE{Am@Hp&Z8`gVX8Vqp!vf znkfWDk2Qp&9D{6_0E~b_iR%ulS7(B)#=+D+(NP* zgg;p%cR0{#pm#81{R|ut_cLdaUnsSk8Zrs;;ZV6o&bp!Y`CAd+oZ?3ubo*Hp?Dv*u z5Qf+ae8qCvvM8inZ^62z^ZY+ut=IWrE_S81bf~tY=}Vp%i=&u}&qZi2j2rjYvp8FG z(fw0&vW9XCQWY^Vvafe9{jqhojG@;o698uwjoc&Fe|eCxh<6b-EBYikXfSC=QN+2Y zX=MBrU~;pECuR_rTO-@iruSLn$KwPJUrAIj!t6`1i!i@5bk&yHf0j|e&a7^~@jMAHyqKst0Ty>dSYuUr!tS#n`T3+B{h2QUT1oV{kiLG`NxmbB?2JKj9{D`nNAjw zS2?Nr9_5bq>A5z_wP>`NL}1O}e3#ls!V%6X$pFu;t<@tDz=uHL2UT`<{^Y@3NY(oF z>sNvCIKdu)uEhdYISxv>{3XAEz(hkCDo0J&jf=Wedb531{%&>YQ?t5z@L|S4_+5#) zkedNW84TG^>BBxG_;Xps#ECmo>$+R)kAT(<;><~T_G~Wofe+0TRFHuj!eQLu!V(e| zJ(*pAn&BbJfEJ9^WG&coT7al#Zsbj)Xf;K!1@?H7%RgB1jruAUDV|)O0Fu@@XqSKe zx)Po;Z+&LB%Bb_8-ki#Q0j>&;;;Zd%+kfL)LlHdBVf?e~1X3=~ZlyQx9lm+FZ&VzT z-*Y4!e@ zS)4w4yA;IZDP?I2R>cgUbHMLWML;`2%PcuRS#|zA);HwJjFMXA)u=H=l=PR^&H?@&)mK}TlLY{{tQ&qtq$dxhaa6hB4R zlh-eSR?dF<{xsuC!QNTqr&iW>d?arKu-QnEvllN;kb%gb0N3h+JULl3{LcrvJPOFM zbOA~^69t5PXz@3`G(C%roh1v91`^8i@igrkU{VeCXx`q2 zv%B)f>hC1-`ZmnX`1jZI0har7|NC2Btk;D9@w|9q4e$K>wesHp4ds7JcKNOQP51xa ztU@um-MIgk-|RjSL@0S)|5>Z{Qo-{-z54fm5LQ0?4q5@iCco6}rYfS%xJZsfTpsoGr~ZpMZc)$QL32e1v!Ii!N?9m!AK~ zXxvr*@+z~#`3rhfn9`}Su?SbU2PCwsz|$mL(zk60j)C4&eZaD(kKbyKQqKeUq;tgI z#YM5yVw4K4u0nLMB#66{UJ;%MyrCUi{l0R=^HHwJipl@{CaCN#XjC$*pl9v|a1Bw{+};Q*1xbr@nm;PJRo(dN(PH0^wctNBANt$P311a``u znV5hpOQP+-t}#*y;fNP9z;~qc1yu6^973VRl zz{yhi-Z#XroVt2cK{GhusWgsD{YmmH=XugdL}q22EdYfQ%m+Pn90%rrBFUYf)7}4J zo2Y$JQBjgi)U2a)o5O-qS5gP`7Y73dB3^!}=U# zp2sqjN|bT)KFI2b7&BrlwaK5~kQ*$t8!>!1>{EB`Fn#~+Ijah_iH=!B%TM=*edxu? z0CLr*kK$J_;%siUpqj8Ge)U6!4ppO=ef!XV2Xl}-*WaCH{sBMs8SuYD@UG&n-lhfa z2HyNTY99}Q`)386A~`oD+<9!-vLz>`H^=l0)};Ui@+9Uat|ue&L0hKPmI@LE zFiorEgSi(jUhFwEi>x36xvj zCb))xM^X7_5vR14JsmFW9N}?LF>M6%C=$a_nF@J_n)MS8OsY=p#LQcV$aN$z@ngf# z0>~w?B!l5*N^s)j)w*Q`lTyh&irp&Qmn=ii0OF>kTAJD>jjhQ+aN}jr%NZ< z39=4dPhQ;#_;iv%NlXDKyp+>7?xzjh0BGpI<;%Z7DiZ7z8oo$$-=+uet9<_?C9d`F z(4K_qdPZua~%)ZYnf1>Mu0gc4e*3ns#NMT}^C6sOt$PUg2oC+lxa$D1c`Yem4H-8MIs}YB|UYf zGiyA*BHpp|q@2IM_%o3BMHDTnh(Re729&g?g`dWfllw+iP3EZeR5mQz$}x$&RIg=? zW49=4d!sudF`EsjHc({&PQ~Gz!@k4TM6&DxX^zRtK&KUS_ka=t`cwJVHAQ}w^QEZqbA(+lu{)pIU;4RF>%?V6H} zuyI?qYPC>j*3R-zjaCT4metd5{Yfe^=&*`I(Ui5ahK174sl-TZ5>L~Rbh=NE$;vyp z1(4)$>fpgBB?24^lhf?MiY0b+wiCCpXsy)(d%rq|C1O8~oYG zj$bF&z0&)G$X>Ya8H^A-)AJ>jn>vzOz*jX*&9pEL8IsHrH25u5u{2F_B4&FAAP@lW zdi{D~sp@kwNgda|+kOOamqn~oeXQPIy?eLB*Pmci|9)vKOlUYciC`_{Ay&DaX0|*| zsL&pszg3Ejw4VoNJ06hZfWt)%t5Z0KDYsC>4AIse*6+E|$8K37B1l=RC6J~};!6Bp zrt4gq)LIx|PTL~^U$<=DJmvD-X^EW7%6IZdB9qv?z(nVh>h$9-Wv}mr6YtngzGo+m zAOD7QOo;4PbPaKKv~5@y$&QIZ*@jzEFR|TO^?KwZ(;P0ZdFs^j>DjH3aj6zX*=Tu1 z9{|KL9oA&exn~BiHW9G2J)wZxF7rEj^eEsISvW9hO#Zq9{D9tr2hVD&X^AbPzZM6_ znJg;ySM)!X4*v536j7}X)W6bO{r$wsbgOs&SS1IrxNNyLiP!vNl`Cp6-Uz1acUO*k ztDPFw;bQ=;n<0dJsh%JNGEm$AfF)D^iK2^Z=rdCF?fdsj?S~J$m_0Y8r|uj-kSW5z zbP6-kV{on}-@7LYmY@LveeccJ#yVEmP*G(G*}$lE{&=ze!EDbu z)6SwRPe4|49O)C2zNfMk_oIoGV;LT@>z!M5zGsa{jeBbiE@(*L#eiV$9W*>VrbpD9 zTKIM^y8q$*`^dk$PrQ+rh<>-@h_xB{i5&W~I`k&VYIUygw#PWBlQ|WDQ(Crd8w!44 z!`4tUi3tFL2uCL1dtkJHAekQ=9U|4I#jLy(w=zP{^~T;gF=qNbmOk5`pGz{5p6_U% z=Qril*CQYV2bo#eURHq1V45aqeT)J~DRk_@ zLD@pr@c0D{6GY1-k3>ndEJ*P@&YUU#XMq^AiQLD<#l^9BHhHiq;uxK$V@$+)c)W7U`w2ImtQ5`@aM62l9*Kg0 zEA2Ea#wS!n(LEBuLw>RCWsT|v$3B~SA0&TY3@)$Ku;CO3O_#t`uh|1|0S_%B#%$y` zf=*O^T8SUppHW9d-ReJCN{WZ|*JDJ(iFdZ~(5`^l|m4=uvmlH0#@! z$WQ>c|@e(^9_lqcTsmX~oYL6doF};N!DjD1^E+5*$!Qhd&Mx;n^=$Z~XUJYPFM6)NmBsJ!OC>2oPy#+JW{z6iK7@ zTVA=R1&GIVh*!|Gg=d{PG|E1I-gve9_h%gp-s$-ZwQ#uOOal=ikx923w)zye5Jc@C zf)tYn*7wA6cAxS-7T+QP#>xKVhF_LkhIlV3-nzG#L<6LErDtL7>S7ksDtd@^zmzQ? z^|3NJ$nC0nAH7&>W7vpVKZbFjkpCNiRY4c%|W4w^^-fk1Hwms zd0tCsZMuPH@O3%WU_-mDstkoIT#5eRjGf3z*wcXWJrTMpyQSmlEXqjH_0WKIw^9-k zBoP97R@`jAEMV3>hy^=D?N|6vl`^zet z_o;w%B;z?fhfJ&jB;SVB4$tpFcrGhUjUwMJ0)BfW;3dyZ1cv1D=@d3V2riB-Yc)$F z^ODS*Jt$f|>>XC}fJ$hSBG{(k++zM^*1C$ff#BewWOI^e3(ASg))o?7zJ;zA-&3VW zjvjr)D-aUZQxvf#OJ-ub0*7bIz89&M^^;;iPUTq_hLS*6VIL)|?4uCr1!eC?}T6kEf(RF4a9@ zo<)|%ey*UKjHGu8h@8`|NzG;Ko0?(J9=2j{*eve37#>mxL)x~N*kt{Y&57hi25NHW z)VaC>T?4sWmTuojCM78kZ0%dPHIeTr%UpQR0Ls)N{-lb^6Rl+kMTQ&#yj*vR3j~OE z(FVFY)(?(vlDz+4wuTyq3Z*Mpfir#$+MRA8Q5G8vm?0zDE6|geN_Hz2%cK&joOP@T zzf%qkp^dUgj4P-`ZKz?xm^&aELCLy=7o#4Kbr8l3N#jGE|16r14%6EzqE4OfX(!`5 z7Z2l8A2D4{78l3Ol)P`Biu|d4!^iJxCb}(7+dzTkP`mJ+r#W<>?vpQ^k>F@A+G$vH zDfFr04BqW}!c*!^EyPkxTE$!IGS`Q7&zM^T8~0VpnLRYW}tHI zaDisXYNWfn`}kW+vAYt22!`T~>{y62nJP)T(x~ImjEsH&M!AX37ln&rl}IePKo-C7 z?m>@Jhi$HG%-L-gE&vvBWhNG5nIeNjIw(B5JRu?Qz@iQ&zr8#K?jsFj6v5+V(+h;l zd0RmteH%U@qaQjh;ovjyaS?J_dP4>2r+%@e+qBiKeyHzcKwbAqjR|+xS3cRbUt_^U z_OUL~AOk8PQJCW6YRD@&$8Rkv+GmS6k83?aSXj({5a^PI#8~WiNjv(QEEvELM-svr z2tlPbC+#aHc_Karvj7%HTj=Uc_UU)jIMOVX>M0yXSk|TB9+a@cr@=M24PE(Wo_G9z zPWpXU7i+aM9QN1ueOKqXE0@}BiX+EO&dZz4XQV?l?$3RpxUw<#zmR6QaVLfQ`QJdR zc3IesU0JO^46E_oCuxrV)6Ox<-rco_Ib66ovdQWnOYi<;+HJ@m0$|7-J4?T1`{uQ0D~pM`oc6=?O(H@GNwZTsKvRk&zvxBMT!Riw4?{2$k#&v&30zftUy zJyJf-ddnP}dh$OC#m5-z+Qle9@G0dYa<;V;P0dT#ooz{6&FJKO0LVko?kiu&vKH8r z3e+DOc|0kdQ`;RM{R)eVrCJc^K0W=Pxa->crMPCZtC0e5frQ)xFyJWA=_rxs@TEzY zKTj)RHLVqvIh&mw+3D2fjI*}V&o_$8X4w|S>u4t$_R&F+QmsL#{tuB$Vb4`1H+gY%? zw>R+h?ceY4%e2Fo4=RV^8|W#S{Deu9dgcbBKq6Rp5u{cMQwypFRK=mXiR|u#(9nMr zSe!KmT2EQ_Wk0KS7J_4XDm#C?TCcQO4P_yJWvkNnTNN`-^1+XtM_ajQohp407}{~{ z$;>@V;SWUoCG4nj5t)^mLBP(W)wFt7eJ%7`#awTu`pv?}!x*E2fs8>VAqaLfQnTv zBqPV2k+x(GZ#bH&&PO@^{P_TfUmK!&cSu7Qt?P@mY5$Uwf6 z9W)I&i<8NGQ<2}TqH_<%V|d&+dEG8Q6V8Vzcv;?jEDu9jb#$M7X#RjwjUaeiZR7qNbTu zTsV(5XZ?C4hSHZeZ=^T}f*P(C{u95DyguGL0>|lD971qJl?| zn9R?-1sOtqK9dtnSzGvu|3%oDz}1}hZ$C4JF=H?WBgQsU2w6h*Fk@yqiAb`?kfmhF z8j0@N?Kvf~WQ>X=ds)VMkB|_hQue)USwm6(_jNLJpJATYov1?9&ydMk^$}wY~k#T1Nzn zLZ+mu(niTEwt@#4eMq`6ZW{G$O1}{SS5FQ*-xn^YO!%c+dJI)^s4b`VoJ!O^+Fr3c?wv8v(uc7)K?%QGS^*KpfpFMrr8{I#! z;9{s|IuB6rZ`yKH_Bimkrz0&hF)bjII-pXDf#mzSEM!4zG#t0_YR7ILSFKvLNhzDW zKW#O(1N^`hsB#TZc44<#Q#vS`2#BJfAC8EHW-fjbJ}>DCY4zmJMN zg!)4hbCk}ZR?_9vn(^R2If-)!7unRfRgPfVwa>H&1-2o)3*yY*F(mkG(>sFZr_nMv_*nxv^V3O#ryO%ZX6wolFVkkFi;66grn zG30hC-8=q&e@>c7PF)AYuyijV+i341c*s$-_{IogkyJT78Bb&|sdB}Nzw^%OE<1Dy z8<4aO{d#HkhcHVUzFP8OI#1f{Wn{|y^SACHt5Gb}NhiSBfc#SUq;aKgaKU);V;ChW zlh=_ImaL3kM`b$@TNO82W@e7zCT$Ih1$YoPVblQl`8J4vrWbfG_;_zBYiXBFeWb<> z;J({6hk%;1P*0Xj&V7u>z-~Q&U_t8`-jX6l;0_Oy7OAT|XM9X#>nNVMCM`^a$Q{ZZ zuQy0IyMSyv4#$!ze{ysrSa);Co!To{MH$iNqR6r z4BRapWn{MrPGnrUS0!o8$Xms)AO!igNa<1orSRU}(R;Hzd zQAp*CbZry#Q8c19kq+dJArF68eMrVSV4RC9EsyBA`pfquCG+n8_ zaAva?tIi0}h-;Wq373*A$aC~)C7X4$XEZpW?7!NC2s13MJOfyC4qEV~A47W5hu%W{ zzaA()l>O*g=-0&73E)(?e9B5mad70Mmhhc(tu0r{1z!KHLk`x{ix_Q{u(3~nuveie zDuX};&d{E|qNISuoWms@XMa-4Y?u3$ph5LhJUeGZl`Lyv^Yu^_wkhjLsaMy(bI5St zwFEbJFU3b%*fzT{(&ieD^Jdf#eACK6PD|BfD}<&ZGiPvN z#yI*GvA7qJHKq2-M$<(OsjShBsF>)92O43HwNiQ9{nU%a!{YjKy zmBWMW(sWTf_Fi$lGx*yk5K<>&&VK@T4hxMS$$4{-)fY!uEx(c~`mf;+-N&GH&F zpv(c$5&?3_nGG($?v!l7Gmg)G&9jZoTRjA}k(PZ8I2o5t5J^-3t-0BNUD!;IeJJyk zy?9H_|9Ei;`4iA&v)9M@cbu7)H_(jeX`jLHawg#dDUyS?BquuWM^9$CD7&N9Mp{c3 zD)}5TvN3VP{5%N7_Wt{;sh|BRJ!~pSAMBgx1H_? zvJ6cNfTbCx%(BVrkdhkt%v!tL@}Ug`EeT=gIq~24@RyIOkMNyAWhl`NC9%fLrA6@@ z!;-2eJhTHodqxC~^8TKzx~*>(6P}0l><|*yau;JbILVXCCO*9GTCGe%W&mjuU5|=P$)XqSp(H(FN3C+U7T*JHXjTmUf4H2A_YH3vv)OMwKNe?kTDg6I)lgLb0x9RQ_*Y z&IvenHK$A9&Nhf*erMC~T$eG2tV_4ILAO_>&z<6p1M!Z=F4PwI3VWU%)R+7P9HdE5Q(~-GP<(H0ouMcufNEirrNag8}Q59d)jS2T41?&P|Jt?>v zfjqo+v@7@nXsv^G#MY(5)>0v|og@`M220OIS_y|rezl5;Aq{323oZxLIBR4Qom z^_LyTaif}wo*rsE(yH40+57mM77?E{c=z(+XW&!W1QiLg!TZK!VQS+qI#fU!eQxxM zm2t)6X9-6<*<%{^#U zW2CF9yo>QWAI~Jx%sny{1jDf9NCxV7E|U`-Z6637TZh_-vy|7_HyLz+laod*d+=-dOs>{{pxg62Vg(_6aYLaItaZ{Vo=!dg;zE!qBk9v9! zrcxSnY2P4;FVQkbVXWk66k7*l&ed-KKXk*Bb7hiXs zs5=%!itIM~2n}~HAlHgM)e&=$c6+N7N&k$b-HUfFUY6_8Ff-A@!^7j@#dC)(+f0a% zB4OyEUp+USF5Cl%;^R~@n?l5)@cA$>-Q$h6_>;GI2F^_EczR`e#;-fv@{9K0PfTt~ zG5g?r%A!uQpkGWIAfzsC7<2x_3wnmawdp(N&AqcN*&!8pGx*F%a0bCL zU{B80)vwJfrb*{yti=IAwK|tHLP{kF6ro}&C@EsrQ>GX_``aCGO-w#@>Qt9&Tc6bF zbN(STj1y(uw3}NJ;#{(G=IvbTtq-)ct5Z(V+H>_08_OA6gJT z!x+FPF>3jl7t;w<3if6D2Z88k11j5uR$7&!sYo!0dVLQzxqsf;y+rLgz5)6-9((J) zHEK7WePh~9znU&}!v-^pP6;~A=CNj79H-fAt6(4G#5H(=?vljXotq!`{1}`VknHa` zdPGdwX^bFOHw2ZYDw)6JeJwF1tvh(E+F7qm_{OF|m3&S+;RTQ zSy1}E_hPUs$BiRs)o>$KG^CBQ&^RyTEVJ`i#N0!3WB|mY)p~h`whyBHHKg#=TzeW2 z{&BQTkC+jV*cAeS~}3KRzXAJ(kSRg%ewL|(lnMiS`^ zjvUWYxn5XU_%J^gG6PSVUw3r4pX1MkOpOOvah$V%#eYSON&8^=Z57EgC3M7tVI%0? z0H1A$;#2eIN(@19I0*+AzsSzz;dDr$*6jqE*Nz>qdB}etkO8-%NYF_M3DyVua7lV2 z)G<7aDbsiQ{=5h%xTVI~w$}C5M)A4(&Tx4~S*jr_SD*I+>n%CpzxDeS)*YEvxHS+tMH=PFw3{P8Ms?KWi$=U@i{vYjR__kvK@iGBNp4azz|-mQ zCM+M<$<=)NsZ^|{pXjzuQJx%k*s0CKUL6keK^%>I;*nWZcD(H5l*H7ENMtG2ghSx@ zFW&`C7|G&PIrgayP_mdknjluukCnO`gadUx8_t8qz8-|1XK zfZaw)+l%BW_B0?bbEHWii7h(h%0JFxLkFzRFB9ZZBADRnebCU0czQqD1>xt>506%X#2M#B-U zB9~MmODXO`3_{Hq8t6oacnxq-Lkscb1GwSx8}KNxp1w~P2RyTfSVe%7ZUM48*CY$Y zh&#!#B@hnyl18(WD+`$qN>8|OW*j}n_ct!Cl8)Ap0uFNG;4R{Yj^Sxb^Wt)4jbAh51>USt<^qmrtCT8}7NP&bK!`TWpU(C#zY}*ojY`e= z_1Be&o7IlryOsf%N9lfS`Xd zvF3}pzD&~5aEN8>wX2Y~B4-V$e614-2U_Rt>M~S?v`u{ffhcU8XWyr~62@Uu;x}~6 zd~fE5HM^yt&YTL2xkRBGuNQALNZ^*?UPVA9j5|#L#(x=UZKFf%=+JI-S(a6NB?1=+ z>kM#J*yViQ#uRbjx*qf+f{4@KFYV=aJ~&eniR~-V+>r*9nL_V$T z@%(;w=I5cP6f~!oMFaMiDN+C~&Rosmd-6bx1)vIX7RI3Whxn1MRL-XzCOUvdDDLq2 zOi8)cOzv)1wPR4~;zCu117TBQyYdEX59isjhD<_HYV}aj7vj1oq1`~1pYn{Lkuyl= z9UVX)8Hc02x^0fP#`o#hkuF)N$pm$2UVT3Uk|Ofa`^NBu3S@>C>nVX46BgSh-o+t- zOXNx&_Z^Gh-b;8DfqT$dlJs(?*B~7c zd!w@`>CO!oQ+@&om3@rWtK7Op+b|aCFJc#%U(yRTpGY+=s`4Dt(AQ=w^sZgIa^Q+b zv@+lWYRUj(e=O=V4`U38KpFeA1%{lTFHh+&IM0!_LLcRhnc zpe_q$ht=(q)JZbe+uO!3Q)+u;$!hKdw#_7p$#@YnD8I^9#%r984C?K&TqJepiS0jLP6}#1vb!h~PC_c21h88@>I4lBY0%61XGd3f_`&@L z>ckt53~~0k67MNvJ1j#skRTg0YSid-%PYsfZ(5Z?W90>Hw55*QesA(8Q_tHxAESGa z^5fbiM_+83-uvpIJ9j4gU&^EtmU@@)AXGWn&9akG1BV2n^)&KPKjL?sAwG`|GKGqW zc}QCXzYH3Nc1|8f!&tnBWlEhIGQE;Ga;OWqg51{0d+tpgab66I8#@36Qs94l*LBX{0@4xSu zp7Ig>rIrAJXH>!tOL2gSPT}{gn9$Ihlvy0+kl^56dkTFoZTj(KQmV#GLHSj&MGb3$ zUP56X-EChDxD<4F@hWb+)#cP9i_Qf2l1;?f>LJz;9-u_d>i;0VSr3<_q{WGOeteko z`klF%1z&|moL#!~I_cudNP>^sploR%3c z*srzn_2^Z#_^adkt(|Q2nEx!z##pujc$$V)z~5R7x82WjH#DNl(76wS2Xxm|KTkHm zRs?{Z92WsMWF)BTNei0;R0jDbaVBtK4E6Qi2PO7-d+aDboTC#MF}?TLt@|7JUVWNS z(EW0zal)F<(mUIAI(F^m`pM%xVt;tN?2%)s&E>{!>|eV0o#pm+D&@Fn_&em>2oMr6 zr<#7Ehh4`Bus-X+oP&@9-ET6AqmkxZqY#q$BZSoHY6<7`6#qNxFH7pw7yr2&x9Iak z3&ejq3(OkF(#UtrsERr_h?yk_Kvg}A3e|7XqJ{N;l|KWQL>05z##qY>_HkbozfR!d z2uHh4J$js)Sg%CiN6ZtO<`{qd)p1KRJu9#O_zgN6aj3RU*!vIc4*w;p;7he_yDs`m zNl`oYfywv?le8Y*znJFclWo5aE#p`Y(gI>m1ER(VX0T*b#*C0(=IXy*o&OE0yZ4M| zpkg4*2tB)k=F0B=BOVrq8F??O~M)~-d^%p6vYl{ zsIe5F(mq5&4|hki-xFc5Z!-FKO)o+@pb|~z2evc8&zN?95$4o#a40-&kGKBU?DN+? z;k3mwkdChI?jo8RskAw|!{aZUJlS0xGxMmh0#`ylFn^yJ?sKf)-2*ouCqS5-M)#-E z+5ny=1VBK{nKv)whabLj`bqcZ^`8Ty8zvY_NYad~Ls~6*$uWCg0oaCs-tgc?5?WL9 zh{rmrC|^WL{>v-SmMT~?I=+sSBG6xxSy>%{fF-DToe*&Cvw@e?1RsZR5fK>%ai!~( zzupc1`yh+YZDG)KCGRMWQbv!l$OFhLj6puEa)vRNr_9_RWiF``Vg4KfDq5y`SX5 zzX9-_3@Q77@T=NLn%|ad&CdAcxIEXINll0cbnlViMA=}L!sw(UK-IH zf~xZiR8W$hN%{xi&&1yvvdA4i@M5bQXv>+BiJ{#10(bYx*gbMqB%*X}P|5UC#s9}+ zv`uK)2PiUcbK4qt?AZx{7D>rGib_n*X(XDpCcS#pU5;cwHBZ#(-QM`Q$iBk^i4G=Nz?6G-=k^ubBuJexyY$duYe2`N z-2NU(;8*U6cimujfS6mKiwTV6C<_63(Sb^Paq$TFn%(T>-N5Yq1^rs7YpLfLwNJ*5 zMr8}1GuVp6Z_s}|8^@KNf9Ts<^kV0I(1C%Qxs9v9&BoeQG+@s+Xn`TGBP)&6k zo(;O$TrF8f_Kxk}8(DG?%|pUW@XicIn?lNqT#dMtH(!+;Ef0K=4}!K4 zd~30>`dqA_f~8Iq-qSh+9%c0&N6Ce~k?#m~!e z!6Zc^A|tWp@HIrqZWPWb^fJ z;G|&#J;v#szQxYPKIs_@Y_LmciIo@l5+myICeH|Nc^d0Ea5%=7NV-0|a^^|d%8;5U z&wj^Avj$GKFh~)R=|Fj!T9wCkUdFQ@$~*LcDzi3#aLzj2!|a1QHv5MQ>hgX2;INs8 z1Zzh5ov@E+2uQZX3idy7PEWI&64ZXvWV=obUgX@Nvk%+nFzrzUQ{Pqwx(DaYv(E3M zS+C!^_1U(a&40)lV!D&Jd^1R7f155k7mx!+>5b_g+;y z#^CK|a{p^S$tmmA#iH-~-0HQylB8Wg8Mncf@S;0I);^v`W0?FTZriqr-(PFLD>u=k zU44BQBkIQVE|tv;IRj5EeXky}imQbXQDZ@n(tR6R1rwmX$!W$2I=mRpr~9hr3r>%6 z=9b*B*lOYuCQ>0Gz%&k=RVg^Dj+JF@yW-bQY0OKrKQWCi-FUu`S9c=K~b6*=1-AWmu)Fz#V#>;~u$4MsHzSiV`EjYpEYb<6AyEop0A zHV-<#4@4dx8=Ru&%WJ={)=w z*5gxPC6Dkb=c>;C2033>w+J17r^x;C;rfCrO=drPb??3tG8vplj7J&yDl zXtvOe*+mv#pv)ADWT%q9rqZq&T+z1C^Hd*Kq%~HY_O3=@VPF2^wXeEvw+p1v--`)A z=KfFoxwPGB|4$=+bG7>n68CJ#`wWI1>awJiUw`;$Xk;5Qq@-#DkHuDS?AhES+4NZ( zB-MoLaBVU6uPed-di*%Z>7(mly9mq-TMQ*glME$>$;zi>!C5+g_KeZjhthmH{q(s( zt${$+Wh0L-ZC0)M2Ztkw*C znz?7gY``@S|)%~TV#d5U@M z6EV$wK;2G8xzR(7u#461pgm!m>Hb@lDpbIW#YJN1^s;6C_pSq5BzrF|p*oOev>{{k zt7hg$YjlHg>b^ebI6d6sF=sa)?0@BdKSZyvx&QD$vZQ8OWh5(Et-%N=mk*5g3uR5n`<&j_0HwqQtkI`_>bmMOh9tu{Y#x?<*QflkX&r)o>Y zNpZ2m-Oud1Bl@|=l&w&q!f%Y1NcSii2g0rEIXTf3g!Uol zhguWe)oFmnwC!HKG=Zh^Fz&x*-FfX3Tl(8|qG%T6K*8%p0>N5!{2qHo1~iI+oOTmP z-5o_k6s01)0Aw^GqiHU1j4`oI&2F9J#0~Mz2LOrIP-~3>5vcM>iK~0)Y)wNPAhP0F zj1fGrCSQSS^RssPu8A|p+iG{)3UQA*RL`YEMBByyKPkZ=?h1?7upLCpZk9y9`D0mj zCD0-S@7)_6c&bfI%hu_@jCUZgBy(jvB+!LJZXl{5Q7p=VtfF%gDJ$ z_rb!Lbs(`g?Ke6&1t3E1Kb%&XUYujhTF*E4Y(T>SU9$gv3^$^nyP>39~?$bl+4)B zvf6wNDA0wfcu1WB8I2cxA1LV6Z(hm%c7ax)=>t{t?|YRuKtf-zp%DcaE?khCFmr@% z6jLqEWsr(Tn?FRImJLq>$C7H4CL)%YdWT;qt*@00n1wowxV2_&!sWPOPdx+ckzE7L zP9?(w$KQPTl5Qtjpa|?)RQO7^Wbh+LPoCUUwKFMMgr>am5#FT#qX%h`ygsR<08Cni z)7(Kam9+td)dth}q)Y_RjOj5&Y`nqJyeIVvPiycl3#0p~j$-;)Vj{p|li=>$l$?~W zX^?4^w-u|8Uf;?+_8UUWYUimAi+&}9qAnN3$^f4Mn>;!gQmB(-E1Lrjn?B-Zv3H5N z^=t^cq1VlCzxsF$YLK@X;dOTPr%45R{FmUyR@a_vz%ypyC7eTTffH2R z+b z=mAr?Hj82ZM*kDb{gaGlJhS?%(v(V_1(iZHN>oGfB$E(kbC;YRg+l!&iB(&Ar5swe z+U((@oZ6OG5-aP|T!A&Z3U8-k`7%LX$JQ{<7p7as#_YjHEu(l{&h`Puh;6CP@UK75 zXHhB%vS+FXto2{AF&YJvNArL;5-Dl6F)}Ifi8X7E{X0?Xg#+I!eozc`@o zZ}bIBcx8|V8{^6%d$TE1+?Si5y_X$hF09O@ePp~)6D16lXUA;X=Sr7vn&e^!>D^&X ztLiq6{>kj|JTUqoR~$$QI1g;xD1wKcX@F1lj0K*rKX$~|NQHFStil|c_*+W$%^YS9 za=>&zTPrB7ZmvvesUZOr5H7^)HLF!fHY~Mb5wYKxhEKizq;cr+EX8TX#NO zT>rZVw#{Qjvr&p_*(kziugolj(OnNeP3|4({i9;vfnz?duiNavGZl9X{0`lNm4Qtk zRFB=ayRg?-tSB9?>2g75YR*ddJVJwA97A~q)A>D$4E^ETEV4rVA52z{o=+vp6C-K3 zP30`J3~^Sle@~B>tf~vUsqSlV^T6mF61rR~gdE7?J$cn&&9IZ-Ke?wtR_glEaP`8l zz5E%q)3O)lCVLtqsF6q0fx34nRl5#_NUMN*SFal9EYj;c+P?P149svH*gH@*30y3( zok-chzDTB^OslzlbQi*;WA|)+C)x~cED6xKBEVkYKjm=S;0(NsM4zdQ>Jb! z+xAA{8jM37?bsQ_tWbOPPfr8f(Gm>Jh!8(y8!M8WY~xKuQ^vg|o(Dfibziw2R1y=G zp|lFDMhO-}NQb72+kKJ;-H1pm`LbxxcF-X0%97HXE6|4y=l5HGR{Y=a@VRH6vqED~0bEH<%{g=Jc?3H2jQ8{YGWaWPDG2-g`*v z;*!0;tncEI;q5;KC|z=J2r9AF6Wc)^(WeR5WhAD`t$x63sOPj(g3sFUGxd*wpZ?`f z>UU3@ulK)vG}kK3Xi@yo{6?{`@6?F_tPo1gNbW!3az7^IRfYn6939o7L5mh{LL5nL zvVc$qK-iI@)c|~EsSO3)GPEMoaP^W`cS0WMS$25&;48&gx#csMFqz#68V%({^a1=O z^?rnJ66M+Ws#^{`3}ge6P(m4hv(xoHZd}9 z6gYVGIYp=&(e()#OB!W8j}UYl;p!yKP!ZXC!5y!>ygcczBreGcl`$pA1aDmeoK)ff^VXL5=}H<^V{p3995MNBEX4V&XJ-hwD%=+fEkIh!g8C%%6_2q#Hjm1&f| z{OJR_Jtq@0yN)|L>Y4Fz`kl5PEO+nY{;!a9bAs2dln5Wggfu88H$vu^8*3>|T1|)TRNDVbN z%_1+h7<;}o_hYc&Wub>?A!NgG`_$?mC}ksHtu&Q1;uT9*MeG<{DZ=pW=}qyGO{Agk zPebvfiWN@b%C6*oL)C#(y?PM@HEe?xfh?vpX-MHdNNfTGFq$pZx(Aw+1F$AeI1x&f zA5fj$@vHofjQn~-$UlI$&zQD6#x&_gh#?sr#qjb^%LU>@bbI#c69P1hRvdR+wK2s zqa5AoDq}d)QIARE0SXGd8TU4gM<)Wfc)j=up(@*QTwL!~d?sLjJt0gDMYPqH@xXN7c`@62q zKt_e+ufXpduDI)XCkCVVd=!?4M)!h8VlLUyb&Y99P7dOZ==w3B;NGcI zr>$4VLc>~l5ot!8qqVV4d~)bzQYac|6Q}Y_Q-W>rRwgj~PQtKm{msCwzmoR7mV?w< zk9qntM9mi$$Ya4r87pzxrqM&V3< z4kXFjlR8zM9{S-C1+Ry1Y(4h$C0;hOtrJv9g^NiA029^+CEnorC-q_n>qR2zmaKa{ z7;cqkw7d=I_(VT_`Q?|+=P()2M1ny(=APmnX5D%gk72~wM};o!raxIXGe4)Eyon%< zJ}6(F0=dQL$CJMPp5$0xufja5lsrN@$tLdDH+vlWEwp6oqzac#-`q3Q;lpGqZJa{m(FiTB~vahceBb_6vA+K(aMPu%}2Pjy4Y zUI?7+tdptP`@CDKtl!3m;e~CcRZG)@85r`Hk+$EMLx&E@M91|dqnMMZ7Ulk=7$^m` zVfZPwb$Rc$JVy#finC|@2IM)8;|C9%Rv58;_Plv5#(an{o3Lh;*vL@3nW~n8XOJsr zNYKDkj&t@-yMrO&`Yr^dci}EuT}&;Kvn*N&O%&`eDjWb`xTw+}T{;104ySxl;0s(@*CuZujKJftVOogwvQU<$wU+f^5OlxQTY1N~QZ#N$!*(kbXh_?~#QvtIt6fBq@4w8O;X z(i>w$wYbs%rvPBF^mc)t0xFEjpAPp-HJ&_!*}0T6w`6+5C^RLKB)p_(7b!a_qVXGE zEpUb>+SsUL2@i7HARMnDCi+m;N%x7i&sj->=R&(`NzZ8wj*J~WdimvjJ0DBBR5xcu z`#U*#*Rg|k93Q@Ocs0p1B61hCJie@D(vznoB^d?8oYaS^J^U|obdTzsXv66|0$s}v zR|;AEHj3nyT;p6)5d3bh1dFaWidGHhiJ}a4$52_jFXu)+8^`*$?5eU&;Qid7G)0wj zho}ZAG4Hb)HBK}rb!hh7XJAxi7Fe?2=kBm+t^;4Zy441C^Jk8Y79H$wD3=~jos*i( zzOqOl07Qhhta|s*bq?B1$ByUbe?0f9)p2Zv|uygA@!xV`^Q6wSxs(r^5{gM(=4;j^?H7 z*2Bl~8xfkS#qeXdHIOo(Xh;h7+)poOU!rGBV$uvKZ7+;d3pZk}8j zgG4~B;(bBC$+M95g?B(@?}hKI(oC|PYyD2ddYH%=_i@F{`{h~rI{Jkm`r4ARrf z$5T$Uc(t}Xy=q~-SnEzS_u3jpJKJCF!P)mh9i4_RLkM~lf$DBplKeTQdE5=?@u~T zX(gmiV+o+9V5t<+KloT@43DL&PKB*VT&Jt=^_7y?AZIYGcs@t}IhGNub31YyE`GnZ zL0BBGV@t_Pm85n{-UeQ?sNPVTJ>iLwRo{pJE^iY(I1ILjePIX zB})h%7T4yMlNc5scOB@C}(K;go@1?3k0KE24HF!RYbAGCb(dGAik zo>cjATuQ{O$|0ZE?%Q+AcRhT5`}MufJ=X`iZEhXkyUlXSwu#9BY6YJ^_md<;50$zzJ`r2hjCK>q7xN*4#yAlh?zd{wrXSLTG;~Fq(3m9Q_QX9=qbwI|ifXW4 z0|kU*%aaV?qC^Wx^h_Z!^`_I7Q57qoXwBrruHAW;p80z5OC^`%&192bw^~_V2O6NK zoqlz8y%vm$z-G1U_VthQ0K!j&PB$G)Qw@?{4%S2UFZIW=BTt(m=UxJSVr$UYK5Kts z-OkC5lhkM9;NU?l?f%$b_viAJA${!*(2Z;)MVmJ3ty(pB_bZj2XfCp8!17UtSsSQ6 zGC0My+cIak+{sXdU}16IHnKq=!`qs#jBLh}9M@>@QacaV^*72fd$81mMvR!1Nh-csG{=|J}tZeu)jv!YV_K0$-|9CquOAxrkcZL z(aCp{SD{Ze+2Yar2nNv54dIg6N_k6kThY2Fv3&8SP2-krp`);=cU4;qMA$YZ^AkHuF_|oH3 zi{2ak72Kri0?u5@hNpRh4F-eOJdi7G`{i?iWb@8;g2?Lw#|9bujXjmhExX@q`qbSW z=H(sgeJrQKR?mc69VWId+5YpIiy!CqZ1c>@Yi4-Hgtk96^7YT1(EXPOeXrzpw`9w&^S%oXg-PkgL ze$H@pXI9lCX;nY#bIw1caHBoiLUJ4HzAQO8Xvm+4$s zSY&$kwF`ufMi4xSVDDUjwUsMfkg)LWVJAk&sT?8WH$T1q`Um)*1H%$Gdj|5*8H_Sd z`tipfClzqhybt~Qv0a$6@QTP+$k?469eW_So&P$(n|?m3 zF(9bwtgWo-H*40D;z~rd)?e|A01`67GvII&WWGi@S*$T9!7gvPKIF_HW>rr!&G8gk zPehc|&1o}7Z0lDySdVlMMV;@NVa$O8lNzu62a6j2v~Rt44;?-%$4yBRwF;a7e9Y4>(bJAi z0JZv~eUV9;vz4hC`f4rFEyHPSg~A}TtIki3?bCTr;QzrTa(^doyislLqm zP=2mlA4|KSX)N2QA_ zS-^tbV4^@lH-H044SFX zta4Mw@5$}#-l4&*OKT?FH@L1F{emxb9r|LHDaTsWKIikurZ+i38x25)&JNxE%a`DN z0+%G_kt-CSL1iRv)P5RG<0wn;vX3jiqFm+wz3)IbA6LuYxREGaH5$6kWAF5S_s{O4 z5XaMHaqO<}S%uRE{5a_nd;OM2oxqGlEFGnq9L+^@vvyoi{|%NDMNVrT6B^{&rZ#Jz zGRs%9u&CO&ZQHhMzdE`#$v3f0jW0tV>?_^x>ADrO79QJG&oJ%8k%@|1*NTc&I)Ogh z40Z=_Vc$OU)pTiG5rnLb>_E}fWY2>yLvJ+;b1vV_uG2RVC{MIe4`hwzB^}`u%g!@a zlruj$279McqnPW|bhid->3dy$d^X^}*O2)$g7EPwfB0pCB42v+if(6OQ*%DjmG^-G z334f-|F)MtpsJei02YYPGFNvTedmuABp5`Ea|ilnXUK{dBE>DC2eP)hRnWt~>R=TVZkZ%&@_hwP^@poj1>jw6fvRjq+Z& zU_tymVIK?-tOhF8Y)@z&IY;|0f#1>rF|AGP0|PR(vWFF)5Yx-n{^IA}rFIUOFk!27 zPp3}xAUt}W-2owFe_<-6X{Cx4jZp#MhVrewaZ3VuLY48pTdA{11(3PcH)yX-abR!s z@Ep~tJOZ#v7*nl@*&BbBbjmXj-SVB+OCK2~i*+to?p&96Ud_@q)D2;^uo*2k(bNq0 zOoS25IbN&BHOB*w$Qpp3=0DrBD|ig zJ1%!D8&h^X0~-Kb^3nR{p4r8<==&WqJHTLU9vhS{{pTDc5C8kqux!^-qX33!JyZ*? znNnOr6@SA9eK6*jgSoO7r`@XY_E&!cGFMqFt_=xK1AG|49ltRT_ZBZ_mHVcf4}$O) zGpYV*ihD&3BfwVt#qcr_)}cg;WF>Pf0PpBZqGS?2awCY4?6{ypm0N<)WAR**uQZksjSGQ!;@5kImvK#b1uWgfiXeH(;qEqlnS;tqYDJAei)qoM z)Wumv0c*Z|eKsDv-}m9PVL_vfAFNa;ga9-8TsPKuuQ524_|KdF?M+&qSyot`>*`#6B@8*EnP->UsX7O?AQ zZeW=*Wl*0Sa~&L;G|ag!Y~Fv@jW_#&qb<2?e(2TY_eBtorrghm?UHpIK4Kd{R7>o-!DT(vXoCWV=OtHp!MmZ4?G1J7xI z0C&sIQ32mFoI)icZ=-L_l@*;f5={=kdrR3&xzQkXA4@3ZWd8(@Im@^2pJ}Bx-WYXK z_sI;2V*_~hFUN+k#o_*nCoU|Hu6;9_-${ipr>!v;j@Hf~Ak`+*^04%~ulX??)86=w zqxot1KGGUB9=vY@<(}k_mU(-pJ&vO2ciTLqv%dd&(1P5P$^UCp{=OK?3ALRNFIyWx z;Erz3h5s0Cznm7*_Kjb|8~F`N)Nc6+4KG35EFHU_^e&%fx&ryPzIj07ojW0}z|+nP^BKZXutv<8i1y}*_sZws zYp{ZBV*iinGs^JLGC}g(^#WLw9}Wm`lhUP2^8@@pB@F6BW|NkE|NZw8Vt--sC4PND zxZqrm55T77g+)Z^;p%FXnt*@+vdzMSi`f^!{7>+Rs2XcE}!N{<00*KRCu{k^Sg+Pb~{2mTj#)w(;ub3(aqCmacB-gQFTlFNVP zi7{51dar-EZy)(x*_q|xdGPxNP4m~RDEF>K&{q>8?96_HMbmPL=6`KJqNbl+V5ZN^ zQ-7E>JGJ}x`kiX`5)@n&Md`f*%zk9T{L9I78oaxr z-PDs+QXP#>A@#r8w#)3xN9X%6xV3wd*8t})xb}w*by46{J|K(i4iC#f`BtIfx6KMF(Wql8M( zSvS9jcwrDMRTYocXqDNmX&KcQ@>U2|Q3H#;yEJE#6^`$=sB zso@M0*P6+dFC;3wC7C`X?@*9jxso*5_xwwBj&w`&%9@?1Kyvvp3mc-aFo`>8;z97f zr%ALnfKth={PZ7Yn$6l7&DvXD9!rajB>U74=dI1&%B}^glS-aH;bSk;Kn+j*koAu* z8?Pi<0FL$MfYO~sN)(|Tqw&A|^b_aRBZYYvxjwA<)2vv#yk%szT?u&3bb7kQthg8p z5L%0#-wEy#&&*x`ee=qiSC>b^zoXA#)=@eVDv#+m7WctQd@L%bOWv1gDH46`<-zjS z=@}k~*t|aAB*nm1$*8p&Zb#M*E`w3qv)(!(i%SqnRjcFL=$W+xnHSB8#vb218Wc;+ znIr;`QsgAL&cgnYBbfANvA|FT+98m=<0B?U1zfv&73E)J%IUYC2<>JChdRX!Nb`Hu z5fem2SYard`SvTaxnm`dJ0EB1cl*)e$i)Q+DY^aQd2c@(53)vY(wbp-rEQY?Ac9_- zHIrK+heL@ZIkF<*2n=tY+#h;=g5%O)i4!P})en2&UPy)uwPK@G7s-~%D7+I6nxstF z^YaoDcD+u|KE){F{|?i_*3?0cu9$L3Kq1t)PMw1j9u~p!%%Pea5;Z)cDnta&u+f0=Zjyyl(uJ(^=&hZSC(F#sFNR*G}^f3EDzw2$zdYe!LO8qhG;~|7%I1f1snk_#; z7uyFknyw4(dR)vuD&aVv(6V?KN;DAV-BqYiXtENTNrTfjC$*MiA9N?MD^ftL(zy1`pKfF6{ zS!>t}>Hs_MV$_SW0m%qQ)0E>sn6QiSsiU5D{TlqFRPXnky8l7bU+2$Hqd!)fPzB3q zC&BnwdaV-}5Sc8p$WxkSlQFi&$B}*=^|7!1dM?q=8PP&wT3Xtpsrl6Jb~S>RE?oy; zA+;v+0!tXCX~L(YjbP5=((~Uwl4k`zyQ6@^+te_9&dbfd+cx~2LTMoX8jt|I7=CuD ztc@U9wFb8X8}aPten*sp&b!cI;U$)RsjH(#bux1N84ijhwW!yS1p^MsjcJMo{AAK0 zb=uez#jU1d9z8~+AluuKt#5*`NYr(D^^IS_b>u%IdY=%xV(fm{3@Ko!go}Ro(1gA| zaI^zQV9-E2x&Sj;r|zTLOoggxIraX)j?yldtvpmul1&<_O;>MV| zyNy25oo)YM&rnTB?I-JnxqD~k8UJP2yed*oeM3%@(IeH=$Y4HcUfGF$vi{%DA$ zO7C;L*wz+#=#pg#BzrzkFwL?W)kp4q`6x+s1crxeQUCSZv^Q@V_i$Y5SyKY>8>rq1 zP1q8`IIpNZ#Ij}(4J6OG`#VJhp(m>woS1@l_V&^r7gvM}KO^=q2h$mSs(|FEfX5JU zvisiScRH%s*>mSEJw98!7ZyFcq2$hUSP@;na%DA97!R&n;m=GZZKtupuBL^RtnEAv zebmI}@rMNfve6Yw_-GAY@mZ&_Fcs_Ru?-=KwEdhKCIgHx^!{@!oU$Q$`w&sVf34yF zIK92W8PiCHs4Nz>7>0`j?WDFv8KyR{TIw9-HW=92eVFqW93i@10>&Egyeb!G8mF46 zj`O$k=y9ScqC|Z`&Z-b$MY*;q%~`lGdHnq+=W9$HI`nr8DXMF$7?hOE31w(U>C-p= z2;e4==3(ZEFl49Q$0 ziFC-TPe*|El)RI}d9}Ph6>viP0(g$CP_VK>7fHl~={Lhhw(v@ca7+5`Nc@5T48eG#xI0vZt>{3M1R+EwuBv!@lT?>7+-=y36)PlOBwaWf0iF5p)o!}V zmWKzd`Q{_?e?MajoX;i|X!=gZrG>d?*X{&3N4L5Wt*ZVFiHd#}_26nK6gGE%=bg1Z z8hxgngP)}%)Rv?aEYi~&hDBD=o#XH(S3#9fh@7+tH$^@QygLdho>DgnM#MnuZCZ=o zmhgO=;UstDW?~vJsMNX=KAAKU+G#ZT?uSkG$W=oOK08)aGx1iQaCdHlETr{v>7ek~ zuSNk(DV3{-hybaq7$! znqv!a8~We4fYs8xTY>X)=FUy!8IL4N>y1*3Ys|-WJ}GoLzNwAL+S=Q+8}J9DRYnTb ze)TzDV&GbYUGsoVH85Lu2AD_7VJW(Fv|Tzp4T)%1xwRl~RI{cPkKniZ6lc%&)#xtD zpxtZ*4WD*1ti{y)T=Geiy-$qB;9#9MWV(I$3?h9{>wI zN>l6`u8EQk`m2* zKpAErE>{srCR@*=6zZUVlV%7n2n96Spdgym;RPHc*4Xyc3P56V*SHfjGBBIFDO3e; zDr-&WjNG{VU{?GOjb|9((^{11!?%)_ zxKOBU&2N9!i5o@HHrC*3$lPQ_RPZ7b8zT@sOH~x=z(fJ^jZhv3%AXDzLeqqXhyT5n zU00QNc;0FD?Ac(Idn79~iLrV2Qu@%=#(AaQ48LkWA=-rOM1$2f9df|Jw(sUn6tlhf zu$q{Tc_r+Ivb8%o5TveYgB3oz{CB<6s@0B31sC?(HfV5r(TnjFadN$PY4`ApId!f) z+Mf_Ml`7ir{gn;@{V5LsE~nq$b<1_k9p+O#rroH_UN<)ekCRg_JorY-IM;Uxtb{gQ zo@TNoRu$oYksBdD^_*Rt`IChEdZlz> z*ug381bzcVXhj{Fr`{X|3;pSwxvYyaAV-MX{faE8HK*Fdvl6ufGa+6MAMRw@;@u?@OPruF%Hzr&0=fuQ4Jj@zO&!W~6$Ay7*i)%#% zdo){&q)dL{8m#4~Bzz-ibZ=E^fC`e`=Ul^66Yz$ucsAD?ts(B-dg9|*wOS+` zKFoH;RPX&n_n2Xd(iWY(K?;{O8}@U=_O2x(vgxYmN!BU|l8a7k`Cx|StNE*3Jt#8k zoQT)F5niwBZ+y8P9St2Z1+?{rT@R{RT6#VGa+~cWdQ#v8*(KuqV_g4Er(+dMAfQ>i za?tG?&aR;oxJ%~b0!};u<9T`6D@aDPac=ygz z2ORU=MKwD1(!P~B1E=^8VwpWs3qG1l(Lbz_aw+L zZr_Mn3(m($vloQ>7PO5q)<-5dIfd?@RPWFB9c<1q&e!jA(-jOXTe4KC{`WW3#2d)D zZm(sVLRucx_yMTyin6WF6~1~lcyC_TXptSzB9sJ;>23!w9FNduj)r_d-5ig-Jelo( z0}TR^+Q~?}Fx;!-r0gKt`kFzB5kkE_~GNVCk06amPzQ1R2u=KdEY?OWn8 z-N2l}9}*J5z*1%QB*aT^i%!(_sY~>Ppwl%9Y`WT=k13eeJ$Ga4pvB8gv%2l~)O55v zuqH|ZZyXA(ym+qN3mCT;MJO~zpu_F3`K;+K`<)r6USLU+C6jLbKF!U|ZTDcRK%UTQQ0W}&kppDY>a*w23^ zeActL+!Z}=BqId@{jHj2HF*fy*Du9lGi^EP7hh<0hE@87(3 zE!g`~`b$XM%&c3pBHOQ(+cRa`wwCc6oX|^0aLSleF#Oq0y>Y(>HK8%7l;h}Muspue z`QD>&df}v6M|H*0T)m|7=7B{fz{uvM(x25aecpu&$ZHL0Ex`b8pWU*C$}HuT=28LS z>*t(aU+yx)%Q@W*Ge>{=Bm<`Q^5x5uj`&ZX?qtNGEw^DgD$=A|m4Mpf-QM2}{xEPw z;U-|yuDg4c@TJ$bJXwn(6VY73oYU#0@Ho#onVUq?w7XlqTD4`jZ-v~l9d7wkkq#^; zZp`EX^lU#DIQ6Uecz$+evs4vKJr=RFZ4ODS%#P0%uJ?Mmk@3xY~gP88dDyZ3pzojU6vO# z{?CYRl=pPb-@Pyax`a`a3aER_v9e;O$45+|Ss5~`qw(ssYx2^SzS(&H(#GlECL z7(r^yCg8;noV0*3HHJ$gSWu}ocU`B>IA;8bl$!>!l-n>gB=Q~2V`_7C%F6XW!&Pm~ z+Rwbq^eaEH1zniF2SW&MpB?~M7pAPbP6ot8oGF7JW@gD@pStZfWsc(M%;#51({iP5 z?ZA#S24aNZY*JrYZTX`;vG_Eun(=7%jjJLKh9>EU|;chPZxt^We3e{$$j$Te2HSMPI~t z=^Zt6C^tx%4q(^|eE(Su)6+hTj$*TpJ~P}HU}}Sj8s&)P|0&#G^vY$#wl*iKdAErr z?8_a1$7M3OLU*ZmrcTudXfgS=n~pxML7N(NO>we(Y{+`;7z$0f7KXM)BbXuHFV_r# ziq=R?`1IR0z$y)GYz!QNG|BuSmo&7Lv*0Nn?nuebFgdIhh9DrC7KX4%v%43ztKjPX zwQEO4C4LORC4f{S2vCSSx|5)3_(!w!sx!opJ|70HzcCQL?4I8|_Xx+2fseW&f=F$i z+>N4^jR0r2uJjtWrVXB7{crS0t$-xd52r2#GRoxo}HRmkh$_ za(hDFu}XAE8xPw7@v>(GzG*ad>a!NZjv%KdUBqyDbftKEVq5_<)S%|vZ~lkYd7!RU zqo`3D%0^9N10d>os)iYr)_Z~uuC;7Oho>T3IY+4-m`Ab^?MFist0r-|sVX_py6(hp z=Kg%)7k+9qnm>tCY{PP?f73a~luIG&b7`2QgW(>Ybe+;@hg0Gq`)sc*(ev()@;iHX zc1#G}MdqAj|C^KE_bWNRF!SQe_M2{~A<4UV195MkYcIETIN8xNZOw#2bid(GvRBM{ zH77%we^L6)b3^{D8M(h*iR%tFMc+=Y`qUql#q;`ir`?*lIcZU1o6iz!^g7unF{gsh z=Uaz#HW+Vx6+h|Nng!Rq{TIFW;ravF3;#b_fAB{egV(X#0l~m*09SW_-rk~Y?GEU} zvK6+`$gtM!+O6%k!`%JVa}}fn%(UuZrd7!a8bi~yD`(DWovh7nBwezk8Cet}GANB|&ULCR(Pc+YN?Y*5o_ zKo*(^5Q0Me(g#bFH(vnN8Tn`X(1Z`-*kYjq6z*qO=1oj+UtFZ)gmczD2oz{?PwMWJDxQp zCL6Ad$^+^j@br1*%n16xfe8;hi_wBpztNAIyWi{zMG#>2`hlevn##_<{;6hj`+jxB zWydf51~7xlU~8gj=fvx8xbBVyzBL1Xwo!``J+~&B`zBu>!Pk~D<8B*e_HHcfhXi{D zif|l?-23l8L*AXz#{7+0Y_@%EhEn`3e#g&xh6T<;-fUV*f{QE?U4gC|F4)=u$dL@D zdrTWT{~s<>N0eRRn8eY@s**O{z&ARj9U-Rk$ZK?6pglT}>i;mATG#Muv^Z%DcFmq(mP(qr#8-Pk+$GqdlQQBD>+D*CJw1K%} zBdS7U34n?^Bnf)U#FLG$UEp1HM#h1U1k#TgHw_&&4AwTzxqv$65|C~$@V~c?t)~Sh z6^0am+UEDG*0K(paqKoQYf4J-AQ7EiEU=3n#>>zJ@ry#{P2 zv~D^u3EQOW7}Q?C=&nQRdYkHGiB$lia`i-SeB{KewwN378tO<=aOW-PG z=AJre-;94bBFyP)l_iW(BMONz!B*PT-fb3iBqGYUL_moaG_L4Tw9(9V3=i@W!i3=3 zr|d56O<%Lj+DvU_(J9YzO2e6G12m-}KNqHR(|D4F!7yyA3G`?Tm7iVWKxwM)?GGiZ zKy83h&7+;gia04MSQ*833s};q-{(B!P`dx}P6fNcm~h9|(J~&Tz`;tX2QjVJ)42;5 zW^iNVu17_G?Cr-b$cl1Bb*p8bFA8Z&vH4}+^@}(()4er3M8zG?8Utqj?<}T#H|lSB zrDsNQm9db5WUQm{5kjFq7e)BB4_&d zka>vAC9_mQhJ*%FMKsA6GKLT`6+$AEq`?>^R2pb9H5rPMB-Q&lvevp;_wRY1_mB74 zw#T;J_gah8b$ze%JcfPWkNx=h8@rkj691aKS{PIg{)n^0z<-6>mUO*d{W+;PK z3HVjF?&i#&<@3)!+ue1D@leRYobHD!1}e<(a0U%>4}bvN(X4mJxXWBKnd!yPWy$aQLwD3W zLYL6>8#z%XiLxC)ywoNl>G|`ABs7|@Dg3v%JkR`HEN*(GBJIkiJNIKBKYWl*OSByS z()}Po`ZNa*hXV|6PKm3KN&NLDZ6xd1?*HpyRs7@AOMY+n9#j?586tBMGD^m$!HnOI z`n7NwvI;@bYPvBLdj^EqILr5`!GXq(h0}$3fWeM~Dxiz%acaab#R96YZq0;=bo0Xf z&dZ!d^a*SVA?wC7lPShvZdV~asrDcJ_kBo$xKb61AfFekKV{yDZF@2zPg1FP;>bEc zmH+$tE@uWqIwD5Wd|u#%(1zQJp!QzuSH_%m-B#tLE&1=jIcm(9NOn9DhLBG3^>G2WZ3)AISAi@%JU z*dYDuPH|V}va>>@PeLDwaEAj)`A4-hVVcrhxJdo3zusKu4kj2{)v;ui z2t&Yagf2MeL~kb~t&EIg81+BQpSUd`6N0XtV^w=jMff6E6Tbd)*r;NefGI64m5NG= zzOxnF|KDHGm9eW!kSP?;0wwhP?No=?B%3OC(bJ|=9W&uqQFg2hWPvX#Lpd{k-}Krq zQh3GOO>`%qTkI#=kj4sc;`pgmzq}+xnGD&$UCF3DyG8XWath(%vw7{RzdoP2J*%JU z2w4#5P>x(g=(K2??Ia;oJX3z%DY3gN*GcQwWTdc?=^TW#V*H=|ay{c6`oU=7B9mQqAa_}C=i;OHpA5K^%i(a@gt{a4j9vTNlQ zOpS_V2m2AS-X-s82tkCYQAxva3xd zy0KC@PqCPeH}w+c6a{J2_!#I^YDesNtu*SF)<-K-0u=Qwkyx0y3mq$YodpRHy&xYv z`r5l^m*=_?-$ZGjiv@u7M3-OEsq-d8trIinLW-hd)+1VT8Y zSWa~LCBA97H`Q;}hF+6{Ipdv$M#Vlh<~0BNLLTSwh$kJ`%wDr}1TEr658ToFzk`ZL z|7Gf=8&EGnN^`yjLo!jmqM`F@+}-HkMF+CrFv>^?g#hc^>+rn-Fk)a{9F=;{?M&rZ>tV1KzF zYr124nPr6S%@J!_`e6oVUrdiymh`#L$L`nq-6__eZst95bk{Rx%hsw*p5DXMIQ4Z{ zW`dJ`R(|72acPSZN9?Z=y!%yBt&UInED2q5^6kr|6XTprcewm%a^+auvtmmfdn_uigeJFqav(6hAQHrCT0bDG z>$k7T`Id2mCX~eYi0I5J#MBizN7oK+{wy*2M{(TMM3|Q1r74eAIjGF+li0m~V4sg= zD*Yn6_v@!?o#I&L81sGQdB-tTo%ViAYBy_0-=G}lJ|B$(+x4-Dx@!5vHstu>ntgOi z4!*va{Qi66BFiUP#u~;$C!LwF+4YYVdPz$Y`JEY^Cwfl0?`^jBX_$RZ=RvP5g7i7` zrLR$gC6rZu?&sgBNE@zk$$+iA0+?c<+0&uB93+R-DP#d%*(efKz=Rub~)gWr_d zO}3qJT&uq%-Kh~9w37==zj!3)kGY?x>3b;U?dZ|h?Yzxe-zYX%lNwN#*1i1<$LMQc z(+a{`q_|}y_Zu|9e*ShHgM`e7t1KT^{qstx$KjZuqDiA|s|IM7%o?5feSM`b0) zj;_3U_0&#VkFVc@Z%Cp|2;U?1=tRIjp?PTek~Ux)x;UZhvSKs@k>c;M+%|9z2d7 zI>G*juZl*5>Y~A+lLPY;idtHCkFr+1mr|_xU~78Fhb=)?&)+!&`pphGQ+)E`tm*#y zi|1cVNp{$ic(F^`JBI_kkMsyhH?!@u=-!OM{a-iC&6!oOeC7U-YsE{`A6sdJr(QN1 z{6sZ1Zq34F8Z(YQiOWsPbak~hI6L!2hWVc@U+-6L==rgDe!~#cRl4@;I^43+)#)|m zxo402M<#7uYU}Bm6zsI9)t5YDyW1CM8*JZSyma=Bz~T{@KZop3Ot`LlV4p+1hl+%< z*1P@7?YipZwrZt6XTz-MO#4AkmPXhct)1Yy{YGHogJ5IpT7%2e2TuKD^jh<%O4Sdq zFL+w;|G8#EE&PH`T+CS6wbxXgUO5JP$4%Nea>c;w-V@qwaji9N>i7`>Sy3PQCXU_u z+{16TmU&m1kUfGs3aELC%I<2VGc;S!_AAOHm zJbYNR>Z0i<5BnZjoBe{{X72eqD@vtF2S59RJm2LegNpLgRt4BCRo1@Q<=B?<&v*8ibL4P*asIt~QJvZqM3kf#QONrh@t0V*jn?{?~iBK2kwuTW|baSno+o^I}ax3>frXrQmNKCgxVX#c0`q)YWuc z!#^7~+8-UH`tX|eyz>RAT@1A@m#+=|WZA7@eUn9Jva_#xk3O~6P1(e|%9_&!4=q=Q zHa+C~xJjpo+F8NwFONs(R#y8`VzJJ9hg^D~>cQ$Ndk=dR$u;#|Av zkQPO2v$V^qxY!Tsf3efxnVO0Ge|nrlg0Xdh$F8oAZjJ6$@p8I;zIntXPQBsow;xUq zDEbfmw8YLFwJC=X(^_ywT*5P3YLqefwj(ky>Pb zXzRpr{$Bij|n*H+!j!KUUS`{}i;hjVOqb67Sq_2Gv=91sH zC@ID=@8_t=5WDf2k1kGJG=+*!s(Pp*Y1c5V3b=Q*-ZgX*Mt>@>AF+^Km_?e4#%Vb3 z@N7Jid5^b{AOW2&PMzqo+q9tIraL^z)@1vH9!p=|dimo@$k>dOy#vavnr9vRs5vfc zPT1>fVs|v-TT0}Sy0_~8qTs4vO%~%W5}SThVb!{9mw(7U?<29gZudi8U2neOjk$e` zX(r2O?YQ;R1F4mF(S?5W872{RWi(veQQ-gdn0JKQ9M3hHMtIN2KKS|Q^Ia#AEO()W z6hjvT0!QOs{ZhZS(TQ|Anrd^EDj3!fAW7u2TrwG3I!}Kmdx#-@D~!sC2229YjQqzy zf$x*FIl4<_Sv`qfMU3V!4-%Jmk$T|yAeM+CI>2CX2YVObSLnh{cw1>9k6d6|?|pgZ zRY1HM<1F^6&mQD*>X$(AvMz{(HUhm6T9`9H%Q@I{Os7quGw|jIfES3_9R0v_1R3Zj z;XDTE2$w3>S|I4}(R>7&Y8O3%!cMT)URHu|uY(S|0jA-n(n@dfFVA4XgQv-#TefH+ z?n)S2*vk+rDtB496P2$7{2`C!r|VrGtGC+i?>uzJtw~I{#D1Aa3Z~Lc2f0H67*RYT z!%`leo0lkUm=<{OaQQFq>(V7NQ|CtmAXYeE@-T*eds(^VK&w$Q?Ldos#>>mAdb4rv z0)NW$?4SdI8o9UAF3~DCX|gD%K=OKUUTBn0{a?PV_M3GbE0gu_`!CQ!O)vh-I_|-_ zj*gD47Yt}78*xI*%k0I*ReX;@jv6&?EF$^+>a&fcR(98x=mHWyy}DyP=vR9J|9ht&DrkMKq!fo? z)xBxsM(yumuH~XiT=_YS!JL2;&0l`n?mBywP(bX>e7zRKm}#?r^uSWych#>TU+g@P zBXlz20fon_cxZ_l7T5KezxC4o4<5VkrBwp6ov|#m_%waUMIN_+79u3;Zu@KNtTFdW zq(Y3!8PF7wJ(^+JKWKcUTP76$lbp$_ zIzMMud|n{VVtSO+(I_-V9FK(XU7nc0+CE_WQe^AgzW`8!{`=WMbhN*Ko zjqFJzNN3xQ+^l_L+Q2sWKTaL^63SZU9Kh{5>IiC#CQWS5oBpzD(@HzSHVcThnNfuq zh|;AXm`Mwm3rvp3U%WAQC{~UkpKjN}Hv0R>y=~kY0tJKog9rJE_jVm&N%WUpKT222OF2#1E<=Z&o}O~~ zok~9TKVN{uzma$daqLCC4s3Of(nl7WAv!V3IS^x4WIHT3*!x1c6I7cc;yxKmfy3(p z{iS;h^M#^XXw1|rJS6ec68#D<-v+On z^Q9&xCbES09hil_h+5Fea3*s08yLrcr*@FDT#%JM!}An(-@WLHZI33q^F!6|A&yZ8 z5de=$edmPWO2jat#2|mf2QZx}qX!~W0l`3);RQ+BMO{7DbLT@5c+vNRc8=zC!~)ww z&+R!qhf^3kjK2t|tKBCoUi*!|I6E6ZVFIsRy!v_g-paOUSssl{w!6+Crytj){`9^KqWJxlOKb7xZiz`6 zc~EYZU`b4{beZHiWXxtyi*B=T=n3=7*%0^LBf<+OJ>WG>CuNIFiTj?&e+p>#fO$Vy zS(&bq#a;LX$sz_(AVSE9HRG8Z(*}do$j1l!OmLTlEDv}JpmQP+!6vOow5R6a_)t>a zIbP*?=1#j{pCSXBd=l@)l!nJBKCy=1jR}${BlMwcF*uT0j2yrku`?H2LN0z+%uyrv=NF_0U0tvxnJgtjW&H(?a*k*%1g##7+`C8Tavj)DVd+sqH-c3A*Ko!lXcwO$Z#B zFHDU^a*9P}NVkF&n*j;CViz^|RE3-{-eAst;8N8YW!Fy2Y(aj0EIRnUEGHc}z$^5~ zO$=}3yTpk!)D1gt>&6VlSsULa_%Scbv41Ej>H-UegR7{snxt=c&tY;L){ zG?7Z-VK`N6CYD>4mz4_l2oVdMIT4lF4bz!pc9pJJK<)z?HB>JfJ7$d0i)rd*gCX6y zneMLTq;SE*d{@;|XkCnPT;(E$dyR(%YMdF#;eAjjLJ*xQyS>B`9J$>Sn^}l7B`d;# z>)f!O=0crSxAo8qG7HJqzJSgd=ibfC@I%l&>`K(wzg?mkCePMYdRKnRu>)N1n4)}W;-ebEric~l*op! zgXchs#slB$iNE$oVZte3IAx21K6xSz;B@*e3gI+Qjkx@ceGDDv%yG}5!X-AmUiA}m}BJ1;_j}A5vBOk(| zzwKO`tPmW5LIlEw8_MR!U9}v{Se{h1>nc|_=g-`~3ZE^zI-u7pMpvK6Q~+SPYN}7> zaul0VA2P6PbS^k6jsSU*knj#@CxQElSJ1mBr@(wV=RCGPFdt-Db~%WAlPf`{iTCf$ z1T1@UVo-v|{@7`{fQk%WYUnU7K1$Ff#q=ZLS-% zx;aq<4~sK5oN>c$4ZXgYo8lhu_U!`6h$xHWJtEIWC0l}s0IREOEW&r z8y`M=z@zfIUW6aVL99YJAVB&(awe|7wYH&>st?Gxd)^r-ti-vu+g3&FQ4`s|H_i?B z@B@|~huy>16{C)ey-e;iWwhIie6|>C9GP#Cf>9@yK->p*UrGx+2d8^%ti#blSDQMm zTj!}x2Gi7koViW~EAwJ9#_MYoA6rI@hsKsQ zrHOvJZW*6tFqaCr3%y1B#_H5hLUTv;k|vt+gt^m)SGkh*$YDAS8y40lS!T~TPrB26 zKa3k#SMmn?Ks-KTh01WlfFTT@NtY%9K#@_b>K?$8;GN!-NiS{)OC~72*K|V^7k~6C zL$k8p98rTG!cwE7Po6&?7@cRfka6^!K1+Cx2W-xGRPw)%r3McM9o5x%o1dS|^?IGB zDV=c;|9A_GrB=Ythxy=;tsjA`%y5VT`ND3gq*Y2R^wi zB#=s?CbFujs*XYw;RZVH$rOmNZbZ!HQGY5F%XeK!J#^IH*v`&&2^&|j@w=5@K-%h{ z1C8|HQ#)INZAJ-g%6W{eWMwBY0_3r)SSY|G(_6*UzciBOdkzbuP^IbOJ0g?j3~sf{ zc|1CiLT_Qo(vmOV&>WfRu5sY3$%3@rjt&Tw-4@KyXw$Xifj0UJoWK3`O4|@hNg(^# zpzX1Qxl++;6M@}*OV4NztV`05zBDItMGZ_wRnN!QwiKO&q}M_Yngt4Rx7?eHR@&H% zzt&;|SsWz^iYHIs1=}C3#?in`cc9DJ$?hxr&Fttr>gU;UcDX}75<3r_Sd#y;;zaD? z(CM=P#{7oKw4qy;@X_C}-QYc|XF6HdV{o)a0Wn4VhK;T-pLZ^JgGgVb5a(=*;7jA^ zB3w|+8FdujK@=`{BEQEI5I0>}yM4h)>-yYVl?rp>#?q@NFv)@!QyU~%(vDLIOcRm3 z)!PS_nx4{`x^azJ>lb8GQi?o4K4-2EhgfuZNYZDxHRj;`^e;#lYB})<$Y*%&e_84kn5zo<$$p|`S_Iq}sOy^AF&3W0fCC%R4 z!JTy~qpRJ6OJ|X^s_wmHlMW}O5V1m+0Rs%hhjnQ2+lJtMXDRITi>BeQ5zRtJ(TZ^# z3n~26u&9xFzYe$b+&cL5q=q+a04SSE()zha>^&U3w%;^704x)XizDNFbX4`hZDmXIr#;}Zb#i@;2 zZU2;F{p6u+gt~MCh8Ja)wj9g-Jw5(hi|K;K_H>+Vn)kSHWyasas#bOJbQ1{>8x7Ah zM5KMg`pjDD95Ig|RLp2;i>X1B%US6~Y|Hj7p^ks9UNi%NysS791u&sh)|1JUJvFYl zwsg>0cJuR?&8GVG*hEg67JpsJZ;zjQC@(T4niGM(!WMb2iSB4y8Vnandq^j(sBZ_KM`*A+s z;Q!Aw3#R()cF#6&pZR2-Q}onw@wj0U{RTs?1eg5_?O7BUP_zuKE`|H}k_22+c+#@& z#bb%AzhWFf+E3ro^p-CB3g1jVGZEq9b2DxYi?<$LT^niD=y}NDP#An!*(o*$v=KAX z=A2RO;3Zl_uC;8`VZr%Ai`HXqBu128|6BO>ve#a#DTBPbXl>hfj2dO($`o{L-O?B7 zt@@I?#CF%iZCuo@)QWUy{wc*vH4aJam)Rm7(llNd`E6fr3+DP|e4r5J$c)@^q^M_j zSFw-WK*!&~*)fneDXZKB+c52Jq$35M^aB-(uKe{5y<94m2wsCD;$OJzU_daD3IX`x zNamfm5)jGLtg9Lud$dC>clFulsE}s{1T%m`C-faOG%m%r-_J&55;j+tu8QQCpd@>t z(Ls12F=C#2Nk3Pjo9Q>0tg>B5I>u~odP2ON!5}W%RWIR$6{5H`bW&%&!|Q63`;{wK z7UL(cW?^VWbc;)ay;B_57TJs3MB9yVcF%Kz!bxL=G%|}y*OjgXa=0p!$hb*)m6Ux! z5#ke>7X}n2=&wxUQjY4Dkk_@VQ)L^I6RhHd6!L*h9K>hQhHg4KvMoIHLt-SOBlUZK&_9?p>(jfj! z4vGM4Fr}*1t4A%+$N?A7z2-~fCdI87Bmm=UMjy>$U$T2%UfIi{u-vp&MNMZeP7e){ zV#cgzS_*`{BG&|v7&sz(dJY$T3nVD%~W*`BBt8J;l?R?-_Z`hBDrG{Ah z%bdeshKUk$M580qLc)ZIFSkytti)p$eq~zx9ZMj&VIJ$XRovJ)6*7M=24Z}PnDm}+R9T}pnUz;! zfl!TMJAaybL8Rr~ii-%%%r#gVQQ^fNtyM(GiwA6*4jLg-8e)^dWU*M7_|_EDh}i0_ z9gfovVVX4VZ!APiad-|QEAqZEjZWk$sDnHl5O{ssIh5_F_=B#U>((VS(Mx?UfGjOjtv9k;=GG$^o?eutAy(l|}WS<`22)b1)yvf7t z0m7%9Cx4vu??YGBB>b>gP~FyXFWC!-B4`HIYCyYSb-X* z%FLI}xWG|4gq}jQvG`W36fh4*(b$ye*3j3NE?w$2=Xw#u7d3TXju7r5eWzD2A&g$Z zfwke~^m$BB_#{0#a6;8=N|I!IxPJ3>I44o60bW^E1@3*Z7P? zj4c8u>8vonVV$`aQ8=N~F$cGI?Y@Ye5wN!?rvSDp0bzwOcdU&4_c04f6PM3Tkov9ytki}Y2aHhuH40nHwFaVkomk|h#5b4D+ za_CIq!UGp?Em^Wu(Nm`5)h#|PbWrkDvV{QJrs8aJo)&aP!7TNV?5~pANZt+V6FK7q zNE?qXGl@{ss4AgN`Y^RN$<0fEU_i2GM9eFtNqN{&SWEvP#`@HMV&)?890479ZPJww z8<^{qDL&dHb9jgE5CNtbdC1@ZC+n)Ji?*yL8Cv$uMBoy0tJqlT>8|h zYr384VjvUS+?~Frk;-CQz+cdT?z$a5+$NiWa+oSmXVc_Nz>1t;0wtamrvENd-V3*-xr7m8I+f zxvyJ;c8RHqfVRAMcW-4`L&k8tY}9lQzG9if!E^Is5M`moEE)R6j>g+)BFOgep+kqd zXQ;?d8=#kNzN2d?wDd20ySZqh%iQb47Sn@2++zZ$&yAK$^7-~9{oJG+GYABkqxUXN zWXB>^Q*`Xj*rTcsfFX^#v^MP%s2Y8+*PZ;bKAODY7)B`;=^T0VLGvINQQYytCKq(x z{OGDh;zVL>)YW4M6rmJPL%f-q8CzVgQl*Mon>OL*<^1#3N8wO$I zac)xN(cFLlvJ4#S16o$TzaED^POs_pbjNzlG)B3}I#iC$%yU&3eOzQXwb03H$Q)cQ zo;NR|tJJ-*w~q$%Q{p0q=ism|ecn|l(LT-nPxQTxL$c|PXA&@Q8cp>>QLS_1O4<){ z)zs)X3RwKaYJ1s^{RVQ-K~~3&v+gyly(nu{W$g?YjDCq_(xtXh1*T&oLu)Ck&F%bZ z*?Y$#xm|GBHCl!wLjS)yh4>yTvXv&W5S2=;YWr z``&ZD46T1!-sZ21`m$F~f-rgL0e()XSUSluFuT(n);yjeHI2C8ro zkwD@d_vVBa?e*6Shx`$nA8Klds2-_hbcA~<-07(cqraBPv^_SVISy>Je-h>!cvDnqZEUTI_5hd3L7Vh z>Ta?SR47lfrg(vq##zUr(I)Og$xB$^8* zChZ?KP*O>C@EPeQ(tT+#arF|{bzyKxJmPsw279L4BSf}@rz8)Tc<^cRKc~9EfvJtc zdEtD-k+BD1Jf%4Fxqs#pL-9Z}bmDV-n8T;=61#D{c=|ByFWHr+bMuU0*tgfUz zSKq@Z4Gp!-F0$xV<< zrq4Hh!Hi@v4^WyD$e44io*?o7!QFi~wPwy5hK0mU7E(%^-fh3lnb)%3sbbGD+I4@` zb*h4{la|CIR*V@KpN}GCA+Ugx@?6?|xNV zg`W~CPDo2BuOMeLZ3gU876H~T6KF$>2VA`W^7@Nb5zkmM_fv>y8zTNt^!v)DV}}u|-SPkkIc7gCh8PT*b)6gL^Ihnaq4Sr5_3-R$WXs zxHxMHH4W%JRq@@Q8=|z-$C}F7;MdEc)O8~%{^2%cf3F`=P8=x)wrtFwI?vh}S@C5& zLq{6)&7dB8;wn@WT3z&h83<`*Li9k`DfD=qWV{J_qcUl=*lhuoALR%`WLJh{!q2^< z%x4mVPD9K>CY20j(3(F-32^LO>Wj-0{gzFHZoy+cDyQe%o?ZTcS%sUYG z_s7OJ-sOt1=2^Mr_!_cTI|J68%H7>w|$X-rTcPzAL z>=#$eO-K?MY;nLSgkvjSDo=%gA@MC3P%fG=0?|Wy1!B%rsyFr;z3q0dKjb)|l?W(t zmXGpIgG?R%wPpNPHrJ5Y93!DGoQ>8((4g=}UF3m4=L#CYPQ7^IUzZ194%N z%mH9v#!5Y(@7VN>lY)qQ^jG`=+S3vEwXCdUnP?yGG^Ic~%ba{?*ap}d0|tPm(KtB< z3(qYyA3s?dY*jN7jOe;(b-Ow0?(eeyRVagXzHm-rSc@6x6iO12HLooz9xJx(z+c@R z!%A`oA4RuAkekd?U&%gOQ7Z`qP9x2*S{r-&lG;TFJfSnRK_4d+G@V+OO4}PsG!_X0 zZRcK$R|L19rZVH@$=Xoz`8E3&w%a(`XK^Nw?ZL)&IUNK!r)@WoP6o6A8|AoI>gMuz ziZ?%bjPUp@TZ|!kU8PbI-xaA35fn^~4kM}y2!bCh74r@Wa#DY?wn#z{F1sE}E%|UI z7Js3AJM0lTt#F|vYhuj)IYlbEOLVRjXotJ@)BQPE`%8bzd*{HSJYMMHy*5B3I<7N-Y>rL zICS0Q-Rn#x&WR=hT#ZuYiB*5cg*ZpWvoZrb!5sS-Ch{kp``AQ%mUdwQtBb`b1D654 zkFqBjZt*-%*!#v@&z7D4sG6f$ZSC5%Vl>2I)rof!+j9s+Y9raiooklfS&$yyg*d&t z2k?f^ivtjiSce9q+(du`gwt6jGBZUAK#g-saMfcUlEs zuGM_zpTT2JC0U%jtlDdO)l-)(=5N(ZNV0ggxZ9RZhL?+KCQb3up7J6(ubxwI`rwPp zm$%uLmspU}KKJnlr&UV>QgT0RH!^VsC!l?(oRgC?0C+E@)(9J$K8oy*GhU&|$bNpM z6UE_XUZx4pp=Ymo)eAN$g^i}~`{&omA#R0jH5G=2hI7&6X-}FomJ^!bK=ZIyrINzg z*S9;w;ve2qw*q`vCeE8be?9NMl7c$$z>*_S3tBNJM_*>qCB@}^qC509Oo%pgmG+Z(D6$-%LI^mZ-Ms{a{e#d5oVlsFJm2GWW z4{crD-sUZFm1v5)(u=FK4X3y#TU=f5@RQ?X7cQxxSl}+M|1t%e61H}ZXDBE6rbF?c zVo;S*zm3JvC0NdXV*X$rH}GJ;CCx$PBQ`5G)da<~yh5)4TYtwXsKjzoGcZfHnE5isQ7U)x0N-8+RKzY6{>zLf_kW?^da~ zsS@P2qIR;n30Gm*;^ctt#I2gv*49C@VCFaN+oxRIy(C3;!6sCz9y_@zeUriAGH&>* zD^G@wWb5)=GqcVSy}=z6?=tqp`B#(C#Xgov^-+G@;`nXC^hf>7;Ze&5)@`VTk-$H{ z_Em?Uav`3#>({R_K;9V2wpZENHKt9Qw&tc?eBh_);4Wt4d-slV*mq#hp0<-v*^OkV z6+48T{H>GpBFk5-Sa;}9GkRB9H*oa*19R-sztP!T#+;%!x|Aw2{PyiiXlrW}8Vo;VCIC6g}tNuCgJpYkwl|26&@e&!0ELFjY(Z zB=0lJ`5_!Ce1~xu;V~g6wtjsScKKHU+=MT>ycLCvwkapng;xP7r-H%eEx^~e)j!{NYd)Z9{+Mz! z11sEpxN20ZFIPClPA{)C6%vG#=dGv8Ms#d0!_dKqcg)Czbk8H zK2^A3wsMQ-2vT$wou<)u$kCTXY4bR}s2jx1YWJpbd1|*9P@k7P8eD5R9_8rfUp$RI zVXJWY16KJzzalHMQb$R#Mq4p`M(ecD?(e_Bttu%vy!Qc%6bpu0K8}3$OxfA4N7d&D zURpUns$Tm1k9zg0D_DW7V`6N)|EN0JM{eUt4n6l;wPVNEe3EPYJ*m{Vy~9nu6LXXl z((If`4YICI2@t!u#&Nwa4FEVUZ$Gl{+Bs)u1s%WAuGYiG$jb!+Zjo^&P(`J3eOj%5 zeq|`p11ZYwuXlYOxU4k@rn}$!0c*>K&<41@$w1250(8>wnt3Czsf{#McR=9Vv(UM# z;$>NBNwprqNDSzNZqW%{=40GZ3}UjcTz!VWsv`O$HixZ5`Mx3XkP1qdOP4R-qSD=S z@SxhHNxHgsc)Sf-wmjPD{nlN(B4n>$zwX_-)g}XOI(4c&{$Fa7Fm=1dhyMB7XOlG^ z^(&E~uERFm_b>U9Q@MKe>fL5tuTNhbf6&h;VaCdw*s73t!->z7U)zA~GZz+QcfUKY z(tpk#|CS#Pg}s$3_c%X@y0jKm>T+1kA0+c{EI_}+kn5?$7&g688&)PUjnLMv0bJw) zV`%~wB4(mH6HiQ5>&cdN2ZT_0x%3bvdbe7IRK?8y4`I@H~8K4WWh&Guw zZyki_DLOK!`s#?9wxiLI90Ia4h(dplr)OOS{>2^WemC)u-teq3V6K*%9=0nr*!B&z z=ff%OHy+XG)$0}(B0Ui;vN)#^4`-~oc^70{^&2*^Ho*x))8&+pAQ?_d$cX|8;QFXqK6o`3)Fw`NtQ1u_A6P!060?!-rL< z!(6EQ!_(7SQ{&x&9+A*DD|l5MO4$AObDo|C2>HgNi@9tv7Vh{JJh4nm$0U!h>ZE@c=K-s+jM?&fyRFbvC6bO~1-ouh%KJHkC zR~FVGiatwo3%qvi9$5~Cyxy!3yGC58d(Z$q`Kw}88ua4gk%I$Pw4-{hYGGl~&G=?h zO0AKgsw?2?f)Xp4aM@~7)Vf@`(i{EDMzYQczBuX1qI!hc)9$ON$k3uyCC6-|6`9B1 zRGV_L3ek8IS<8HO8GOTf)JfRaHYC$Tz{p(2YgbLV<_|^|p5ISI{X&SWj$C1LSlBQM zi@lwaCsZBVc4PgjEvuC78gW9g`+0Okvbvhu+2bR7COzM`?Rh6VUCXPjr$24m^ls*v zPE}jpw%<`QC(YjAtEM7*+DZN4E^~KBhpcL1Q=@o2zyGG}K+*N`Mi*>N(<`SPEzO(# zyvW(sq}=|1lkJ}|B`y(XZ+2Q~@gXx)KYWj}dbh+Mx2&S=yT$#O(ln}&Ig?LCMPs$J z6jYYe+VoPsZ{?_RX;wj6uuXKN=kJj|&4hGNT~WLB(3{Tf+n&6+K>w-IZ^0+#uykS{ z(n3_IJ?;_2Kch>1%@&((*_XDc)w|qF@lTxMUt;Vtc`V=f5-ePuk?hPzK3}!vpA9m4 z^{e(<{Iz=`nSemuJ-g=?wbqA}Z502cW&>7QEB_m?C5{L|1N!64Rfl^P{=jv z(xj1!5Zm!J2mSVlPTtNW*Q2$8iqRf27^EVDNPtZ^whN{KH&9tbd_#HxWG(6M8!fe)}CAzaduX~F>1{)WethTcQ%dCWoA{^B36Fq*5 z{dE+UDc2j2AY5EsYhAc-;pLk*8&FELpc*mHVI|Gq;l5wLen?nk$3WlEVCEu6a1Maa z1|>dccuukp!YA{%qL0Zg2>P3X`d_+wbrhrQJ)sxw-n(bk?M_5Qb-1b3#rYj%pb{jJ zl5o~VYpR6%WZ?AxFOeC5Hk-hxUs7@zz4>#5IE0?n^ZAq4H5mev>)b+W!tYmSggI%# z;kXZP2vo7C5hP#9JRv z1R!4gQ3Mc#M14*;l@%d(B%tRf48jp$%p*=nYJc{6$M)@`X#J<4t@-jUz7|i5ZRPC& zVs|kCgmL(=^nf}3#Nrl%Y-&ipBSSW5*6a_3RALBmR*)X~7^_QFK$#&WksKm;0eMG? zE<4Ku z?Kf&cZ(gF$x*imC0wto_RIeKBSKfkRiaoHc02!+(yiT8%>1;XN2uYoRdD;#f*h~r^ zv=X2yRgq&gQ%q)`h5QwW>RXhZFBxnKNFy4*A@6;I5`Ojib-PK&dd{oF0L07Juh&By zD^sCex5@bvdxNbA8D4UiIBSz<>Qi5kw>CpDqkQ-i#iZKwbIOXB)G-GKt?a@90Qgvh z?bIv2capzptyQ@xTbbyqhf-_W%rt}`myq?ZUbAK%jSEd~IHjUwQZH}sTb#X9EuedL zNj(OtmyJg-leEjh2DYp8uM3k3bEA0l=-aoMjdVBs8dKHku?a&N3b;GqYTHRFLmdWy z!|uZDL{3!xodkG^*9g{^TU$??c^~P?6h?ivGX?A0OO8)3dLV((zktTlLEoV!{DXp$ z_NO+TMS|f{eHxoF^bOP$8D{K$)}nqU=xlG%zuJR`Uv=xY8nT7r&7zO;H%1QST+oR& zr8RQ1uPwiW74eeoO?y=WP+GrhS3Q(L&X5ap0TdO-uQ}9)-tAtyW;U`wLqkIjb#Xa++HTW#J;nWx>oLvgdP^<@ zmuNmiobmw!%T`us0pxu`a5<0A)s(3I!p^|gsH!5HsmLMSRaL7{Pow=9D%PP1ljrl| z*s9Q)cl;UOM4#^OIhbU%9hn3PCuH$1er&H@pnL%y^aEJKDXdfK!AG|QV~`){z2I@ ztfwIF>ER`=cdGnWBV@EinY10(nI%U2tr%7(5y*TrwN@cpT#WAy8DQ*xV2bpYD|=8? zg4Q$zUK=w~VHPw4y2Avusd*f#j!pDmuMtpI^PB0m_u~PqfvXX94Gz0|cx;2ytOcwo ze$+3XD)BV#P-ySw4mdNov&?cq2y{_N69P6zh4!FGja3;7+#G({upj#5fsJT$Ylg29M}W_o`D4qilngtF#GZK^hF<<{Z^l-)e}?s_jjhi@6(6Zq$nNB z$$6WD?76j|1n^@o=}ELFOqd>3lpWK4!;MqbH6r#@*JyoFO%F zvb9(;{zwQpcVXGfESnOgc}{Avb;FG&MDq6#t)Kl-+J_^FQ4HnYe5K)wiAxIa^6-{~D6k8w)wT+QU7kx)y#5R3nauQSLJntnh{BqTJ3N*d0}j(|JO*j;%F zzvb0mvKuT-5B2ReC-JM%%$aWhB=0bUy5iHz;TL8G*5!;gAK6QBC>e^x709Fc<>}=O zv6x1uh7LySR9h`f8xmZKVvGEIosJ==Hrb*CQI6SPGhSZwiuRj}TK@acih7lOo$Mf| zPQ8RtAel!z(erJ9ic&%yRVWtS&-~H0zOwR0?wZ-`P~f0eLslPOnbu^EgHrF_y~m?l zwkRWmW-i=orP?XkZx=N>F!xq&FL{09k~AQVUQj6q4sZq`cmY&I&;I=*s47;F z9CsA^`1n-ouiUiP{3_0mJX`tK-2=Fl(Z5XT_}cQa`XlB7&Y0I3iomEBs>Oj5^lA30 zDlQm@H0L2~1}_y${XI)T?iJ1=oQxmHX2(Cpg6-)lr9*eB%f~=7X+E+eWyn12fw!<4 z?m--=k0KnU#5w&|{CsIOuU^bTt1^ZfNan7u&C|BD>}Fr*>H2?XsT2QikjrJGz=HoR z;1X--glQ)yRiQC@lk za-zv7G$FU!C!6AVTOUU74#nSOh*uUdwqoV^EjfK)pLid%rhZU3BTsBB)j4qa4({G7 za`_l5D-ELTcDnjkNOne2FZW4^VY+=^_nD1R)Qq7yWSY8`&=M}6$(s>W)t~u`CwXhK zzy6Ge4%y@?$=c9bJ=dg9S-C^1g*sgsmOgV4c6@Rt3iqzm>( zt!)CFeHK0NtUM6@s+15EpttUhDn0rY*zye@J&#iiFE+q0oLKPCjYu(k_38l(KyHI@ zng{7p?Oti<+~2$QlS4!=!G53`!+w0KEZ7g!0kc#Mc_w2i)t<%HZKhF$7W^Aj-eu}| z!LzJLF;y8M(S2!*q$7u~)wAfCKt>vTFu4m8su$)S}ixX_Ym3pGFr~hsW4DSs5 zCiWy?&Yyf=lzq5<{kqiTlqWTXb0Qz#dGVq-mE=9R=ci1CIlQRZrcE0Kq5cl@XZ8R5 zb3GsO9fFFnv9SQ1REGoW;E4rello@;Dj0_DKKB|0B z)4rd;e@J|9QAWtOBbdn34o?;1pZTaY}>g%&Wpy0`%<0KpTblacaH zyx8&~(OWYmS4W;Y-?*K+dR;sf)Bu{9y4`@5p~bz@p_6O09W}iA@bXiW)f5X>mFEZS zHBt+y_4u^jqoFY~=btJ)yT0zNHjd+-bph=N2ujRdU$^a(73ER(d6A*jZw@Y*GcTn` zk-N5dT?IyFa4BuSW4+b3U&AVJvK`=L-&1u)<+W~gD&O-49Jd`+x^o&~*#Y*m~cA43A-H*Q`PuOZvh8$3U81Ky7{ePRzU6Hx~ANIP{NS z5v~RerlhU@{-yK2ag6X&Ep2b!x2EDLm$K59bA(5w-$AS|_kYl77)lfV!8<86qo# z9}8Oi8`L_*DCNzDZ_ZA;WkHpNPRRJ_JY~C%Ra^ds(D7a}y8rvH{vXD}w)`)_wc5LP z?{+5X3r&%%`upxA*!r88%WkXsc&ACtlAt9Pr`0V1G zuCKfQ*G6{B7VZ1@6~cejJkVoKHHBsBjYe?04HV(9YhV?3s1}qI^zp-~e%9U7`TbRU zZQG&rMd%rYK=gE^i3ig;V{!x)m#ED9H&^-nrS4I;tbKeW@tLDDOreEF;p*urYRb%m z_f^aVU0c?zA9&%fzZ{dAYPm2|Inqb3k^&Lz$=*l%o{MhDPfnGuO$;oQ9jlxdkoc(mUd$ywX z*Ga>x&+R#SS?;8LA-0QvsrQ%eDrwbhTgA;ByV*`|rAd0_>*XA|MrkVT)~f7?==d!;#ogrMoQ6^nstkYom_vXbk~+GX$yx8 z%V_CYAeVYAce_II+*-V&)AM0wz@)MJTQ^jqjcID znC9w=?9|{Ql^p<@zy8v0(y!XnQ&i$-jjnasSYJRE7$+sgIPX76#`h6$DHNbx!zn*% zY3Tj>+4Jv*M$~9qbI_K+8KXv30qf!OR%&DYO@UwmE9=S%l$z@hPoZmWGIyVzA$Y0c z(8$U4V-DYO+;#Tl^78gw2Beg4*^<5FZo#xPlS7&H+%3xH24!lxo5?@^`mYdMpYhLo zYxM&t6~PqUek%&$bt6s)lSWOkiGt=eoW5m^-yr%?qn0)LTKiwWrT5^{|CPA?UvcWB z1G?ubn?}rgI`%NI9@ae#kjHEiNV-Xnop82v9=Uav{Sgz~DX=6RfK+zAiF$&&?ZRzI zE8Vx{S;oTjh9jgHpibyUw0imKl@e+Osnf~p-^i4osiP~UbU7J#;6%ycfG4D)j10bW zT}0G(`Mf7H0A*-o1biGG5pgCwfCH1(nmVyY&6*=f^z#-is!k3UsFOy0gt@u-aF?Ql zHPoB};P4_TgZ>Bz2*`IlC8!on=AZ&jdF=7`lA1$rb+;S*cJ%X<5c?)0H`gAHR96@~ zdy|bdiGeX1}=Bl`-2|aCcIEAsYUM z>O)G_km6d_v-_Q%Rd8+=KhTBuAvy)hHCCRe!8afLSd`x{YDo}g>Tk1?u>2ji!&K1! z<2PxuiVZ8Rd2;Z5r>&P0FWhf+*gC3Py8HN1=D-qG`-ZBi)nI5@v3t#{4^UI`La3w! zT9(2Fqmj(9XGd+CIPq&3_~dJ7@8YI)VoL5jbzDZ@%4c}t$65#rp7`B}h&S|?18j3! zFfrsz6L1`Osko7P%wU~FC-;K~w;&l0F}vXFn-PX!edB`%4{p}my=9A<%AP%YJ{1%Y zYO5&1@87S6<3WvDwMH5mHchxV+x4#tYI%HF1C;UCR6gnJ>r1mmPx1*kN8IkDv_g7S zi8$mN1t>L~YR%>%h_%L(H%z^iWw) zbhszI1i-q&StOI=7cE^{7uSTN1<{XcCYtON2%0+o@yarWUj)aNMn)0-_;Fo1eRb-L zUc6Y1m$4oX3exTfi?Z8B2drmatk-IYGd@5sHVoqR)o4q0UPqxN@&-!W2rxb1wsY1q zHeFU%nK?)4O{6L0H}jW0-WJdyw&(DORU;T+WhPXr^*_1h@Y{e{6DCZ!qWLw{skAxB zg5pbSdtZ2$?Ci7FWx=az_x$?(VUmTu$*$-z4My!-1}0inu_tc!52 z8vmiVot!*)ZRHN_P8Jyf0215JDcZnmbpZowv#~zJ`A8nnaBzfLiq|}I^_SY|>I$Jh z?yav{>nMv|!6cB8ho zc76Js17UmdG0Ir+EH8kl9BFEeK5R#EY+tc4NOxFXRjeS$2Ck^VTmj4lC9lPyQ_^Dh z?7L`{CQUk>@kRMR0Ea8yHT_eswPg}xDmjDb1yS7W2L#zu zzg@wsl?;Q#ggV*>%@SGR%4VB_)K#X<+0p#~ud5cCDkUvs1jC1{{-i}8z@R&<%$6S^ zP;0#mqIziG-)FIEbG_E;iW-E521rVKg^?30IcLY6*)`?;n&hU|r3*gvJ%2f3p~>fm z)us%MjIO-r%NLthA3t_iY@rjfcr-~zrvZtnfnpRLX}-p+lOs9=?of)2i;FMoky=}t z#XD~orVn0Qc9Y^LW}wKDZohb8q^Qq6p$D!#x7G|flIuF-$=az-L)F@}JNZKwL6{i6V7-=fgxRB`xqPbH_^>7&R$r4Hz&W zyYyp0D84r*)9jCIDpoo-bcSE_ruzA%uRpY^1kA4Q^Z)^w=8j^RLiag+_O|+RzRRPP z0|PTK^XVk6Vb-BjeUzDFY^>PG@XG4Kcl*b9@c4K5_}zEc^JumDQ+>a}Nd|SbG+?I*~%I1yl&pNCdF)seinR7^&YtkVQO~XO- z<1-t5y>TSj3YU*jP#!BE8>lH~r-u!5e)Q^vmtAS7+SznB=;D!tD#c5fl1)KDL1UWx zgpxLd6yvR#WEcc3O%ABTsJS97%*3x(;-ve=SDIT}KCWe1TkDJaKu9f+!L(K|D9=1_ zlIfS9&S$;0{R+yfMydeP3av3?#wd*)A*J&g=^+G)R}_}CI7k1^pl}`MqG47eZmg=d zy7WiKjhi>W=7A_|5=}wqgojnU_kT;HUS)UjaoU!PRa;DWwH${Dbeyf7b8j#$qP#B8{kXg1cU|MRfA$L?_FNyODV_ z{Jvt;vO|cN6$-Jpyi$njelP@lCB=0b4i#5wiL-4$)gCrU<;-+{xmZe3wI1@=DL56A z*}i-<$7>B&Z}j-_k5smPoNbG*WAU=tiBFy!8#Wfk4B=;Wt|;od<|w|+*Q6T^?5>hy z+-k1j?97mxj{?v1U-F`gLb5vO^r`-(y!R=5S1>(EBz)WvlsL;7T%XwyR3E2x4HA?DKQ^@G{^A3wWtX}+J*6{sV!kXjp7b^c${ReGk;K;t$ey*N{Yh3 zmn;8(^7r%WA*4K6kK;I>9{D^f6Jj-AMBwdLKA zw8{^SXU%GX`mT~x#MKo7sZvuW2TSOh;;+dpoji;K`fqQaqHLpN5|@s88^w^uE_ z>~cPQsLg{r8kG@pv;6s`d6k)AR%)6sfs@=v#2EY;GUAuSGWe%vQdn~@GHtKtZ9me= z0HUh%+Ir`TXB@EZIAll*xXellQA|Mi4vOWUv>Xr4rBQxQ%HrDUiyZ1;aBL4~H zpI(hj3AU|=G67-veaKUhsd7iFQffL>ZV)!```>}#NE381*>$$Ey`or0S9fn5qk;;p zfbR3Xi>KPPi;aFt`<1V6{J|dS7pmkT&X4+!0ef^m*0v$nB1Yw;-LR4An)Tz?+^l}` z*i@fUe#Rl6(0Llpm@$vfy15xbzWR`$!1%98Iz-?-!C|G@r*MzV}~se_yTl7LURIE4W@~QL-^YF(?k|t|8Fu zSPC@oHpF@x==SGBDyvO9T^siAGe%|dpUc#~|Ja{eIL)}_OzXqLex9m!8=n^Ykn)GS zjMY24f|jc3{akz8!?Yk;g`ecrn}D1Zx9-DpY7jG6R?a4c*!NewB=&}jrNN%ll*$zM zxA_l+uCA`)P^w!0#ZMJf?j!l1l7d~tYFASg?{E9B>3z{SWE)+j(t}U~XG{d<1GRrD zEF{Vebbhon38leSJWr)JPzVD6i~gg|5j9sbrRd~=eB~>tI%{Et>T5RRz%`Sgn*-4U z^gvQlrA>)Rp2!z1lX^;ZKXOKLQqpmLm?#wisH-ZzWgcGEX5)kO@N~taH8 zfq~U`Kksm8XsknLO+d1FR!{^Y|B--!07uvm&jN`vH5CInxT|MPyvlbkpyVn~UuGn`v-tw1NQd?mS{*%ZdY_R?a5QydZSjh%3HUX@uu{Lv(=vPZb;OT zh5pj=Fng~EhmFr~cXPx@gXh|X8pzJMCb5T(73Dia+KIkV;Y=*K0~&HRLmULk4j1;# zikJ?OfP};B?V{NUJ=(Xg%!3eUc>IJ3pP|=IApIWGeecQ{gB>$&XQ^#=-L}mw-61bI z(CCx<;4s62k-Dt(0UesDG7gKDFU4g~o30&cD0jc7fL? z6Ap^JMUMN0KBCh@KNs|F;Mty$5C8S1n6gn)PzHzpKjoc?ThIC5?>}YBo-s(-Vyt6} zNF=H3W8bAh){=-&+GZ@-QkH}^QI^P_MynB(qR>WBs1yxRLW$1fW#)Izec$IFIM;RN zx_{R-_wPr2zn{;0dA*+73tdUXDH)Aj9KSyA8{NpmyjT2K8ZhDJn98~b!AX2hzeqXV zn+*OJ+i|aPX))VaO->c1+-;?#e5IJ@a1VAwSEA!|lv>TFbM{0l)tNoamt}Ah)4tYw zVD8(kTeo%JY8;gEP3DKA&h|IE-WQI;`eTI*8LGQq(**wJ1Plh({~Zo3;egzin=k$6 zJ)&H>$yHyxL!sVJKe+5_f1A@67lxGda(`D-15}T2LFMMxxt6oVKBXuKh(*(0S{z$e zHRSX^Rm+zDR4uD=uk}XOtfH`AvyKEEKHOuI`6D#Sn<^GfGBrKqlc{dxuu>UXU!<6Q zp285+QxGToh3rlD74-l7cdpm@igvo!BcI7QrtFAt?dB@m)SsmJ9=5{vf&?k^i%~(V z#6zB#Vz9(DQ|A4-R$GSDS4{OA+T7@s#!Y&ys>Kjse!1${w_s~G1`Or-D4!p>Lb&@Dw$5x+4^rVUPA^a zo4t19M%S%opFYi_RyZ`VJ5TpPq+P-4jLt~51l#5^+zLCKl9BP~kx*f*(A1VonPuM+ z2fp*}m_>~DwT^A%Qx#=i71xF$ma*%tIkCPlPkrsDzvnAP@Ua4|z^$n$%;srEN#|MB zma^T?{(f>Y-K;`-r5aQ;BzgaFCiP-kI_T>aA8S`Uh$sdYz)Mi|A2O&{P7ok*wH1^8 zeiq%%=v7|9sZ*z_BdV)=L?)@sp5fIO#8wpl*UM_5fd!Njd>%{t!PD_hRZX}EjL=pU zd*lyZe$P8P-K|~Gr<%qCzaPtbY<2Zv{Z_?(&|4}-Ma$>Tynk+5uWh)6Yu)%%mx(ZS z-g)HA`{cLZBZ*Sdx7-*+fLLwgz|xL~{ajFfyV`f+NTyQ?{%m6UB=vRNnPZ00ExG$F z7>wNx-B^WVVq)T0il4i67g=(YJH1N>7HD$S*grXP{kgaG7{D;oaGd2pPKD-CL(A!-$L1q!e}S|9GT|Gw$ENe|6-$HGmVg0p%v`{%@Oc zuy$t=6SiLo16Oe>i`&2;hUQ*>GC-l4Tmbh@# zgnxU`W`orqnqfXwgIoOzYwg;Xw;QvlPEeAsYsg!dRz?LUZjb8}qy^z#HBI?Ca_T&j!NkD_oy46BNQn9iae zi0S@lOSjG>2&i++HCi-JFH4z58!m+lk(oE?Z*a|s!h?oKgaOhgum|??eO|37Ee}EF z_5!d-^gdW=J&`2Keida26}xbP@*bEZ>f($qbY=GWCtA5DBGW;IiY)4yjr>&e^=lyo!W|q{Yq_Ih zHh)n4N2g;10{4)V(Pr%j6$NO;<^CV!)H7JUtDj02dCdRipVEMecg`L&Ye3is0JD-LfDd=cw<2tD?l*EVfy-ud8>|Xk#GI;Pw^-y zi*ipCSc~rLZUa7O#PUGi-juy5a5Pt(gn(Q%A8TwZ>RfsCRAzrYeAtQziN9d>Dfy2T z9>F@?1kjlO!v()O!|utI29S^2anSVb+gIfB7>KM$Hx`@h=(2ms$)YC!Jb+p1Aj^G`dwQKFP^x!%(;7nwP)o*YHB(e1qIi%32+6+ww-*T> zSIs}OKi+dH3XIzeho1aiJ3B8-x{C-RL%?_Vue@~#lHr67^_p%Rc-Y^6FE`ARM4;ji*hwFA@=gAWq8$zXLq>jlwCwS4|?wl;t$y z;t3#_WAq2CSBsETGz^_QVZvxc#g`G1J08OmL3<~frl(Z)h%v?QqeDbUY%2bICE6)dQMn-f-kDa->86<>iU*W`Mb<9#n_~cKb9JE##1hCZ2XpBo zZZf>*OYj{%=|9z}GzW;(F_)+G0Vb%Vk6~B$k^vK3{o^IRH~W&fMZy0$v2MXJULwG> z@4QP2sm5MuRv9oa1A(#Cw2PK;#250OMNL4Q;P2wMp*I;gxG+@jKWn*FtUglGd@YJ* z0v!4aWq}PuuqbJrkkd45rh&VI(c{^TT=_%dyrno4twM)kYZa88f*Ej_$hx6BwlSEM z*`0ER!}1UMD@I@h*w!EbOl>(su4n`2FY3?);FN732Odx%1BW^3S)Q^q8`?IVZoz!``hkWnVU-!oC-O9SI@sU=%A*ETop`W#oC4cS+-Po#_F*>e?Y|Fsm$tw3Ipp|+UBDGDsse@#%w%> zF*$yxKECR?jKfFO{R?-NggVq$0JFmf#}ew4|qSxf6z47LrF zhXl*TrZTPRXJs48S+rSM?Ss^fshpcCD4nL258AcudG(|NY<0-b;8}ef(t?m%FEm7F zmT{zjO|4=C{ft5l4u@a2a=+%nqXk)c z5Y?==g-hNzyL=GZ=_nI98XIwP9`tzmr(+rXHuQFQJ)#9skClg6%+shnFL0G9eX!aA(dJeJ~(*YEip{+A9YEL*PJ;gDnOl|q+me2jg-@1JH4)&JR2cQ*xW8>gjS-jmiorE2yZ(GDyr zUnmN!r#%tpM4N4Zx-k2T_?itmP7G#>$6~JF@c33$_qUi;VT>dX;fFVJ+Rpsm;P_PA z?p|I~M|)IQY;Nau@c_)K{>D|Cvm-^${%sW7rMev8tGb-?u$DEc4C-953WePdFV=%U z*PZV}=Pix6x~hcTo1He~6ltggnDMywezis=VML<|AuQ2@F{@DJNt5hFk?k4$*qo1q){uIF zZT*212`O9Yxe_@t#U>%dv2g8er*^7o*h;KgU!Ol^n0Zk59 zNref`-0|$mbz8sD%c>}tWF5PI3U-eB_Q}lCfHpZnN(%mzz1g+L?j?q zIFOs$w{Lgh@)NVMV4C77;!LyojV`M-TcN2UBosr0cCo1C&+?<_KfS$d4)APb19BG2 zKmn;cM0tinlDkC2VU+NfnH%a1iPX@id8>!feD8Yn_r@d>gH|svwPk>mT7&^m$yjxwI27}_1F685K1#f%!!TSS{p(*CV`idlQ z1ia?x>Z$DHYO=m(_UGDQr3*W_Xt6=ai{W&#*IMX!2e4wKk#`usn~f}2kFpaYQ|qZu z?3?OoIhPvl7`UNV=T!yuy6~3$%Bn^>f0;%xLUkWJ{V*ugq0cwS7fE}*jCuktnHUvy z(u5XdaK7Vn)u!w=#WHZAuxD2%KUTdlcwkub4@;obIkJih&weD0_u%*m6MC0=@3Oyl zWO1W{A(Y`<^W=>+8GgvPHJNs^$?c8lXdz)!7jFRH%rqIskcm9r9r zBzd|sF3N;W1ul(1;-&1Q{$={nNU@ee#0bVEeFo|@hmF3qNK;q@ZBg&hc+TIKJ@w-$ zA{s>?1KH~i14M->L<6X$OyWB^=~eA@(i{BIIVl540S$O9P#eSq!33}W?&IHU)caZ= zw?~g26}?dBzCG#ij7l|$yAi#U2g?~CU6X{9KPcG=v)u)AdV<4);<7KjO2NFpLnJal z059%^#Ym)wXH>m=rlm*;?dgd3a}4LZL~y99yK~=*m|1a`A{1~|e_txpBt&lm1-3Tq zejYwaq5v1IS+frzcYIas2y)G=DBy3TS>mxJLaDj2T8B}t3vKfSrkK?SOl3DsfJV$c z@S4`jdm*?fCTxz;?FR#*|09DR|6>)H{hb@plp9w(RG=g}ydvp6kf?~-M!FF$YDwAQ z3ciV!+}q!O;e*Ak_qUMVhvkgx_;-{h?KTW64?wN;k`k3zr?I0wprh?}Ng7#6LzmAM z1r3*@|D8U1W*?u@taI%I!{DyC$6y9q*Z(ckHZ4wCSD>;X%FPnF*qeULF;%vDmIx zt=h%O=Re$`2?zbK&F{OQPE+9=xaX%=?s*&Tvfqt8FL|f0J6cVCb1Q~B&#tR(SH-}s zIFjx9l3I|OM640iDiNo?;t-yO`i7+3=WI_Ns83P{VoFTt!*LhbSGgal2-Mv$*4fzr zrg6imgK-H!r{R^p{mgdI4N4L2`hFzC(ZKe`l-B@td!pZZ&Q&1bP|bs4Hjj0kZ>OE) zJ2Zvw$cnCw9L-m`@@%fp#O~Z$?$nRZxp4$^v50pK-#=yUe9F9A&tFUwmI6<#VuT|V zYU=!8TTn~oS-}X-@|U0m0wVw&Q!H)cTL$x*tyjU_hznKh?C=XO62_c}e9Bh_ApD(o ze(TPi(Blc(gc@v8+tTp(=X}tf%UeFLQEm!K-qP?(m-HMREG1Gsk?P*tkjK-Z#B1L^ zX?&#n&b(;Z8KM?vbOQ2{OSXAK7OXO-tQ?`dU)()5fI&sOP-2VCj&{Ai zx8wRS;ONH$U@o+4M;S)8O34-jkn1~=jRCYJc42VJiedNWhn_mMgnNotwUTQ-E@|v9 zFRFj76JVZSR$mc6kyY)w2~tUUYHnvYCm@`bE0>R@P8SDR$^KtVhq00XT#UP45?}Uw z7Lq@JgFu}D6_u^Au_+>c$u35?gi`9ecbz2;ao*gx5+~jblbqy{Nmu~ohvcpJ2Ly1U zs|eWlVk7ax3-V7-4N|>1A|rYmPP+VR!Kz`NAK+G%y(sdv8E3bFN-S(UqtGv=*B&kc zn0Q5DtVfk=IcTi?UQ^TZD^zYTDLy%bZt+%!XsRwP#+;4$ZspsN#hjxWgO@iWyRR9G zocyH|?E5cFgpqHt|2v$DfH-tO`GL>2sUy`vKzoxWWXP-2JVKO_NCrFrfw(Z9zka=o zUtq;^9qtP-jU{C*WugJzOO$FuARa|oksUn+C#g}zkKVUmzZc{K(14bj zdlbk1v>-9F%>ehUOXe()f1IvpMcsH@pb8m>q%U-DJh^tU*)Y{*V$$rvf*Zkd@U&wg$m2p_|O5 zeiVNUeZWl;wLny*dxW0!JE8L90*_x9c{D7|1f zv5HMrAd&oj603ivo5@`H_}9C83S~T#j`Nqbj?aKP2)#Rs4<)fISp>T66NW z-1))O%MXK18ex%{J|zg$64o!6!V*kiJ61=o#2vs~Z_KBz`8@Nkq1ZhQugU<#-ih;Ixsyrop)!YKFinN{RUxj914%uI<&a8Jn zJfa3L_>wv#l@;c_-;U`{Snt<#?M~Gbs-NM_lMDCPG7Mos#+iNXoIkuA6OfuqWQ_ki z7JkJ?`YQ%;HKi)*N&M8s{psF1zQpQVIo4X)7Tz8^W=vy*1_(Oq*I#)LD5+iBwr!=h zhV6K>(Chl26-(|d>uU8Z&c!T09??vwUicf0yMP_~7IU!#iz-&>f^7OWi=gUM0p+~X z1*FgR{csL}1`5F*M+(2}6qu;9Rk|ezg-@y#D3((OqrS}*bwW)RBx%g;<8y)@PSw4a zd6aI_3zw$w45o^@Bu+j|72I&)Tn?(LcPr>fTm4YyO3op&t?RH_(4fmYG>!QWyD^@p!39bjg5QiTwttWn0@g=!RxSUFtuM8yI6GO z{=Iuej@Sq_3qLq*-mKQBQ6m*2NJehMmJ#W+5t)xV4_j+ws|B-{b^Rg2ZR|`F%ZILc za%If-cg%K|PhFb_(qb;SRruc>J9fnKfXU5Yri1x^2IGBl_yZfuRHiW3TwD-I65MUz z>1(gtzPt|xt=YYx`b@SKU4z($!yQAvc3=zq42)C}6;g5d)B08O@n0VJEae~Nt9JT1 zKr|91+i)+pMTcWQ|6I0*Xqo8-JcsRl5?PhC%)C-o_w@2vqtO4^$DPl)&b564AkX@; z{;*-Z679NpR99Vl^cdT%Q|OG|loqG2hvWaM_z(}E;B+y4cH{~$ngyvQLAeG)_7nb7p~_2bv1%9xPvYe1`{YRPrGs+;pq`;1MGTT}0tDntR<(Q+r=zGSm2p6Cd5! zM-Xpgg(Ik~{_e?9Nu-91+6CI2>M*@(;c_-V2IFq15ytTMipn2RlTRrmz#i-%;zGia zZ@RoqB2J9^v9!Zs{12xRSvb9)mrjm~@tK0J-q%Oj#Nc0P%R_ale8)OBT#nf5Oh7d2 z;Wh{R$5-i2Bgxcn5b-S++Bv+#wT?lveIwDpS8EhLf8NE?a#G&GW5@cyVfG6uByuV1 za6nb3pOCHomGm1Jlr0(XCNYoNJtwvd;SyCGxv?8%*^Yz!Eog2%~PSXwZ z97KiSQ+TA89bx}Xlv^1Y8E1;eJKW(*uQ8g#VG=Ovliv`V9?DLP1*HpL$voz&q~jY7 zqTF)Q-~GYXw%Ihw(dx%1_H3veUgr3S2u9_oo?dewo=$-gs*C-9Rb$|p&GMKVefFps zw6U+9(Hv&U(QBqu;%V+9P`RQ;dF82fb%*X!t$~Z&VE}YjoftZj00`#$jaJm@7+(_a zCgMpvU-b63u}K&E-HvDWb)x-jyZR^=E$TC?s--z7RPWPy+!$ZDMDT2!$*ukhuqNPZ zMM6G*!OW_o$eHY7XxwKCP@z4ja~c=45mdmrXSMg=@TV&rmeMerGDXq3V@D=Xu_by2 zMa$fotIZs$>Bf*RKmSCwU_0t(?~;+Eb(}Yko>SN(PN!g##n6VVxwc|h)+#<760(z} zmeTSDMQZ5fg(eqg1LOWOO07NEjU-0a4%BCQh%;kgRox*wTa(-Ru>DD3eleS8S#%Jx z5;p?-Q>J|0EZUj!VTziGQ{T#X>C?PFv3cRMI7p7L# zP}Wqq_PA48RP>9m62b{=OO9D?7VjDxW7oX1;Xl;e8U^wvkE@p#uC21WUEzlw1BWg_p*-99~1#vOM<3ZrpKbG}>{c`B)y9Fl9%1uj4l9? zy+JX~ZnPXsW-EEO5`)!Pk!nuUadx?%_++@0QgRfoj+c1uPf#YSCOaz^2e@T_?LWiH z6J&EYr%ZCvIfz6dk>$|SVJcQY>Uw+opNLNI4Vnrlwg+e>rQ{~Rz6sngv#Erm)wY8d zsp~tSke^&}<%x!S{v>Ld@EDV&&t6=a$cr`$eLd+bNYcvK8|QP6r9JR^V-|?McmL_K zEdy_?ePOKsjCv;Pl?y{!RNd3EqjM3t$J$mz>$qEcDO{QrUXxw$symNfhih|d&YeW% zkePwovH_7L1F|})Tp5w}j}6EFJ16slqwlyt?V|uWihA`(*@EA}H{R>Ko?~-@Tf0|# z?f~6FPoY*lHjfO^h*!NCObl_bm!8Yp?dkH^`H>9pQdydR#I^Rz<-gWpk4VXL!rij9 zav4&AJ}OIx);>&nkf=Or{@AVV+uSnetjf*Vl_4KbW|Pn#<5ZRM+?L$Y-<2*hI%{jC zdaKxTv@5@M9)~FHP+9SK&p?iYh`3#~&;P1d9XvMMrsvtwbX-2U3F>z#h>P^Db1s}3 zT24~f{)@%4w2p<5VBx0>+Oi_0=2h^uw^5xMwznpVK=na>ui4hg&)>bfurXu4J=)TU z$h%kG6vW3XT>uh-*MFlsZtMO@?@N{b=J&^A@|h8%;h%n_e8d2)dayh;1|e-%pdKRP_V>SC=W6yks>i+XA8xRh+}>G!Q+8iw zir0cI{b#;i+@);msf@@*F=s>&#v%rE4CHV&7B` zJ7Z({V2pLM<7FdIZ5%G4SllCnNJFaGD8<$DbL{T9I^mP4HY@_8Uh#ft`z;;4IV{tX zt9~hSsh{<=CE)vm+%4{JCSkSeS2d)ew`L+J?bGM;X8Za+(^pUad%m&RRGH%3mT>3A zR0ic8vo82t-QwMK|HeR0@Kpg9iLse87XW^a--^Ezet!MDNvYe_!Hh|m;`<{P8Q9Tn zY1x~aFY*k?&`*nL&=UGza{UFy4k?HFuU~7XTiK9)3kg}bDD&Y`gm=Zr>Bqas(a1Pu?m}Z zsSMt2xML^};4_bWa+&klXj9vgyFAU*@aOC8*9)D=Np`Sb^tWkwCLQUGc6}W9lhHoS zDQ9Q&@u{fS#--(y&`Mtd$#Or!D^@@CsahPbZjAb8x^mRB0o6z5s|h*BbMtd&drn;t z_i@f;)7*_Fnw#EqtFm7PHudTG+_SS%TK4<+EpTS+^mD*FbL~$WyO=aPJJjHQ?xHzC zu*7OcUlSIX@BJ26c}2(ljn?Y0Sz7wbUDQ^Nw^Cjk5~_YQe|>Cb^fAiU+|L&@#r)y$ z$~Rxtj`lH4_l9EUWw+}Z;-VLw z=sV*($FFrjXHCsMr)5DQ5eL+%@BO4y`}BsldB#PyR_v;#CRXI+PVT{Vrgz-g1Uk=Sp+GJ}tcv6V=-7B3= zH`A^PI}zS!uu0jbjRy`MEPk7u`#!~c_~C*Qb(7LxdrqLN;@5_MMdVu5X;i4yW-9oP z7v?&-=M(qixL^(UvaWXRID{VFKFhI{!Kt5x6M;!?8EJS0)!?>rhl ziXwlF_nC8-AHa_kf4LGDV|C9pEUK=!NHgujiYE)k1kGL&R^d~9vNcO zR#}874UBW|=<5GVJ}xqZddkOl&&+HsFwvMUPtXeQyy{6lL#D2g|9rRFCi{Hvx>#x3y=j^$Nw4IZ>$!4y&w@wDJO zBvT=$B@oyTGRE4viSk2WAX+NONOg(YI((5|^MhSl!uNrzXrLI7{vsh$s1ATI!-g!U z5cmyqfg*|Kk~4q?&bx%2U&z#+(R>{V>UWGfc(66%o`wodX#?bS>la%5u;?i_Ib<&b zu`KLo;xnr)FYFc&c+sECj+WePQ3TQnFR0{FoMZT%!FwGM?cL^ND6U3Ednk4MY&6)C z|EQ)d$~^F=#u6J=(i37DzX#2h3AL+d{#Z^1b9g)$7DYiDzCUdP!w3{bD0v|EiVaZt zRNoDhOU%6YbbfRV*gAneGw?g?AgGM0A{sL~?eHKP*{(V|t(2mV^O-3!Fz@Z&-Mft! zE$TupXiHR!yn)fd(Z{iMntHxiqix%UMs6K}>3uJ{ zg?^mh{z|#ZgX+9wCW3vBW_dVeoO$OLW>2*rD%uAyTc&u(6sixLGu9#d-xTp?&t5fj zBIS-6O3C!9H{*$Kuj#6Mk1WltIN`DU`Q-RDRK#>9>9NS(M2;j#!lp+yn zgGt3qubS`WK_}or2D0CC$SKw{c$u^~T-87BV7BS{t?fPgGY!3fWIO zBZjYG{m*tk6W8`0stzA3-^BRoeSEqO88TZWGZlHumD^yqnH4f#vN61+|6LUnp_sJX+_U%NI=x2)du z{Xk-V{VvFljusB$^gZk!4<7WAWlkF0WR#WC0aoAdtMR^V%h>r`?7q>(Npd$pv%=%Q_dP#nGar@dg~gHMUeoSdSZoxKeO^~8_fe=6smr?bRu`ptfn^w(db-)`}K zsomAdxA@LL6$JF&zPLu32fUu;VUkkQ@O0*g1Go*6blP;S)H|YkcwKh>0)Xc5js8Q6 z4n}OTSrxV6*|}8KZn0kN?DU>2a!x>#cXu1QFR0yd_&!j_ucv2*=K6UD2l+R1>lv#P z#SOHzQsb*j$+C3i#$CJd=?YC$`nntE;$EEAJb6J|byiOI*wMXQ$b0E8%D+uV17zf~ z>a@;V&9qG3?7dLd_x133IhW@A#P+M2?4bSq^U$*w^x8M~oL*pd_$L>K9+6)+ZcH6= zEs!yAzrAz0$jPuE+P|%_nwFMdhpcf~HmlF3I+&Y#PYUT2SrN5D(cUH+@TaNUKJsQ- z$r&s642}MbNkP6zo6ODcHVytd%!lzaElqm4VAk1&oLRu;PMtbce7H)^ye7ZRm5ia8 zu4W4(7UK2pdU4#bz$%Lq+W&OyhYyC)b>B?|jK4NEz}uOuHvQbd^GP-S&@$soe9kAV zT3&@Vn^fB^)#pPjyx~u?@(;8m978c>Yp(C$s~+X`A}pjP%uY?+wkqNAw^>moANU(3 zEB*GywtzrCQ0_9ZGx7onUCl9j9d{RcC#yo!KCPAUrlW6CJEK4)oha+&AT~Vx;7Y$&#b~3P6>J90Q~+%8(pdLw^SY?f(v8aw`ZX%v zVep$H2!TQWeQ#<)J3n}|0{^R~W2nczFsn(g_jEhmb75Djvw=nK_dMEg^k}wRD!8!2 zZ^zG9H;*-HWh|-W$(5^ZZLf|TdS+i?kLKoVm}776D<;uq>0X^zeq2*n{i|ELWmrq) z>aTe~yeo^+k$6IO&ouL*CQbD0h@K@vTfOuxd{B3iN5*je%IHT@cH2O-ouJ`a!Wp`R zE4*T$DqvqbZeo_t`hMroP^Xq$Jl}jiyZ$ze+&1VL!9@6=q?=nJR<=-E{5atwcl#a) z_+7v%O}Jo%=@Ep`u&#FfWte==%T<6{g3Xu$u!$UYVafX8&1S7$eO_av(>2_{vq!j| z$#)-kO~+mRXkkzbje$mjY~XC-&ek@>7Ubx>z+}x@@Wg4f|=;JL4zn<8u3Mr_FDVa{4%4II$Dd==M3cADyPd((lwQia1Es z2P2CP-S(~tD)3afqUB~7t>NjYzT{q9xZRB>_YWf4ULaws{~&{<4J~|Tz6M(!FX(%O@g6ZlX-cd;A8AIlyG-1llt95&!fMz|i z?NX8^y)LGk)TsV^?fQ!b&G!d+s_cC>6&HLjcfA4)Bd1nLU?QY|E-bq|*E;0jL67P% z`wR^u`@bLSe85+{3rs3(WH~`gZE>WheyFg|)418pv|J^0#@SDs;PCQ4&d}=__HSC3 z_>x4kND&A7G#xX8GkB3u_=^@V-d}RLo)}MnGKf zVZwCRZ+`z=MObcLW67k|riqM@z4@mlL6=pdK*UB_$E(4o=XSSN=Yd5Pya$RsTN1>%d0;coQ2 zs7gcgi|3Mmz>B>*R1}zvMWfzw`+hT=K@zGdQQ(6;wLeUf#4ncoYkxSd7pr{1;)hR*0PHeeOlPLV>s#zVVp98=jz%PR;jd>!FPM$O3W6KT_s3u` zDrKR?mG63{H_2scF?euQ9_uHtq5$!ajPTfJrH3x0r#iM4SV-X@JIxs<+-1ZopQv38 z+;I^kjFyHlkMa3=7o}$H>{Vn5u7*iniWH5kCJUm%;_34JG{xi8qdDFk=_dkT}t6$3? zZ-*c^EdjE(v)^7&gy2d@z;lxM>GOFC8m2HDJ=gzf=#?A?|NTq#3g&TSh>3_IseYHM zHs1-#drlX3i(U4BI!+{jM){ozs%(AV7-S8PB2-s7YMwRq$iUGy7FK16{IQ}nk1!^B zK^PQQ52sW;Ft=D2kN)4xn!p0u%QDV@-@b#Bs2fHABdd9bp4wAZ@%VMmHuf9xd zp-xmuKEFwb{_zJb0FYhvj#me2e5#0g)y_=&xf&j->69MKiOkW>oz93%|s+ zWAlC7x5dBeWZ9`p2q+l=@6Msl)bv)gUvMfKan-N<7LFP}AC>bqb%50eerd=k2qL;k z&9X<9JJ$&e(=C4gW$CRN^;X2OrtI)s(f`i=XFa2b?XpiHVU~rKL^}%aHCyySAk|`l zha~#I+IkcFB3xd&sPj>~N$e;(=7P$(Q&Q#> ze1Agls5oZ0CB(%exfhJ~Vy%uMo}-bLs#xi@3SYeV9S*aeLT%~6(b?XAu63{Z>~_c1 z6*SEOuR|}NH#awzj9|}dfV74RGv$TS*4Sh>dO5 zj=1>~KL5;pc!?M5yl0Frb^H?AA^Q#tx=)_&y2}46O=Pj->H|EbdJilcLHh57W>%t- zdiNin_;+5SCSfMDeJx?HC9|7=y$nB#u%;CDX&<6O%HKf$p!~m(tt-Q@kb|y|S#9-a zRFwF+q%%{VgSFZh?vF8u^Vqg+>4@|@cgDOO);)`bV!iS{=lsf#>n2uxIc+nt=1tHW zmwM(&S8EMDipJ=hW8Q$dQSrWENlV*^<3|!1kdK7#x5#Zr0Ms6Q-WsjrezP>cw~J3q zbii+A2HN#y;KOsbuS@5!O?Gg8G1XiG2yp0vdnIVEUtifF>0kjjN~n!DU4N>wuxZ=PW^@I5f-+|`-)zElWvgx>Sg|gEb zDhPSsWO#=Y#vUaDFo>Oa>ZiNvCzJ;=;t^$g@4QG8D=RCRGgAH5mNDRepL%<3s}IzRa|%R4X1 z=fHszo8o3o(YNjU6!)KU6P?CY^Kok?*v$;Yz`zNRb0i3H656E(Mz0JP0x&r5O3;v1 zCG*UBOz~G!j_9xNAzXxQAPeb-E9Su4xMLs?5X{wepef#3FUFVMw+2!Z=5X|fl0(5& z9j09@b98W);84iivpOS(4_{z7WkQnfu338hEn?0%9HlceMaG+dUMWe)KRP9joNfMv1tKQh{(J7iAH-t z()^w$o<=j~Sgl#JpXoM!EuFTrNyor=aqsSp?_`>ih;NdH35FUukkDNB> zQV_yj5G{_b{21oMd7+o~6(Q7ESB-(ooy3C|5z$651!cQUxmP?$vPH(A2HYkCJ9NxA zUi)DVL&JceSH@WWB{bzOuEw3v)H}gp z-aKt#&P+xU@HmRZ*s_!JF}adxmi~wPoBv3-Kb-Wq+)UFXH$UG(uy{nO5YDK-cObqE zU>2#4wf}1E4Q`R_j>9Fk{D~Fcxp$g!a5_4d2oT|U;~N%^%T86+V*l1z`~2p@J8TJ? zhhXWKSEP4}`1YDLzU1ZYT@}N!VSuxv{mIDP__BX73Z{k>B z7QHc?yS9z7+B(R-h!62%BZ1*hnw8#vtX1Nqp&JXbKaVUsCNHD24O1mfY|6U|;uoZJ zVfGh8m=4L&YEaI7#jk8FVMezoer-p!HQvcloBCmI2mKj_QxcPs_OQpsV3cwU${u<> z`pu`Mp97XfFnCuc>0D+iNW_s)`@@N^$kB@X4-MXgxV3fUJY!_su5Rpa=&}Ie{$#@z zW5$jRE9POGSGv$n-v)*+w#o#ske00IL!yI^uXfc4IORiq=g% zsk`#79>!RE^J9iCQ8Bjji%axxKem?%$VwNaWV&%%k+Z5P0(Jc3`(!;9+XK{kG6Oe6 zG8>GS9xj9Szm0x*3^P)0gK1}06rbw9e4M7;GxTB@d-rgF-u?OKjRCw-DRMtC=hqt;WhfOa=UqTf46>&oF>+9e&dH~5~tP|MAKf=gK5zf9t={;W5^ zBBmGiR}7orizILDKe@SC?*V69AWBQk_%D<6&UiO&-nO z=AJ6s2CgYcn{4GyFo|Q7pPwH<&x7itI#xRtH%_Uk4{Bnm)gsn!MZV#@M$SQr9f)o@zU95p7zD~c z!|=THvx}QP$=8xACuJ2asePO@#eGN6>sf3BFdoBkom-2hEX;hs_^K49irM;ouPGa# z@1-LuxJGBeMSLR**fRYBfxyp>?Dc!v$ytih=+*?&(X(=TuoW+iQf_lX+W$OhPW3Yl zBelX$8B|0%@Ce~aL(ZZ;>zGxcv85wH@TcZ{?$@uM7l&~r;&15qA#O+@MgJjl9k^?@(d}Er zIpf?Rk!+8o20%~^_t1KfL@Y7Y{2kVU40qtu%J^F#{{%wm+yTOp&4->F^y(|s$1otx z?ZhUh6gJO$@+6qLTN05mtRE|Un=BXFBEA7A)EMT7pv|En!>1i2Jan+v78{l1UIML-rq04#!V-uTgHv> znKHD_={oPF8pnry@Z{UoITt$091ptXU;sf*JtA@p7|kXIR_5=tLYg6`zi!2=J0D`n zW5m2_0Inlbz=af{|NcT1x{lBRK@~J5c4sZ=sFGj;BFaMMMsCl!e2J;fe1u-(mK{%} z!`=y77@_6}xz_B}|VdwixSZ{tUdecUPdK5W1;m3ua>&lbpXYLwcVsADD8Q)E$ zNGeNP%*|jBrl&4~`FZiKBT;Iv1A6k*ZY{A5c(i4akC6Zn9A1)h!y4et2&6%9hiAFAOk(<62RMgg4n|rAzB^Uo(@suc7U_>B{KmfG$vVqN4H+`z z4*irEv%w$5n+J)uRb7>&xLaIr*IM*mx|av~0vc6jMm}dKXy}Z_Tt?2Ho~#(=T00EH zz~^+?U{lt^B`ybM3To19{Mc!{QXtS-grl<-6VL=-d-rSg810I(H^f!{`%me}O4?m3 zg1<2}w&%8NI=n-l`lMuGx>@)6op}d+`}!!x^Ihj-XIzVHv4 zK=$m#MO(E~ey10G$uTdH?>deZMSryL$e&3h$A5kq?p%hPb)e|2r(EOCcRv|?OQyC{ z+t*hR9z^h8Yz_CDGTokwQEeR)u@QJLVq{~3jxA!7{>weQGucQcCKHM! z1QA0o>M0~zUZS)Uvk6jL6ATy_AnH{29cprr-F}i^jB9s*==UGIcrc<5Q++4{1TvY( z<~F=@zt&J1ETCY?${kaHiVOdOxO9(AWO^PoG);bF3KuLy8${q5qTeA^*lGga1AO znfNbl25kKIKV`bT+5epI(yGm%|4)8cU85yu8-8tOb~8iap9y28m|iem^vC}Kn}dEL literal 0 HcmV?d00001 diff --git a/runs/frontier-fidelity-envelope-v1/patches/0001-vidur-large-payload-model-prediction.patch b/runs/frontier-fidelity-envelope-v1/patches/0001-vidur-large-payload-model-prediction.patch new file mode 100644 index 0000000..56eac1f --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/patches/0001-vidur-large-payload-model-prediction.patch @@ -0,0 +1,84 @@ +diff --git a/frontier/cc_backend/backends/vidur_cc_backend.py b/frontier/cc_backend/backends/vidur_cc_backend.py +index ca1983a..0c57f05 100644 +--- a/frontier/cc_backend/backends/vidur_cc_backend.py ++++ b/frontier/cc_backend/backends/vidur_cc_backend.py +@@ -882,2 +882,21 @@ class VidurCCBackend(BaseCCBackend): +- # Fallback to analytical if not in cache +- logger.debug(f"num_tokens={num_tokens} not in cache, using analytical fallback") ++ # The precomputed lookup is capped at 100k elements, while realistic ++ # TP payloads are commonly much larger. A cache miss does not mean the ++ # measured-data model is unavailable: predict on demand and memoize the ++ # exact payload instead of silently switching model families. ++ with self._cache_lock: ++ model = self._models.get("all_reduce") ++ if model is not None: ++ features = pd.DataFrame({"num_tokens": [num_tokens]}) ++ result = float(model.predict(features)[0]) ++ with self._cache_lock: ++ self._predictions["all_reduce"][(num_tokens,)] = result ++ logger.debug( ++ f"predict_allreduce: data_size={data_size_bytes}, num_tokens={num_tokens}, " ++ f"result={result:.6f} ms (ML model, on-demand cache miss)" ++ ) ++ return max(0.0, result) ++ ++ logger.debug( ++ f"num_tokens={num_tokens} not in cache and model unavailable, " ++ "using analytical fallback" ++ ) +diff --git a/tests/unit/test_vidur_cc_large_payload.py b/tests/unit/test_vidur_cc_large_payload.py +new file mode 100644 +index 0000000..7e87aa7 +--- /dev/null ++++ b/tests/unit/test_vidur_cc_large_payload.py +@@ -0,0 +1,50 @@ ++from __future__ import annotations ++ ++import threading ++import unittest ++ ++import numpy as np ++ ++from frontier.cc_backend.backends.vidur_cc_backend import VidurCCBackend ++ ++ ++class RecordingModel: ++ def __init__(self, value: float) -> None: ++ self.value = value ++ self.features = [] ++ ++ def predict(self, features): ++ self.features.append(features.copy()) ++ return np.array([self.value]) ++ ++ ++class VidurCCLargePayloadTest(unittest.TestCase): ++ def test_cache_miss_uses_measured_model_and_memoizes(self) -> None: ++ backend = object.__new__(VidurCCBackend) ++ backend._cache_lock = threading.RLock() ++ backend._num_devices = 2 ++ backend._predictions = {"all_reduce": {(100000,): 0.1}} ++ model = RecordingModel(0.321) ++ backend._models = {"all_reduce": model} ++ backend._analytical_fallback_allreduce = lambda *_: self.fail( ++ "analytical fallback must not run when the measured model exists" ++ ) ++ ++ data_size_bytes = 2048 * 2048 * 2 ++ expected_elements = data_size_bytes // 2 ++ first = backend.predict_allreduce(data_size_bytes, num_devices=2) ++ second = backend.predict_allreduce(data_size_bytes, num_devices=2) ++ ++ self.assertEqual(first, 0.321) ++ self.assertEqual(second, 0.321) ++ self.assertEqual(len(model.features), 1) ++ self.assertEqual( ++ int(model.features[0].iloc[0]["num_tokens"]), expected_elements ++ ) ++ self.assertEqual( ++ backend._predictions["all_reduce"][(expected_elements,)], 0.321 ++ ) ++ ++ ++if __name__ == "__main__": ++ unittest.main() diff --git a/runs/frontier-fidelity-envelope-v1/plot_mock_envelope.py b/runs/frontier-fidelity-envelope-v1/plot_mock_envelope.py new file mode 100644 index 0000000..8d21765 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/plot_mock_envelope.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Render the preregistered fidelity-envelope figure prototype.""" + +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np + + +OUT = Path(__file__).with_name("mock-fidelity-envelope.png") + + +def main() -> None: + workloads = ["F0\n2048/1", "F1\n512/1", "F2\n2048/128", "T1\nexact trace"] + variants = ["A0 analytical", "A1 native measured", "A2 measured+fix", "A3 +batch profile"] + # Prototype values are deliberately marked as MOCK and encode possible, + # distinguishable outcomes only. They are never read by result analysis. + regret = np.array( + [ + [12.5, 14.0, 18.0, 25.0], + [11.0, 13.0, 16.0, 22.0], + [7.5, 10.0, 12.0, 18.0], + [2.0, 4.0, 7.0, 12.0], + ] + ) + + fig, (ax0, ax1) = plt.subplots( + 1, 2, figsize=(11.5, 4.5), gridspec_kw={"width_ratios": [1.35, 1.0]} + ) + x = np.arange(len(workloads)) + width = 0.19 + colors = ["#7f7f7f", "#4c78a8", "#f58518", "#54a24b"] + for index, (variant, color) in enumerate(zip(variants, colors)): + ax0.bar( + x + (index - 1.5) * width, + regret[index], + width, + label=variant, + color=color, + ) + ax0.axhline(5.0, color="#d62728", linestyle="--", linewidth=1.5, label="5% gate") + ax0.set_xticks(x, workloads) + ax0.set_ylabel("Worst selected-config regret (%)") + ax0.set_title("A. Rank fidelity across workload complexity") + ax0.legend(fontsize=8, ncol=2, frameon=False) + ax0.grid(axis="y", alpha=0.25) + + consumption = np.array( + [ + [0, 0, 0], + [35, 65, 0], + [100, 0, 0], + [100, 0, 100], + ] + ) + bottom = np.zeros(len(variants)) + labels = ["measured collective hit", "analytical fallback", "batch-profile coverage"] + stack_colors = ["#4c78a8", "#e45756", "#54a24b"] + for values, label, color in zip(consumption.T, labels, stack_colors): + ax1.barh(variants, values, left=bottom, label=label, color=color) + bottom += values + ax1.set_xlim(0, 200) + ax1.set_xlabel("Coverage counters (normalized; separate axes by mechanism)") + ax1.set_title("B. Profile consumption, not just final rank") + ax1.grid(axis="x", alpha=0.25) + ax1.legend(fontsize=8, frameon=False, loc="lower right") + + fig.suptitle("MOCK / preregistered layout — values are not experimental results", fontsize=12) + fig.tight_layout() + fig.savefig(OUT, dpi=180, bbox_inches="tight") + print(OUT) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv b/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv new file mode 100644 index 0000000..85fedab --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv @@ -0,0 +1,25 @@ +time_stats.all_reduce.min,time_stats.all_reduce.max,time_stats.all_reduce.mean,time_stats.all_reduce.median,time_stats.all_reduce.std,rank,num_workers,size,collective,devices_per_node,max_devices_per_node +0.08288000151515007,0.08288000151515007,0.08288000151515007,0.08288000151515007,0.0,0,2,4096,all_reduce,2,8 +0.0793600007891655,0.0793600007891655,0.0793600007891655,0.0793600007891655,0.0,0,2,32768,all_reduce,2,8 +0.0713919997215271,0.0713919997215271,0.0713919997215271,0.0713919997215271,0.0,0,2,65536,all_reduce,2,8 +0.08056000247597694,0.08056000247597694,0.08056000247597694,0.08056000247597694,0.0,0,2,131072,all_reduce,2,8 +0.0865279994904995,0.0865279994904995,0.0865279994904995,0.0865279994904995,0.0,0,2,262144,all_reduce,2,8 +0.07135999947786331,0.07135999947786331,0.07135999947786331,0.07135999947786331,0.0,0,2,524288,all_reduce,2,8 +0.07321599870920181,0.07321599870920181,0.07321599870920181,0.07321599870920181,0.0,0,2,1048576,all_reduce,2,8 +0.09025600180029869,0.09025600180029869,0.09025600180029869,0.09025600180029869,0.0,0,2,2097152,all_reduce,2,8 +0.08083200082182884,0.08083200082182884,0.08083200082182884,0.08083200082182884,0.0,0,2,4194304,all_reduce,2,8 +0.10891199856996536,0.10891199856996536,0.10891199856996536,0.10891199856996536,0.0,0,2,8388608,all_reduce,2,8 +0.1703840047121048,0.1703840047121048,0.1703840047121048,0.1703840047121048,0.0,0,2,16777216,all_reduce,2,8 +0.25539200007915497,0.25539200007915497,0.25539200007915497,0.25539200007915497,0.0,0,2,33554432,all_reduce,2,8 +0.1021759994328022,0.1021759994328022,0.1021759994328022,0.1021759994328022,0.0,0,4,4096,all_reduce,4,8 +0.12694399803876877,0.12694399803876877,0.12694399803876877,0.12694399803876877,0.0,0,4,32768,all_reduce,4,8 +0.09161599725484848,0.09161599725484848,0.09161599725484848,0.09161599725484848,0.0,0,4,65536,all_reduce,4,8 +0.08580800145864487,0.08580800145864487,0.08580800145864487,0.08580800145864487,0.0,0,4,131072,all_reduce,4,8 +0.09867199882864952,0.09867199882864952,0.09867199882864952,0.09867199882864952,0.0,0,4,262144,all_reduce,4,8 +0.09646400064229965,0.09646400064229965,0.09646400064229965,0.09646400064229965,0.0,0,4,524288,all_reduce,4,8 +0.08377600088715553,0.08377600088715553,0.08377600088715553,0.08377600088715553,0.0,0,4,1048576,all_reduce,4,8 +0.1128000020980835,0.1128000020980835,0.1128000020980835,0.1128000020980835,0.0,0,4,2097152,all_reduce,4,8 +0.08755199983716011,0.08755199983716011,0.08755199983716011,0.08755199983716011,0.0,0,4,4194304,all_reduce,4,8 +0.12361599877476692,0.12361599877476692,0.12361599877476692,0.12361599877476692,0.0,0,4,8388608,all_reduce,4,8 +0.20030399411916733,0.20030399411916733,0.20030399411916733,0.20030399411916733,0.0,0,4,16777216,all_reduce,4,8 +0.2924960106611252,0.2924960106611252,0.2924960106611252,0.2924960106611252,0.0,0,4,33554432,all_reduce,4,8 diff --git a/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.manifest.json b/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.manifest.json new file mode 100644 index 0000000..6ea2791 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.manifest.json @@ -0,0 +1,15 @@ +{ + "output": "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv", + "output_sha256": "9d693fd406616b599e57bcde66c980c7fc2831b3acf37d3eb633cec80ea0070d", + "payload_contract": "size=num_tokens*hidden_dim*2_bytes", + "rows": 24, + "schema": "frontier-vidur-allreduce-materialization-v1", + "source": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2/allreduce.json", + "source_sha256": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3", + "target": "time_stats.all_reduce.median", + "tp_coverage": [ + 2, + 4 + ], + "unused_stat_policy": "repeat critical_path_median; std=0" +} diff --git a/runs/frontier-fidelity-envelope-v1/remote-sync-marker/README.md b/runs/frontier-fidelity-envelope-v1/remote-sync-marker/README.md new file mode 100644 index 0000000..bf2fe4b --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/remote-sync-marker/README.md @@ -0,0 +1,2 @@ +This directory is the gpu-fleet synchronization marker. Experiment code is +synchronized to dash0 through the project Git branch before dispatch. diff --git a/runs/frontier-fidelity-envelope-v1/run_flashattn_composition.sh b/runs/frontier-fidelity-envelope-v1/run_flashattn_composition.sh new file mode 100644 index 0000000..03afe43 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_flashattn_composition.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -euo pipefail + +TP="${TP:?TP must be set to 1, 2, or 4}" +case "${TP}" in + 1|2|4) ;; + *) echo "ERROR: invalid TP=${TP}" >&2; exit 1 ;; +esac + +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +MODEL="${MODEL:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +CAMPAIGN_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROFILE_SCRIPT="${CAMPAIGN_ROOT}/../frontier-qwen30-vllm020-profile-v1/profile_vllm020_flashattn.py" +LOG_DIR="${OUTPUT_ROOT}/logs" +PROVENANCE_DIR="${OUTPUT_ROOT}/provenance" +BATCH_SPECS=(2q512 4q512 8q512 16q512 2q2k 4q2k) + +mkdir -p "${LOG_DIR}" "${PROVENANCE_DIR}" "${OUTPUT_ROOT}/raw" +exec > >(tee -a "${LOG_DIR}/composition.log") 2>&1 + +if [[ -z "${CUDA_VISIBLE_DEVICES:-}" ]]; then + echo "ERROR: CUDA_VISIBLE_DEVICES must contain the fleet-allocated GPU" >&2 + exit 1 +fi +IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES}" +if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then + echo "ERROR: expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2 + exit 1 +fi + +echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=${MODEL} runtime=vLLM-0.20.0+cu129 operator=FlashAttention3 tp=${TP} batch_specs=${BATCH_SPECS[*]} profile_script=${PROFILE_SCRIPT} output=${OUTPUT_ROOT} expected_wall=3-8m hard_wall=900s hard_gpu_cap=0.25_H20h" +date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" +nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader + +test -x "${VENV_ROOT}/bin/python" +test -f "${VLLM_SOURCE}/benchmarks/attention_benchmarks/runner.py" +test -f "${MODEL}/config.json" +test -f "${PROFILE_SCRIPT}" + +git -C "${CAMPAIGN_ROOT}/../.." rev-parse HEAD > "${PROVENANCE_DIR}/aituner.commit" +git -C "${VLLM_SOURCE}" rev-parse HEAD > "${PROVENANCE_DIR}/vllm-source.commit" +sha256sum "${PROFILE_SCRIPT}" "${BASH_SOURCE[0]}" > "${PROVENANCE_DIR}/source.sha256" +uv pip freeze --python "${VENV_ROOT}/bin/python" > "${PROVENANCE_DIR}/pip-freeze.txt" +nvidia-smi --query-gpu=index,uuid,name,driver_version,memory.total --format=csv,noheader > "${PROVENANCE_DIR}/gpus.csv" +printf '%s\n' "${BATCH_SPECS[@]}" > "${PROVENANCE_DIR}/batch-specs.txt" + +timeout --signal=TERM --kill-after=30s 780 \ + "${VENV_ROOT}/bin/python" "${PROFILE_SCRIPT}" \ + --vllm-source "${VLLM_SOURCE}" \ + --model "${MODEL}" \ + --output "${OUTPUT_ROOT}/raw/flashattn-composition-tp${TP}.json" \ + --tp "${TP}" \ + --batch-specs "${BATCH_SPECS[@]}" \ + --warmup-iters 5 \ + --repeats 10 \ + --profile-kv-update + +test -s "${OUTPUT_ROOT}/raw/flashattn-composition-tp${TP}.json" +sha256sum "${OUTPUT_ROOT}/raw/flashattn-composition-tp${TP}.json" "${PROVENANCE_DIR}"/* > "${OUTPUT_ROOT}/artifacts.sha256" +nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader +date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ" +echo "FLASHATTN_COMPOSITION_COMPLETE tp=${TP} cases=${#BATCH_SPECS[@]}" diff --git a/runs/frontier-fidelity-envelope-v1/smoke-report.md b/runs/frontier-fidelity-envelope-v1/smoke-report.md new file mode 100644 index 0000000..f55c31b --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/smoke-report.md @@ -0,0 +1,15 @@ +# Frontier measured-collective smoke + +日期：2026-07-17。设备：local CPU（Frontier simulation only）。Frontier commit：`d9cfeb6d8791fbf2f295dd9744c56a666171776e`，沿用既有 dirty compatibility patch set；A2 额外 patch SHA256 为 `35cc6be846589faf8cb5fa3ce5fdfe0aee8f086ba7dbb5dbcdc677148f19a3c8`。 + +固定 cell：Qwen3-30B-A3B BF16 profile-v2，`TP2/MNS8/MBT8192`，`ISL=2048/OSL=1`，64 requests，8 req/s，prefix off，TTFT SLO 1256 ms。 + +| variant | CC path for 2048-token payload | TTFT p50/p95/max (ms) | pass rate | +|---|---|---:|---:| +| A0 analytical | analytical | 122.2406276 | 1.0 | +| A1 native Vidur | `4,194,304 > 100,000` elements，lookup miss 后 analytical fallback | 122.2406276 | 1.0 | +| A2 measured + direct miss | measured random-forest estimator；exact payload memoized | 120.8898909 | 1.0 | + +A1 与 A0 的所有 TTFT 数值完全一致，验证 measured CSV 虽成功加载和训练，但没有参与该 payload 的 prediction。A2 model 对该 payload 的预测为 `0.09877793 ms`；同一个 TP2/2048 row 的 measured critical-path median 是 `0.10891200 ms`。A2 相对 A1 的 E2E delta 为 `-1.3507367 ms`（`-1.105%`），与每层多次 collective 的累计量级一致，因此通过“execution path 必须变化”的 smoke gate。 + +这个 smoke 只证明 profile consumption；不证明 ranking 已恢复。下一步必须运行完整 A2 TP×MNS surface，并与已冻结 real capacity比较。 diff --git a/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py b/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py new file mode 100644 index 0000000..3f5a79c --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import csv +import importlib.util +import json +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).parent + + +def load(name: str): + path = ROOT / name + spec = importlib.util.spec_from_file_location(path.stem, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class FidelityEnvelopeTest(unittest.TestCase): + def test_materialize_allreduce(self) -> None: + module = load("materialize_frontier_allreduce.py") + rows = [] + for tp in (2, 4): + for tokens in (1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192): + rows.append( + { + "tensor_parallel_size": tp, + "num_tokens": tokens, + "hidden_dim": 2048, + "payload_bytes": tokens * 2048 * 2, + "critical_path_median_ms": tp + tokens / 1000, + } + ) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / "allreduce.json" + source.write_text( + json.dumps( + { + "schema_version": "qwen30_vllm020_allreduce_frozen.v1", + "rows": rows, + } + ) + ) + output = root / "all_reduce.csv" + manifest = module.convert(source, output) + self.assertEqual(manifest["rows"], 24) + with output.open(newline="") as handle: + converted = list(csv.DictReader(handle)) + self.assertEqual(converted[0]["num_workers"], "2") + self.assertEqual(converted[0]["size"], "4096") + self.assertEqual(converted[-1]["num_workers"], "4") + self.assertEqual(converted[-1]["size"], str(8192 * 2048 * 2)) + self.assertEqual( + converted[-1]["time_stats.all_reduce.median"], + str(4 + 8192 / 1000), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/runs/frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py b/runs/frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py index 10249d7..ecd0e74 100644 --- a/runs/frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py +++ b/runs/frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py @@ -48,6 +48,10 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--requests", type=int, default=64) parser.add_argument("--rate", type=float, action="append") parser.add_argument("--config", action="append") + parser.add_argument( + "--cc-backend", choices=("analytical", "vidur"), default="analytical" + ) + parser.add_argument("--allreduce-csv", type=Path) parser.add_argument("--timeout-seconds", type=float, default=900.0) parser.add_argument("--resume", action="store_true") return parser.parse_args() @@ -182,6 +186,37 @@ def knobs(config: Config, paths: dict[str, Path], cache: Path) -> dict[str, Any] } +def configure_cc_command( + command: list[str], *, backend: str, allreduce_csv: Path | None, cache: Path +) -> list[str]: + configured = list(command) + option = "--cc_backend_config_type" + try: + index = configured.index(option) + except ValueError as error: + raise ValueError(f"Frontier command is missing {option}") from error + configured[index + 1] = backend + if backend == "analytical": + if allreduce_csv is not None: + raise ValueError("--allreduce-csv requires --cc-backend vidur") + return configured + if allreduce_csv is None: + raise ValueError("--cc-backend vidur requires --allreduce-csv") + configured.extend( + [ + "--vidur_cc_backend_config_all_reduce_input_file", + str(allreduce_csv), + "--vidur_cc_backend_config_cache_dir", + str(cache), + "--vidur_cc_backend_config_k_fold_cv_splits", + "6", + "--vidur_cc_backend_config_num_training_job_threads", + "1", + ] + ) + return configured + + def find_metrics(run_dir: Path) -> tuple[Path, Path]: systems = list((run_dir / "metrics").rglob("system_metrics.json")) requests = list((run_dir / "metrics").rglob("request_metrics.csv")) @@ -231,6 +266,10 @@ def main() -> None: args.profile_root = args.profile_root.resolve() args.python_deps = args.python_deps.resolve() args.output_root = args.output_root.resolve() + if args.allreduce_csv is not None: + args.allreduce_csv = args.allreduce_csv.resolve() + if not args.allreduce_csv.is_file(): + raise FileNotFoundError(args.allreduce_csv) rates = tuple(args.rate or RATES) selected = list(GRID) if args.config: @@ -274,6 +313,12 @@ def main() -> None: run_id=f"qwen30_prefill_{config.name}_r{rate:g}", knobs=config_knobs, ) + command = configure_cc_command( + command, + backend=args.cc_backend, + allreduce_csv=args.allreduce_csv, + cache=args.output_root / "cc-cache", + ) write_json(run_dir / "command.json", command) environment = os.environ.copy() pythonpath = [str(args.python_deps), str(args.frontier_source)] @@ -389,6 +434,15 @@ def main() -> None: "coverage": coverage, "sha256": {name: sha256(path) for name, path in paths.items()}, }, + "collective": { + "backend": args.cc_backend, + "allreduce_csv": ( + str(args.allreduce_csv) if args.allreduce_csv is not None else None + ), + "allreduce_csv_sha256": ( + sha256(args.allreduce_csv) if args.allreduce_csv is not None else None + ), + }, "config_results": config_results, "capacity": capacities, } diff --git a/runs/frontier-phase-factorial-v0/test_phase_factorial.py b/runs/frontier-phase-factorial-v0/test_phase_factorial.py index 4172926..0ceb170 100644 --- a/runs/frontier-phase-factorial-v0/test_phase_factorial.py +++ b/runs/frontier-phase-factorial-v0/test_phase_factorial.py @@ -43,3 +43,20 @@ def test_kendall_tau_b() -> None: analysis = load("analyze_qwen30_prefill_fidelity.py") assert analysis.kendall_tau_b([1, 2, 3], [1, 2, 3])["kendall_tau_b"] == 1 assert analysis.kendall_tau_b([1, 2, 3], [3, 2, 1])["kendall_tau_b"] == -1 + + +def test_configure_cc_command(tmp_path: Path) -> None: + surface = load("run_frontier_qwen30_prefill_surface.py") + base = ["python", "--cc_backend_config_type", "analytical", "--other", "x"] + analytical = surface.configure_cc_command( + base, backend="analytical", allreduce_csv=None, cache=tmp_path + ) + assert analytical == base + profile = tmp_path / "all_reduce.csv" + profile.write_text("header\n") + vidur = surface.configure_cc_command( + base, backend="vidur", allreduce_csv=profile, cache=tmp_path / "cache" + ) + assert vidur[2] == "vidur" + assert "--vidur_cc_backend_config_all_reduce_input_file" in vidur + assert str(profile) in vidur diff --git a/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py b/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py index 0d420e0..16f1b68 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py +++ b/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py @@ -114,7 +114,7 @@ MOE_METADATA = ( def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--linear", type=Path, required=True) - parser.add_argument("--attention", type=Path, nargs=3, required=True) + parser.add_argument("--attention", type=Path, nargs="+", required=True) parser.add_argument("--moe", type=Path, required=True) parser.add_argument("--router", type=Path, required=True) parser.add_argument("--allreduce", type=Path, nargs=2, required=True) From 1fa203384f9da23d7908fa7cca464aef6411c88e Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 09:55:09 +0800 Subject: [PATCH 097/148] Add batch-aware profile and exact trace preparation --- runs/frontier-fidelity-envelope-v1/.gitignore | 1 + .../prepare_exact_trace.py | 302 +++++++ .../profile-v3-batch-final/allreduce.json | 824 ++++++++++++++++++ .../profile-v3-batch-final/attention.csv | 151 ++++ .../attention_true_mixed_fused.csv | 31 + .../profile-v3-batch-final/linear_op.csv | 37 + .../profile-v3-batch-final/manifest.json | 57 ++ .../profiles/profile-v3-batch-final/moe.csv | 73 ++ .../test_fidelity_envelope.py | 21 + .../freeze_frontier_profiles.py | 15 +- 10 files changed, 1511 insertions(+), 1 deletion(-) create mode 100644 runs/frontier-fidelity-envelope-v1/prepare_exact_trace.py create mode 100644 runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/allreduce.json create mode 100644 runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/attention.csv create mode 100644 runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/attention_true_mixed_fused.csv create mode 100644 runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/linear_op.csv create mode 100644 runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/manifest.json create mode 100644 runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/moe.csv diff --git a/runs/frontier-fidelity-envelope-v1/.gitignore b/runs/frontier-fidelity-envelope-v1/.gitignore index 008f888..29cb6f8 100644 --- a/runs/frontier-fidelity-envelope-v1/.gitignore +++ b/runs/frontier-fidelity-envelope-v1/.gitignore @@ -6,3 +6,4 @@ simulator-a2/ simulator-a3/ fleet-state/ fleet-artifacts/ +profiles/profile-v3-batch/ diff --git a/runs/frontier-fidelity-envelope-v1/prepare_exact_trace.py b/runs/frontier-fidelity-envelope-v1/prepare_exact_trace.py new file mode 100644 index 0000000..405376c --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/prepare_exact_trace.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +"""Prepare exact Qwen trace replays with block-16 prefix identities. + +Prompt text is written only below ``private/``. Public manifests and Frontier +fixtures contain lengths, arrivals, session IDs, and deterministic block IDs. +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +import platform +import socket +import time +from pathlib import Path +from typing import Any + + +CSV_FIELDS = ( + "arrived_at", + "num_prefill_tokens", + "num_decode_tokens", + "session_id", + "block_hash_ids", +) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def threshold_name(value: float) -> str: + return f"u{value:.12g}".replace(".", "p") + + +def token_payload(tokens: list[int]) -> bytes: + return len(tokens).to_bytes(2, "little") + b"".join( + int(token).to_bytes(4, "little", signed=False) for token in tokens + ) + + +def block_identity_records( + token_ids: list[int], block_size: int +) -> list[tuple[int, bytes]]: + """Return parent-sensitive identities and independent collision witnesses.""" + parent = b"FRONTIER_EXACT_TRACE_ROOT" + records = [] + for start in range(0, len(token_ids), block_size): + payload = token_payload(token_ids[start : start + block_size]) + identity_input = parent + b"\0" + payload + parent = hashlib.blake2b(identity_input, digest_size=16).digest() + records.append( + ( + int.from_bytes(parent, "big", signed=False), + hashlib.sha256(identity_input).digest(), + ) + ) + return records + + +def block_identities(token_ids: list[int], block_size: int) -> list[int]: + """Return parent-sensitive identities with the same prefix equivalence as vLLM.""" + return [identity for identity, _ in block_identity_records(token_ids, block_size)] + + +def root_sessions(rows: list[dict[str, Any]]) -> dict[int, int]: + roots: dict[int, int] = {} + for row in rows: + chat_id = int(row["chat_id"]) + parent = int(row["parent_chat_id"]) + roots[chat_id] = chat_id if parent == -1 else roots.get(parent, parent) + return roots + + +def update_digest(digest: Any, values: list[Any]) -> None: + digest.update(json.dumps(values, separators=(",", ":")).encode()) + digest.update(b"\n") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--trace", type=Path, required=True) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument("--served-model-name", default="qwen3-30b-a3b") + parser.add_argument("--sampling-u-max", type=float, action="append", required=True) + parser.add_argument("--max-model-len", type=int, default=40960) + parser.add_argument("--source-block-size", type=int, default=64) + parser.add_argument("--runtime-block-size", type=int, default=16) + parser.add_argument("--batch-size", type=int, default=16) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + thresholds = sorted(set(args.sampling_u_max)) + if not thresholds or thresholds[0] < 0 or thresholds[-1] > 1: + raise ValueError("sampling thresholds must be in [0, 1]") + if args.source_block_size % args.runtime_block_size: + raise ValueError("source block size must be divisible by runtime block size") + if min(args.max_model_len, args.source_block_size, args.runtime_block_size, args.batch_size) <= 0: + raise ValueError("length and batch arguments must be positive") + + import transformers + from transformers import AutoTokenizer + + rows = [json.loads(line) for line in args.trace.open() if line.strip()] + roots = root_sessions(rows) + eligible = [ + (index, row) + for index, row in enumerate(rows) + if int(row["input_length"]) + int(row["output_length"]) <= args.max_model_len + and int(row["output_length"]) > 0 + ] + tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True) + prepared: list[dict[str, Any]] = [] + identity_to_digest: dict[int, bytes] = {} + source_to_runtime: dict[int, tuple[int, ...]] = {} + runtime_to_source: dict[tuple[int, ...], int] = {} + identity_collision_count = 0 + source_to_runtime_conflicts = 0 + runtime_to_source_conflicts = 0 + length_mismatches = 0 + source_hash_count_mismatches = 0 + started = time.time() + + for start in range(0, len(eligible), args.batch_size): + batch = eligible[start : start + args.batch_size] + encoded = tokenizer( + [row["prompt"] for _, row in batch], + add_special_tokens=False, + padding=False, + truncation=False, + )["input_ids"] + for (source_index, row), token_ids in zip(batch, encoded, strict=True): + token_ids = [int(token) for token in token_ids] + if len(token_ids) != int(row["input_length"]): + length_mismatches += 1 + source_hashes = [int(value) for value in row["hash_ids"]] + if len(source_hashes) != math.ceil(len(token_ids) / args.source_block_size): + source_hash_count_mismatches += 1 + identity_records = block_identity_records(token_ids, args.runtime_block_size) + runtime_ids = [identity for identity, _ in identity_records] + for runtime_id, witness in identity_records: + previous = identity_to_digest.setdefault(runtime_id, witness) + identity_collision_count += int(previous != witness) + + blocks_per_source = args.source_block_size // args.runtime_block_size + for block_index, source_id in enumerate(source_hashes): + begin = block_index * blocks_per_source + relation = tuple(runtime_ids[begin : begin + blocks_per_source]) + previous_relation = source_to_runtime.setdefault(source_id, relation) + source_to_runtime_conflicts += int(previous_relation != relation) + previous_source = runtime_to_source.setdefault(relation, source_id) + runtime_to_source_conflicts += int(previous_source != source_id) + + prepared.append( + { + "source_index": source_index, + "chat_id": int(row["chat_id"]), + "session_id": roots[int(row["chat_id"])], + "arrival": float(row["timestamp"]), + "input_length": int(row["input_length"]), + "output_length": int(row["output_length"]), + "sampling_u": float(row["sampling_u"]), + "prompt": row["prompt"], + "runtime_block_ids": runtime_ids, + } + ) + + args.output_root.mkdir(parents=True, exist_ok=True) + threshold_manifests = [] + for threshold in thresholds: + name = threshold_name(threshold) + public = args.output_root / "public" / name + private = args.output_root / "private" / name + public.mkdir(parents=True, exist_ok=True) + private.mkdir(parents=True, exist_ok=True) + selected = [row for row in prepared if row["sampling_u"] <= threshold] + frontier_path = public / "frontier.csv" + real_path = private / "real_requests.jsonl" + vector_digest = hashlib.sha256() + with frontier_path.open("w", newline="") as output: + writer = csv.DictWriter(output, fieldnames=CSV_FIELDS, lineterminator="\n") + writer.writeheader() + for row in selected: + writer.writerow( + { + "arrived_at": f"{row['arrival']:.12f}", + "num_prefill_tokens": row["input_length"], + "num_decode_tokens": row["output_length"], + "session_id": row["session_id"], + "block_hash_ids": "|".join( + str(value) for value in row["runtime_block_ids"] + ), + } + ) + update_digest( + vector_digest, + [ + row["source_index"], + row["arrival"], + row["input_length"], + row["output_length"], + row["session_id"], + row["runtime_block_ids"], + ], + ) + with real_path.open("w") as output: + for row in selected: + output.write( + json.dumps( + { + "source_index": row["source_index"], + "arrived_at": row["arrival"], + "input_length": row["input_length"], + "output_length": row["output_length"], + "session_id": row["session_id"], + "runtime_block_ids": row["runtime_block_ids"], + "body": { + "model": args.served_model_name, + "prompt": row["prompt"], + "min_tokens": row["output_length"], + "max_tokens": row["output_length"], + "ignore_eos": True, + "stream": True, + }, + }, + separators=(",", ":"), + ) + + "\n" + ) + threshold_manifests.append( + { + "sampling_u_max": threshold, + "selected_requests": len(selected), + "request_rate": len(selected) / 600.0, + "frontier_csv": str(frontier_path.resolve()), + "frontier_csv_sha256": sha256(frontier_path), + "real_requests_private": str(real_path.resolve()), + "real_requests_private_sha256": sha256(real_path), + "row_vector_sha256": vector_digest.hexdigest(), + } + ) + + failures = { + "input_length_mismatches": length_mismatches, + "source_hash_count_mismatches": source_hash_count_mismatches, + "runtime_identity_collisions": identity_collision_count, + "source_to_runtime_relation_conflicts": source_to_runtime_conflicts, + "runtime_to_source_relation_conflicts": runtime_to_source_conflicts, + } + manifest = { + "schema": "qwen30-exact-trace-block16-v1", + "status": "pass" if not any(failures.values()) else "fail", + "execution": { + "host": socket.gethostname(), + "python": platform.python_version(), + "transformers": transformers.__version__, + "tokenizer": type(tokenizer).__name__, + "elapsed_seconds": round(time.time() - started, 3), + }, + "source": { + "trace": str(args.trace.resolve()), + "trace_sha256": sha256(args.trace), + "requests": len(rows), + "eligible_requests": len(eligible), + "model": str(args.model.resolve()), + "max_model_len": args.max_model_len, + }, + "block_contract": { + "source_block_size": args.source_block_size, + "runtime_block_size": args.runtime_block_size, + "identity": "BLAKE2b-128(parent runtime identity, exact token-id block)", + "unique_runtime_identities": len(identity_to_digest), + "unique_source_relations": len(source_to_runtime), + "failures": failures, + }, + "selection_contract": ( + "eligible universe followed by source sampling_u threshold; no length " + "selection or output override; original arrivals/order preserved" + ), + "thresholds": threshold_manifests, + "privacy": "prompt text exists only under private/ and must not be harvested", + } + manifest_path = args.output_root / "public" / "manifest.json" + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + print(json.dumps({"status": manifest["status"], "failures": failures, "thresholds": threshold_manifests}, sort_keys=True)) + if manifest["status"] != "pass": + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/allreduce.json b/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/allreduce.json new file mode 100644 index 0000000..aae1f0a --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/allreduce.json @@ -0,0 +1,824 @@ +{ + "environment": [ + { + "backend_env": { + "VLLM_ALLREDUCE_USE_FLASHINFER": "1", + "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm" + }, + "gpu": "NVIDIA H20", + "model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B", + "torch_cuda": "12.9", + "torch_version": "2.11.0+cu129", + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + }, + { + "backend_env": { + "VLLM_ALLREDUCE_USE_FLASHINFER": "1", + "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm" + }, + "gpu": "NVIDIA H20", + "model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B", + "torch_cuda": "12.9", + "torch_version": "2.11.0+cu129", + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + } + ], + "frontier_consumption": "diagnostic_only_in_base_profile_only_run; measured lookup requires a separate CC-backend injection ablation", + "rows": [ + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08288000151515007, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 1, + "payload_bytes": 4096, + "per_rank_time_ms": [ + { + "max": 0.4872959852218628, + "mean": 0.1310015980154276, + "median": 0.07679999992251396, + "min": 0.06217600032687187, + "std": 0.12790721677293843 + }, + { + "max": 0.4402880072593689, + "mean": 0.12842560112476348, + "median": 0.08288000151515007, + "min": 0.0655680000782013, + "std": 0.11220176524616535 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.0793600007891655, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 8, + "payload_bytes": 32768, + "per_rank_time_ms": [ + { + "max": 0.12716799974441528, + "mean": 0.07871360033750534, + "median": 0.0759200006723404, + "min": 0.06032000109553337, + "std": 0.019331314939874535 + }, + { + "max": 0.12380799651145935, + "mean": 0.08059840016067028, + "median": 0.0793600007891655, + "min": 0.06217600032687187, + "std": 0.01687088356254092 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.0713919997215271, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 16, + "payload_bytes": 65536, + "per_rank_time_ms": [ + { + "max": 0.12697599828243256, + "mean": 0.0767391998320818, + "median": 0.07078400254249573, + "min": 0.05910399928689003, + "std": 0.018775178979463278 + }, + { + "max": 0.11430399864912033, + "mean": 0.07594559974968433, + "median": 0.0713919997215271, + "min": 0.06124800071120262, + "std": 0.015719922150631036 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08056000247597694, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 32, + "payload_bytes": 131072, + "per_rank_time_ms": [ + { + "max": 0.1037760004401207, + "mean": 0.07954559996724128, + "median": 0.08056000247597694, + "min": 0.05955199897289276, + "std": 0.0135697420393132 + }, + { + "max": 0.10608000308275223, + "mean": 0.07971520014107228, + "median": 0.07593599706888199, + "min": 0.06028800085186958, + "std": 0.015319849772775456 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.0865279994904995, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 64, + "payload_bytes": 262144, + "per_rank_time_ms": [ + { + "max": 0.14470399916172028, + "mean": 0.09121599942445754, + "median": 0.0865279994904995, + "min": 0.06441599875688553, + "std": 0.024893837894277456 + }, + { + "max": 0.12438400089740753, + "mean": 0.08531199917197227, + "median": 0.08031999692320824, + "min": 0.06364800035953522, + "std": 0.01878029830059533 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.07135999947786331, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 128, + "payload_bytes": 524288, + "per_rank_time_ms": [ + { + "max": 0.11753600090742111, + "mean": 0.07606079950928687, + "median": 0.07135999947786331, + "min": 0.05843200162053108, + "std": 0.01755519771639284 + }, + { + "max": 0.1103999987244606, + "mean": 0.07607359997928143, + "median": 0.07073600217700005, + "min": 0.05721599981188774, + "std": 0.016904445220949783 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.07321599870920181, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 256, + "payload_bytes": 1048576, + "per_rank_time_ms": [ + { + "max": 0.11740799993276596, + "mean": 0.07749119997024537, + "median": 0.07203200086951256, + "min": 0.05862399935722351, + "std": 0.017701381594822835 + }, + { + "max": 0.11382400244474411, + "mean": 0.07733759954571724, + "median": 0.07321599870920181, + "min": 0.059039998799562454, + "std": 0.017188890882557036 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.09025600180029869, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 512, + "payload_bytes": 2097152, + "per_rank_time_ms": [ + { + "max": 0.13980799913406372, + "mean": 0.0950367994606495, + "median": 0.09025600180029869, + "min": 0.06815999746322632, + "std": 0.022829835127539378 + }, + { + "max": 0.14764800667762756, + "mean": 0.09710080176591873, + "median": 0.08720000088214874, + "min": 0.07152000069618225, + "std": 0.02500574954063789 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08083200082182884, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 1024, + "payload_bytes": 4194304, + "per_rank_time_ms": [ + { + "max": 0.15574400126934052, + "mean": 0.08760640025138855, + "median": 0.07846399769186974, + "min": 0.07097599655389786, + "std": 0.024487311423551025 + }, + { + "max": 0.16284799575805664, + "mean": 0.08963519930839539, + "median": 0.08083200082182884, + "min": 0.07100799679756165, + "std": 0.026047320562445356 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.10891199856996536, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 2048, + "payload_bytes": 8388608, + "per_rank_time_ms": [ + { + "max": 0.1582079976797104, + "mean": 0.11541439890861512, + "median": 0.10836799815297127, + "min": 0.09062399715185165, + "std": 0.018285136316576037 + }, + { + "max": 0.1578879952430725, + "mean": 0.11537599861621857, + "median": 0.10891199856996536, + "min": 0.0960640013217926, + "std": 0.018246538626977286 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.1703840047121048, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 4096, + "payload_bytes": 16777216, + "per_rank_time_ms": [ + { + "max": 0.19327999651432037, + "mean": 0.1707327976822853, + "median": 0.1703840047121048, + "min": 0.14815999567508698, + "std": 0.014211056022719618 + }, + { + "max": 0.19276799261569977, + "mean": 0.1658592015504837, + "median": 0.16379200667142868, + "min": 0.14560000598430634, + "std": 0.013840249648693638 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.25539200007915497, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 8192, + "payload_bytes": 33554432, + "per_rank_time_ms": [ + { + "max": 0.2807359993457794, + "mean": 0.25750079900026324, + "median": 0.25539200007915497, + "min": 0.24624000489711761, + "std": 0.008925204570802302 + }, + { + "max": 0.2863999903202057, + "mean": 0.2585055992007256, + "median": 0.255280002951622, + "min": 0.24371199309825897, + "std": 0.012059738582656496 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.1021759994328022, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 1, + "payload_bytes": 4096, + "per_rank_time_ms": [ + { + "max": 0.9770879745483398, + "mean": 0.1974783968180418, + "median": 0.10099200159311295, + "min": 0.05913599953055382, + "std": 0.2660581738745569 + }, + { + "max": 0.892799973487854, + "mean": 0.18164799660444259, + "median": 0.1021759994328022, + "min": 0.06435199826955795, + "std": 0.23947520188197013 + }, + { + "max": 0.6467199921607971, + "mean": 0.15839359983801843, + "median": 0.10100800171494484, + "min": 0.06800000369548798, + "std": 0.16617013141866102 + }, + { + "max": 0.6725760102272034, + "mean": 0.15686400160193442, + "median": 0.10044800117611885, + "min": 0.06063999980688095, + "std": 0.17523222161300497 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.12694399803876877, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 8, + "payload_bytes": 32768, + "per_rank_time_ms": [ + { + "max": 0.6659520268440247, + "mean": 0.1927648030221462, + "median": 0.12694399803876877, + "min": 0.07609599828720093, + "std": 0.16697784531907736 + }, + { + "max": 0.695360004901886, + "mean": 0.19356480240821838, + "median": 0.11726400256156921, + "min": 0.0796160027384758, + "std": 0.17588096678867862 + }, + { + "max": 0.5939840078353882, + "mean": 0.1868800014257431, + "median": 0.12379200011491776, + "min": 0.07427199929952621, + "std": 0.1463231714943902 + }, + { + "max": 0.6635839939117432, + "mean": 0.1874335989356041, + "median": 0.12014400213956833, + "min": 0.07526399940252304, + "std": 0.16680959677760304 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.09161599725484848, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 16, + "payload_bytes": 65536, + "per_rank_time_ms": [ + { + "max": 0.12665599584579468, + "mean": 0.09443839862942696, + "median": 0.09124799817800522, + "min": 0.06431999802589417, + "std": 0.0205690155775906 + }, + { + "max": 0.1303039938211441, + "mean": 0.09712959825992584, + "median": 0.09161599725484848, + "min": 0.07648000121116638, + "std": 0.019072048129173236 + }, + { + "max": 0.13836799561977386, + "mean": 0.09821119979023933, + "median": 0.09148800000548363, + "min": 0.0727040022611618, + "std": 0.021314066545189116 + }, + { + "max": 0.12992000579833984, + "mean": 0.09272959977388381, + "median": 0.08931199833750725, + "min": 0.06406400352716446, + "std": 0.021360803830354765 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08580800145864487, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 32, + "payload_bytes": 131072, + "per_rank_time_ms": [ + { + "max": 0.13449600338935852, + "mean": 0.08960640132427215, + "median": 0.08299200236797333, + "min": 0.06796800345182419, + "std": 0.020633597898445998 + }, + { + "max": 0.14735999703407288, + "mean": 0.09248319901525974, + "median": 0.08580800145864487, + "min": 0.05913599953055382, + "std": 0.025960234928829564 + }, + { + "max": 0.13705599308013916, + "mean": 0.08947199806571007, + "median": 0.08460799977183342, + "min": 0.0634239986538887, + "std": 0.02122838946992339 + }, + { + "max": 0.13846400380134583, + "mean": 0.0859104000031948, + "median": 0.08308799937367439, + "min": 0.05974400043487549, + "std": 0.022373631424433445 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.09867199882864952, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 64, + "payload_bytes": 262144, + "per_rank_time_ms": [ + { + "max": 0.1279039978981018, + "mean": 0.09748799875378608, + "median": 0.09678399935364723, + "min": 0.0655359998345375, + "std": 0.02315719144614622 + }, + { + "max": 0.1356479972600937, + "mean": 0.10018239840865135, + "median": 0.09532799944281578, + "min": 0.06185600161552429, + "std": 0.02330700253650042 + }, + { + "max": 0.13142399489879608, + "mean": 0.09778879955410957, + "median": 0.09492799639701843, + "min": 0.06560000032186508, + "std": 0.02307579212430045 + }, + { + "max": 0.1276479959487915, + "mean": 0.09611519873142242, + "median": 0.09867199882864952, + "min": 0.0642239972949028, + "std": 0.02273612181973431 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.09646400064229965, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 128, + "payload_bytes": 524288, + "per_rank_time_ms": [ + { + "max": 0.14060799777507782, + "mean": 0.09770880043506622, + "median": 0.09115200117230415, + "min": 0.06950400024652481, + "std": 0.024752645089849798 + }, + { + "max": 0.14377599954605103, + "mean": 0.09824960008263588, + "median": 0.08999999985098839, + "min": 0.07103999704122543, + "std": 0.025403407389046027 + }, + { + "max": 0.13680000603199005, + "mean": 0.09993600100278854, + "median": 0.09646400064229965, + "min": 0.06790400296449661, + "std": 0.022297424273985882 + }, + { + "max": 0.1391039937734604, + "mean": 0.09769919961690902, + "median": 0.09601600095629692, + "min": 0.06835199892520905, + "std": 0.023899922100804445 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08377600088715553, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 256, + "payload_bytes": 1048576, + "per_rank_time_ms": [ + { + "max": 0.1464959979057312, + "mean": 0.09276160076260567, + "median": 0.08377600088715553, + "min": 0.06777600198984146, + "std": 0.024751086465295658 + }, + { + "max": 0.14319999516010284, + "mean": 0.09080640003085136, + "median": 0.080400001257658, + "min": 0.06796800345182419, + "std": 0.023365718881708488 + }, + { + "max": 0.1382399946451187, + "mean": 0.09063360020518303, + "median": 0.08193599805235863, + "min": 0.06627199798822403, + "std": 0.02372618650854302 + }, + { + "max": 0.14313599467277527, + "mean": 0.09044799953699112, + "median": 0.08128000050783157, + "min": 0.06652799993753433, + "std": 0.02462486862033686 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.1128000020980835, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 512, + "payload_bytes": 2097152, + "per_rank_time_ms": [ + { + "max": 0.1600320041179657, + "mean": 0.11426239982247352, + "median": 0.1128000020980835, + "min": 0.07657600194215775, + "std": 0.030240087702350687 + }, + { + "max": 0.15881599485874176, + "mean": 0.11206399947404862, + "median": 0.10628800094127655, + "min": 0.0772159993648529, + "std": 0.029933135345483627 + }, + { + "max": 0.15612800419330597, + "mean": 0.10761600062251091, + "median": 0.09860799834132195, + "min": 0.07689599692821503, + "std": 0.02634237020678647 + }, + { + "max": 0.15865600109100342, + "mean": 0.11094079986214637, + "median": 0.10979199782013893, + "min": 0.07583999633789062, + "std": 0.028619217028542445 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08755199983716011, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 1024, + "payload_bytes": 4194304, + "per_rank_time_ms": [ + { + "max": 0.11999999731779099, + "mean": 0.0843871995806694, + "median": 0.08755199983716011, + "min": 0.06332799792289734, + "std": 0.016638543890716024 + }, + { + "max": 0.1218239963054657, + "mean": 0.08518079966306687, + "median": 0.08032000064849854, + "min": 0.06393600255250931, + "std": 0.019900899429956945 + }, + { + "max": 0.11849600076675415, + "mean": 0.0843968003988266, + "median": 0.08702399954199791, + "min": 0.06297600269317627, + "std": 0.017031908773433545 + }, + { + "max": 0.12300799787044525, + "mean": 0.0846304003149271, + "median": 0.08139199763536453, + "min": 0.06195199862122536, + "std": 0.020106523698622265 + } + ], + "selected_backend": "nccl_fallback", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.12361599877476692, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 2048, + "payload_bytes": 8388608, + "per_rank_time_ms": [ + { + "max": 0.17155200242996216, + "mean": 0.12168959975242614, + "median": 0.12055999785661697, + "min": 0.09609600156545639, + "std": 0.022565485532483928 + }, + { + "max": 0.9246399998664856, + "mean": 0.19978560134768486, + "median": 0.12361599877476692, + "min": 0.09715200215578079, + "std": 0.24230694305662265 + }, + { + "max": 0.9317439794540405, + "mean": 0.20037759989500045, + "median": 0.12327999994158745, + "min": 0.09603200107812881, + "std": 0.24450903278000383 + }, + { + "max": 0.9321280121803284, + "mean": 0.19875840097665787, + "median": 0.12230399996042252, + "min": 0.0950080007314682, + "std": 0.24519252220785093 + } + ], + "selected_backend": "nccl_fallback", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.20030399411916733, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 4096, + "payload_bytes": 16777216, + "per_rank_time_ms": [ + { + "max": 0.3261440098285675, + "mean": 0.20826880186796187, + "median": 0.19974400103092194, + "min": 0.1409280002117157, + "std": 0.051397264855718945 + }, + { + "max": 0.3248000144958496, + "mean": 0.20548800230026246, + "median": 0.1979840025305748, + "min": 0.141184002161026, + "std": 0.04980002399656717 + }, + { + "max": 0.32547199726104736, + "mean": 0.21280319690704347, + "median": 0.20030399411916733, + "min": 0.14127999544143677, + "std": 0.05304243085685509 + }, + { + "max": 0.26047998666763306, + "mean": 0.1969312012195587, + "median": 0.18079999834299088, + "min": 0.14057600498199463, + "std": 0.042174123638424224 + } + ], + "selected_backend": "nccl_fallback", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.2924960106611252, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 8192, + "payload_bytes": 33554432, + "per_rank_time_ms": [ + { + "max": 0.45052799582481384, + "mean": 0.3013375997543335, + "median": 0.28273600339889526, + "min": 0.21334399282932281, + "std": 0.07214223764527236 + }, + { + "max": 0.4466240108013153, + "mean": 0.2968191936612129, + "median": 0.27796798944473267, + "min": 0.21241599321365356, + "std": 0.07400678240849741 + }, + { + "max": 0.3830080032348633, + "mean": 0.2946112036705017, + "median": 0.2924960106611252, + "min": 0.21084800362586975, + "std": 0.05323627615746332 + }, + { + "max": 0.4609600007534027, + "mean": 0.3054272010922432, + "median": 0.289792001247406, + "min": 0.21062399446964264, + "std": 0.07608482904865672 + } + ], + "selected_backend": "nccl_fallback", + "tensor_parallel_size": 4 + } + ], + "schema_version": "qwen30_vllm020_allreduce_frozen.v1" +} diff --git a/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/attention.csv b/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/attention.csv new file mode 100644 index 0000000..a8101ec --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/attention.csv @@ -0,0 +1,151 @@ +time_stats.attn_input_reshape.min,time_stats.attn_input_reshape.max,time_stats.attn_input_reshape.mean,time_stats.attn_input_reshape.median,time_stats.attn_input_reshape.std,time_stats.attn_kv_cache_save.min,time_stats.attn_kv_cache_save.max,time_stats.attn_kv_cache_save.mean,time_stats.attn_kv_cache_save.median,time_stats.attn_kv_cache_save.std,time_stats.attn_prefill.min,time_stats.attn_prefill.max,time_stats.attn_prefill.mean,time_stats.attn_prefill.median,time_stats.attn_prefill.std,time_stats.attn_decode.min,time_stats.attn_decode.max,time_stats.attn_decode.mean,time_stats.attn_decode.median,time_stats.attn_decode.std,time_stats.attn_output_reshape.min,time_stats.attn_output_reshape.max,time_stats.attn_output_reshape.mean,time_stats.attn_output_reshape.median,time_stats.attn_output_reshape.std,n_embd,n_q_head,n_kv_head,block_size,num_tensor_parallel_workers,max_model_len,batch_size,prefill_chunk_size,kv_cache_size,is_prefill,attention_backend,is_mixed_batch,mode,seq_lens,total_tokens,max_seq_len,min_seq_len,avg_seq_len,equal_seq_len,seq_len_variance,seq_len_std,seq_len_cv,is_chunked_prefill_sample,chunk_start_token,chunk_end_token,total_prefill_tokens,profiling_precision,model_arch,quant_signature,measurement_type,is_true_mixed_batch,prefill_seq_lens,prefill_kv_cache_sizes,decode_kv_cache_sizes,num_prefill_seqs,num_decode_seqs,decode_batch_size,total_batch_size,total_decode_tokens,decode_avg_kv_cache_size,batch_composition_ratio,batch_spec,projection_policy +0.0,0.0,0.0,0.0,0.0,0.01414399966597557,0.028863999992609024,0.019705599918961526,0.01771199982613325,0.005157200849836681,0.047968000173568726,0.07046400010585785,0.05810240097343922,0.05810240097343922,0.007477463486041561,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.04947200044989586,0.020412799902260303,0.01635199971497059,0.010107497379722417,0.046560000628232956,0.08323200047016144,0.05587520003318787,0.05587520003318787,0.011126758739503428,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.022207999601960182,0.017033600155264138,0.015312000177800655,0.002819235991970241,0.05104000121355057,0.07692799717187881,0.056396800279617305,0.056396800279617305,0.007481982178637539,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015072000212967396,0.022272000089287758,0.01706880023702979,0.01616000011563301,0.002460889579319197,0.06931199878454208,0.0838719978928566,0.07432000041007995,0.07432000041007995,0.004777766433175866,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.018592000007629395,0.028543999418616295,0.02095999978482723,0.019183999858796597,0.003198175496053494,0.12179200351238251,0.15408000349998474,0.1307712011039257,0.1307712011039257,0.00858807797538298,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.027775999158620834,0.03385600075125694,0.030131200328469276,0.029680000618100166,0.0021152558103575215,0.32678401470184326,0.3450239896774292,0.33396480381488797,0.33396480381488797,0.0045872424917606375,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.04438399896025658,0.05084799975156784,0.046540799736976626,0.04531199857592583,0.002277905811237223,1.0959680080413818,1.1151360273361206,1.0999775886535645,1.0999775886535645,0.005694403246120485,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.078015998005867,0.08691199868917465,0.08114239946007729,0.08019199967384338,0.00292795706334475,4.070400238037109,4.113152027130127,4.087088012695312,4.087088012695312,0.013660567012509554,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.05167999863624573,0.022115200012922286,0.017583999782800674,0.010340094822340818,0.05196800082921982,0.09011200070381165,0.06328320093452933,0.06328320093452933,0.012557341255467452,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01583999954164028,0.026623999699950218,0.018927999772131443,0.017376000061631203,0.003514650316260619,0.06681600213050842,0.07993599772453308,0.0725280001759529,0.0725280001759529,0.004343502558613716,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01648000068962574,0.030880000442266464,0.01945280022919178,0.017967999912798405,0.004096211183007485,0.1311360001564026,0.1546880006790161,0.13908160030841826,0.13908160030841826,0.007511906874366178,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.03558399900794029,0.020851199887692927,0.018559999763965607,0.005235911594130716,0.32950401306152344,0.350271999835968,0.33912960588932034,0.33912960588932034,0.006027400986663648,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.019328000023961067,0.040608000010252,0.022790400311350822,0.020704000256955624,0.006113051965778337,1.1415679454803467,1.1518720388412476,1.144483208656311,1.144483208656311,0.0032332311374389127,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015807999297976494,0.030688000842928886,0.019596799835562707,0.01774400006979704,0.004343384771033462,0.0,0.0,0.0,0.0,0.0,0.049056001007556915,0.07580800354480743,0.05948160067200661,0.05948160067200661,0.009031541471446955,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.02486399933695793,0.01923839971423149,0.018079999834299088,0.0032282528537266424,0.0,0.0,0.0,0.0,0.0,0.05142400041222572,0.07353600114583969,0.059328000620007516,0.059328000620007516,0.0073307807735143084,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016543999314308167,0.03977600112557411,0.021379199624061585,0.018511999398469925,0.006593576176246171,0.0,0.0,0.0,0.0,0.0,0.0488319993019104,0.06435199826955795,0.05479039996862411,0.05479039996862411,0.005672522998491864,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01635199971497059,0.02844800055027008,0.019267200119793416,0.017952000722289085,0.0035068687666949577,0.0,0.0,0.0,0.0,0.0,0.049855999648571014,0.07798399776220322,0.05986879989504815,0.05986879989504815,0.01043914754878828,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016383999958634377,0.026079999282956123,0.01923519968986511,0.017791999503970146,0.0032161974331284568,0.0,0.0,0.0,0.0,0.0,0.058111999183893204,0.1045759990811348,0.06708480007946492,0.06708480007946492,0.013479022462646494,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.04057599976658821,0.019100800156593323,0.015455999877303839,0.007512281243011577,0.0,0.0,0.0,0.0,0.0,0.05363199859857559,0.07782399654388428,0.06090559959411622,0.06090559959411622,0.007544620176348091,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.02191999927163124,0.016684799920767546,0.01563199982047081,0.0024293118621811216,0.0,0.0,0.0,0.0,0.0,0.0629120022058487,0.07891199737787247,0.06891520097851753,0.06891520097851753,0.005472695665695425,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.038943998515605927,0.018313600029796363,0.01561600062996149,0.007127270260115769,0.0,0.0,0.0,0.0,0.0,0.08675199747085571,0.10662399977445602,0.09391999915242194,0.09391999915242194,0.006988099589086635,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.054687999188899994,0.021439999900758268,0.01539199985563755,0.012052764849597775,0.0,0.0,0.0,0.0,0.0,0.13488000631332397,0.1528639942407608,0.1431359991431236,0.1431359991431236,0.005436271464033599,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.041919998824596405,0.01899839974939823,0.015343999955803156,0.007989843526623287,0.0,0.0,0.0,0.0,0.0,0.06176000088453293,0.08374399691820145,0.06747519969940186,0.06747519969940186,0.0066067747128778,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016256000846624374,0.11353600025177002,0.042761600017547606,0.028960000723600388,0.029104301538020762,0.0,0.0,0.0,0.0,0.0,0.09734400361776352,0.14422400295734406,0.11392960175871848,0.11392960175871848,0.013198594600417867,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.034143999218940735,0.018918400071561335,0.01643200032413006,0.005500943993080684,0.0,0.0,0.0,0.0,0.0,0.12918399274349213,0.15087999403476715,0.13807999789714814,0.13807999789714814,0.007658330538677587,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.03641600161790848,0.0198208000510931,0.01780799962580204,0.0057264128169845765,0.0,0.0,0.0,0.0,0.0,0.22099199891090393,0.23904000222682953,0.2293503984808922,0.2293503984808922,0.004861342907006028,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.035840000957250595,0.018908800091594458,0.015792000107467175,0.0064374817924757475,0.0,0.0,0.0,0.0,0.0,0.10134399682283401,0.12201599776744843,0.10896319895982742,0.10896319895982742,0.006336330809165179,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.03417599946260452,0.017846399918198586,0.014800000004470348,0.006495007539635255,0.0,0.0,0.0,0.0,0.0,0.13126400113105774,0.15561600029468536,0.1389280006289482,0.1389280006289482,0.008381472811075022,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03519999980926514,0.019168000388890504,0.01600000075995922,0.005995522477654695,0.0,0.0,0.0,0.0,0.0,0.21728000044822693,0.2343679964542389,0.2231455981731415,0.2231455981731415,0.004720730646739123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.03670400008559227,0.018441599886864425,0.015023999847471714,0.006596535162793127,0.0,0.0,0.0,0.0,0.0,0.39321601390838623,0.4524799883365631,0.4058080047369003,0.4058080047369003,0.01578349755088941,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.022143999114632607,0.016912000067532063,0.015168000012636185,0.0028156089295136347,0.0,0.0,0.0,0.0,0.0,0.15587200224399567,0.3079040050506592,0.17838079929351805,0.17838079929351805,0.04355865575265927,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.02112000063061714,0.015516800060868263,0.01488000014796853,0.001940870731593904,0.0,0.0,0.0,0.0,0.0,0.21587200462818146,0.23561599850654602,0.22250880002975468,0.22250880002975468,0.006181951170646666,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.039264000952243805,0.023609600123018028,0.02131200022995472,0.007236979625711548,0.0,0.0,0.0,0.0,0.0,0.408735990524292,0.470335990190506,0.4336863994598388,0.4336863994598388,0.01844662383160074,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.025407999753952026,0.016227199975401164,0.014960000291466713,0.0031617375441736185,0.0,0.0,0.0,0.0,0.0,0.7412800192832947,0.7627840042114258,0.7464000046253203,0.7464000046253203,0.006112167448837547,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01462399959564209,0.02812799997627735,0.020652799773961304,0.021359999664127827,0.004308706957613102,0.028383498565450627,0.039859687970646644,0.032492258074592426,0.032492258074592426,0.00453597266208597,0.029312501176103633,0.04116431058787463,0.03355574193327539,0.03355574193327539,0.004684436757701648,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.034143999218940735,0.023171199765056372,0.024255999363958836,0.005908614918096971,0.03333159243114438,0.038935341782478095,0.03580428402241854,0.03580428402241854,0.002082270297044095,0.03633240903369937,0.04244065945634484,0.03902771507087562,0.03902771507087562,0.0022697354261490147,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.02611199952661991,0.019635199941694735,0.018112000077962875,0.0038298050749257795,0.04189529417991216,0.057484239920526384,0.04744885718421094,0.04744885718421094,0.004779830748455743,0.051672703037266184,0.07089975418195164,0.05852234134479412,0.05852234134479412,0.005895334539786378,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.029311999678611755,0.018662399891763926,0.01673599984496832,0.0044017112162725355,0.04322973959325901,0.05049827064705393,0.04535414343408448,0.04535414343408448,0.0022944156000240697,0.08733025617719538,0.10201372836398578,0.09162185574241775,0.09162185574241775,0.004635047631846023,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014592000283300877,0.026367999613285065,0.016336000058799982,0.01515199989080429,0.003415353455946402,0.06031842775160765,0.06618323188375198,0.06274415549817247,0.06274415549817247,0.001985647289644255,0.14768157653992678,0.16204076249052324,0.15362064543185072,0.15362064543185072,0.004861590945216247,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.02454400062561035,0.017167999967932702,0.016191999427974224,0.0028685016454498436,0.09128700688359712,0.09689150775996329,0.09360396051475776,0.09360396051475776,0.0013477542744916764,0.2740889887523892,0.2909164873209846,0.2810456356995366,0.2810456356995366,0.004046628526808561,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.01881599985063076,0.03097599931061268,0.024598400108516216,0.024848000146448612,0.0037539437391565975,0.1035249255866932,0.10603132147437412,0.10399797220840973,0.10399797220840973,0.0007025109429056814,0.5652750707893444,0.5789606938874117,0.5678580377517648,0.5678580377517648,0.003835906384194897,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.018624000251293182,0.043487999588251114,0.024460799992084503,0.019600000232458115,0.008922081629781394,0.196169204945307,0.2076140047945799,0.2008444429250931,0.2008444429250931,0.00394350953292805,1.1196708009293268,1.1849940417370974,1.1463555573610091,1.1463555573610091,0.022508285530529783,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.027135999873280525,0.03667199984192848,0.02945920005440712,0.028256000019609928,0.0029446678408169553,0.37757279619664613,0.3898113624476372,0.38075520430942184,0.38075520430942184,0.0036600203831042254,0.2522831942132049,0.2604606493092597,0.25440958703617433,0.25440958703617433,0.0024455194930253126,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.0435199998319149,0.049695998430252075,0.04502719938755036,0.04395199939608574,0.002035785660169308,1.1048984388245497,1.1189053886476108,1.1112371236754128,1.1112371236754128,0.0050220696724624985,0.1395495077239122,0.14131859607071193,0.14035008841031085,0.14035008841031085,0.0006342911944856293,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.018783999606966972,0.024288000538945198,0.020627199858427047,0.019567999988794327,0.0019457052717059358,0.09455999732017517,0.12185599654912949,0.10207359939813614,0.10207359939813614,0.00753346544014261,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,2,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512]",1024,512,512,512.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.027295999228954315,0.034752000123262405,0.02943360023200512,0.028528000228106976,0.0023159160681123767,0.14416000247001648,0.15904000401496887,0.14979200065135959,0.14979200065135959,0.00480624675866005,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,4,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512]",2048,512,512,512.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.043455999344587326,0.05084799975156784,0.045500800386071204,0.04411200061440468,0.002490556062831049,0.24454399943351746,0.25865599513053894,0.2504959970712662,0.2504959970712662,0.003778494140301353,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512]",4096,512,512,512.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.07673600316047668,0.08374399691820145,0.07970559895038605,0.07873599976301193,0.0022907976135004057,0.445248007774353,0.4758400022983551,0.45409599840641024,0.45409599840641024,0.009133230340383306,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512]",8192,512,512,512.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.043296001851558685,0.051072001457214355,0.044972800090909,0.04399999976158142,0.002451216824441962,0.6147199869155884,0.6290879845619202,0.6204927921295166,0.6204927921295166,0.004344846739788608,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,2,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048]",4096,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.07737600058317184,0.09123200178146362,0.08094720020890236,0.07980800047516823,0.004065264798368611,1.1744320392608643,1.1887680292129517,1.178323209285736,1.178323209285736,0.004395876469319665,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,4,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048, 2048, 2048]",8192,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.02364799939095974,0.01809599995613098,0.016352000646293163,0.0035481127058959038,0.04822399839758873,0.08566399663686752,0.05961279980838299,0.05961279980838299,0.011445665413968877,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.020864000543951988,0.01668160008266568,0.015664000064134598,0.0025769063833097584,0.049695998430252075,0.08057600259780884,0.05882879942655563,0.05882879942655563,0.009515126108519331,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.028704000636935234,0.017430400010198355,0.015056000091135502,0.004349294613335555,0.049855999648571014,0.07366400212049484,0.05459520071744919,0.05459520071744919,0.0069610920757925574,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.03161599859595299,0.017788799852132796,0.015711999963968992,0.005005385723318659,0.06102399900555611,0.08179199695587158,0.06650560013949873,0.06650560013949873,0.006947995595801105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.04416000097990036,0.019200000166893005,0.015488000120967627,0.008561241323364038,0.08054400235414505,0.09196799993515015,0.08607039973139763,0.08607039973139763,0.004145329035483754,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.023391999304294586,0.019667199812829494,0.018463999964296818,0.0022577669687832585,0.18729600310325623,0.20585599541664124,0.19546559900045393,0.19546559900045393,0.006339068824303663,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.026240000501275063,0.03446400165557861,0.028297600522637367,0.027088000439107418,0.0025148441522922374,0.5754240155220032,0.5889919996261597,0.5800191938877105,0.5800191938877105,0.003858869273829596,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.04182400181889534,0.047200001776218414,0.043036799877882004,0.0423360001295805,0.0017073405772076728,2.063199996948242,2.0787200927734375,2.067151999473572,2.067151999473572,0.004959651271127835,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.05648000165820122,0.02270399993285537,0.017935999669134617,0.012377915150727689,0.049536000937223434,0.07196799665689468,0.056015999615192415,0.056015999615192415,0.0070476637552742884,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01408000010997057,0.021344000473618507,0.01611520005390048,0.014928000047802925,0.0025499884993961702,0.07072000205516815,0.2642880082130432,0.1588256008923054,0.1588256008923054,0.053220347086102376,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01360000018030405,0.03014400042593479,0.016975999902933837,0.015056000091135502,0.004715159697364111,0.08393599838018417,0.11036799848079681,0.09160000011324881,0.09160000011324881,0.007911669434472792,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.020767999812960625,0.016233599931001663,0.015392000321298838,0.002202615907995427,0.1998399943113327,0.22070400416851044,0.20855360180139543,0.20855360180139543,0.00689307200230667,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.033440001308918,0.018684800155460833,0.016048000194132328,0.00553991005639174,0.6043199896812439,0.635807991027832,0.6126143991947173,0.6126143991947173,0.008745933953408096,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013824000023305416,0.03359999880194664,0.017811199743300678,0.014944000169634819,0.005926140483565472,0.0,0.0,0.0,0.0,0.0,0.045471999794244766,0.07539200037717819,0.054758400097489356,0.054758400097489356,0.010253548506101549,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02409599907696247,0.01641279999166727,0.01473599998280406,0.003347158638713987,0.0,0.0,0.0,0.0,0.0,0.0461760014295578,0.07529599964618683,0.05460800044238568,0.05460800044238568,0.009748937798340135,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.028416000306606293,0.018611199874430894,0.01598400017246604,0.005209875093863647,0.0,0.0,0.0,0.0,0.0,0.048128001391887665,0.07897599786520004,0.061353600397706036,0.061353600397706036,0.010488153655157845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.025728000327944756,0.016092800162732603,0.01512000011280179,0.003300888123052605,0.0,0.0,0.0,0.0,0.0,0.04864000156521797,0.07648000121116638,0.05810560062527656,0.05810560062527656,0.009473544218404408,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.03551999852061272,0.018588799890130757,0.016064000315964222,0.006071057437992447,0.0,0.0,0.0,0.0,0.0,0.04822399839758873,0.07862400263547897,0.05660480037331582,0.05660480037331582,0.009565394213730401,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.028831999748945236,0.01699519995599985,0.015024000313133001,0.004285000744868188,0.0,0.0,0.0,0.0,0.0,0.04854400083422661,0.06719999760389328,0.05778240002691746,0.05778240002691746,0.006852125554679805,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.0144640002399683,0.024927999824285507,0.0161183999851346,0.01521599991247058,0.0029774808524673907,0.0,0.0,0.0,0.0,0.0,0.05379199981689453,0.08966399729251862,0.06270079985260964,0.06270079985260964,0.009983591277092696,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03001599945127964,0.017648000083863736,0.01547200046479702,0.004585126309484848,0.0,0.0,0.0,0.0,0.0,0.061664000153541565,0.07692799717187881,0.06704320013523103,0.06704320013523103,0.005500191798490629,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.026367999613285065,0.016947199776768684,0.015039999969303608,0.0037951566103550205,0.0,0.0,0.0,0.0,0.0,0.08799999952316284,0.111455999314785,0.0964031994342804,0.0964031994342804,0.007397541615558088,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01369599997997284,0.021247999742627144,0.015359999984502793,0.014640000183135271,0.0020942770950814317,0.0,0.0,0.0,0.0,0.0,0.051711998879909515,0.07065600156784058,0.058054400235414506,0.058054400235414506,0.006633034815910025,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.0307839997112751,0.017148799914866685,0.015039999969303608,0.004882374686312284,0.0,0.0,0.0,0.0,0.0,0.061919998377561576,0.07843200117349625,0.06628479920327664,0.06628479920327664,0.004962852689801192,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013887999579310417,0.02969600073993206,0.017500799987465142,0.015232000034302473,0.00467273319705314,0.0,0.0,0.0,0.0,0.0,0.08819200098514557,0.11097600311040878,0.09493440166115762,0.09493440166115762,0.007509235042985577,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.022112000733613968,0.01751680001616478,0.016047999262809753,0.0030306621792915785,0.0,0.0,0.0,0.0,0.0,0.13065600395202637,0.15110400319099426,0.13857279866933822,0.13857279866933822,0.00750137841249771,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.04012800008058548,0.01892479993402958,0.01550400024279952,0.007459545267816961,0.0,0.0,0.0,0.0,0.0,0.06278400123119354,0.08259200304746628,0.0704512007534504,0.0704512007534504,0.005979055984382744,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.02236800082027912,0.016332800220698118,0.014928000047802925,0.002784044363186731,0.0,0.0,0.0,0.0,0.0,0.1003199964761734,0.1279360055923462,0.10921279862523078,0.10921279862523078,0.00862769617046716,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0225600004196167,0.016128000058233737,0.014800000004470348,0.002958953332547708,0.0,0.0,0.0,0.0,0.0,0.13116799294948578,0.14812800288200378,0.13783999979496003,0.13783999979496003,0.005696384361148053,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.029983999207615852,0.017468800116330386,0.01508800033479929,0.0047379550962483065,0.0,0.0,0.0,0.0,0.0,0.217631995677948,0.2447360008955002,0.22715839892625808,0.22715839892625808,0.008831828221847138,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02304000034928322,0.016512000095099212,0.014512000139802694,0.0035026774828624254,0.0,0.0,0.0,0.0,0.0,0.11020799726247787,0.12307199835777283,0.11600959971547126,0.11600959971547126,0.004667637266950902,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.042080000042915344,0.017510399967432023,0.014607999939471483,0.00823117883530167,0.0,0.0,0.0,0.0,0.0,0.15702399611473083,0.17587199807167053,0.16399359852075576,0.16399359852075576,0.006588299074676393,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0208320003002882,0.015299199987202883,0.01462399959564209,0.001916768086505386,0.0,0.0,0.0,0.0,0.0,0.21779200434684753,0.2415360063314438,0.2260768011212349,0.2260768011212349,0.007251352080685236,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014015999622642994,0.021088000386953354,0.015619200188666582,0.01473599998280406,0.002013227452302141,0.0,0.0,0.0,0.0,0.0,0.3959999978542328,0.4152640104293823,0.40332479774951924,0.40332479774951924,0.006942401431914052,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.03379200026392937,0.020595200080424547,0.019600000232458115,0.006548754248881344,0.026192623739694512,0.040006882507168426,0.029155180178492036,0.029155180178492036,0.00408828748438241,0.024591375524545756,0.037561119537986146,0.027372820355088746,0.027372820355088746,0.0038383559348575944,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.020896000787615776,0.015318400040268899,0.014431999996304512,0.0019640312960926966,0.029349018208693862,0.06236262941356679,0.03714313592014963,0.03714313592014963,0.009618313791872569,0.028826981462526914,0.06125337308649041,0.036482463672774496,0.036482463672774496,0.009447230957011863,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.032127998769283295,0.017286399938166143,0.014479999896138906,0.005536495446636193,0.03273085874558354,0.04394578491937082,0.03563837490653034,0.03563837490653034,0.003429601730256567,0.03488514202593899,0.04683821345079978,0.037984025407085474,0.037984025407085474,0.0036553316361902684,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.021503999829292297,0.015078400075435639,0.01425600005313754,0.002238435975478849,0.042100813549974185,0.051784144690147756,0.045378692890289625,0.045378692890289625,0.003184109940650555,0.05111518843748309,0.06287185663927425,0.05509490773570219,0.05509490773570219,0.0038658725544299132,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.02687999978661537,0.017667199857532977,0.01566399959847331,0.003864811846387173,0.048475323773821306,0.05599956978723733,0.05123499252968345,0.05123499252968345,0.002427379820423941,0.08429268135885387,0.09737642835214408,0.0890914090616175,0.0890914090616175,0.0042209177332077005,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.01462399959564209,0.03359999880194664,0.019804799742996693,0.016032000072300434,0.006750519908145474,0.07121508474579985,0.07292308367136396,0.07194410845270183,0.07194410845270183,0.0005500065915943844,0.14760091249712767,0.15114092373010238,0.14911189243564577,0.14911189243564577,0.0011399477384397005,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.021856000646948814,0.016835200227797033,0.015263999812304974,0.00283854448975065,0.08146338272142935,0.08560865714384096,0.0834932625520734,0.0834932625520734,0.0012259635905347874,0.2782486219401307,0.29240733788179385,0.2851819365989658,0.2851819365989658,0.004187435731481665,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.04495999962091446,0.02143679987639189,0.015856000129133463,0.009709987872683342,0.1194459208702178,0.12302524755181463,0.12053885718696096,0.12053885718696096,0.001005118119341339,0.5597220650459199,0.5764947443228802,0.5648435507167102,0.5648435507167102,0.004709970715400788,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.018112000077962875,0.026496000587940216,0.019318400137126445,0.018432000651955605,0.0024249605112359905,0.19770253574610402,0.222811797868348,0.2054811520619412,0.2054811520619412,0.008000944254868043,0.13941746080159492,0.157124211777114,0.14490284788179197,0.14490284788179197,0.005642170080515992,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.02595200017094612,0.03155200183391571,0.02727359998971224,0.026575999334454536,0.0016260948764843166,0.6014684881116659,0.6095472988212,0.6046757700946376,0.6046757700946376,0.0023537435563177303,0.11325152264578045,0.11477269562841545,0.11385542721485654,0.11385542721485654,0.0004431903698023628,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.03359999880194664,0.01814719969406724,0.015375999733805656,0.005678506740662529,0.06774400174617767,0.07891199737787247,0.07312640026211739,0.07312640026211739,0.003826583051422731,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,2,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512]",1024,512,512,512.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01788800023496151,0.028543999418616295,0.021340799890458582,0.019504000432789326,0.003718627037219618,0.09548799693584442,0.1327359974384308,0.10823359936475753,0.10823359936475753,0.013230635292043864,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,4,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512]",2048,512,512,512.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.02627200074493885,0.030368000268936157,0.027184000052511693,0.02643200010061264,0.0013545679205210022,0.14364799857139587,0.1597760021686554,0.15008639842271806,0.15008639842271806,0.005367723415171222,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512]",4096,512,512,512.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.04150399938225746,0.04726399853825569,0.042950399965047834,0.04224000126123428,0.001720147501765378,0.2433920055627823,0.28963199257850647,0.2549152016639709,0.2549152016639709,0.013450991019687407,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512]",8192,512,512,512.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.025887999683618546,0.03407999873161316,0.028303999826312064,0.026367999613285065,0.0028877495368841042,0.325439989566803,0.3441599905490875,0.33442879617214205,0.33442879617214205,0.005693597811441922,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,2,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048]",4096,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.04137599840760231,0.05084799975156784,0.044828799366950986,0.043087998405098915,0.003560363865797613,0.6110399961471558,0.6421759724617004,0.6204223990440368,0.6204223990440368,0.011214490370794758,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,4,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048, 2048, 2048]",8192,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.022784000262618065,0.016435200069099664,0.015519999898970127,0.0023688301421469523,0.04879999905824661,0.09139200299978256,0.06054079942405224,0.06054079942405224,0.012152608702448775,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014816000126302242,0.02844800055027008,0.017008000146597625,0.015696000307798386,0.003941466294662679,0.047807998955249786,0.07356800138950348,0.05626560002565384,0.05626560002565384,0.00842179125412236,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.053408000618219376,0.019353600218892097,0.01532800029963255,0.01138302400841626,0.048448000103235245,0.0785600021481514,0.0556256003677845,0.0556256003677845,0.009348274502616908,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.03139200061559677,0.01823679991066456,0.016159999649971724,0.004860071238302512,0.055424001067876816,0.08505599945783615,0.0640383992344141,0.0640383992344141,0.00921448636178623,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.026528000831604004,0.017324799951165915,0.01536000007763505,0.0036004822686428305,0.07660800218582153,0.0942080020904541,0.08209280073642732,0.08209280073642732,0.00515895587669752,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.021247999742627144,0.016403199825435876,0.01532800029963255,0.001995897022647934,0.11395200341939926,0.15113599598407745,0.12431039959192276,0.12431039959192276,0.011164431123683732,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.017184000462293625,0.028672000393271446,0.019865600019693376,0.017823999747633934,0.0035798491315929977,0.3171840012073517,0.3341119885444641,0.3261695951223373,0.3261695951223373,0.005111046452878918,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.024639999493956566,0.030400000512599945,0.02656640000641346,0.02556800004094839,0.0020189846603237303,1.0648640394210815,1.0828479528427124,1.0715327858924866,1.0715327858924866,0.005558639263049851,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014976000413298607,0.021695999428629875,0.017305599898099898,0.01593599934130907,0.0025718046517268054,0.04956800118088722,0.07932800054550171,0.06228480041027069,0.06228480041027069,0.01027160349757827,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01539199985563755,0.03580800071358681,0.019686400331556796,0.017136000096797943,0.005918387811304003,0.05158400163054466,0.10713600367307663,0.06364160068333148,0.06364160068333148,0.015832957809696766,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02223999984562397,0.016672000009566545,0.015887999907135963,0.0020934945946034563,0.06521599739789963,0.08902399986982346,0.07520959973335266,0.07520959973335266,0.007913840684102929,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015231999568641186,0.04028800129890442,0.019971200078725816,0.01646399963647127,0.007351305886577286,0.14467200636863708,0.16844800114631653,0.15363519936800005,0.15363519936800005,0.008174435899956223,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015519999898970127,0.02304000034928322,0.01775679988786578,0.016784000210464,0.0025077243712082584,0.33740800619125366,0.35343998670578003,0.3445120006799698,0.3445120006799698,0.00445648463590358,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.02393599972128868,0.01809599995613098,0.01654400024563074,0.002998393102466254,0.0,0.0,0.0,0.0,0.0,0.0504320003092289,0.0843840017914772,0.060083200410008426,0.060083200410008426,0.00986959318572296,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.030047999694943428,0.019510399922728537,0.017680000513792038,0.004065185265963332,0.0,0.0,0.0,0.0,0.0,0.05004800111055374,0.07036799937486649,0.059315200522542,0.059315200522542,0.006537768821329647,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.02393599972128868,0.018035199772566558,0.016512000001966953,0.0030667689116777724,0.0,0.0,0.0,0.0,0.0,0.05100800096988678,0.06735999882221222,0.058387200161814694,0.058387200161814694,0.005832787739241381,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.026208000257611275,0.017500799987465142,0.01646399963647127,0.0030666103306165714,0.0,0.0,0.0,0.0,0.0,0.05023999884724617,0.07254400104284286,0.057254400476813315,0.057254400476813315,0.006890853653068151,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.027327999472618103,0.0179776000790298,0.01648000068962574,0.003783417447531392,0.0,0.0,0.0,0.0,0.0,0.04819199815392494,0.07100799679756165,0.05621119923889638,0.05621119923889638,0.007824391484124725,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.036448001861572266,0.019136000238358975,0.016704000532627106,0.006076530095624803,0.0,0.0,0.0,0.0,0.0,0.047488000243902206,0.08441600203514099,0.0588383998721838,0.0588383998721838,0.011275940726332522,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015135999768972397,0.033952001482248306,0.02119360016658902,0.018000000156462193,0.007136646614305304,0.0,0.0,0.0,0.0,0.0,0.04931199923157692,0.06992000341415405,0.05755840018391609,0.05755840018391609,0.007297587694315226,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.034272000193595886,0.0204415999352932,0.017311999574303627,0.00695404701803013,0.0,0.0,0.0,0.0,0.0,0.05215999856591225,0.0735040009021759,0.06117440015077591,0.06117440015077591,0.007381118436894922,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03743999823927879,0.02012479966506362,0.01775999926030636,0.006181045509079057,0.0,0.0,0.0,0.0,0.0,0.06355199962854385,0.08508799970149994,0.07019200026988984,0.07019200026988984,0.0062246580442117845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.032607998698949814,0.01959999995306134,0.017487999983131886,0.004882475068460749,0.0,0.0,0.0,0.0,0.0,0.04918399825692177,0.0865280032157898,0.05973760038614274,0.05973760038614274,0.011546847942113974,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02675200067460537,0.017430400010198355,0.016560000367462635,0.003243135094239819,0.0,0.0,0.0,0.0,0.0,0.052671998739242554,0.08367999643087387,0.06238719932734965,0.06238719932734965,0.011207824780630104,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.03612799942493439,0.019327999837696553,0.01688000001013279,0.006058251425153085,0.0,0.0,0.0,0.0,0.0,0.06364800035953522,0.07878399640321732,0.06935679838061332,0.06935679838061332,0.004515588328677643,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.03711999952793121,0.019670399930328132,0.017152000218629837,0.006165994646522264,0.0,0.0,0.0,0.0,0.0,0.09071999788284302,0.11507199704647064,0.10252480059862136,0.10252480059862136,0.008782544051535657,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.026528000831604004,0.018396800104528665,0.01601599995046854,0.004279788048986283,0.0,0.0,0.0,0.0,0.0,0.05331199988722801,0.07977599650621414,0.06076480001211167,0.06076480001211167,0.008761579122069606,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.02643200010061264,0.01726400004699826,0.015855999663472176,0.003210008417242029,0.0,0.0,0.0,0.0,0.0,0.062431998550891876,0.08246400207281113,0.06970879957079888,0.06970879957079888,0.007016341676068233,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.02223999984562397,0.0166015999391675,0.015887999907135963,0.00204913969129354,0.0,0.0,0.0,0.0,0.0,0.10127999633550644,0.1141119971871376,0.10621120035648347,0.10621120035648347,0.004029318311754605,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03363199904561043,0.019305599946528675,0.016671999357640743,0.0055445582307981234,0.0,0.0,0.0,0.0,0.0,0.1319040060043335,0.15839999914169312,0.14040640145540234,0.14040640145540234,0.007998041073596942,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014368000440299511,0.04447999969124794,0.018908800091594458,0.0157279996201396,0.0086814937461721,0.0,0.0,0.0,0.0,0.0,0.06428799778223038,0.08982399851083755,0.07349760085344315,0.07349760085344315,0.008605284512197258,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.028672000393271446,0.01890560006722808,0.016080000437796116,0.004797584627815021,0.0,0.0,0.0,0.0,0.0,0.11123199760913849,0.14115199446678162,0.11942399889230729,0.11942399889230729,0.008513394706791027,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.024927999824285507,0.016915200091898442,0.015216000378131866,0.003374147499442635,0.0,0.0,0.0,0.0,0.0,0.15887999534606934,0.18111999332904816,0.16934399753808976,0.16934399753808976,0.007415181260761123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.026944000273942947,0.017737600207328796,0.015199999790638685,0.00415598714375255,0.0,0.0,0.0,0.0,0.0,0.2192319929599762,0.23472000658512115,0.22809920012950893,0.22809920012950893,0.004730841376327335,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.05052800104022026,0.020320000313222408,0.015520000364631414,0.010604522177924678,0.024270629882498958,0.03340247625954076,0.028446344104128624,0.028446344104128624,0.0032330369099793834,0.023697370291069768,0.03261352722995356,0.027774456325453972,0.027774456325453972,0.0031566742680923386,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.033663999289274216,0.019987199828028678,0.018240000121295452,0.005522929139253732,0.0259194055660947,0.03719755183990719,0.029916029687899703,0.029916029687899703,0.003966666590554318,0.027104595853449518,0.03889844644729374,0.03128396954022015,0.03128396954022015,0.004148046317967881,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014592000283300877,0.03299200162291527,0.019840000104159115,0.016207999549806118,0.006546343008726814,0.02587869595769926,0.03763167265431482,0.030174939058162802,0.030174939058162802,0.0037303581782277364,0.026473304070195713,0.03849632587654989,0.03086826083864964,0.03086826083864964,0.003816069654529222,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.013824000023305416,0.021023999899625778,0.016304000187665223,0.015584000386297703,0.0023236165806545476,0.031810621525966996,0.04156949936878106,0.0346749350032807,0.0346749350032807,0.003130209181143985,0.035677378270900374,0.04662250161636451,0.038889864871740294,0.038889864871740294,0.0035107033960828727,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.02175999991595745,0.01569600012153387,0.015008000191301107,0.0020882666168036863,0.038211712107062853,0.07212229256520057,0.04364509673334097,0.04364509673334097,0.009671953074025085,0.0476442859917874,0.08992570455184198,0.05441890342616107,0.05441890342616107,0.01205947791784038,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014368000440299511,0.033824000507593155,0.017628800217062236,0.014864000026136637,0.005791495403857738,0.05214261250030033,0.06266261508706669,0.05677791295527661,0.05677791295527661,0.0030298223079651514,0.08648138506878382,0.10392938682791132,0.09416928531647478,0.09416928531647478,0.005025126612204489,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.031199999153614044,0.017900799959897996,0.015343999955803156,0.004974475661316249,0.06411958891421111,0.07405276123263956,0.06793047918211377,0.06793047918211377,0.0033918658556700006,0.14058441263169497,0.16236323591492058,0.1489399211274489,0.1489399211274489,0.00743678300375353,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.03855999931693077,0.018495999928563833,0.015647999942302704,0.006918530056761627,0.09872985549401277,0.10683454583043753,0.10185316839884552,0.10185316839884552,0.0020802254037042074,0.2743261390166379,0.2968454508984596,0.2830044295482752,0.2830044295482752,0.005780016596065092,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.030400000512599945,0.01977920001372695,0.01726400014013052,0.005350883464206169,0.13918872472233365,0.16279522855335207,0.1501912864839173,0.1501912864839173,0.008992185529011513,0.11892328861766266,0.13909276049083735,0.1283239123428725,0.1283239123428725,0.007682951884956939,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.01727999933063984,0.02223999984562397,0.01819519978016615,0.017680000513792038,0.0014397298270620873,0.3728044181625443,0.38295504353701676,0.3761264483787333,0.3761264483787333,0.0033660272332490977,0.07967557017401883,0.08184495665371805,0.08038555277807365,0.08038555277807365,0.0007193856241088455,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.034591998904943466,0.01941439984366298,0.01756799966096878,0.005541380584373771,0.05951999872922897,0.08268799632787704,0.06715519949793816,0.06715519949793816,0.006657802116288364,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,2,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512]",1024,512,512,512.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.03283200040459633,0.019180799927562477,0.016543999314308167,0.0052187911233635975,0.07199999690055847,0.08675199747085571,0.07749439924955369,0.07749439924955369,0.004849874741156772,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,4,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512]",2048,512,512,512.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01724799908697605,0.02284800074994564,0.01928640007972717,0.018400000408291817,0.0020641390157565697,0.09676799923181534,0.12310399860143663,0.10618879944086074,0.10618879944086074,0.008982738207839057,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512]",4096,512,512,512.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.024224000051617622,0.03315199911594391,0.027436799928545953,0.027328000403940678,0.002679154874546328,0.14716799557209015,0.16412800550460815,0.15470399856567385,0.15470399856567385,0.005423454433987419,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512]",8192,512,512,512.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01833599992096424,0.04806400090456009,0.026252799853682517,0.022672000341117382,0.00878438440429033,0.1844799965620041,0.2072959989309311,0.19359359890222552,0.19359359890222552,0.007663539820967659,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,2,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048]",4096,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.024480000138282776,0.04979199916124344,0.029081599973142146,0.025679999962449074,0.007227422044689197,0.34147199988365173,0.37968000769615173,0.3583200007677078,0.3583200007677078,0.011804422100726231,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,4,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048, 2048, 2048]",8192,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median diff --git a/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/attention_true_mixed_fused.csv b/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/attention_true_mixed_fused.csv new file mode 100644 index 0000000..153bb52 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/attention_true_mixed_fused.csv @@ -0,0 +1,31 @@ +num_tensor_parallel_workers,batch_spec,num_prefill_seqs,num_decode_seqs,total_prefill_tokens,total_decode_tokens,decode_avg_kv_cache_size,attention_core_mean_ms,attention_core_mean_as_median_ms,kv_cache_update_median_ms,pure_prefill_reference_mean_ms,pure_decode_reference_mean_ms,projected_prefill_mean_ms,projected_decode_mean_ms,projection_sum_error_ms,representation +1,q64_8q1s512,1,8,64,8,512.0,0.06604800000786781,0.06604800000786781,0.021359999664127827,0.05810240097343922,0.0600041144660541,0.032492258074592426,0.03355574193327539,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q128_8q1s1k,1,8,128,8,1024.0,0.07483199909329416,0.07483199909329416,0.024255999363958836,0.05587520003318787,0.06090559959411622,0.03580428402241854,0.03902771507087562,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q128_16q1s1k,1,16,128,16,1024.0,0.10597119852900506,0.10597119852900506,0.018112000077962875,0.05587520003318787,0.06891520097851753,0.04744885718421094,0.05852234134479412,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q256_16q1s2k,1,16,256,16,2048.0,0.13697599917650224,0.13697599917650224,0.01673599984496832,0.056396800279617305,0.11392960175871848,0.04535414343408448,0.09162185574241775,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q256_32q1s2k,1,32,256,32,2048.0,0.2163648009300232,0.2163648009300232,0.01515199989080429,0.056396800279617305,0.13807999789714814,0.06274415549817247,0.15362064543185072,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q512_32q1s4k,1,32,512,32,4096.0,0.3746495962142944,0.3746495962142944,0.016191999427974224,0.07432000041007995,0.2231455981731415,0.09360396051475776,0.2810456356995366,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q512_64q1s4k,1,64,512,64,4096.0,0.6718560099601746,0.6718560099601746,0.024848000146448612,0.07432000041007995,0.4058080047369003,0.10399797220840973,0.5678580377517648,-1.1102230246251565e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q1k_64q1s8k,1,64,1024,64,8192.0,1.3472000002861022,1.3472000002861022,0.019600000232458115,0.1307712011039257,0.7464000046253203,0.2008444429250931,1.1463555573610091,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q2k_32q1s4k,1,32,2048,32,4096.0,0.6351647913455962,0.6351647913455962,0.028256000019609928,0.33396480381488797,0.2231455981731415,0.38075520430942184,0.25440958703617433,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q4k_16q1s4k,1,16,4096,16,4096.0,1.2515872120857237,1.2515872120857237,0.04395199939608574,1.0999775886535645,0.1389280006289482,1.1112371236754128,0.14035008841031085,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q64_8q1s512,1,8,64,8,512.0,0.05652800053358078,0.05652800053358078,0.019600000232458115,0.05961279980838299,0.055968457407185014,0.029155180178492036,0.027372820355088746,6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q128_8q1s1k,1,8,128,8,1024.0,0.07362559959292413,0.07362559959292413,0.014431999996304512,0.05882879942655563,0.05778240002691746,0.03714313592014963,0.036482463672774496,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q128_16q1s1k,1,16,128,16,1024.0,0.0736224003136158,0.0736224003136158,0.014479999896138906,0.05882879942655563,0.06270079985260964,0.03563837490653034,0.037984025407085474,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q256_16q1s2k,1,16,256,16,2048.0,0.10047360062599181,0.10047360062599181,0.01425600005313754,0.05459520071744919,0.06628479920327664,0.045378692890289625,0.05509490773570219,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q256_32q1s2k,1,32,256,32,2048.0,0.14032640159130094,0.14032640159130094,0.01566399959847331,0.05459520071744919,0.09493440166115762,0.05123499252968345,0.0890914090616175,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q512_32q1s4k,1,32,512,32,4096.0,0.22105600088834762,0.22105600088834762,0.016032000072300434,0.06650560013949873,0.13783999979496003,0.07194410845270183,0.14911189243564577,-2.7755575615628914e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q512_64q1s4k,1,64,512,64,4096.0,0.36867519915103913,0.36867519915103913,0.015263999812304974,0.06650560013949873,0.22715839892625808,0.0834932625520734,0.2851819365989658,5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q1k_64q1s8k,1,64,1024,64,8192.0,0.6853824079036711,0.6853824079036711,0.015856000129133463,0.08607039973139763,0.40332479774951924,0.12053885718696096,0.5648435507167102,1.1102230246251565e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q2k_32q1s4k,1,32,2048,32,4096.0,0.3503839999437332,0.3503839999437332,0.018432000651955605,0.19546559900045393,0.13783999979496003,0.2054811520619412,0.14490284788179197,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q4k_16q1s4k,1,16,4096,16,4096.0,0.7185311973094941,0.7185311973094941,0.026575999334454536,0.5800191938877105,0.10921279862523078,0.6046757700946376,0.11385542721485654,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q64_8q1s512,1,8,64,8,512.0,0.0562208004295826,0.0562208004295826,0.015520000364631414,0.06054079942405224,0.0591108573866742,0.028446344104128624,0.027774456325453972,-6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q128_8q1s1k,1,8,128,8,1024.0,0.061199999228119854,0.061199999228119854,0.018240000121295452,0.05626560002565384,0.0588383998721838,0.029916029687899703,0.03128396954022015,-6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q128_16q1s1k,1,16,128,16,1024.0,0.06104319989681244,0.06104319989681244,0.016207999549806118,0.05626560002565384,0.05755840018391609,0.030174939058162802,0.03086826083864964,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q256_16q1s2k,1,16,256,16,2048.0,0.07356479987502099,0.07356479987502099,0.015584000386297703,0.0556256003677845,0.06238719932734965,0.0346749350032807,0.038889864871740294,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q256_32q1s2k,1,32,256,32,2048.0,0.09806400015950203,0.09806400015950203,0.015008000191301107,0.0556256003677845,0.06935679838061332,0.04364509673334097,0.05441890342616107,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q512_32q1s4k,1,32,512,32,4096.0,0.1509471982717514,0.1509471982717514,0.014864000026136637,0.0640383992344141,0.10621120035648347,0.05677791295527661,0.09416928531647478,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q512_64q1s4k,1,64,512,64,4096.0,0.21687040030956264,0.21687040030956264,0.015343999955803156,0.0640383992344141,0.14040640145540234,0.06793047918211377,0.1489399211274489,2.7755575615628914e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q1k_64q1s8k,1,64,1024,64,8192.0,0.3848575979471207,0.3848575979471207,0.015647999942302704,0.08209280073642732,0.22809920012950893,0.10185316839884552,0.2830044295482752,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q2k_32q1s4k,1,32,2048,32,4096.0,0.2785151988267898,0.2785151988267898,0.01726400014013052,0.12431039959192276,0.10621120035648347,0.1501912864839173,0.1283239123428725,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q4k_16q1s4k,1,16,4096,16,4096.0,0.456512001156807,0.456512001156807,0.017680000513792038,0.3261695951223373,0.06970879957079888,0.3761264483787333,0.08038555277807365,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation diff --git a/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/linear_op.csv b/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/linear_op.csv new file mode 100644 index 0000000..2803f02 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/linear_op.csv @@ -0,0 +1,37 @@ +time_stats.emb.min,time_stats.emb.max,time_stats.emb.mean,time_stats.emb.median,time_stats.emb.std,time_stats.input_layernorm.min,time_stats.input_layernorm.max,time_stats.input_layernorm.mean,time_stats.input_layernorm.median,time_stats.input_layernorm.std,time_stats.attn_pre_proj.min,time_stats.attn_pre_proj.max,time_stats.attn_pre_proj.mean,time_stats.attn_pre_proj.median,time_stats.attn_pre_proj.std,time_stats.attn_rope.min,time_stats.attn_rope.max,time_stats.attn_rope.mean,time_stats.attn_rope.median,time_stats.attn_rope.std,time_stats.attn_post_proj.min,time_stats.attn_post_proj.max,time_stats.attn_post_proj.mean,time_stats.attn_post_proj.median,time_stats.attn_post_proj.std,time_stats.post_attention_layernorm.min,time_stats.post_attention_layernorm.max,time_stats.post_attention_layernorm.mean,time_stats.post_attention_layernorm.median,time_stats.post_attention_layernorm.std,n_head,n_kv_head,n_embd,n_expanded_embd,vocab_size,use_gated_mlp,use_qk_norm,attn_output_gate,num_tokens,num_tensor_parallel_workers,padded_n_embd,padded_n_expanded_embd,model_arch,is_step2_mini,share_expert_dim,share_q_dim,measurement_type,profiling_precision,quant_signature +0.029184000566601753,0.06780800223350525,0.03157280012965202,0.030736000277101994,0.005874173435341216,0.033215999603271484,0.04825599864125252,0.03443359974771738,0.0337119996547699,0.0031825678429048183,1.438431978225708,1.505568027496338,1.446228802204132,1.4429279565811157,0.014128607642643025,0.538752019405365,0.5440319776535034,0.5416463971138,0.5420799851417542,0.0016099306164432847,1.0073280334472656,1.0163840055465698,1.0098415970802308,1.0081279873847961,0.0032980457197329728,0.04022400081157684,0.041728001087903976,0.04091359991580248,0.04081599973142147,0.0004728505416767937,32,4,2048,768,151936,True,True,False,8192,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.01360000018030405,0.06784000247716904,0.017755200061947106,0.0179840000346303,0.00837681421401443,0.01836800016462803,0.03651199862360954,0.019606399815529585,0.018719999119639397,0.0038821958848767424,0.7512000203132629,0.8208960294723511,0.7566704005002975,0.75382399559021,0.014793092586577971,0.28995200991630554,0.29337599873542786,0.29135999977588656,0.29150401055812836,0.000998381071258815,0.5149760246276855,0.5169600248336792,0.5160208016633987,0.5158880054950714,0.0005038652783291977,0.02051199972629547,0.021503999829292297,0.021067200042307378,0.021104000508785248,0.00023542855306902367,32,4,2048,768,151936,True,True,False,4096,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.0080960001796484,0.04281599819660187,0.011092799971811474,0.011039999779313803,0.005489135752949302,0.012223999947309494,0.026335999369621277,0.013187199970707298,0.01247999956831336,0.0030236510562153375,0.3928639888763428,0.45372799038887024,0.3976895987987518,0.39528000354766846,0.012905176716136006,0.1547199934720993,0.15884800255298615,0.15712319910526276,0.1573439985513687,0.001165121945135037,0.26633599400520325,0.268095999956131,0.26719200164079665,0.2671840041875839,0.0004242740199415328,0.013024000450968742,0.013887999579310417,0.013489600038155913,0.013520000036805868,0.0002382817554057738,32,4,2048,768,151936,True,True,False,2048,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.00825599953532219,0.04755200073122978,0.02398160002194345,0.01961600035429001,0.00874683840888523,0.018880000337958336,0.03868800029158592,0.021590400114655496,0.0208320003002882,0.004057015751746236,0.24316799640655518,0.2710399925708771,0.2521967992186546,0.25065599381923676,0.007998418528894075,0.09644799679517746,0.19120000302791595,0.10407840013504029,0.09963199868798256,0.020043722414992166,0.13600000739097595,0.18892799317836761,0.15760480016469955,0.15760000050067902,0.0112223677907762,0.009664000011980534,0.010015999898314476,0.009836799977347255,0.009824000298976898,9.016971952948177e-05,32,4,2048,768,151936,True,True,False,1024,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.017376000061631203,0.044704001396894455,0.026318399980664254,0.027312000282108784,0.007565344034680866,0.01836800016462803,0.03014400042593479,0.021276800055056812,0.020655999891459942,0.002632977855097951,0.1438719928264618,0.17132799327373505,0.152497598528862,0.15012799948453903,0.007947150319625347,0.1430719941854477,0.19305600225925446,0.16630879789590836,0.1685439944267273,0.014628024163894684,0.08899199962615967,0.10467199981212616,0.0943599995225668,0.09374399855732918,0.003472669494074113,0.007615999784320593,0.007935999892652035,0.007769599952735007,0.0077760000713169575,7.680004540222077e-05,32,4,2048,768,151936,True,True,False,512,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.016992000862956047,0.0544000007212162,0.02573199989274144,0.026016000658273697,0.00803323401720434,0.018432000651955605,0.02348800003528595,0.020648000109940768,0.02062400057911873,0.0013939985047930988,0.10220800340175629,0.1361600011587143,0.11850560046732425,0.11684799939393997,0.010637754898360304,0.16710400581359863,0.21110400557518005,0.19078560024499894,0.19409599900245667,0.013714352646558832,0.06265600025653839,0.0740479975938797,0.06842879951000214,0.0690080001950264,0.0031292240446560557,0.00979200005531311,0.033440001308918,0.01864320016466081,0.017280000261962414,0.006957998188876383,32,4,2048,768,151936,True,True,False,256,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.017152000218629837,0.04396799951791763,0.025907999789342284,0.02598400041460991,0.007714554737915267,0.018400000408291817,0.03667199984192848,0.024132800102233887,0.02112000063061714,0.006120348733354326,0.10678400099277496,0.1363839954137802,0.1193264003843069,0.11583999916911125,0.009838647443214228,0.17017599940299988,0.22748799622058868,0.18853759989142418,0.18433599919080734,0.015728078443174653,0.04569600149989128,0.06652799993753433,0.05192639995366335,0.05151999928057194,0.004516605694976449,0.021856000646948814,0.026335999369621277,0.02384479995816946,0.023599999956786633,0.0011301153013314744,32,4,2048,768,151936,True,True,False,128,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.017343999817967415,1.0683200359344482,0.05748240072280168,0.029504000209271908,0.16366824814254827,0.018688000738620758,0.3317759931087494,0.03685439983382821,0.021151999942958355,0.06767422466731164,0.10255999863147736,0.9434880018234253,0.16412640027701855,0.11956800147891045,0.17964606281728834,0.1714559942483902,2.1306240558624268,0.3011296011507511,0.1926399990916252,0.42310127734378766,0.03574400022625923,0.6859520077705383,0.08389280084520578,0.04279999993741512,0.14321647071615612,0.020479999482631683,0.1831360012292862,0.033024000097066165,0.023856000043451786,0.03470987082429836,32,4,2048,768,151936,True,True,False,64,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.016095999628305435,0.05142400041222572,0.026363200135529043,0.02676799986511469,0.008637725852473854,0.01849599927663803,0.03577600046992302,0.022193600237369538,0.020848000422120094,0.004615266181181815,0.10540799796581268,0.15014399588108063,0.12211520001292228,0.11896000057458878,0.012772013396624768,0.17315199971199036,0.21478399634361267,0.1881632000207901,0.1873439997434616,0.011761164657572015,0.03481600061058998,0.058079998940229416,0.042200000025331974,0.03969600051641464,0.006461281521769989,0.02143999934196472,0.038816001266241074,0.02466559996828437,0.023856000043451786,0.0035418033748569927,32,4,2048,768,151936,True,True,False,32,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.015904000028967857,0.03753599897027016,0.023127999808639287,0.02527999971061945,0.0054230027890305385,0.018400000408291817,0.03001599945127964,0.020488000102341176,0.020096000283956528,0.002585688394137697,0.10355199873447418,0.1438400000333786,0.11536479964852334,0.11124800145626068,0.011136028294919255,0.16502399742603302,0.2072959989309311,0.18646399974822997,0.19075199961662292,0.013296437761247597,0.03142400085926056,0.05215999856591225,0.03888959977775812,0.03750399872660637,0.0057399302067536314,0.020128000527620316,0.04064000025391579,0.023937600292265417,0.023648000322282314,0.004144197400898248,32,4,2048,768,151936,True,True,False,16,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.015200000256299973,0.037151999771595,0.02144480012357235,0.02195199951529503,0.005576364091933697,0.017952000722289085,0.0226879995316267,0.019934400077909233,0.019952000118792057,0.0011812499470458758,0.10063999891281128,0.13468800485134125,0.11595199964940547,0.1207519993185997,0.011974301621092394,0.16332800686359406,0.20748800039291382,0.18162400051951408,0.1767839938402176,0.01474120743720334,0.03222399950027466,0.043455999344587326,0.03829439990222454,0.03859200142323971,0.0031378131240041122,0.020959999412298203,0.03587200120091438,0.023795200139284135,0.023423999547958374,0.0031339489695198443,32,4,2048,768,151936,True,True,False,8,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.014944000169634819,0.04022400081157684,0.02162720002233982,0.022463999688625336,0.006017219317026815,0.018464000895619392,0.026528000831604004,0.021264000236988066,0.020896000787615776,0.001934095017586082,0.10156799852848053,0.1703999936580658,0.12565439902245998,0.1244799979031086,0.01641776722785409,0.1653759926557541,0.23865599930286407,0.1969360001385212,0.19223999977111816,0.02058903050274278,0.03254399821162224,0.06752000004053116,0.0443536002188921,0.041519999504089355,0.00984829071098989,0.020096000283956528,0.040031999349594116,0.026934400014579297,0.02478400059044361,0.00562071702648593,32,4,2048,768,151936,True,True,False,1,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.7531200051307678,0.8461440205574036,0.7618160009384155,0.7576479911804199,0.019464233617982506,0.2922559976577759,0.2985599935054779,0.2953856036067009,0.29576000571250916,0.001575901077689139,0.510047972202301,0.5140479803085327,0.5121696025133133,0.5123839974403381,0.0012263137313476844,,,,,,32,4,2048,768,151936,True,True,False,8192,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.39529600739479065,0.47279998660087585,0.40216960161924364,0.39825600385665894,0.0162749796240819,0.15625600516796112,0.16211199760437012,0.15959519892930984,0.15988799929618835,0.0011493418942396922,0.2635200023651123,0.2642880082130432,0.2639120012521744,0.26392000913619995,0.0001903593401384135,,,,,,32,4,2048,768,151936,True,True,False,4096,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.2531839907169342,0.32419198751449585,0.26924319565296173,0.26049599051475525,0.01657194262368116,0.10051199793815613,0.2375359982252121,0.12411200068891048,0.10311999917030334,0.039466165428540506,0.15881599485874176,0.19289599359035492,0.16896959990262986,0.1640480011701584,0.009162416086418127,,,,,,32,4,2048,768,151936,True,True,False,2048,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.15302400290966034,0.1780800074338913,0.16284480094909667,0.16113600134849548,0.00740355661356144,0.14521600306034088,0.20233599841594696,0.17807039842009545,0.1796799972653389,0.013908448560094403,0.0907519981265068,0.09750399738550186,0.09460479989647866,0.09478399902582169,0.0017965487667361475,,,,,,32,4,2048,768,151936,True,True,False,1024,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11740799993276596,0.17958399653434753,0.13262080028653145,0.12878400087356567,0.014146060532423056,0.17468799650669098,0.21161599457263947,0.1910431995987892,0.18966399878263474,0.01270903647700971,0.05926400050520897,0.07577600330114365,0.06530559975653887,0.06404799968004227,0.00479458462514625,,,,,,32,4,2048,768,151936,True,True,False,512,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11260800063610077,0.17209599912166595,0.13610880002379416,0.13809599727392197,0.017675922458993677,0.18729600310325623,0.26822400093078613,0.20815680101513861,0.2078079953789711,0.01700718593658771,0.04499199986457825,0.06537599861621857,0.0507551996037364,0.04787199944257736,0.005727123232692158,,,,,,32,4,2048,768,151936,True,True,False,256,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11257600039243698,0.23452800512313843,0.13866880126297473,0.13964799791574478,0.026590047697062115,0.1828799992799759,0.24751999974250793,0.20735519900918006,0.20670399814844131,0.01500438131586336,0.03654399886727333,0.05593600124120712,0.0408239996060729,0.03892800025641918,0.004864409450710354,,,,,,32,4,2048,768,151936,True,True,False,128,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11734399944543839,0.18726399540901184,0.13890240006148816,0.13308800011873245,0.019086167340006257,0.16991999745368958,0.2443840056657791,0.19185120090842248,0.19257599860429764,0.018017536159219673,0.03033600002527237,0.04956800118088722,0.038387199863791466,0.03750400058925152,0.00500897018702887,,,,,,32,4,2048,768,151936,True,True,False,64,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11420799791812897,0.18115200102329254,0.13938880078494548,0.1393439993262291,0.018141313962922536,0.1693439930677414,0.221343994140625,0.19187839925289155,0.19438399374485016,0.01430752121272052,0.03017600066959858,0.06019200012087822,0.03866560012102127,0.0364960003644228,0.0074199790031205266,,,,,,32,4,2048,768,151936,True,True,False,32,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11276800185441971,0.16223999857902527,0.1331360016018152,0.13305599987506866,0.01432493740801146,0.17187200486660004,0.23625600337982178,0.19287680014967917,0.1913280040025711,0.01754628831589704,0.029823999851942062,0.07574400305747986,0.03900959976017475,0.036927999928593636,0.009277465083962879,,,,,,32,4,2048,768,151936,True,True,False,16,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.1130559965968132,0.19814400374889374,0.13379519879817964,0.12531199678778648,0.020074427302248836,0.16841599345207214,0.21139200031757355,0.1893615983426571,0.19092799723148346,0.012418720676762711,0.029503999277949333,0.07558400183916092,0.040144000016152856,0.03728000074625015,0.010223239196498205,,,,,,32,4,2048,768,151936,True,True,False,8,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.115167997777462,0.2640640139579773,0.13805920109152794,0.13118399679660797,0.03132849683515479,0.17103999853134155,0.2977280020713806,0.19899839907884598,0.1976960003376007,0.028928046763067948,0.0297279991209507,0.05990400165319443,0.03834720011800528,0.0363520011305809,0.0070885330713495905,,,,,,32,4,2048,768,151936,True,True,False,1,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.3940800130367279,0.46483200788497925,0.39923040121793746,0.3957759886980057,0.015108903906233569,0.16022400557994843,0.1634880006313324,0.16203359961509706,0.16228799521923065,0.0010220720912414007,0.26073598861694336,0.2627840042114258,0.26138080209493636,0.2613760083913803,0.00041640363494172775,,,,,,32,4,2048,768,151936,True,True,False,8192,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.2536959946155548,0.27161601185798645,0.25960480123758317,0.2577280104160309,0.005142017404245015,0.09849599748849869,0.10281600058078766,0.10057279989123344,0.10063999891281128,0.0008836232237268523,0.13913600146770477,0.17606399953365326,0.15875840038061143,0.15988799929618835,0.010058952112389172,,,,,,32,4,2048,768,151936,True,True,False,4096,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.15142400562763214,0.1780479997396469,0.16139679849147798,0.1602879986166954,0.0070656055731385115,0.14601600170135498,0.23343999683856964,0.17913119941949845,0.179967999458313,0.023065274309176566,0.09388799965381622,0.12108799815177917,0.10317599996924401,0.10073599964380264,0.007364345618470178,,,,,,32,4,2048,768,151936,True,True,False,2048,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.12198399752378464,0.18406400084495544,0.13770400024950505,0.13232000172138214,0.01623164885687898,0.1773120015859604,0.20688000321388245,0.1870912007987499,0.18535999953746796,0.00825628058448234,0.05766399949789047,0.06739199906587601,0.06187200043350458,0.061216000467538834,0.003004312467037863,,,,,,32,4,2048,768,151936,True,True,False,1024,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11737599968910217,0.18614399433135986,0.13109439946711063,0.12771200388669968,0.01372168725934724,0.17587199807167053,0.2699519991874695,0.19926720038056372,0.19075199961662292,0.02327478982490538,0.041728001087903976,0.06224000081419945,0.05000480003654957,0.049375999718904495,0.006185850944273961,,,,,,32,4,2048,768,151936,True,True,False,512,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.1159679964184761,0.1610880047082901,0.12988320142030715,0.12494400516152382,0.011865375783184065,0.1767680048942566,0.27379199862480164,0.21035519987344742,0.21275199949741364,0.023206964017550125,0.037087999284267426,0.062144000083208084,0.04439679980278015,0.04283200018107891,0.006155226392511492,,,,,,32,4,2048,768,151936,True,True,False,256,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.12015999853610992,0.158720001578331,0.13573280088603495,0.1343199983239174,0.010666830836014414,0.1737920045852661,0.22127999365329742,0.20261440128087999,0.20321600139141083,0.011153965849067301,0.03587200120091438,0.04944000020623207,0.038265600241720675,0.037328001111745834,0.0030431580113575636,,,,,,32,4,2048,768,151936,True,True,False,128,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.1212799996137619,0.15139199793338776,0.13183839991688728,0.13014400005340576,0.00851207547022104,0.17257599532604218,0.2250880002975464,0.1872655987739563,0.18193599581718445,0.013638284975248818,0.02969600073993206,0.0525440014898777,0.03840640028938651,0.03444799967110157,0.007598511754254174,,,,,,32,4,2048,768,151936,True,True,False,64,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11270400136709213,0.1624639928340912,0.13508000001311302,0.13180799782276154,0.014243564451606775,0.17052799463272095,0.23715199530124664,0.1953311987221241,0.1966560035943985,0.01500670718978398,0.030239999294281006,0.05270399898290634,0.03807039987295866,0.03742399998009205,0.004906165602021684,,,,,,32,4,2048,768,151936,True,True,False,32,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.12319999933242798,0.17919999361038208,0.14059039913117885,0.14156799763441086,0.016217662982107223,0.1701119989156723,0.21987199783325195,0.18903039917349815,0.1870879977941513,0.014117854507841239,0.030368000268936157,0.06406400352716446,0.04045119984075427,0.03710399940609932,0.00878942205983628,,,,,,32,4,2048,768,151936,True,True,False,16,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11296000331640244,0.16761599481105804,0.13843199908733367,0.13814399391412735,0.014925286935582404,0.1711679995059967,0.21561600267887115,0.1906527981162071,0.1876479983329773,0.011803992622137974,0.0306560005992651,0.09216000139713287,0.041129599791020155,0.036847999319434166,0.013739721468154687,,,,,,32,4,2048,768,151936,True,True,False,8,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11929599940776825,0.2008959949016571,0.14535359852015972,0.13971199840307236,0.024769473061576282,0.16803200542926788,0.25123199820518494,0.19573760256171227,0.19366399943828583,0.023196011081705884,0.03049599938094616,0.08089599758386612,0.04739360017701984,0.0453919991850853,0.01286389846889463,,,,,,32,4,2048,768,151936,True,True,False,1,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none diff --git a/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/manifest.json b/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/manifest.json new file mode 100644 index 0000000..eff69e5 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/manifest.json @@ -0,0 +1,57 @@ +{ + "attention_tp_coverage": [ + 1, + 2, + 4 + ], + "environment_contract": { + "dtype": "bfloat16", + "frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "hardware": "NVIDIA H20", + "model": "Qwen3-30B-A3B", + "tensor_parallel_sizes": [ + 1, + 2, + 4 + ], + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + }, + "inputs": { + "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-composition-tp1-20260717-v1-20260717T014550833983Z/artifacts/artifacts/attention-composition-tp1-v1/raw/flashattn-composition-tp1.json": "ce461be92a5059e2d168b2cbf21b55d9a55ee71c3b0586a3e63cd8288938168d", + "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-composition-tp2-20260717-v1-20260717T014551831545Z/artifacts/artifacts/attention-composition-tp2-v1/raw/flashattn-composition-tp2.json": "997d67fb3a28b488c3aa4945ae7c6d4ecf8a08726ee3cb4652d16c221ff395ad", + "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-composition-tp4-20260717-v1-20260717T014552897206Z/artifacts/artifacts/attention-composition-tp4-v1/raw/flashattn-composition-tp4.json": "046cffd4c11d8a36513167ec25d3e1ca016324715679ea688a58d4a71dfe62b2", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931" + }, + "outputs": { + "allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3", + "attention.csv": "8af0206ffe8f515c124194ecb589a0b8d640c40f59b131dc65c1fe1ad221c007", + "attention_true_mixed_fused.csv": "43ef4be90bddc9aeac6dbbe339feec24162cd1f2129a08fbd959e6ee4eaf5f60", + "linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "moe.csv": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0" + }, + "profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-fused-mixed-total-conserving-pure-prefill-batch-composition", + "projection_contract": { + "allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity", + "attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target; true mixed rows use a total-conserving compatibility projection", + "attention_pure_prefill_batch_composition": "Direct FA3 measurements for 2/4 requests at query length 2048 and 2/4/8/16 requests at query length 512 for each TP; included only when batch-composition attention inputs are supplied", + "attention_true_mixed": "The directly measured fused total is preserved in diagnostics. Frontier's two targets are projected by the same-TP pure prefill/decode reference ratio, with projected prefill + decode exactly equal to the fused total; the split is a schema compatibility attribution, not an observation", + "linear": "Frontier profiler using vLLM 0.20 CUDA operators", + "moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero" + }, + "row_counts": { + "allreduce": 24, + "attention_frontier_compatible": 150, + "attention_true_mixed_fused_diagnostic": 30, + "linear": 36, + "moe": 72 + }, + "schema_version": "frontier_qwen30_vllm020_frozen_profile.v3" +} diff --git a/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/moe.csv b/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/moe.csv new file mode 100644 index 0000000..e5bf507 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/moe.csv @@ -0,0 +1,73 @@ +time_stats.moe_gating_linear.min,time_stats.moe_gating_linear.max,time_stats.moe_gating_linear.mean,time_stats.moe_gating_linear.median,time_stats.moe_gating_linear.std,time_stats.moe_gating_routing_topk.min,time_stats.moe_gating_routing_topk.max,time_stats.moe_gating_routing_topk.mean,time_stats.moe_gating_routing_topk.median,time_stats.moe_gating_routing_topk.std,time_stats.moe_shuffling.min,time_stats.moe_shuffling.max,time_stats.moe_shuffling.mean,time_stats.moe_shuffling.median,time_stats.moe_shuffling.std,time_stats.moe_grouped_gemm.min,time_stats.moe_grouped_gemm.max,time_stats.moe_grouped_gemm.mean,time_stats.moe_grouped_gemm.median,time_stats.moe_grouped_gemm.std,num_tokens,num_experts,num_experts_per_device,expert_parallel_size,routing_runtime_path,routing_assignment_policy,routing_weight_policy,routing_uses_router_logits,gating_runtime_context,gating_runtime_context_impl,router_topk,hidden_dim,expert_hidden_dim,use_gated,num_tensor_parallel_workers,total_routed_tokens,model_expansion_ratio,tokens_per_expert_avg,tokens_to_experts_ratio,expert_utilization,min_load_ratio,load_imbalance_cv,max_load_ratio,load_entropy,load_gini_coefficient,load_distribution,seed,moe_grouped_gemm_backend,measurement_type,profiling_precision,model_arch,quant_signature,router_median_nonadditivity_ratio,projection_policy +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.33926400542259216,0.405023992061615,0.36780479848384856,0.36507199704647064,0.01690507644474779,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,1.1943039894104004,1.286784052848816,1.228384006023407,1.2273280024528503,0.02832547242381263,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.3984375,0.0,1.346291201783626,4.0,5.59375,0.661865234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,1.679744005203247,1.766144037246704,1.7095808148384095,1.7015680074691772,0.02438921262998535,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.625,0.0,1.015504800579495,5.0,6.15516433212955,0.529052734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,2.1576640605926514,2.2921600341796875,2.2097824096679686,2.188944101333618,0.045572321842012986,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.875,0.0,0.6343057228182637,2.5,6.64370748444639,0.35369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,2.422368049621582,2.5130879878997803,2.4516672134399413,2.434159994125366,0.03278900287381846,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.984375,0.0,0.4921254921257382,2.25,6.817190042344769,0.272369384765625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,2.2217600345611572,2.289599895477295,2.2571327924728393,2.263375997543335,0.021660416089449488,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,1.0,0.125,0.3486861500690843,1.875,6.908192310183997,0.197662353515625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,2.18668794631958,2.318079948425293,2.2218016147613526,2.211087942123413,0.035380897213135316,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,1.0,0.4375,0.2525504668006971,1.875,6.953347743053017,0.1410369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,2.2291839122772217,2.3929600715637207,2.2908096313476562,2.2804640531539917,0.04479348924786221,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,1.0,0.65625,0.15765965680164504,1.5625,6.98229848728205,0.08779525756835938,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,2.0678720474243164,2.1297600269317627,2.0837119817733765,2.0779199600219727,0.017880044357986735,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,1.0,0.625,0.12169081635504074,1.3125,6.9892029662356325,0.06879425048828125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,2.916032075881958,3.0819520950317383,2.9805248022079467,2.9656319618225098,0.05482195799019572,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,1.0,0.796875,0.07935434147688751,1.1796875,6.9954297964750305,0.044734954833984375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,4.421599864959717,4.535359859466553,4.486294317245483,4.497056007385254,0.036990243787549344,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,1.0,0.8203125,0.060849326483103046,1.17578125,6.9973188375859685,0.033740997314453125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,7.302591800689697,7.402751922607422,7.354758310317993,7.3464319705963135,0.032142662400335566,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,1.0,0.890625,0.0412323087266341,1.08984375,6.998772433185578,0.02334284782409668,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.35280001163482666,0.39692801237106323,0.37662720382213594,0.37196800112724304,0.013401318050665304,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.4692479968070984,0.5523840188980103,0.5134752035140991,0.5100640058517456,0.02291433464135784,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.34652799367904663,0.4119040071964264,0.3789471983909607,0.38550400733947754,0.02073945105803335,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.31462401151657104,0.7456960082054138,0.38617280423641204,0.34545600414276123,0.12230201266253077,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.32521599531173706,0.419871985912323,0.36325119733810424,0.34968000650405884,0.03161798848223672,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.289792001247406,0.4663360118865967,0.4091839998960495,0.41655999422073364,0.0446001986506615,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.3761279881000519,0.4416320025920868,0.40686399936676027,0.40540799498558044,0.02257778769899645,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.7172480225563049,0.8663039803504944,0.7723807990550995,0.7591840028762817,0.04164772379451242,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.0195519924163818,1.2216639518737793,1.1253888130187988,1.1453600525856018,0.06548005322243594,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7490559816360474,1.9644800424575806,1.8529024004936219,1.814303994178772,0.08042565617327288,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,3.2479360103607178,3.385279893875122,3.296070408821106,3.2800960540771484,0.04525529026442046,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,6.344799995422363,6.517856121063232,6.464438438415527,6.478623867034912,0.05116674443145098,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.2648000121116638,0.325439989566803,0.28852800130844114,0.28390398621559143,0.01933778377635077,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.7347840070724487,0.862496018409729,0.7769344031810761,0.769216001033783,0.03290485328796285,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.421875,0.0,1.346291201783626,6.0,5.652114648336087,0.636962890625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.9198399782180786,0.9646080136299133,0.9412063956260681,0.9411839842796326,0.014939365085478117,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.5703125,0.0,1.118033988749895,5.0,6.008641773518898,0.580810546875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,1.2796800136566162,1.3484159708023071,1.3006976008415223,1.2929120063781738,0.020807998177176254,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.8828125,0.0,0.6959705453537527,3.0,6.60872850615583,0.38055419921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,1.3630399703979492,1.4430400133132935,1.3909215927124023,1.3892319798469543,0.022335744492366926,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.984375,0.0,0.5201036555341637,3.0,6.798826509158851,0.28302001953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,1.27948796749115,1.3904000520706177,1.3176063895225525,1.309440016746521,0.038060887827312775,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,1.0,0.25,0.3511282039725661,1.875,6.91002266305238,0.1970977783203125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,1.264415979385376,1.3145920038223267,1.2791999936103822,1.2753440141677856,0.014130605249568332,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,1.0,0.375,0.24692938483248605,1.6875,6.955481130775285,0.13909912109375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,1.3081920146942139,1.347648024559021,1.3292255997657776,1.329967975616455,0.014558863679016933,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,1.0,0.625,0.17143053326165383,1.5625,6.9786675275754035,0.09520339965820312,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.242751955986023,1.3112000226974487,1.2747935891151427,1.266207993030548,0.021093073517695057,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,1.0,0.78125,0.11000099875256815,1.296875,6.991308871213679,0.062183380126953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7388160228729248,1.8077759742736816,1.772764801979065,1.772704005241394,0.021056644077284283,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,1.0,0.78125,0.0864630150197678,1.1796875,6.994552526394139,0.048796653747558594,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,2.6563520431518555,2.7063679695129395,2.6785055875778196,2.6791679859161377,0.01639963463052944,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,1.0,0.8671875,0.06127686514721937,1.16015625,6.997291583027146,0.03497934341430664,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,4.386879920959473,4.452256202697754,4.4108480453491214,4.406303882598877,0.019768161791937636,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,1.0,0.884765625,0.041723768525324195,1.1171875,6.998746434318934,0.02298593521118164,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.24208000302314758,0.4028480052947998,0.3041536003351212,0.277103990316391,0.05660721484881584,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.2447039932012558,0.30502399802207947,0.26446720361709597,0.26265600323677063,0.016744548364435372,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.2337920069694519,0.2881599962711334,0.26074880361557007,0.264384001493454,0.016469850143940968,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.23369599878787994,0.28591999411582947,0.25465920120477675,0.25385600328445435,0.01593204652619194,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.2295999974012375,0.26556798815727234,0.24674240052700042,0.2497600018978119,0.010345732066199003,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.21721599996089935,0.29020801186561584,0.2394208014011383,0.2346400022506714,0.018747330357768585,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2717759907245636,0.305184006690979,0.28813759982585907,0.28809599578380585,0.01183422600545854,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.3917759954929352,0.43772798776626587,0.41130879521369934,0.4131519943475723,0.012992473640805227,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.6176639795303345,0.7009919881820679,0.642767995595932,0.6330719888210297,0.024074084919938756,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.0820800065994263,1.1674879789352417,1.1034304022789,1.0977439880371094,0.0233067292981566,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.9809919595718384,2.0415360927581787,2.003715181350708,1.992751955986023,0.022066076434645737,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,3.790112018585205,3.8651199340820312,3.829139161109924,3.8230879306793213,0.025177464160110564,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.212351992726326,0.24383999407291412,0.22760000079870224,0.22723200172185898,0.01050568575837594,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.47494399547576904,0.5184000134468079,0.4920704007148743,0.49169600009918213,0.011991064701471855,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.3984375,0.0,1.3919410907075054,6.0,5.570159765557392,0.667236328125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.6360960006713867,0.7004479765892029,0.6608384013175964,0.6572319865226746,0.020416877242438597,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.625,0.0,1.0307764064044151,4.0,6.138251855282827,0.5382080078125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.780896008014679,0.8301439881324768,0.80346559882164,0.8030399978160858,0.016801230312128875,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.859375,0.0,0.6903350635742038,3.0,6.5943747091218174,0.38067626953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.8607040047645569,0.9195200204849243,0.8783008038997651,0.8751039803028107,0.01719059253115595,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.9765625,0.0,0.49410588440130926,2.75,6.814452474347134,0.271270751953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.833952009677887,0.894752025604248,0.8619967997074127,0.863215982913971,0.018716378851797198,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,1.0,0.25,0.33693529145074724,2.125,6.9186075263155535,0.1867218017578125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.834879994392395,0.8871039748191833,0.8651552021503448,0.8638879954814911,0.015262430894400969,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,1.0,0.375,0.25567294018677456,1.8125,6.952441049154937,0.14349365234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.8518080115318298,0.9097599983215332,0.8810272097587586,0.8751040101051331,0.017005819633271906,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,1.0,0.625,0.1747801353218523,1.53125,6.978069554482723,0.09820938110351562,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.8470079898834229,0.9010239839553833,0.8694015920162201,0.8716959953308105,0.016138881171190216,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,1.0,0.65625,0.1158122428154187,1.3125,6.9901908183358845,0.06445503234863281,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.1698240041732788,1.2311359643936157,1.1888479948043824,1.1890720129013062,0.017978797794492758,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,1.0,0.78125,0.08347181893108634,1.1796875,6.994921772573154,0.046871185302734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.7702720165252686,1.8097599744796753,1.7919103980064393,1.7956640124320984,0.012641295676021557,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,1.0,0.78125,0.06866734477822484,1.20703125,6.996602562938728,0.03801727294921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.968672037124634,3.0278079509735107,2.9899007797241213,2.9824799299240112,0.01816414122631667,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,1.0,0.8984375,0.04399546833977376,1.126953125,6.998607314922362,0.024699926376342773,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.19574399292469025,0.2512960135936737,0.21939200013875962,0.2199999988079071,0.017532156418212565,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.20483200252056122,0.24006399512290955,0.22215040028095245,0.22433599829673767,0.00969639786132892,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.20559999346733093,0.24726399779319763,0.22126719802618028,0.22207999974489212,0.01328093478485499,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.20003199577331543,0.2301120012998581,0.21453119963407516,0.21598400175571442,0.010402239855151332,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.19551999866962433,0.22972799837589264,0.21238719969987868,0.21488000452518463,0.010706095835489097,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.19420799612998962,0.2903999984264374,0.2211231991648674,0.21476799994707108,0.025955008886196004,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2290560007095337,0.289247989654541,0.2543327987194061,0.24939200282096863,0.01663236753503115,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.30831998586654663,0.36953601241111755,0.3324000000953674,0.3288639932870865,0.018617999572156707,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.462911993265152,0.5497919917106628,0.4893856018781662,0.4816960096359253,0.023348152887178286,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,0.7662079930305481,0.8717759847640991,0.788454395532608,0.7744799852371216,0.03174746482604812,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.363935947418213,1.4143040180206299,1.3812703967094422,1.3798720240592957,0.014770450075530007,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.5507519245147705,2.680704116821289,2.579859209060669,2.566223978996277,0.03673283558365955,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize diff --git a/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py b/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py index 3f5a79c..2326466 100644 --- a/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py +++ b/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py @@ -25,6 +25,27 @@ def load(name: str): class FidelityEnvelopeTest(unittest.TestCase): + def test_block_identities_are_parent_sensitive_and_prefix_stable(self) -> None: + module = load("prepare_exact_trace.py") + prefix = list(range(32)) + left = module.block_identities(prefix + [100, 101], 16) + right = module.block_identities(prefix + [200, 201], 16) + self.assertEqual(left[:2], right[:2]) + self.assertNotEqual(left[2], right[2]) + changed_parent = module.block_identities([999] + prefix[1:] + [100, 101], 16) + self.assertNotEqual(left[0], changed_parent[0]) + self.assertNotEqual(left[1], changed_parent[1]) + + def test_root_sessions_follow_parent_chain(self) -> None: + module = load("prepare_exact_trace.py") + rows = [ + {"chat_id": 10, "parent_chat_id": -1}, + {"chat_id": 11, "parent_chat_id": 10}, + {"chat_id": 12, "parent_chat_id": 11}, + {"chat_id": 20, "parent_chat_id": -1}, + ] + self.assertEqual(module.root_sessions(rows), {10: 10, 11: 10, 12: 10, 20: 20}) + def test_materialize_allreduce(self) -> None: module = load("materialize_frontier_allreduce.py") rows = [] diff --git a/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py b/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py index 16f1b68..da2a16d 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py +++ b/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py @@ -638,11 +638,17 @@ def main() -> None: args.output / "moe.csv", args.output / "allreduce.json", ] + batch_composition_augmented = len(args.attention) > 3 manifest = { - "schema_version": "frontier_qwen30_vllm020_frozen_profile.v2", + "schema_version": ( + "frontier_qwen30_vllm020_frozen_profile.v3" + if batch_composition_augmented + else "frontier_qwen30_vllm020_frozen_profile.v2" + ), "profile_id": ( "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-" "fused-mixed-total-conserving" + + ("-pure-prefill-batch-composition" if batch_composition_augmented else "") ), "environment_contract": { "hardware": "NVIDIA H20", @@ -674,6 +680,13 @@ def main() -> None: "ratio, with projected prefill + decode exactly equal to the fused total; " "the split is a schema compatibility attribution, not an observation" ), + "attention_pure_prefill_batch_composition": ( + "Direct FA3 measurements for 2/4 requests at query length 2048 and " + "2/4/8/16 requests at query length 512 for each TP; included only " + "when batch-composition attention inputs are supplied" + if batch_composition_augmented + else "not included" + ), "moe": ( "Replicated gate and fused top-k plus TP-local modular expert kernel; " "expert measurement already includes prepare/finalize so shuffling is zero" From 0c747448b6d6a31ab9a66985f6d64ed824ff2b86 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 09:58:26 +0800 Subject: [PATCH 098/148] Normalize frozen profile CSV line endings --- .../profile-v3-batch-final/attention.csv | 302 +++++++++--------- .../attention_true_mixed_fused.csv | 62 ++-- .../profile-v3-batch-final/manifest.json | 6 +- .../profiles/profile-v3-batch-final/moe.csv | 146 ++++----- .../freeze_frontier_profiles.py | 7 +- 5 files changed, 264 insertions(+), 259 deletions(-) diff --git a/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/attention.csv b/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/attention.csv index a8101ec..cf785bf 100644 --- a/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/attention.csv +++ b/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/attention.csv @@ -1,151 +1,151 @@ -time_stats.attn_input_reshape.min,time_stats.attn_input_reshape.max,time_stats.attn_input_reshape.mean,time_stats.attn_input_reshape.median,time_stats.attn_input_reshape.std,time_stats.attn_kv_cache_save.min,time_stats.attn_kv_cache_save.max,time_stats.attn_kv_cache_save.mean,time_stats.attn_kv_cache_save.median,time_stats.attn_kv_cache_save.std,time_stats.attn_prefill.min,time_stats.attn_prefill.max,time_stats.attn_prefill.mean,time_stats.attn_prefill.median,time_stats.attn_prefill.std,time_stats.attn_decode.min,time_stats.attn_decode.max,time_stats.attn_decode.mean,time_stats.attn_decode.median,time_stats.attn_decode.std,time_stats.attn_output_reshape.min,time_stats.attn_output_reshape.max,time_stats.attn_output_reshape.mean,time_stats.attn_output_reshape.median,time_stats.attn_output_reshape.std,n_embd,n_q_head,n_kv_head,block_size,num_tensor_parallel_workers,max_model_len,batch_size,prefill_chunk_size,kv_cache_size,is_prefill,attention_backend,is_mixed_batch,mode,seq_lens,total_tokens,max_seq_len,min_seq_len,avg_seq_len,equal_seq_len,seq_len_variance,seq_len_std,seq_len_cv,is_chunked_prefill_sample,chunk_start_token,chunk_end_token,total_prefill_tokens,profiling_precision,model_arch,quant_signature,measurement_type,is_true_mixed_batch,prefill_seq_lens,prefill_kv_cache_sizes,decode_kv_cache_sizes,num_prefill_seqs,num_decode_seqs,decode_batch_size,total_batch_size,total_decode_tokens,decode_avg_kv_cache_size,batch_composition_ratio,batch_spec,projection_policy -0.0,0.0,0.0,0.0,0.0,0.01414399966597557,0.028863999992609024,0.019705599918961526,0.01771199982613325,0.005157200849836681,0.047968000173568726,0.07046400010585785,0.05810240097343922,0.05810240097343922,0.007477463486041561,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.04947200044989586,0.020412799902260303,0.01635199971497059,0.010107497379722417,0.046560000628232956,0.08323200047016144,0.05587520003318787,0.05587520003318787,0.011126758739503428,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.022207999601960182,0.017033600155264138,0.015312000177800655,0.002819235991970241,0.05104000121355057,0.07692799717187881,0.056396800279617305,0.056396800279617305,0.007481982178637539,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.015072000212967396,0.022272000089287758,0.01706880023702979,0.01616000011563301,0.002460889579319197,0.06931199878454208,0.0838719978928566,0.07432000041007995,0.07432000041007995,0.004777766433175866,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.018592000007629395,0.028543999418616295,0.02095999978482723,0.019183999858796597,0.003198175496053494,0.12179200351238251,0.15408000349998474,0.1307712011039257,0.1307712011039257,0.00858807797538298,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.027775999158620834,0.03385600075125694,0.030131200328469276,0.029680000618100166,0.0021152558103575215,0.32678401470184326,0.3450239896774292,0.33396480381488797,0.33396480381488797,0.0045872424917606375,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.04438399896025658,0.05084799975156784,0.046540799736976626,0.04531199857592583,0.002277905811237223,1.0959680080413818,1.1151360273361206,1.0999775886535645,1.0999775886535645,0.005694403246120485,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.078015998005867,0.08691199868917465,0.08114239946007729,0.08019199967384338,0.00292795706334475,4.070400238037109,4.113152027130127,4.087088012695312,4.087088012695312,0.013660567012509554,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.05167999863624573,0.022115200012922286,0.017583999782800674,0.010340094822340818,0.05196800082921982,0.09011200070381165,0.06328320093452933,0.06328320093452933,0.012557341255467452,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.01583999954164028,0.026623999699950218,0.018927999772131443,0.017376000061631203,0.003514650316260619,0.06681600213050842,0.07993599772453308,0.0725280001759529,0.0725280001759529,0.004343502558613716,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.01648000068962574,0.030880000442266464,0.01945280022919178,0.017967999912798405,0.004096211183007485,0.1311360001564026,0.1546880006790161,0.13908160030841826,0.13908160030841826,0.007511906874366178,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.03558399900794029,0.020851199887692927,0.018559999763965607,0.005235911594130716,0.32950401306152344,0.350271999835968,0.33912960588932034,0.33912960588932034,0.006027400986663648,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.019328000023961067,0.040608000010252,0.022790400311350822,0.020704000256955624,0.006113051965778337,1.1415679454803467,1.1518720388412476,1.144483208656311,1.144483208656311,0.0032332311374389127,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.015807999297976494,0.030688000842928886,0.019596799835562707,0.01774400006979704,0.004343384771033462,0.0,0.0,0.0,0.0,0.0,0.049056001007556915,0.07580800354480743,0.05948160067200661,0.05948160067200661,0.009031541471446955,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.02486399933695793,0.01923839971423149,0.018079999834299088,0.0032282528537266424,0.0,0.0,0.0,0.0,0.0,0.05142400041222572,0.07353600114583969,0.059328000620007516,0.059328000620007516,0.0073307807735143084,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.016543999314308167,0.03977600112557411,0.021379199624061585,0.018511999398469925,0.006593576176246171,0.0,0.0,0.0,0.0,0.0,0.0488319993019104,0.06435199826955795,0.05479039996862411,0.05479039996862411,0.005672522998491864,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.01635199971497059,0.02844800055027008,0.019267200119793416,0.017952000722289085,0.0035068687666949577,0.0,0.0,0.0,0.0,0.0,0.049855999648571014,0.07798399776220322,0.05986879989504815,0.05986879989504815,0.01043914754878828,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.016383999958634377,0.026079999282956123,0.01923519968986511,0.017791999503970146,0.0032161974331284568,0.0,0.0,0.0,0.0,0.0,0.058111999183893204,0.1045759990811348,0.06708480007946492,0.06708480007946492,0.013479022462646494,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.04057599976658821,0.019100800156593323,0.015455999877303839,0.007512281243011577,0.0,0.0,0.0,0.0,0.0,0.05363199859857559,0.07782399654388428,0.06090559959411622,0.06090559959411622,0.007544620176348091,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.02191999927163124,0.016684799920767546,0.01563199982047081,0.0024293118621811216,0.0,0.0,0.0,0.0,0.0,0.0629120022058487,0.07891199737787247,0.06891520097851753,0.06891520097851753,0.005472695665695425,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.038943998515605927,0.018313600029796363,0.01561600062996149,0.007127270260115769,0.0,0.0,0.0,0.0,0.0,0.08675199747085571,0.10662399977445602,0.09391999915242194,0.09391999915242194,0.006988099589086635,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.054687999188899994,0.021439999900758268,0.01539199985563755,0.012052764849597775,0.0,0.0,0.0,0.0,0.0,0.13488000631332397,0.1528639942407608,0.1431359991431236,0.1431359991431236,0.005436271464033599,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.041919998824596405,0.01899839974939823,0.015343999955803156,0.007989843526623287,0.0,0.0,0.0,0.0,0.0,0.06176000088453293,0.08374399691820145,0.06747519969940186,0.06747519969940186,0.0066067747128778,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.016256000846624374,0.11353600025177002,0.042761600017547606,0.028960000723600388,0.029104301538020762,0.0,0.0,0.0,0.0,0.0,0.09734400361776352,0.14422400295734406,0.11392960175871848,0.11392960175871848,0.013198594600417867,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.034143999218940735,0.018918400071561335,0.01643200032413006,0.005500943993080684,0.0,0.0,0.0,0.0,0.0,0.12918399274349213,0.15087999403476715,0.13807999789714814,0.13807999789714814,0.007658330538677587,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.03641600161790848,0.0198208000510931,0.01780799962580204,0.0057264128169845765,0.0,0.0,0.0,0.0,0.0,0.22099199891090393,0.23904000222682953,0.2293503984808922,0.2293503984808922,0.004861342907006028,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.035840000957250595,0.018908800091594458,0.015792000107467175,0.0064374817924757475,0.0,0.0,0.0,0.0,0.0,0.10134399682283401,0.12201599776744843,0.10896319895982742,0.10896319895982742,0.006336330809165179,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.03417599946260452,0.017846399918198586,0.014800000004470348,0.006495007539635255,0.0,0.0,0.0,0.0,0.0,0.13126400113105774,0.15561600029468536,0.1389280006289482,0.1389280006289482,0.008381472811075022,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03519999980926514,0.019168000388890504,0.01600000075995922,0.005995522477654695,0.0,0.0,0.0,0.0,0.0,0.21728000044822693,0.2343679964542389,0.2231455981731415,0.2231455981731415,0.004720730646739123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.03670400008559227,0.018441599886864425,0.015023999847471714,0.006596535162793127,0.0,0.0,0.0,0.0,0.0,0.39321601390838623,0.4524799883365631,0.4058080047369003,0.4058080047369003,0.01578349755088941,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.022143999114632607,0.016912000067532063,0.015168000012636185,0.0028156089295136347,0.0,0.0,0.0,0.0,0.0,0.15587200224399567,0.3079040050506592,0.17838079929351805,0.17838079929351805,0.04355865575265927,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.02112000063061714,0.015516800060868263,0.01488000014796853,0.001940870731593904,0.0,0.0,0.0,0.0,0.0,0.21587200462818146,0.23561599850654602,0.22250880002975468,0.22250880002975468,0.006181951170646666,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.039264000952243805,0.023609600123018028,0.02131200022995472,0.007236979625711548,0.0,0.0,0.0,0.0,0.0,0.408735990524292,0.470335990190506,0.4336863994598388,0.4336863994598388,0.01844662383160074,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.025407999753952026,0.016227199975401164,0.014960000291466713,0.0031617375441736185,0.0,0.0,0.0,0.0,0.0,0.7412800192832947,0.7627840042114258,0.7464000046253203,0.7464000046253203,0.006112167448837547,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.01462399959564209,0.02812799997627735,0.020652799773961304,0.021359999664127827,0.004308706957613102,0.028383498565450627,0.039859687970646644,0.032492258074592426,0.032492258074592426,0.00453597266208597,0.029312501176103633,0.04116431058787463,0.03355574193327539,0.03355574193327539,0.004684436757701648,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.034143999218940735,0.023171199765056372,0.024255999363958836,0.005908614918096971,0.03333159243114438,0.038935341782478095,0.03580428402241854,0.03580428402241854,0.002082270297044095,0.03633240903369937,0.04244065945634484,0.03902771507087562,0.03902771507087562,0.0022697354261490147,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.02611199952661991,0.019635199941694735,0.018112000077962875,0.0038298050749257795,0.04189529417991216,0.057484239920526384,0.04744885718421094,0.04744885718421094,0.004779830748455743,0.051672703037266184,0.07089975418195164,0.05852234134479412,0.05852234134479412,0.005895334539786378,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.029311999678611755,0.018662399891763926,0.01673599984496832,0.0044017112162725355,0.04322973959325901,0.05049827064705393,0.04535414343408448,0.04535414343408448,0.0022944156000240697,0.08733025617719538,0.10201372836398578,0.09162185574241775,0.09162185574241775,0.004635047631846023,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.014592000283300877,0.026367999613285065,0.016336000058799982,0.01515199989080429,0.003415353455946402,0.06031842775160765,0.06618323188375198,0.06274415549817247,0.06274415549817247,0.001985647289644255,0.14768157653992678,0.16204076249052324,0.15362064543185072,0.15362064543185072,0.004861590945216247,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.02454400062561035,0.017167999967932702,0.016191999427974224,0.0028685016454498436,0.09128700688359712,0.09689150775996329,0.09360396051475776,0.09360396051475776,0.0013477542744916764,0.2740889887523892,0.2909164873209846,0.2810456356995366,0.2810456356995366,0.004046628526808561,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.01881599985063076,0.03097599931061268,0.024598400108516216,0.024848000146448612,0.0037539437391565975,0.1035249255866932,0.10603132147437412,0.10399797220840973,0.10399797220840973,0.0007025109429056814,0.5652750707893444,0.5789606938874117,0.5678580377517648,0.5678580377517648,0.003835906384194897,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.018624000251293182,0.043487999588251114,0.024460799992084503,0.019600000232458115,0.008922081629781394,0.196169204945307,0.2076140047945799,0.2008444429250931,0.2008444429250931,0.00394350953292805,1.1196708009293268,1.1849940417370974,1.1463555573610091,1.1463555573610091,0.022508285530529783,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.027135999873280525,0.03667199984192848,0.02945920005440712,0.028256000019609928,0.0029446678408169553,0.37757279619664613,0.3898113624476372,0.38075520430942184,0.38075520430942184,0.0036600203831042254,0.2522831942132049,0.2604606493092597,0.25440958703617433,0.25440958703617433,0.0024455194930253126,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.0435199998319149,0.049695998430252075,0.04502719938755036,0.04395199939608574,0.002035785660169308,1.1048984388245497,1.1189053886476108,1.1112371236754128,1.1112371236754128,0.0050220696724624985,0.1395495077239122,0.14131859607071193,0.14035008841031085,0.14035008841031085,0.0006342911944856293,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.018783999606966972,0.024288000538945198,0.020627199858427047,0.019567999988794327,0.0019457052717059358,0.09455999732017517,0.12185599654912949,0.10207359939813614,0.10207359939813614,0.00753346544014261,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,2,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512]",1024,512,512,512.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.027295999228954315,0.034752000123262405,0.02943360023200512,0.028528000228106976,0.0023159160681123767,0.14416000247001648,0.15904000401496887,0.14979200065135959,0.14979200065135959,0.00480624675866005,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,4,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512]",2048,512,512,512.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.043455999344587326,0.05084799975156784,0.045500800386071204,0.04411200061440468,0.002490556062831049,0.24454399943351746,0.25865599513053894,0.2504959970712662,0.2504959970712662,0.003778494140301353,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512]",4096,512,512,512.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.07673600316047668,0.08374399691820145,0.07970559895038605,0.07873599976301193,0.0022907976135004057,0.445248007774353,0.4758400022983551,0.45409599840641024,0.45409599840641024,0.009133230340383306,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512]",8192,512,512,512.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.043296001851558685,0.051072001457214355,0.044972800090909,0.04399999976158142,0.002451216824441962,0.6147199869155884,0.6290879845619202,0.6204927921295166,0.6204927921295166,0.004344846739788608,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,2,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048]",4096,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.07737600058317184,0.09123200178146362,0.08094720020890236,0.07980800047516823,0.004065264798368611,1.1744320392608643,1.1887680292129517,1.178323209285736,1.178323209285736,0.004395876469319665,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,4,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048, 2048, 2048]",8192,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.02364799939095974,0.01809599995613098,0.016352000646293163,0.0035481127058959038,0.04822399839758873,0.08566399663686752,0.05961279980838299,0.05961279980838299,0.011445665413968877,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.020864000543951988,0.01668160008266568,0.015664000064134598,0.0025769063833097584,0.049695998430252075,0.08057600259780884,0.05882879942655563,0.05882879942655563,0.009515126108519331,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.028704000636935234,0.017430400010198355,0.015056000091135502,0.004349294613335555,0.049855999648571014,0.07366400212049484,0.05459520071744919,0.05459520071744919,0.0069610920757925574,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.03161599859595299,0.017788799852132796,0.015711999963968992,0.005005385723318659,0.06102399900555611,0.08179199695587158,0.06650560013949873,0.06650560013949873,0.006947995595801105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.04416000097990036,0.019200000166893005,0.015488000120967627,0.008561241323364038,0.08054400235414505,0.09196799993515015,0.08607039973139763,0.08607039973139763,0.004145329035483754,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.023391999304294586,0.019667199812829494,0.018463999964296818,0.0022577669687832585,0.18729600310325623,0.20585599541664124,0.19546559900045393,0.19546559900045393,0.006339068824303663,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.026240000501275063,0.03446400165557861,0.028297600522637367,0.027088000439107418,0.0025148441522922374,0.5754240155220032,0.5889919996261597,0.5800191938877105,0.5800191938877105,0.003858869273829596,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.04182400181889534,0.047200001776218414,0.043036799877882004,0.0423360001295805,0.0017073405772076728,2.063199996948242,2.0787200927734375,2.067151999473572,2.067151999473572,0.004959651271127835,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.05648000165820122,0.02270399993285537,0.017935999669134617,0.012377915150727689,0.049536000937223434,0.07196799665689468,0.056015999615192415,0.056015999615192415,0.0070476637552742884,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.01408000010997057,0.021344000473618507,0.01611520005390048,0.014928000047802925,0.0025499884993961702,0.07072000205516815,0.2642880082130432,0.1588256008923054,0.1588256008923054,0.053220347086102376,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.01360000018030405,0.03014400042593479,0.016975999902933837,0.015056000091135502,0.004715159697364111,0.08393599838018417,0.11036799848079681,0.09160000011324881,0.09160000011324881,0.007911669434472792,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.020767999812960625,0.016233599931001663,0.015392000321298838,0.002202615907995427,0.1998399943113327,0.22070400416851044,0.20855360180139543,0.20855360180139543,0.00689307200230667,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.033440001308918,0.018684800155460833,0.016048000194132328,0.00553991005639174,0.6043199896812439,0.635807991027832,0.6126143991947173,0.6126143991947173,0.008745933953408096,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.013824000023305416,0.03359999880194664,0.017811199743300678,0.014944000169634819,0.005926140483565472,0.0,0.0,0.0,0.0,0.0,0.045471999794244766,0.07539200037717819,0.054758400097489356,0.054758400097489356,0.010253548506101549,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02409599907696247,0.01641279999166727,0.01473599998280406,0.003347158638713987,0.0,0.0,0.0,0.0,0.0,0.0461760014295578,0.07529599964618683,0.05460800044238568,0.05460800044238568,0.009748937798340135,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.028416000306606293,0.018611199874430894,0.01598400017246604,0.005209875093863647,0.0,0.0,0.0,0.0,0.0,0.048128001391887665,0.07897599786520004,0.061353600397706036,0.061353600397706036,0.010488153655157845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.025728000327944756,0.016092800162732603,0.01512000011280179,0.003300888123052605,0.0,0.0,0.0,0.0,0.0,0.04864000156521797,0.07648000121116638,0.05810560062527656,0.05810560062527656,0.009473544218404408,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.03551999852061272,0.018588799890130757,0.016064000315964222,0.006071057437992447,0.0,0.0,0.0,0.0,0.0,0.04822399839758873,0.07862400263547897,0.05660480037331582,0.05660480037331582,0.009565394213730401,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.028831999748945236,0.01699519995599985,0.015024000313133001,0.004285000744868188,0.0,0.0,0.0,0.0,0.0,0.04854400083422661,0.06719999760389328,0.05778240002691746,0.05778240002691746,0.006852125554679805,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.0144640002399683,0.024927999824285507,0.0161183999851346,0.01521599991247058,0.0029774808524673907,0.0,0.0,0.0,0.0,0.0,0.05379199981689453,0.08966399729251862,0.06270079985260964,0.06270079985260964,0.009983591277092696,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03001599945127964,0.017648000083863736,0.01547200046479702,0.004585126309484848,0.0,0.0,0.0,0.0,0.0,0.061664000153541565,0.07692799717187881,0.06704320013523103,0.06704320013523103,0.005500191798490629,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.026367999613285065,0.016947199776768684,0.015039999969303608,0.0037951566103550205,0.0,0.0,0.0,0.0,0.0,0.08799999952316284,0.111455999314785,0.0964031994342804,0.0964031994342804,0.007397541615558088,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.01369599997997284,0.021247999742627144,0.015359999984502793,0.014640000183135271,0.0020942770950814317,0.0,0.0,0.0,0.0,0.0,0.051711998879909515,0.07065600156784058,0.058054400235414506,0.058054400235414506,0.006633034815910025,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.0307839997112751,0.017148799914866685,0.015039999969303608,0.004882374686312284,0.0,0.0,0.0,0.0,0.0,0.061919998377561576,0.07843200117349625,0.06628479920327664,0.06628479920327664,0.004962852689801192,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.013887999579310417,0.02969600073993206,0.017500799987465142,0.015232000034302473,0.00467273319705314,0.0,0.0,0.0,0.0,0.0,0.08819200098514557,0.11097600311040878,0.09493440166115762,0.09493440166115762,0.007509235042985577,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.022112000733613968,0.01751680001616478,0.016047999262809753,0.0030306621792915785,0.0,0.0,0.0,0.0,0.0,0.13065600395202637,0.15110400319099426,0.13857279866933822,0.13857279866933822,0.00750137841249771,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.04012800008058548,0.01892479993402958,0.01550400024279952,0.007459545267816961,0.0,0.0,0.0,0.0,0.0,0.06278400123119354,0.08259200304746628,0.0704512007534504,0.0704512007534504,0.005979055984382744,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.02236800082027912,0.016332800220698118,0.014928000047802925,0.002784044363186731,0.0,0.0,0.0,0.0,0.0,0.1003199964761734,0.1279360055923462,0.10921279862523078,0.10921279862523078,0.00862769617046716,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0225600004196167,0.016128000058233737,0.014800000004470348,0.002958953332547708,0.0,0.0,0.0,0.0,0.0,0.13116799294948578,0.14812800288200378,0.13783999979496003,0.13783999979496003,0.005696384361148053,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.029983999207615852,0.017468800116330386,0.01508800033479929,0.0047379550962483065,0.0,0.0,0.0,0.0,0.0,0.217631995677948,0.2447360008955002,0.22715839892625808,0.22715839892625808,0.008831828221847138,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02304000034928322,0.016512000095099212,0.014512000139802694,0.0035026774828624254,0.0,0.0,0.0,0.0,0.0,0.11020799726247787,0.12307199835777283,0.11600959971547126,0.11600959971547126,0.004667637266950902,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.042080000042915344,0.017510399967432023,0.014607999939471483,0.00823117883530167,0.0,0.0,0.0,0.0,0.0,0.15702399611473083,0.17587199807167053,0.16399359852075576,0.16399359852075576,0.006588299074676393,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0208320003002882,0.015299199987202883,0.01462399959564209,0.001916768086505386,0.0,0.0,0.0,0.0,0.0,0.21779200434684753,0.2415360063314438,0.2260768011212349,0.2260768011212349,0.007251352080685236,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014015999622642994,0.021088000386953354,0.015619200188666582,0.01473599998280406,0.002013227452302141,0.0,0.0,0.0,0.0,0.0,0.3959999978542328,0.4152640104293823,0.40332479774951924,0.40332479774951924,0.006942401431914052,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.03379200026392937,0.020595200080424547,0.019600000232458115,0.006548754248881344,0.026192623739694512,0.040006882507168426,0.029155180178492036,0.029155180178492036,0.00408828748438241,0.024591375524545756,0.037561119537986146,0.027372820355088746,0.027372820355088746,0.0038383559348575944,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.020896000787615776,0.015318400040268899,0.014431999996304512,0.0019640312960926966,0.029349018208693862,0.06236262941356679,0.03714313592014963,0.03714313592014963,0.009618313791872569,0.028826981462526914,0.06125337308649041,0.036482463672774496,0.036482463672774496,0.009447230957011863,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.032127998769283295,0.017286399938166143,0.014479999896138906,0.005536495446636193,0.03273085874558354,0.04394578491937082,0.03563837490653034,0.03563837490653034,0.003429601730256567,0.03488514202593899,0.04683821345079978,0.037984025407085474,0.037984025407085474,0.0036553316361902684,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.021503999829292297,0.015078400075435639,0.01425600005313754,0.002238435975478849,0.042100813549974185,0.051784144690147756,0.045378692890289625,0.045378692890289625,0.003184109940650555,0.05111518843748309,0.06287185663927425,0.05509490773570219,0.05509490773570219,0.0038658725544299132,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.02687999978661537,0.017667199857532977,0.01566399959847331,0.003864811846387173,0.048475323773821306,0.05599956978723733,0.05123499252968345,0.05123499252968345,0.002427379820423941,0.08429268135885387,0.09737642835214408,0.0890914090616175,0.0890914090616175,0.0042209177332077005,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.01462399959564209,0.03359999880194664,0.019804799742996693,0.016032000072300434,0.006750519908145474,0.07121508474579985,0.07292308367136396,0.07194410845270183,0.07194410845270183,0.0005500065915943844,0.14760091249712767,0.15114092373010238,0.14911189243564577,0.14911189243564577,0.0011399477384397005,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.021856000646948814,0.016835200227797033,0.015263999812304974,0.00283854448975065,0.08146338272142935,0.08560865714384096,0.0834932625520734,0.0834932625520734,0.0012259635905347874,0.2782486219401307,0.29240733788179385,0.2851819365989658,0.2851819365989658,0.004187435731481665,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.04495999962091446,0.02143679987639189,0.015856000129133463,0.009709987872683342,0.1194459208702178,0.12302524755181463,0.12053885718696096,0.12053885718696096,0.001005118119341339,0.5597220650459199,0.5764947443228802,0.5648435507167102,0.5648435507167102,0.004709970715400788,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.018112000077962875,0.026496000587940216,0.019318400137126445,0.018432000651955605,0.0024249605112359905,0.19770253574610402,0.222811797868348,0.2054811520619412,0.2054811520619412,0.008000944254868043,0.13941746080159492,0.157124211777114,0.14490284788179197,0.14490284788179197,0.005642170080515992,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.02595200017094612,0.03155200183391571,0.02727359998971224,0.026575999334454536,0.0016260948764843166,0.6014684881116659,0.6095472988212,0.6046757700946376,0.6046757700946376,0.0023537435563177303,0.11325152264578045,0.11477269562841545,0.11385542721485654,0.11385542721485654,0.0004431903698023628,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.03359999880194664,0.01814719969406724,0.015375999733805656,0.005678506740662529,0.06774400174617767,0.07891199737787247,0.07312640026211739,0.07312640026211739,0.003826583051422731,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,2,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512]",1024,512,512,512.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.01788800023496151,0.028543999418616295,0.021340799890458582,0.019504000432789326,0.003718627037219618,0.09548799693584442,0.1327359974384308,0.10823359936475753,0.10823359936475753,0.013230635292043864,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,4,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512]",2048,512,512,512.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.02627200074493885,0.030368000268936157,0.027184000052511693,0.02643200010061264,0.0013545679205210022,0.14364799857139587,0.1597760021686554,0.15008639842271806,0.15008639842271806,0.005367723415171222,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512]",4096,512,512,512.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.04150399938225746,0.04726399853825569,0.042950399965047834,0.04224000126123428,0.001720147501765378,0.2433920055627823,0.28963199257850647,0.2549152016639709,0.2549152016639709,0.013450991019687407,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512]",8192,512,512,512.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.025887999683618546,0.03407999873161316,0.028303999826312064,0.026367999613285065,0.0028877495368841042,0.325439989566803,0.3441599905490875,0.33442879617214205,0.33442879617214205,0.005693597811441922,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,2,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048]",4096,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.04137599840760231,0.05084799975156784,0.044828799366950986,0.043087998405098915,0.003560363865797613,0.6110399961471558,0.6421759724617004,0.6204223990440368,0.6204223990440368,0.011214490370794758,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,4,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048, 2048, 2048]",8192,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.022784000262618065,0.016435200069099664,0.015519999898970127,0.0023688301421469523,0.04879999905824661,0.09139200299978256,0.06054079942405224,0.06054079942405224,0.012152608702448775,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014816000126302242,0.02844800055027008,0.017008000146597625,0.015696000307798386,0.003941466294662679,0.047807998955249786,0.07356800138950348,0.05626560002565384,0.05626560002565384,0.00842179125412236,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.053408000618219376,0.019353600218892097,0.01532800029963255,0.01138302400841626,0.048448000103235245,0.0785600021481514,0.0556256003677845,0.0556256003677845,0.009348274502616908,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.03139200061559677,0.01823679991066456,0.016159999649971724,0.004860071238302512,0.055424001067876816,0.08505599945783615,0.0640383992344141,0.0640383992344141,0.00921448636178623,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.026528000831604004,0.017324799951165915,0.01536000007763505,0.0036004822686428305,0.07660800218582153,0.0942080020904541,0.08209280073642732,0.08209280073642732,0.00515895587669752,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.021247999742627144,0.016403199825435876,0.01532800029963255,0.001995897022647934,0.11395200341939926,0.15113599598407745,0.12431039959192276,0.12431039959192276,0.011164431123683732,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.017184000462293625,0.028672000393271446,0.019865600019693376,0.017823999747633934,0.0035798491315929977,0.3171840012073517,0.3341119885444641,0.3261695951223373,0.3261695951223373,0.005111046452878918,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.024639999493956566,0.030400000512599945,0.02656640000641346,0.02556800004094839,0.0020189846603237303,1.0648640394210815,1.0828479528427124,1.0715327858924866,1.0715327858924866,0.005558639263049851,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014976000413298607,0.021695999428629875,0.017305599898099898,0.01593599934130907,0.0025718046517268054,0.04956800118088722,0.07932800054550171,0.06228480041027069,0.06228480041027069,0.01027160349757827,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.01539199985563755,0.03580800071358681,0.019686400331556796,0.017136000096797943,0.005918387811304003,0.05158400163054466,0.10713600367307663,0.06364160068333148,0.06364160068333148,0.015832957809696766,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02223999984562397,0.016672000009566545,0.015887999907135963,0.0020934945946034563,0.06521599739789963,0.08902399986982346,0.07520959973335266,0.07520959973335266,0.007913840684102929,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.015231999568641186,0.04028800129890442,0.019971200078725816,0.01646399963647127,0.007351305886577286,0.14467200636863708,0.16844800114631653,0.15363519936800005,0.15363519936800005,0.008174435899956223,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.015519999898970127,0.02304000034928322,0.01775679988786578,0.016784000210464,0.0025077243712082584,0.33740800619125366,0.35343998670578003,0.3445120006799698,0.3445120006799698,0.00445648463590358,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.02393599972128868,0.01809599995613098,0.01654400024563074,0.002998393102466254,0.0,0.0,0.0,0.0,0.0,0.0504320003092289,0.0843840017914772,0.060083200410008426,0.060083200410008426,0.00986959318572296,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.030047999694943428,0.019510399922728537,0.017680000513792038,0.004065185265963332,0.0,0.0,0.0,0.0,0.0,0.05004800111055374,0.07036799937486649,0.059315200522542,0.059315200522542,0.006537768821329647,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.02393599972128868,0.018035199772566558,0.016512000001966953,0.0030667689116777724,0.0,0.0,0.0,0.0,0.0,0.05100800096988678,0.06735999882221222,0.058387200161814694,0.058387200161814694,0.005832787739241381,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.026208000257611275,0.017500799987465142,0.01646399963647127,0.0030666103306165714,0.0,0.0,0.0,0.0,0.0,0.05023999884724617,0.07254400104284286,0.057254400476813315,0.057254400476813315,0.006890853653068151,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.027327999472618103,0.0179776000790298,0.01648000068962574,0.003783417447531392,0.0,0.0,0.0,0.0,0.0,0.04819199815392494,0.07100799679756165,0.05621119923889638,0.05621119923889638,0.007824391484124725,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.036448001861572266,0.019136000238358975,0.016704000532627106,0.006076530095624803,0.0,0.0,0.0,0.0,0.0,0.047488000243902206,0.08441600203514099,0.0588383998721838,0.0588383998721838,0.011275940726332522,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.015135999768972397,0.033952001482248306,0.02119360016658902,0.018000000156462193,0.007136646614305304,0.0,0.0,0.0,0.0,0.0,0.04931199923157692,0.06992000341415405,0.05755840018391609,0.05755840018391609,0.007297587694315226,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.034272000193595886,0.0204415999352932,0.017311999574303627,0.00695404701803013,0.0,0.0,0.0,0.0,0.0,0.05215999856591225,0.0735040009021759,0.06117440015077591,0.06117440015077591,0.007381118436894922,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03743999823927879,0.02012479966506362,0.01775999926030636,0.006181045509079057,0.0,0.0,0.0,0.0,0.0,0.06355199962854385,0.08508799970149994,0.07019200026988984,0.07019200026988984,0.0062246580442117845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.032607998698949814,0.01959999995306134,0.017487999983131886,0.004882475068460749,0.0,0.0,0.0,0.0,0.0,0.04918399825692177,0.0865280032157898,0.05973760038614274,0.05973760038614274,0.011546847942113974,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02675200067460537,0.017430400010198355,0.016560000367462635,0.003243135094239819,0.0,0.0,0.0,0.0,0.0,0.052671998739242554,0.08367999643087387,0.06238719932734965,0.06238719932734965,0.011207824780630104,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.03612799942493439,0.019327999837696553,0.01688000001013279,0.006058251425153085,0.0,0.0,0.0,0.0,0.0,0.06364800035953522,0.07878399640321732,0.06935679838061332,0.06935679838061332,0.004515588328677643,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.03711999952793121,0.019670399930328132,0.017152000218629837,0.006165994646522264,0.0,0.0,0.0,0.0,0.0,0.09071999788284302,0.11507199704647064,0.10252480059862136,0.10252480059862136,0.008782544051535657,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.026528000831604004,0.018396800104528665,0.01601599995046854,0.004279788048986283,0.0,0.0,0.0,0.0,0.0,0.05331199988722801,0.07977599650621414,0.06076480001211167,0.06076480001211167,0.008761579122069606,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.02643200010061264,0.01726400004699826,0.015855999663472176,0.003210008417242029,0.0,0.0,0.0,0.0,0.0,0.062431998550891876,0.08246400207281113,0.06970879957079888,0.06970879957079888,0.007016341676068233,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.02223999984562397,0.0166015999391675,0.015887999907135963,0.00204913969129354,0.0,0.0,0.0,0.0,0.0,0.10127999633550644,0.1141119971871376,0.10621120035648347,0.10621120035648347,0.004029318311754605,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03363199904561043,0.019305599946528675,0.016671999357640743,0.0055445582307981234,0.0,0.0,0.0,0.0,0.0,0.1319040060043335,0.15839999914169312,0.14040640145540234,0.14040640145540234,0.007998041073596942,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014368000440299511,0.04447999969124794,0.018908800091594458,0.0157279996201396,0.0086814937461721,0.0,0.0,0.0,0.0,0.0,0.06428799778223038,0.08982399851083755,0.07349760085344315,0.07349760085344315,0.008605284512197258,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.028672000393271446,0.01890560006722808,0.016080000437796116,0.004797584627815021,0.0,0.0,0.0,0.0,0.0,0.11123199760913849,0.14115199446678162,0.11942399889230729,0.11942399889230729,0.008513394706791027,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.024927999824285507,0.016915200091898442,0.015216000378131866,0.003374147499442635,0.0,0.0,0.0,0.0,0.0,0.15887999534606934,0.18111999332904816,0.16934399753808976,0.16934399753808976,0.007415181260761123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.026944000273942947,0.017737600207328796,0.015199999790638685,0.00415598714375255,0.0,0.0,0.0,0.0,0.0,0.2192319929599762,0.23472000658512115,0.22809920012950893,0.22809920012950893,0.004730841376327335,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.05052800104022026,0.020320000313222408,0.015520000364631414,0.010604522177924678,0.024270629882498958,0.03340247625954076,0.028446344104128624,0.028446344104128624,0.0032330369099793834,0.023697370291069768,0.03261352722995356,0.027774456325453972,0.027774456325453972,0.0031566742680923386,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.033663999289274216,0.019987199828028678,0.018240000121295452,0.005522929139253732,0.0259194055660947,0.03719755183990719,0.029916029687899703,0.029916029687899703,0.003966666590554318,0.027104595853449518,0.03889844644729374,0.03128396954022015,0.03128396954022015,0.004148046317967881,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.014592000283300877,0.03299200162291527,0.019840000104159115,0.016207999549806118,0.006546343008726814,0.02587869595769926,0.03763167265431482,0.030174939058162802,0.030174939058162802,0.0037303581782277364,0.026473304070195713,0.03849632587654989,0.03086826083864964,0.03086826083864964,0.003816069654529222,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.013824000023305416,0.021023999899625778,0.016304000187665223,0.015584000386297703,0.0023236165806545476,0.031810621525966996,0.04156949936878106,0.0346749350032807,0.0346749350032807,0.003130209181143985,0.035677378270900374,0.04662250161636451,0.038889864871740294,0.038889864871740294,0.0035107033960828727,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.02175999991595745,0.01569600012153387,0.015008000191301107,0.0020882666168036863,0.038211712107062853,0.07212229256520057,0.04364509673334097,0.04364509673334097,0.009671953074025085,0.0476442859917874,0.08992570455184198,0.05441890342616107,0.05441890342616107,0.01205947791784038,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.014368000440299511,0.033824000507593155,0.017628800217062236,0.014864000026136637,0.005791495403857738,0.05214261250030033,0.06266261508706669,0.05677791295527661,0.05677791295527661,0.0030298223079651514,0.08648138506878382,0.10392938682791132,0.09416928531647478,0.09416928531647478,0.005025126612204489,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.031199999153614044,0.017900799959897996,0.015343999955803156,0.004974475661316249,0.06411958891421111,0.07405276123263956,0.06793047918211377,0.06793047918211377,0.0033918658556700006,0.14058441263169497,0.16236323591492058,0.1489399211274489,0.1489399211274489,0.00743678300375353,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.03855999931693077,0.018495999928563833,0.015647999942302704,0.006918530056761627,0.09872985549401277,0.10683454583043753,0.10185316839884552,0.10185316839884552,0.0020802254037042074,0.2743261390166379,0.2968454508984596,0.2830044295482752,0.2830044295482752,0.005780016596065092,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.030400000512599945,0.01977920001372695,0.01726400014013052,0.005350883464206169,0.13918872472233365,0.16279522855335207,0.1501912864839173,0.1501912864839173,0.008992185529011513,0.11892328861766266,0.13909276049083735,0.1283239123428725,0.1283239123428725,0.007682951884956939,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.01727999933063984,0.02223999984562397,0.01819519978016615,0.017680000513792038,0.0014397298270620873,0.3728044181625443,0.38295504353701676,0.3761264483787333,0.3761264483787333,0.0033660272332490977,0.07967557017401883,0.08184495665371805,0.08038555277807365,0.08038555277807365,0.0007193856241088455,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio -0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.034591998904943466,0.01941439984366298,0.01756799966096878,0.005541380584373771,0.05951999872922897,0.08268799632787704,0.06715519949793816,0.06715519949793816,0.006657802116288364,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,2,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512]",1024,512,512,512.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.03283200040459633,0.019180799927562477,0.016543999314308167,0.0052187911233635975,0.07199999690055847,0.08675199747085571,0.07749439924955369,0.07749439924955369,0.004849874741156772,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,4,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512]",2048,512,512,512.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.01724799908697605,0.02284800074994564,0.01928640007972717,0.018400000408291817,0.0020641390157565697,0.09676799923181534,0.12310399860143663,0.10618879944086074,0.10618879944086074,0.008982738207839057,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512]",4096,512,512,512.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.024224000051617622,0.03315199911594391,0.027436799928545953,0.027328000403940678,0.002679154874546328,0.14716799557209015,0.16412800550460815,0.15470399856567385,0.15470399856567385,0.005423454433987419,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512]",8192,512,512,512.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.01833599992096424,0.04806400090456009,0.026252799853682517,0.022672000341117382,0.00878438440429033,0.1844799965620041,0.2072959989309311,0.19359359890222552,0.19359359890222552,0.007663539820967659,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,2,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048]",4096,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median -0.0,0.0,0.0,0.0,0.0,0.024480000138282776,0.04979199916124344,0.029081599973142146,0.025679999962449074,0.007227422044689197,0.34147199988365173,0.37968000769615173,0.3583200007677078,0.3583200007677078,0.011804422100726231,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,4,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048, 2048, 2048]",8192,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +time_stats.attn_input_reshape.min,time_stats.attn_input_reshape.max,time_stats.attn_input_reshape.mean,time_stats.attn_input_reshape.median,time_stats.attn_input_reshape.std,time_stats.attn_kv_cache_save.min,time_stats.attn_kv_cache_save.max,time_stats.attn_kv_cache_save.mean,time_stats.attn_kv_cache_save.median,time_stats.attn_kv_cache_save.std,time_stats.attn_prefill.min,time_stats.attn_prefill.max,time_stats.attn_prefill.mean,time_stats.attn_prefill.median,time_stats.attn_prefill.std,time_stats.attn_decode.min,time_stats.attn_decode.max,time_stats.attn_decode.mean,time_stats.attn_decode.median,time_stats.attn_decode.std,time_stats.attn_output_reshape.min,time_stats.attn_output_reshape.max,time_stats.attn_output_reshape.mean,time_stats.attn_output_reshape.median,time_stats.attn_output_reshape.std,n_embd,n_q_head,n_kv_head,block_size,num_tensor_parallel_workers,max_model_len,batch_size,prefill_chunk_size,kv_cache_size,is_prefill,attention_backend,is_mixed_batch,mode,seq_lens,total_tokens,max_seq_len,min_seq_len,avg_seq_len,equal_seq_len,seq_len_variance,seq_len_std,seq_len_cv,is_chunked_prefill_sample,chunk_start_token,chunk_end_token,total_prefill_tokens,profiling_precision,model_arch,quant_signature,measurement_type,is_true_mixed_batch,prefill_seq_lens,prefill_kv_cache_sizes,decode_kv_cache_sizes,num_prefill_seqs,num_decode_seqs,decode_batch_size,total_batch_size,total_decode_tokens,decode_avg_kv_cache_size,batch_composition_ratio,batch_spec,projection_policy +0.0,0.0,0.0,0.0,0.0,0.01414399966597557,0.028863999992609024,0.019705599918961526,0.01771199982613325,0.005157200849836681,0.047968000173568726,0.07046400010585785,0.05810240097343922,0.05810240097343922,0.007477463486041561,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.04947200044989586,0.020412799902260303,0.01635199971497059,0.010107497379722417,0.046560000628232956,0.08323200047016144,0.05587520003318787,0.05587520003318787,0.011126758739503428,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.022207999601960182,0.017033600155264138,0.015312000177800655,0.002819235991970241,0.05104000121355057,0.07692799717187881,0.056396800279617305,0.056396800279617305,0.007481982178637539,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015072000212967396,0.022272000089287758,0.01706880023702979,0.01616000011563301,0.002460889579319197,0.06931199878454208,0.0838719978928566,0.07432000041007995,0.07432000041007995,0.004777766433175866,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.018592000007629395,0.028543999418616295,0.02095999978482723,0.019183999858796597,0.003198175496053494,0.12179200351238251,0.15408000349998474,0.1307712011039257,0.1307712011039257,0.00858807797538298,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.027775999158620834,0.03385600075125694,0.030131200328469276,0.029680000618100166,0.0021152558103575215,0.32678401470184326,0.3450239896774292,0.33396480381488797,0.33396480381488797,0.0045872424917606375,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.04438399896025658,0.05084799975156784,0.046540799736976626,0.04531199857592583,0.002277905811237223,1.0959680080413818,1.1151360273361206,1.0999775886535645,1.0999775886535645,0.005694403246120485,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.078015998005867,0.08691199868917465,0.08114239946007729,0.08019199967384338,0.00292795706334475,4.070400238037109,4.113152027130127,4.087088012695312,4.087088012695312,0.013660567012509554,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.05167999863624573,0.022115200012922286,0.017583999782800674,0.010340094822340818,0.05196800082921982,0.09011200070381165,0.06328320093452933,0.06328320093452933,0.012557341255467452,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01583999954164028,0.026623999699950218,0.018927999772131443,0.017376000061631203,0.003514650316260619,0.06681600213050842,0.07993599772453308,0.0725280001759529,0.0725280001759529,0.004343502558613716,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01648000068962574,0.030880000442266464,0.01945280022919178,0.017967999912798405,0.004096211183007485,0.1311360001564026,0.1546880006790161,0.13908160030841826,0.13908160030841826,0.007511906874366178,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.03558399900794029,0.020851199887692927,0.018559999763965607,0.005235911594130716,0.32950401306152344,0.350271999835968,0.33912960588932034,0.33912960588932034,0.006027400986663648,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.019328000023961067,0.040608000010252,0.022790400311350822,0.020704000256955624,0.006113051965778337,1.1415679454803467,1.1518720388412476,1.144483208656311,1.144483208656311,0.0032332311374389127,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015807999297976494,0.030688000842928886,0.019596799835562707,0.01774400006979704,0.004343384771033462,0.0,0.0,0.0,0.0,0.0,0.049056001007556915,0.07580800354480743,0.05948160067200661,0.05948160067200661,0.009031541471446955,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.02486399933695793,0.01923839971423149,0.018079999834299088,0.0032282528537266424,0.0,0.0,0.0,0.0,0.0,0.05142400041222572,0.07353600114583969,0.059328000620007516,0.059328000620007516,0.0073307807735143084,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016543999314308167,0.03977600112557411,0.021379199624061585,0.018511999398469925,0.006593576176246171,0.0,0.0,0.0,0.0,0.0,0.0488319993019104,0.06435199826955795,0.05479039996862411,0.05479039996862411,0.005672522998491864,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01635199971497059,0.02844800055027008,0.019267200119793416,0.017952000722289085,0.0035068687666949577,0.0,0.0,0.0,0.0,0.0,0.049855999648571014,0.07798399776220322,0.05986879989504815,0.05986879989504815,0.01043914754878828,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016383999958634377,0.026079999282956123,0.01923519968986511,0.017791999503970146,0.0032161974331284568,0.0,0.0,0.0,0.0,0.0,0.058111999183893204,0.1045759990811348,0.06708480007946492,0.06708480007946492,0.013479022462646494,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.04057599976658821,0.019100800156593323,0.015455999877303839,0.007512281243011577,0.0,0.0,0.0,0.0,0.0,0.05363199859857559,0.07782399654388428,0.06090559959411622,0.06090559959411622,0.007544620176348091,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.02191999927163124,0.016684799920767546,0.01563199982047081,0.0024293118621811216,0.0,0.0,0.0,0.0,0.0,0.0629120022058487,0.07891199737787247,0.06891520097851753,0.06891520097851753,0.005472695665695425,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.038943998515605927,0.018313600029796363,0.01561600062996149,0.007127270260115769,0.0,0.0,0.0,0.0,0.0,0.08675199747085571,0.10662399977445602,0.09391999915242194,0.09391999915242194,0.006988099589086635,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.054687999188899994,0.021439999900758268,0.01539199985563755,0.012052764849597775,0.0,0.0,0.0,0.0,0.0,0.13488000631332397,0.1528639942407608,0.1431359991431236,0.1431359991431236,0.005436271464033599,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.041919998824596405,0.01899839974939823,0.015343999955803156,0.007989843526623287,0.0,0.0,0.0,0.0,0.0,0.06176000088453293,0.08374399691820145,0.06747519969940186,0.06747519969940186,0.0066067747128778,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016256000846624374,0.11353600025177002,0.042761600017547606,0.028960000723600388,0.029104301538020762,0.0,0.0,0.0,0.0,0.0,0.09734400361776352,0.14422400295734406,0.11392960175871848,0.11392960175871848,0.013198594600417867,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.034143999218940735,0.018918400071561335,0.01643200032413006,0.005500943993080684,0.0,0.0,0.0,0.0,0.0,0.12918399274349213,0.15087999403476715,0.13807999789714814,0.13807999789714814,0.007658330538677587,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.03641600161790848,0.0198208000510931,0.01780799962580204,0.0057264128169845765,0.0,0.0,0.0,0.0,0.0,0.22099199891090393,0.23904000222682953,0.2293503984808922,0.2293503984808922,0.004861342907006028,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.035840000957250595,0.018908800091594458,0.015792000107467175,0.0064374817924757475,0.0,0.0,0.0,0.0,0.0,0.10134399682283401,0.12201599776744843,0.10896319895982742,0.10896319895982742,0.006336330809165179,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.03417599946260452,0.017846399918198586,0.014800000004470348,0.006495007539635255,0.0,0.0,0.0,0.0,0.0,0.13126400113105774,0.15561600029468536,0.1389280006289482,0.1389280006289482,0.008381472811075022,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03519999980926514,0.019168000388890504,0.01600000075995922,0.005995522477654695,0.0,0.0,0.0,0.0,0.0,0.21728000044822693,0.2343679964542389,0.2231455981731415,0.2231455981731415,0.004720730646739123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.03670400008559227,0.018441599886864425,0.015023999847471714,0.006596535162793127,0.0,0.0,0.0,0.0,0.0,0.39321601390838623,0.4524799883365631,0.4058080047369003,0.4058080047369003,0.01578349755088941,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.022143999114632607,0.016912000067532063,0.015168000012636185,0.0028156089295136347,0.0,0.0,0.0,0.0,0.0,0.15587200224399567,0.3079040050506592,0.17838079929351805,0.17838079929351805,0.04355865575265927,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.02112000063061714,0.015516800060868263,0.01488000014796853,0.001940870731593904,0.0,0.0,0.0,0.0,0.0,0.21587200462818146,0.23561599850654602,0.22250880002975468,0.22250880002975468,0.006181951170646666,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.039264000952243805,0.023609600123018028,0.02131200022995472,0.007236979625711548,0.0,0.0,0.0,0.0,0.0,0.408735990524292,0.470335990190506,0.4336863994598388,0.4336863994598388,0.01844662383160074,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.025407999753952026,0.016227199975401164,0.014960000291466713,0.0031617375441736185,0.0,0.0,0.0,0.0,0.0,0.7412800192832947,0.7627840042114258,0.7464000046253203,0.7464000046253203,0.006112167448837547,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01462399959564209,0.02812799997627735,0.020652799773961304,0.021359999664127827,0.004308706957613102,0.028383498565450627,0.039859687970646644,0.032492258074592426,0.032492258074592426,0.00453597266208597,0.029312501176103633,0.04116431058787463,0.03355574193327539,0.03355574193327539,0.004684436757701648,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.034143999218940735,0.023171199765056372,0.024255999363958836,0.005908614918096971,0.03333159243114438,0.038935341782478095,0.03580428402241854,0.03580428402241854,0.002082270297044095,0.03633240903369937,0.04244065945634484,0.03902771507087562,0.03902771507087562,0.0022697354261490147,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.02611199952661991,0.019635199941694735,0.018112000077962875,0.0038298050749257795,0.04189529417991216,0.057484239920526384,0.04744885718421094,0.04744885718421094,0.004779830748455743,0.051672703037266184,0.07089975418195164,0.05852234134479412,0.05852234134479412,0.005895334539786378,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.029311999678611755,0.018662399891763926,0.01673599984496832,0.0044017112162725355,0.04322973959325901,0.05049827064705393,0.04535414343408448,0.04535414343408448,0.0022944156000240697,0.08733025617719538,0.10201372836398578,0.09162185574241775,0.09162185574241775,0.004635047631846023,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014592000283300877,0.026367999613285065,0.016336000058799982,0.01515199989080429,0.003415353455946402,0.06031842775160765,0.06618323188375198,0.06274415549817247,0.06274415549817247,0.001985647289644255,0.14768157653992678,0.16204076249052324,0.15362064543185072,0.15362064543185072,0.004861590945216247,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.02454400062561035,0.017167999967932702,0.016191999427974224,0.0028685016454498436,0.09128700688359712,0.09689150775996329,0.09360396051475776,0.09360396051475776,0.0013477542744916764,0.2740889887523892,0.2909164873209846,0.2810456356995366,0.2810456356995366,0.004046628526808561,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.01881599985063076,0.03097599931061268,0.024598400108516216,0.024848000146448612,0.0037539437391565975,0.1035249255866932,0.10603132147437412,0.10399797220840973,0.10399797220840973,0.0007025109429056814,0.5652750707893444,0.5789606938874117,0.5678580377517648,0.5678580377517648,0.003835906384194897,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.018624000251293182,0.043487999588251114,0.024460799992084503,0.019600000232458115,0.008922081629781394,0.196169204945307,0.2076140047945799,0.2008444429250931,0.2008444429250931,0.00394350953292805,1.1196708009293268,1.1849940417370974,1.1463555573610091,1.1463555573610091,0.022508285530529783,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.027135999873280525,0.03667199984192848,0.02945920005440712,0.028256000019609928,0.0029446678408169553,0.37757279619664613,0.3898113624476372,0.38075520430942184,0.38075520430942184,0.0036600203831042254,0.2522831942132049,0.2604606493092597,0.25440958703617433,0.25440958703617433,0.0024455194930253126,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.0435199998319149,0.049695998430252075,0.04502719938755036,0.04395199939608574,0.002035785660169308,1.1048984388245497,1.1189053886476108,1.1112371236754128,1.1112371236754128,0.0050220696724624985,0.1395495077239122,0.14131859607071193,0.14035008841031085,0.14035008841031085,0.0006342911944856293,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.018783999606966972,0.024288000538945198,0.020627199858427047,0.019567999988794327,0.0019457052717059358,0.09455999732017517,0.12185599654912949,0.10207359939813614,0.10207359939813614,0.00753346544014261,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,2,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512]",1024,512,512,512.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.027295999228954315,0.034752000123262405,0.02943360023200512,0.028528000228106976,0.0023159160681123767,0.14416000247001648,0.15904000401496887,0.14979200065135959,0.14979200065135959,0.00480624675866005,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,4,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512]",2048,512,512,512.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.043455999344587326,0.05084799975156784,0.045500800386071204,0.04411200061440468,0.002490556062831049,0.24454399943351746,0.25865599513053894,0.2504959970712662,0.2504959970712662,0.003778494140301353,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512]",4096,512,512,512.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.07673600316047668,0.08374399691820145,0.07970559895038605,0.07873599976301193,0.0022907976135004057,0.445248007774353,0.4758400022983551,0.45409599840641024,0.45409599840641024,0.009133230340383306,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512]",8192,512,512,512.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.043296001851558685,0.051072001457214355,0.044972800090909,0.04399999976158142,0.002451216824441962,0.6147199869155884,0.6290879845619202,0.6204927921295166,0.6204927921295166,0.004344846739788608,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,2,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048]",4096,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.07737600058317184,0.09123200178146362,0.08094720020890236,0.07980800047516823,0.004065264798368611,1.1744320392608643,1.1887680292129517,1.178323209285736,1.178323209285736,0.004395876469319665,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,4,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048, 2048, 2048]",8192,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.02364799939095974,0.01809599995613098,0.016352000646293163,0.0035481127058959038,0.04822399839758873,0.08566399663686752,0.05961279980838299,0.05961279980838299,0.011445665413968877,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.020864000543951988,0.01668160008266568,0.015664000064134598,0.0025769063833097584,0.049695998430252075,0.08057600259780884,0.05882879942655563,0.05882879942655563,0.009515126108519331,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.028704000636935234,0.017430400010198355,0.015056000091135502,0.004349294613335555,0.049855999648571014,0.07366400212049484,0.05459520071744919,0.05459520071744919,0.0069610920757925574,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.03161599859595299,0.017788799852132796,0.015711999963968992,0.005005385723318659,0.06102399900555611,0.08179199695587158,0.06650560013949873,0.06650560013949873,0.006947995595801105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.04416000097990036,0.019200000166893005,0.015488000120967627,0.008561241323364038,0.08054400235414505,0.09196799993515015,0.08607039973139763,0.08607039973139763,0.004145329035483754,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.023391999304294586,0.019667199812829494,0.018463999964296818,0.0022577669687832585,0.18729600310325623,0.20585599541664124,0.19546559900045393,0.19546559900045393,0.006339068824303663,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.026240000501275063,0.03446400165557861,0.028297600522637367,0.027088000439107418,0.0025148441522922374,0.5754240155220032,0.5889919996261597,0.5800191938877105,0.5800191938877105,0.003858869273829596,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.04182400181889534,0.047200001776218414,0.043036799877882004,0.0423360001295805,0.0017073405772076728,2.063199996948242,2.0787200927734375,2.067151999473572,2.067151999473572,0.004959651271127835,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.05648000165820122,0.02270399993285537,0.017935999669134617,0.012377915150727689,0.049536000937223434,0.07196799665689468,0.056015999615192415,0.056015999615192415,0.0070476637552742884,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01408000010997057,0.021344000473618507,0.01611520005390048,0.014928000047802925,0.0025499884993961702,0.07072000205516815,0.2642880082130432,0.1588256008923054,0.1588256008923054,0.053220347086102376,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01360000018030405,0.03014400042593479,0.016975999902933837,0.015056000091135502,0.004715159697364111,0.08393599838018417,0.11036799848079681,0.09160000011324881,0.09160000011324881,0.007911669434472792,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.020767999812960625,0.016233599931001663,0.015392000321298838,0.002202615907995427,0.1998399943113327,0.22070400416851044,0.20855360180139543,0.20855360180139543,0.00689307200230667,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.033440001308918,0.018684800155460833,0.016048000194132328,0.00553991005639174,0.6043199896812439,0.635807991027832,0.6126143991947173,0.6126143991947173,0.008745933953408096,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013824000023305416,0.03359999880194664,0.017811199743300678,0.014944000169634819,0.005926140483565472,0.0,0.0,0.0,0.0,0.0,0.045471999794244766,0.07539200037717819,0.054758400097489356,0.054758400097489356,0.010253548506101549,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02409599907696247,0.01641279999166727,0.01473599998280406,0.003347158638713987,0.0,0.0,0.0,0.0,0.0,0.0461760014295578,0.07529599964618683,0.05460800044238568,0.05460800044238568,0.009748937798340135,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.028416000306606293,0.018611199874430894,0.01598400017246604,0.005209875093863647,0.0,0.0,0.0,0.0,0.0,0.048128001391887665,0.07897599786520004,0.061353600397706036,0.061353600397706036,0.010488153655157845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.025728000327944756,0.016092800162732603,0.01512000011280179,0.003300888123052605,0.0,0.0,0.0,0.0,0.0,0.04864000156521797,0.07648000121116638,0.05810560062527656,0.05810560062527656,0.009473544218404408,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.03551999852061272,0.018588799890130757,0.016064000315964222,0.006071057437992447,0.0,0.0,0.0,0.0,0.0,0.04822399839758873,0.07862400263547897,0.05660480037331582,0.05660480037331582,0.009565394213730401,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.028831999748945236,0.01699519995599985,0.015024000313133001,0.004285000744868188,0.0,0.0,0.0,0.0,0.0,0.04854400083422661,0.06719999760389328,0.05778240002691746,0.05778240002691746,0.006852125554679805,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.0144640002399683,0.024927999824285507,0.0161183999851346,0.01521599991247058,0.0029774808524673907,0.0,0.0,0.0,0.0,0.0,0.05379199981689453,0.08966399729251862,0.06270079985260964,0.06270079985260964,0.009983591277092696,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03001599945127964,0.017648000083863736,0.01547200046479702,0.004585126309484848,0.0,0.0,0.0,0.0,0.0,0.061664000153541565,0.07692799717187881,0.06704320013523103,0.06704320013523103,0.005500191798490629,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.026367999613285065,0.016947199776768684,0.015039999969303608,0.0037951566103550205,0.0,0.0,0.0,0.0,0.0,0.08799999952316284,0.111455999314785,0.0964031994342804,0.0964031994342804,0.007397541615558088,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01369599997997284,0.021247999742627144,0.015359999984502793,0.014640000183135271,0.0020942770950814317,0.0,0.0,0.0,0.0,0.0,0.051711998879909515,0.07065600156784058,0.058054400235414506,0.058054400235414506,0.006633034815910025,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.0307839997112751,0.017148799914866685,0.015039999969303608,0.004882374686312284,0.0,0.0,0.0,0.0,0.0,0.061919998377561576,0.07843200117349625,0.06628479920327664,0.06628479920327664,0.004962852689801192,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013887999579310417,0.02969600073993206,0.017500799987465142,0.015232000034302473,0.00467273319705314,0.0,0.0,0.0,0.0,0.0,0.08819200098514557,0.11097600311040878,0.09493440166115762,0.09493440166115762,0.007509235042985577,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.022112000733613968,0.01751680001616478,0.016047999262809753,0.0030306621792915785,0.0,0.0,0.0,0.0,0.0,0.13065600395202637,0.15110400319099426,0.13857279866933822,0.13857279866933822,0.00750137841249771,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.04012800008058548,0.01892479993402958,0.01550400024279952,0.007459545267816961,0.0,0.0,0.0,0.0,0.0,0.06278400123119354,0.08259200304746628,0.0704512007534504,0.0704512007534504,0.005979055984382744,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.02236800082027912,0.016332800220698118,0.014928000047802925,0.002784044363186731,0.0,0.0,0.0,0.0,0.0,0.1003199964761734,0.1279360055923462,0.10921279862523078,0.10921279862523078,0.00862769617046716,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0225600004196167,0.016128000058233737,0.014800000004470348,0.002958953332547708,0.0,0.0,0.0,0.0,0.0,0.13116799294948578,0.14812800288200378,0.13783999979496003,0.13783999979496003,0.005696384361148053,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.029983999207615852,0.017468800116330386,0.01508800033479929,0.0047379550962483065,0.0,0.0,0.0,0.0,0.0,0.217631995677948,0.2447360008955002,0.22715839892625808,0.22715839892625808,0.008831828221847138,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02304000034928322,0.016512000095099212,0.014512000139802694,0.0035026774828624254,0.0,0.0,0.0,0.0,0.0,0.11020799726247787,0.12307199835777283,0.11600959971547126,0.11600959971547126,0.004667637266950902,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.042080000042915344,0.017510399967432023,0.014607999939471483,0.00823117883530167,0.0,0.0,0.0,0.0,0.0,0.15702399611473083,0.17587199807167053,0.16399359852075576,0.16399359852075576,0.006588299074676393,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0208320003002882,0.015299199987202883,0.01462399959564209,0.001916768086505386,0.0,0.0,0.0,0.0,0.0,0.21779200434684753,0.2415360063314438,0.2260768011212349,0.2260768011212349,0.007251352080685236,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014015999622642994,0.021088000386953354,0.015619200188666582,0.01473599998280406,0.002013227452302141,0.0,0.0,0.0,0.0,0.0,0.3959999978542328,0.4152640104293823,0.40332479774951924,0.40332479774951924,0.006942401431914052,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.03379200026392937,0.020595200080424547,0.019600000232458115,0.006548754248881344,0.026192623739694512,0.040006882507168426,0.029155180178492036,0.029155180178492036,0.00408828748438241,0.024591375524545756,0.037561119537986146,0.027372820355088746,0.027372820355088746,0.0038383559348575944,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.020896000787615776,0.015318400040268899,0.014431999996304512,0.0019640312960926966,0.029349018208693862,0.06236262941356679,0.03714313592014963,0.03714313592014963,0.009618313791872569,0.028826981462526914,0.06125337308649041,0.036482463672774496,0.036482463672774496,0.009447230957011863,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.032127998769283295,0.017286399938166143,0.014479999896138906,0.005536495446636193,0.03273085874558354,0.04394578491937082,0.03563837490653034,0.03563837490653034,0.003429601730256567,0.03488514202593899,0.04683821345079978,0.037984025407085474,0.037984025407085474,0.0036553316361902684,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.021503999829292297,0.015078400075435639,0.01425600005313754,0.002238435975478849,0.042100813549974185,0.051784144690147756,0.045378692890289625,0.045378692890289625,0.003184109940650555,0.05111518843748309,0.06287185663927425,0.05509490773570219,0.05509490773570219,0.0038658725544299132,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.02687999978661537,0.017667199857532977,0.01566399959847331,0.003864811846387173,0.048475323773821306,0.05599956978723733,0.05123499252968345,0.05123499252968345,0.002427379820423941,0.08429268135885387,0.09737642835214408,0.0890914090616175,0.0890914090616175,0.0042209177332077005,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.01462399959564209,0.03359999880194664,0.019804799742996693,0.016032000072300434,0.006750519908145474,0.07121508474579985,0.07292308367136396,0.07194410845270183,0.07194410845270183,0.0005500065915943844,0.14760091249712767,0.15114092373010238,0.14911189243564577,0.14911189243564577,0.0011399477384397005,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.021856000646948814,0.016835200227797033,0.015263999812304974,0.00283854448975065,0.08146338272142935,0.08560865714384096,0.0834932625520734,0.0834932625520734,0.0012259635905347874,0.2782486219401307,0.29240733788179385,0.2851819365989658,0.2851819365989658,0.004187435731481665,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.04495999962091446,0.02143679987639189,0.015856000129133463,0.009709987872683342,0.1194459208702178,0.12302524755181463,0.12053885718696096,0.12053885718696096,0.001005118119341339,0.5597220650459199,0.5764947443228802,0.5648435507167102,0.5648435507167102,0.004709970715400788,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.018112000077962875,0.026496000587940216,0.019318400137126445,0.018432000651955605,0.0024249605112359905,0.19770253574610402,0.222811797868348,0.2054811520619412,0.2054811520619412,0.008000944254868043,0.13941746080159492,0.157124211777114,0.14490284788179197,0.14490284788179197,0.005642170080515992,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.02595200017094612,0.03155200183391571,0.02727359998971224,0.026575999334454536,0.0016260948764843166,0.6014684881116659,0.6095472988212,0.6046757700946376,0.6046757700946376,0.0023537435563177303,0.11325152264578045,0.11477269562841545,0.11385542721485654,0.11385542721485654,0.0004431903698023628,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.03359999880194664,0.01814719969406724,0.015375999733805656,0.005678506740662529,0.06774400174617767,0.07891199737787247,0.07312640026211739,0.07312640026211739,0.003826583051422731,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,2,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512]",1024,512,512,512.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01788800023496151,0.028543999418616295,0.021340799890458582,0.019504000432789326,0.003718627037219618,0.09548799693584442,0.1327359974384308,0.10823359936475753,0.10823359936475753,0.013230635292043864,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,4,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512]",2048,512,512,512.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.02627200074493885,0.030368000268936157,0.027184000052511693,0.02643200010061264,0.0013545679205210022,0.14364799857139587,0.1597760021686554,0.15008639842271806,0.15008639842271806,0.005367723415171222,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512]",4096,512,512,512.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.04150399938225746,0.04726399853825569,0.042950399965047834,0.04224000126123428,0.001720147501765378,0.2433920055627823,0.28963199257850647,0.2549152016639709,0.2549152016639709,0.013450991019687407,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512]",8192,512,512,512.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.025887999683618546,0.03407999873161316,0.028303999826312064,0.026367999613285065,0.0028877495368841042,0.325439989566803,0.3441599905490875,0.33442879617214205,0.33442879617214205,0.005693597811441922,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,2,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048]",4096,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.04137599840760231,0.05084799975156784,0.044828799366950986,0.043087998405098915,0.003560363865797613,0.6110399961471558,0.6421759724617004,0.6204223990440368,0.6204223990440368,0.011214490370794758,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,4,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048, 2048, 2048]",8192,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.022784000262618065,0.016435200069099664,0.015519999898970127,0.0023688301421469523,0.04879999905824661,0.09139200299978256,0.06054079942405224,0.06054079942405224,0.012152608702448775,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014816000126302242,0.02844800055027008,0.017008000146597625,0.015696000307798386,0.003941466294662679,0.047807998955249786,0.07356800138950348,0.05626560002565384,0.05626560002565384,0.00842179125412236,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.053408000618219376,0.019353600218892097,0.01532800029963255,0.01138302400841626,0.048448000103235245,0.0785600021481514,0.0556256003677845,0.0556256003677845,0.009348274502616908,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.03139200061559677,0.01823679991066456,0.016159999649971724,0.004860071238302512,0.055424001067876816,0.08505599945783615,0.0640383992344141,0.0640383992344141,0.00921448636178623,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.026528000831604004,0.017324799951165915,0.01536000007763505,0.0036004822686428305,0.07660800218582153,0.0942080020904541,0.08209280073642732,0.08209280073642732,0.00515895587669752,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.021247999742627144,0.016403199825435876,0.01532800029963255,0.001995897022647934,0.11395200341939926,0.15113599598407745,0.12431039959192276,0.12431039959192276,0.011164431123683732,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.017184000462293625,0.028672000393271446,0.019865600019693376,0.017823999747633934,0.0035798491315929977,0.3171840012073517,0.3341119885444641,0.3261695951223373,0.3261695951223373,0.005111046452878918,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.024639999493956566,0.030400000512599945,0.02656640000641346,0.02556800004094839,0.0020189846603237303,1.0648640394210815,1.0828479528427124,1.0715327858924866,1.0715327858924866,0.005558639263049851,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014976000413298607,0.021695999428629875,0.017305599898099898,0.01593599934130907,0.0025718046517268054,0.04956800118088722,0.07932800054550171,0.06228480041027069,0.06228480041027069,0.01027160349757827,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01539199985563755,0.03580800071358681,0.019686400331556796,0.017136000096797943,0.005918387811304003,0.05158400163054466,0.10713600367307663,0.06364160068333148,0.06364160068333148,0.015832957809696766,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02223999984562397,0.016672000009566545,0.015887999907135963,0.0020934945946034563,0.06521599739789963,0.08902399986982346,0.07520959973335266,0.07520959973335266,0.007913840684102929,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015231999568641186,0.04028800129890442,0.019971200078725816,0.01646399963647127,0.007351305886577286,0.14467200636863708,0.16844800114631653,0.15363519936800005,0.15363519936800005,0.008174435899956223,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015519999898970127,0.02304000034928322,0.01775679988786578,0.016784000210464,0.0025077243712082584,0.33740800619125366,0.35343998670578003,0.3445120006799698,0.3445120006799698,0.00445648463590358,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.02393599972128868,0.01809599995613098,0.01654400024563074,0.002998393102466254,0.0,0.0,0.0,0.0,0.0,0.0504320003092289,0.0843840017914772,0.060083200410008426,0.060083200410008426,0.00986959318572296,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.030047999694943428,0.019510399922728537,0.017680000513792038,0.004065185265963332,0.0,0.0,0.0,0.0,0.0,0.05004800111055374,0.07036799937486649,0.059315200522542,0.059315200522542,0.006537768821329647,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.02393599972128868,0.018035199772566558,0.016512000001966953,0.0030667689116777724,0.0,0.0,0.0,0.0,0.0,0.05100800096988678,0.06735999882221222,0.058387200161814694,0.058387200161814694,0.005832787739241381,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.026208000257611275,0.017500799987465142,0.01646399963647127,0.0030666103306165714,0.0,0.0,0.0,0.0,0.0,0.05023999884724617,0.07254400104284286,0.057254400476813315,0.057254400476813315,0.006890853653068151,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.027327999472618103,0.0179776000790298,0.01648000068962574,0.003783417447531392,0.0,0.0,0.0,0.0,0.0,0.04819199815392494,0.07100799679756165,0.05621119923889638,0.05621119923889638,0.007824391484124725,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.036448001861572266,0.019136000238358975,0.016704000532627106,0.006076530095624803,0.0,0.0,0.0,0.0,0.0,0.047488000243902206,0.08441600203514099,0.0588383998721838,0.0588383998721838,0.011275940726332522,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015135999768972397,0.033952001482248306,0.02119360016658902,0.018000000156462193,0.007136646614305304,0.0,0.0,0.0,0.0,0.0,0.04931199923157692,0.06992000341415405,0.05755840018391609,0.05755840018391609,0.007297587694315226,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.034272000193595886,0.0204415999352932,0.017311999574303627,0.00695404701803013,0.0,0.0,0.0,0.0,0.0,0.05215999856591225,0.0735040009021759,0.06117440015077591,0.06117440015077591,0.007381118436894922,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03743999823927879,0.02012479966506362,0.01775999926030636,0.006181045509079057,0.0,0.0,0.0,0.0,0.0,0.06355199962854385,0.08508799970149994,0.07019200026988984,0.07019200026988984,0.0062246580442117845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.032607998698949814,0.01959999995306134,0.017487999983131886,0.004882475068460749,0.0,0.0,0.0,0.0,0.0,0.04918399825692177,0.0865280032157898,0.05973760038614274,0.05973760038614274,0.011546847942113974,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02675200067460537,0.017430400010198355,0.016560000367462635,0.003243135094239819,0.0,0.0,0.0,0.0,0.0,0.052671998739242554,0.08367999643087387,0.06238719932734965,0.06238719932734965,0.011207824780630104,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.03612799942493439,0.019327999837696553,0.01688000001013279,0.006058251425153085,0.0,0.0,0.0,0.0,0.0,0.06364800035953522,0.07878399640321732,0.06935679838061332,0.06935679838061332,0.004515588328677643,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.03711999952793121,0.019670399930328132,0.017152000218629837,0.006165994646522264,0.0,0.0,0.0,0.0,0.0,0.09071999788284302,0.11507199704647064,0.10252480059862136,0.10252480059862136,0.008782544051535657,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.026528000831604004,0.018396800104528665,0.01601599995046854,0.004279788048986283,0.0,0.0,0.0,0.0,0.0,0.05331199988722801,0.07977599650621414,0.06076480001211167,0.06076480001211167,0.008761579122069606,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.02643200010061264,0.01726400004699826,0.015855999663472176,0.003210008417242029,0.0,0.0,0.0,0.0,0.0,0.062431998550891876,0.08246400207281113,0.06970879957079888,0.06970879957079888,0.007016341676068233,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.02223999984562397,0.0166015999391675,0.015887999907135963,0.00204913969129354,0.0,0.0,0.0,0.0,0.0,0.10127999633550644,0.1141119971871376,0.10621120035648347,0.10621120035648347,0.004029318311754605,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03363199904561043,0.019305599946528675,0.016671999357640743,0.0055445582307981234,0.0,0.0,0.0,0.0,0.0,0.1319040060043335,0.15839999914169312,0.14040640145540234,0.14040640145540234,0.007998041073596942,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014368000440299511,0.04447999969124794,0.018908800091594458,0.0157279996201396,0.0086814937461721,0.0,0.0,0.0,0.0,0.0,0.06428799778223038,0.08982399851083755,0.07349760085344315,0.07349760085344315,0.008605284512197258,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.028672000393271446,0.01890560006722808,0.016080000437796116,0.004797584627815021,0.0,0.0,0.0,0.0,0.0,0.11123199760913849,0.14115199446678162,0.11942399889230729,0.11942399889230729,0.008513394706791027,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.024927999824285507,0.016915200091898442,0.015216000378131866,0.003374147499442635,0.0,0.0,0.0,0.0,0.0,0.15887999534606934,0.18111999332904816,0.16934399753808976,0.16934399753808976,0.007415181260761123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.026944000273942947,0.017737600207328796,0.015199999790638685,0.00415598714375255,0.0,0.0,0.0,0.0,0.0,0.2192319929599762,0.23472000658512115,0.22809920012950893,0.22809920012950893,0.004730841376327335,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.05052800104022026,0.020320000313222408,0.015520000364631414,0.010604522177924678,0.024270629882498958,0.03340247625954076,0.028446344104128624,0.028446344104128624,0.0032330369099793834,0.023697370291069768,0.03261352722995356,0.027774456325453972,0.027774456325453972,0.0031566742680923386,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.033663999289274216,0.019987199828028678,0.018240000121295452,0.005522929139253732,0.0259194055660947,0.03719755183990719,0.029916029687899703,0.029916029687899703,0.003966666590554318,0.027104595853449518,0.03889844644729374,0.03128396954022015,0.03128396954022015,0.004148046317967881,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014592000283300877,0.03299200162291527,0.019840000104159115,0.016207999549806118,0.006546343008726814,0.02587869595769926,0.03763167265431482,0.030174939058162802,0.030174939058162802,0.0037303581782277364,0.026473304070195713,0.03849632587654989,0.03086826083864964,0.03086826083864964,0.003816069654529222,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.013824000023305416,0.021023999899625778,0.016304000187665223,0.015584000386297703,0.0023236165806545476,0.031810621525966996,0.04156949936878106,0.0346749350032807,0.0346749350032807,0.003130209181143985,0.035677378270900374,0.04662250161636451,0.038889864871740294,0.038889864871740294,0.0035107033960828727,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.02175999991595745,0.01569600012153387,0.015008000191301107,0.0020882666168036863,0.038211712107062853,0.07212229256520057,0.04364509673334097,0.04364509673334097,0.009671953074025085,0.0476442859917874,0.08992570455184198,0.05441890342616107,0.05441890342616107,0.01205947791784038,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014368000440299511,0.033824000507593155,0.017628800217062236,0.014864000026136637,0.005791495403857738,0.05214261250030033,0.06266261508706669,0.05677791295527661,0.05677791295527661,0.0030298223079651514,0.08648138506878382,0.10392938682791132,0.09416928531647478,0.09416928531647478,0.005025126612204489,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.031199999153614044,0.017900799959897996,0.015343999955803156,0.004974475661316249,0.06411958891421111,0.07405276123263956,0.06793047918211377,0.06793047918211377,0.0033918658556700006,0.14058441263169497,0.16236323591492058,0.1489399211274489,0.1489399211274489,0.00743678300375353,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.03855999931693077,0.018495999928563833,0.015647999942302704,0.006918530056761627,0.09872985549401277,0.10683454583043753,0.10185316839884552,0.10185316839884552,0.0020802254037042074,0.2743261390166379,0.2968454508984596,0.2830044295482752,0.2830044295482752,0.005780016596065092,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.030400000512599945,0.01977920001372695,0.01726400014013052,0.005350883464206169,0.13918872472233365,0.16279522855335207,0.1501912864839173,0.1501912864839173,0.008992185529011513,0.11892328861766266,0.13909276049083735,0.1283239123428725,0.1283239123428725,0.007682951884956939,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.01727999933063984,0.02223999984562397,0.01819519978016615,0.017680000513792038,0.0014397298270620873,0.3728044181625443,0.38295504353701676,0.3761264483787333,0.3761264483787333,0.0033660272332490977,0.07967557017401883,0.08184495665371805,0.08038555277807365,0.08038555277807365,0.0007193856241088455,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.034591998904943466,0.01941439984366298,0.01756799966096878,0.005541380584373771,0.05951999872922897,0.08268799632787704,0.06715519949793816,0.06715519949793816,0.006657802116288364,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,2,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512]",1024,512,512,512.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.03283200040459633,0.019180799927562477,0.016543999314308167,0.0052187911233635975,0.07199999690055847,0.08675199747085571,0.07749439924955369,0.07749439924955369,0.004849874741156772,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,4,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512]",2048,512,512,512.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01724799908697605,0.02284800074994564,0.01928640007972717,0.018400000408291817,0.0020641390157565697,0.09676799923181534,0.12310399860143663,0.10618879944086074,0.10618879944086074,0.008982738207839057,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512]",4096,512,512,512.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.024224000051617622,0.03315199911594391,0.027436799928545953,0.027328000403940678,0.002679154874546328,0.14716799557209015,0.16412800550460815,0.15470399856567385,0.15470399856567385,0.005423454433987419,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512]",8192,512,512,512.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01833599992096424,0.04806400090456009,0.026252799853682517,0.022672000341117382,0.00878438440429033,0.1844799965620041,0.2072959989309311,0.19359359890222552,0.19359359890222552,0.007663539820967659,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,2,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048]",4096,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.024480000138282776,0.04979199916124344,0.029081599973142146,0.025679999962449074,0.007227422044689197,0.34147199988365173,0.37968000769615173,0.3583200007677078,0.3583200007677078,0.011804422100726231,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,4,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048, 2048, 2048]",8192,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median diff --git a/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/attention_true_mixed_fused.csv b/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/attention_true_mixed_fused.csv index 153bb52..2dd009a 100644 --- a/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/attention_true_mixed_fused.csv +++ b/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/attention_true_mixed_fused.csv @@ -1,31 +1,31 @@ -num_tensor_parallel_workers,batch_spec,num_prefill_seqs,num_decode_seqs,total_prefill_tokens,total_decode_tokens,decode_avg_kv_cache_size,attention_core_mean_ms,attention_core_mean_as_median_ms,kv_cache_update_median_ms,pure_prefill_reference_mean_ms,pure_decode_reference_mean_ms,projected_prefill_mean_ms,projected_decode_mean_ms,projection_sum_error_ms,representation -1,q64_8q1s512,1,8,64,8,512.0,0.06604800000786781,0.06604800000786781,0.021359999664127827,0.05810240097343922,0.0600041144660541,0.032492258074592426,0.03355574193327539,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -1,q128_8q1s1k,1,8,128,8,1024.0,0.07483199909329416,0.07483199909329416,0.024255999363958836,0.05587520003318787,0.06090559959411622,0.03580428402241854,0.03902771507087562,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -1,q128_16q1s1k,1,16,128,16,1024.0,0.10597119852900506,0.10597119852900506,0.018112000077962875,0.05587520003318787,0.06891520097851753,0.04744885718421094,0.05852234134479412,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -1,q256_16q1s2k,1,16,256,16,2048.0,0.13697599917650224,0.13697599917650224,0.01673599984496832,0.056396800279617305,0.11392960175871848,0.04535414343408448,0.09162185574241775,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -1,q256_32q1s2k,1,32,256,32,2048.0,0.2163648009300232,0.2163648009300232,0.01515199989080429,0.056396800279617305,0.13807999789714814,0.06274415549817247,0.15362064543185072,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -1,q512_32q1s4k,1,32,512,32,4096.0,0.3746495962142944,0.3746495962142944,0.016191999427974224,0.07432000041007995,0.2231455981731415,0.09360396051475776,0.2810456356995366,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -1,q512_64q1s4k,1,64,512,64,4096.0,0.6718560099601746,0.6718560099601746,0.024848000146448612,0.07432000041007995,0.4058080047369003,0.10399797220840973,0.5678580377517648,-1.1102230246251565e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -1,q1k_64q1s8k,1,64,1024,64,8192.0,1.3472000002861022,1.3472000002861022,0.019600000232458115,0.1307712011039257,0.7464000046253203,0.2008444429250931,1.1463555573610091,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -1,q2k_32q1s4k,1,32,2048,32,4096.0,0.6351647913455962,0.6351647913455962,0.028256000019609928,0.33396480381488797,0.2231455981731415,0.38075520430942184,0.25440958703617433,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -1,q4k_16q1s4k,1,16,4096,16,4096.0,1.2515872120857237,1.2515872120857237,0.04395199939608574,1.0999775886535645,0.1389280006289482,1.1112371236754128,0.14035008841031085,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -2,q64_8q1s512,1,8,64,8,512.0,0.05652800053358078,0.05652800053358078,0.019600000232458115,0.05961279980838299,0.055968457407185014,0.029155180178492036,0.027372820355088746,6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -2,q128_8q1s1k,1,8,128,8,1024.0,0.07362559959292413,0.07362559959292413,0.014431999996304512,0.05882879942655563,0.05778240002691746,0.03714313592014963,0.036482463672774496,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -2,q128_16q1s1k,1,16,128,16,1024.0,0.0736224003136158,0.0736224003136158,0.014479999896138906,0.05882879942655563,0.06270079985260964,0.03563837490653034,0.037984025407085474,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -2,q256_16q1s2k,1,16,256,16,2048.0,0.10047360062599181,0.10047360062599181,0.01425600005313754,0.05459520071744919,0.06628479920327664,0.045378692890289625,0.05509490773570219,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -2,q256_32q1s2k,1,32,256,32,2048.0,0.14032640159130094,0.14032640159130094,0.01566399959847331,0.05459520071744919,0.09493440166115762,0.05123499252968345,0.0890914090616175,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -2,q512_32q1s4k,1,32,512,32,4096.0,0.22105600088834762,0.22105600088834762,0.016032000072300434,0.06650560013949873,0.13783999979496003,0.07194410845270183,0.14911189243564577,-2.7755575615628914e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -2,q512_64q1s4k,1,64,512,64,4096.0,0.36867519915103913,0.36867519915103913,0.015263999812304974,0.06650560013949873,0.22715839892625808,0.0834932625520734,0.2851819365989658,5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -2,q1k_64q1s8k,1,64,1024,64,8192.0,0.6853824079036711,0.6853824079036711,0.015856000129133463,0.08607039973139763,0.40332479774951924,0.12053885718696096,0.5648435507167102,1.1102230246251565e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -2,q2k_32q1s4k,1,32,2048,32,4096.0,0.3503839999437332,0.3503839999437332,0.018432000651955605,0.19546559900045393,0.13783999979496003,0.2054811520619412,0.14490284788179197,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -2,q4k_16q1s4k,1,16,4096,16,4096.0,0.7185311973094941,0.7185311973094941,0.026575999334454536,0.5800191938877105,0.10921279862523078,0.6046757700946376,0.11385542721485654,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -4,q64_8q1s512,1,8,64,8,512.0,0.0562208004295826,0.0562208004295826,0.015520000364631414,0.06054079942405224,0.0591108573866742,0.028446344104128624,0.027774456325453972,-6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -4,q128_8q1s1k,1,8,128,8,1024.0,0.061199999228119854,0.061199999228119854,0.018240000121295452,0.05626560002565384,0.0588383998721838,0.029916029687899703,0.03128396954022015,-6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -4,q128_16q1s1k,1,16,128,16,1024.0,0.06104319989681244,0.06104319989681244,0.016207999549806118,0.05626560002565384,0.05755840018391609,0.030174939058162802,0.03086826083864964,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -4,q256_16q1s2k,1,16,256,16,2048.0,0.07356479987502099,0.07356479987502099,0.015584000386297703,0.0556256003677845,0.06238719932734965,0.0346749350032807,0.038889864871740294,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -4,q256_32q1s2k,1,32,256,32,2048.0,0.09806400015950203,0.09806400015950203,0.015008000191301107,0.0556256003677845,0.06935679838061332,0.04364509673334097,0.05441890342616107,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -4,q512_32q1s4k,1,32,512,32,4096.0,0.1509471982717514,0.1509471982717514,0.014864000026136637,0.0640383992344141,0.10621120035648347,0.05677791295527661,0.09416928531647478,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -4,q512_64q1s4k,1,64,512,64,4096.0,0.21687040030956264,0.21687040030956264,0.015343999955803156,0.0640383992344141,0.14040640145540234,0.06793047918211377,0.1489399211274489,2.7755575615628914e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -4,q1k_64q1s8k,1,64,1024,64,8192.0,0.3848575979471207,0.3848575979471207,0.015647999942302704,0.08209280073642732,0.22809920012950893,0.10185316839884552,0.2830044295482752,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -4,q2k_32q1s4k,1,32,2048,32,4096.0,0.2785151988267898,0.2785151988267898,0.01726400014013052,0.12431039959192276,0.10621120035648347,0.1501912864839173,0.1283239123428725,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation -4,q4k_16q1s4k,1,16,4096,16,4096.0,0.456512001156807,0.456512001156807,0.017680000513792038,0.3261695951223373,0.06970879957079888,0.3761264483787333,0.08038555277807365,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +num_tensor_parallel_workers,batch_spec,num_prefill_seqs,num_decode_seqs,total_prefill_tokens,total_decode_tokens,decode_avg_kv_cache_size,attention_core_mean_ms,attention_core_mean_as_median_ms,kv_cache_update_median_ms,pure_prefill_reference_mean_ms,pure_decode_reference_mean_ms,projected_prefill_mean_ms,projected_decode_mean_ms,projection_sum_error_ms,representation +1,q64_8q1s512,1,8,64,8,512.0,0.06604800000786781,0.06604800000786781,0.021359999664127827,0.05810240097343922,0.0600041144660541,0.032492258074592426,0.03355574193327539,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q128_8q1s1k,1,8,128,8,1024.0,0.07483199909329416,0.07483199909329416,0.024255999363958836,0.05587520003318787,0.06090559959411622,0.03580428402241854,0.03902771507087562,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q128_16q1s1k,1,16,128,16,1024.0,0.10597119852900506,0.10597119852900506,0.018112000077962875,0.05587520003318787,0.06891520097851753,0.04744885718421094,0.05852234134479412,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q256_16q1s2k,1,16,256,16,2048.0,0.13697599917650224,0.13697599917650224,0.01673599984496832,0.056396800279617305,0.11392960175871848,0.04535414343408448,0.09162185574241775,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q256_32q1s2k,1,32,256,32,2048.0,0.2163648009300232,0.2163648009300232,0.01515199989080429,0.056396800279617305,0.13807999789714814,0.06274415549817247,0.15362064543185072,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q512_32q1s4k,1,32,512,32,4096.0,0.3746495962142944,0.3746495962142944,0.016191999427974224,0.07432000041007995,0.2231455981731415,0.09360396051475776,0.2810456356995366,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q512_64q1s4k,1,64,512,64,4096.0,0.6718560099601746,0.6718560099601746,0.024848000146448612,0.07432000041007995,0.4058080047369003,0.10399797220840973,0.5678580377517648,-1.1102230246251565e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q1k_64q1s8k,1,64,1024,64,8192.0,1.3472000002861022,1.3472000002861022,0.019600000232458115,0.1307712011039257,0.7464000046253203,0.2008444429250931,1.1463555573610091,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q2k_32q1s4k,1,32,2048,32,4096.0,0.6351647913455962,0.6351647913455962,0.028256000019609928,0.33396480381488797,0.2231455981731415,0.38075520430942184,0.25440958703617433,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q4k_16q1s4k,1,16,4096,16,4096.0,1.2515872120857237,1.2515872120857237,0.04395199939608574,1.0999775886535645,0.1389280006289482,1.1112371236754128,0.14035008841031085,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q64_8q1s512,1,8,64,8,512.0,0.05652800053358078,0.05652800053358078,0.019600000232458115,0.05961279980838299,0.055968457407185014,0.029155180178492036,0.027372820355088746,6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q128_8q1s1k,1,8,128,8,1024.0,0.07362559959292413,0.07362559959292413,0.014431999996304512,0.05882879942655563,0.05778240002691746,0.03714313592014963,0.036482463672774496,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q128_16q1s1k,1,16,128,16,1024.0,0.0736224003136158,0.0736224003136158,0.014479999896138906,0.05882879942655563,0.06270079985260964,0.03563837490653034,0.037984025407085474,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q256_16q1s2k,1,16,256,16,2048.0,0.10047360062599181,0.10047360062599181,0.01425600005313754,0.05459520071744919,0.06628479920327664,0.045378692890289625,0.05509490773570219,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q256_32q1s2k,1,32,256,32,2048.0,0.14032640159130094,0.14032640159130094,0.01566399959847331,0.05459520071744919,0.09493440166115762,0.05123499252968345,0.0890914090616175,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q512_32q1s4k,1,32,512,32,4096.0,0.22105600088834762,0.22105600088834762,0.016032000072300434,0.06650560013949873,0.13783999979496003,0.07194410845270183,0.14911189243564577,-2.7755575615628914e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q512_64q1s4k,1,64,512,64,4096.0,0.36867519915103913,0.36867519915103913,0.015263999812304974,0.06650560013949873,0.22715839892625808,0.0834932625520734,0.2851819365989658,5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q1k_64q1s8k,1,64,1024,64,8192.0,0.6853824079036711,0.6853824079036711,0.015856000129133463,0.08607039973139763,0.40332479774951924,0.12053885718696096,0.5648435507167102,1.1102230246251565e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q2k_32q1s4k,1,32,2048,32,4096.0,0.3503839999437332,0.3503839999437332,0.018432000651955605,0.19546559900045393,0.13783999979496003,0.2054811520619412,0.14490284788179197,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q4k_16q1s4k,1,16,4096,16,4096.0,0.7185311973094941,0.7185311973094941,0.026575999334454536,0.5800191938877105,0.10921279862523078,0.6046757700946376,0.11385542721485654,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q64_8q1s512,1,8,64,8,512.0,0.0562208004295826,0.0562208004295826,0.015520000364631414,0.06054079942405224,0.0591108573866742,0.028446344104128624,0.027774456325453972,-6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q128_8q1s1k,1,8,128,8,1024.0,0.061199999228119854,0.061199999228119854,0.018240000121295452,0.05626560002565384,0.0588383998721838,0.029916029687899703,0.03128396954022015,-6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q128_16q1s1k,1,16,128,16,1024.0,0.06104319989681244,0.06104319989681244,0.016207999549806118,0.05626560002565384,0.05755840018391609,0.030174939058162802,0.03086826083864964,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q256_16q1s2k,1,16,256,16,2048.0,0.07356479987502099,0.07356479987502099,0.015584000386297703,0.0556256003677845,0.06238719932734965,0.0346749350032807,0.038889864871740294,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q256_32q1s2k,1,32,256,32,2048.0,0.09806400015950203,0.09806400015950203,0.015008000191301107,0.0556256003677845,0.06935679838061332,0.04364509673334097,0.05441890342616107,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q512_32q1s4k,1,32,512,32,4096.0,0.1509471982717514,0.1509471982717514,0.014864000026136637,0.0640383992344141,0.10621120035648347,0.05677791295527661,0.09416928531647478,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q512_64q1s4k,1,64,512,64,4096.0,0.21687040030956264,0.21687040030956264,0.015343999955803156,0.0640383992344141,0.14040640145540234,0.06793047918211377,0.1489399211274489,2.7755575615628914e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q1k_64q1s8k,1,64,1024,64,8192.0,0.3848575979471207,0.3848575979471207,0.015647999942302704,0.08209280073642732,0.22809920012950893,0.10185316839884552,0.2830044295482752,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q2k_32q1s4k,1,32,2048,32,4096.0,0.2785151988267898,0.2785151988267898,0.01726400014013052,0.12431039959192276,0.10621120035648347,0.1501912864839173,0.1283239123428725,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q4k_16q1s4k,1,16,4096,16,4096.0,0.456512001156807,0.456512001156807,0.017680000513792038,0.3261695951223373,0.06970879957079888,0.3761264483787333,0.08038555277807365,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation diff --git a/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/manifest.json b/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/manifest.json index eff69e5..511fc99 100644 --- a/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/manifest.json +++ b/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/manifest.json @@ -32,10 +32,10 @@ }, "outputs": { "allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3", - "attention.csv": "8af0206ffe8f515c124194ecb589a0b8d640c40f59b131dc65c1fe1ad221c007", - "attention_true_mixed_fused.csv": "43ef4be90bddc9aeac6dbbe339feec24162cd1f2129a08fbd959e6ee4eaf5f60", + "attention.csv": "03180eec28087a301c9d8c185874c50bdb28b31e5e9af35fcce5a6a5a9e62147", + "attention_true_mixed_fused.csv": "cccf76e6cb6a9359f188b07d4dc2942442568a6e8cf3ff3d77ab3d35a052cecb", "linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", - "moe.csv": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0" + "moe.csv": "dd698113975e5ed77f51dfd8a69aebf764ed651c3f6f34b71914d25d6e001415" }, "profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-fused-mixed-total-conserving-pure-prefill-batch-composition", "projection_contract": { diff --git a/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/moe.csv b/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/moe.csv index e5bf507..311097f 100644 --- a/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/moe.csv +++ b/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final/moe.csv @@ -1,73 +1,73 @@ -time_stats.moe_gating_linear.min,time_stats.moe_gating_linear.max,time_stats.moe_gating_linear.mean,time_stats.moe_gating_linear.median,time_stats.moe_gating_linear.std,time_stats.moe_gating_routing_topk.min,time_stats.moe_gating_routing_topk.max,time_stats.moe_gating_routing_topk.mean,time_stats.moe_gating_routing_topk.median,time_stats.moe_gating_routing_topk.std,time_stats.moe_shuffling.min,time_stats.moe_shuffling.max,time_stats.moe_shuffling.mean,time_stats.moe_shuffling.median,time_stats.moe_shuffling.std,time_stats.moe_grouped_gemm.min,time_stats.moe_grouped_gemm.max,time_stats.moe_grouped_gemm.mean,time_stats.moe_grouped_gemm.median,time_stats.moe_grouped_gemm.std,num_tokens,num_experts,num_experts_per_device,expert_parallel_size,routing_runtime_path,routing_assignment_policy,routing_weight_policy,routing_uses_router_logits,gating_runtime_context,gating_runtime_context_impl,router_topk,hidden_dim,expert_hidden_dim,use_gated,num_tensor_parallel_workers,total_routed_tokens,model_expansion_ratio,tokens_per_expert_avg,tokens_to_experts_ratio,expert_utilization,min_load_ratio,load_imbalance_cv,max_load_ratio,load_entropy,load_gini_coefficient,load_distribution,seed,moe_grouped_gemm_backend,measurement_type,profiling_precision,model_arch,quant_signature,router_median_nonadditivity_ratio,projection_policy -0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.33926400542259216,0.405023992061615,0.36780479848384856,0.36507199704647064,0.01690507644474779,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,1.1943039894104004,1.286784052848816,1.228384006023407,1.2273280024528503,0.02832547242381263,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.3984375,0.0,1.346291201783626,4.0,5.59375,0.661865234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,1.679744005203247,1.766144037246704,1.7095808148384095,1.7015680074691772,0.02438921262998535,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.625,0.0,1.015504800579495,5.0,6.15516433212955,0.529052734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,2.1576640605926514,2.2921600341796875,2.2097824096679686,2.188944101333618,0.045572321842012986,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.875,0.0,0.6343057228182637,2.5,6.64370748444639,0.35369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,2.422368049621582,2.5130879878997803,2.4516672134399413,2.434159994125366,0.03278900287381846,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.984375,0.0,0.4921254921257382,2.25,6.817190042344769,0.272369384765625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,2.2217600345611572,2.289599895477295,2.2571327924728393,2.263375997543335,0.021660416089449488,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,1.0,0.125,0.3486861500690843,1.875,6.908192310183997,0.197662353515625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,2.18668794631958,2.318079948425293,2.2218016147613526,2.211087942123413,0.035380897213135316,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,1.0,0.4375,0.2525504668006971,1.875,6.953347743053017,0.1410369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,2.2291839122772217,2.3929600715637207,2.2908096313476562,2.2804640531539917,0.04479348924786221,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,1.0,0.65625,0.15765965680164504,1.5625,6.98229848728205,0.08779525756835938,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,2.0678720474243164,2.1297600269317627,2.0837119817733765,2.0779199600219727,0.017880044357986735,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,1.0,0.625,0.12169081635504074,1.3125,6.9892029662356325,0.06879425048828125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,2.916032075881958,3.0819520950317383,2.9805248022079467,2.9656319618225098,0.05482195799019572,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,1.0,0.796875,0.07935434147688751,1.1796875,6.9954297964750305,0.044734954833984375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,4.421599864959717,4.535359859466553,4.486294317245483,4.497056007385254,0.036990243787549344,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,1.0,0.8203125,0.060849326483103046,1.17578125,6.9973188375859685,0.033740997314453125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,7.302591800689697,7.402751922607422,7.354758310317993,7.3464319705963135,0.032142662400335566,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,1.0,0.890625,0.0412323087266341,1.08984375,6.998772433185578,0.02334284782409668,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.35280001163482666,0.39692801237106323,0.37662720382213594,0.37196800112724304,0.013401318050665304,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.4692479968070984,0.5523840188980103,0.5134752035140991,0.5100640058517456,0.02291433464135784,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.34652799367904663,0.4119040071964264,0.3789471983909607,0.38550400733947754,0.02073945105803335,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.31462401151657104,0.7456960082054138,0.38617280423641204,0.34545600414276123,0.12230201266253077,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.32521599531173706,0.419871985912323,0.36325119733810424,0.34968000650405884,0.03161798848223672,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.289792001247406,0.4663360118865967,0.4091839998960495,0.41655999422073364,0.0446001986506615,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.3761279881000519,0.4416320025920868,0.40686399936676027,0.40540799498558044,0.02257778769899645,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.7172480225563049,0.8663039803504944,0.7723807990550995,0.7591840028762817,0.04164772379451242,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.0195519924163818,1.2216639518737793,1.1253888130187988,1.1453600525856018,0.06548005322243594,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7490559816360474,1.9644800424575806,1.8529024004936219,1.814303994178772,0.08042565617327288,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,3.2479360103607178,3.385279893875122,3.296070408821106,3.2800960540771484,0.04525529026442046,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,6.344799995422363,6.517856121063232,6.464438438415527,6.478623867034912,0.05116674443145098,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.2648000121116638,0.325439989566803,0.28852800130844114,0.28390398621559143,0.01933778377635077,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.7347840070724487,0.862496018409729,0.7769344031810761,0.769216001033783,0.03290485328796285,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.421875,0.0,1.346291201783626,6.0,5.652114648336087,0.636962890625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.9198399782180786,0.9646080136299133,0.9412063956260681,0.9411839842796326,0.014939365085478117,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.5703125,0.0,1.118033988749895,5.0,6.008641773518898,0.580810546875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,1.2796800136566162,1.3484159708023071,1.3006976008415223,1.2929120063781738,0.020807998177176254,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.8828125,0.0,0.6959705453537527,3.0,6.60872850615583,0.38055419921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,1.3630399703979492,1.4430400133132935,1.3909215927124023,1.3892319798469543,0.022335744492366926,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.984375,0.0,0.5201036555341637,3.0,6.798826509158851,0.28302001953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,1.27948796749115,1.3904000520706177,1.3176063895225525,1.309440016746521,0.038060887827312775,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,1.0,0.25,0.3511282039725661,1.875,6.91002266305238,0.1970977783203125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,1.264415979385376,1.3145920038223267,1.2791999936103822,1.2753440141677856,0.014130605249568332,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,1.0,0.375,0.24692938483248605,1.6875,6.955481130775285,0.13909912109375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,1.3081920146942139,1.347648024559021,1.3292255997657776,1.329967975616455,0.014558863679016933,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,1.0,0.625,0.17143053326165383,1.5625,6.9786675275754035,0.09520339965820312,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.242751955986023,1.3112000226974487,1.2747935891151427,1.266207993030548,0.021093073517695057,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,1.0,0.78125,0.11000099875256815,1.296875,6.991308871213679,0.062183380126953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7388160228729248,1.8077759742736816,1.772764801979065,1.772704005241394,0.021056644077284283,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,1.0,0.78125,0.0864630150197678,1.1796875,6.994552526394139,0.048796653747558594,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,2.6563520431518555,2.7063679695129395,2.6785055875778196,2.6791679859161377,0.01639963463052944,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,1.0,0.8671875,0.06127686514721937,1.16015625,6.997291583027146,0.03497934341430664,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,4.386879920959473,4.452256202697754,4.4108480453491214,4.406303882598877,0.019768161791937636,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,1.0,0.884765625,0.041723768525324195,1.1171875,6.998746434318934,0.02298593521118164,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.24208000302314758,0.4028480052947998,0.3041536003351212,0.277103990316391,0.05660721484881584,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.2447039932012558,0.30502399802207947,0.26446720361709597,0.26265600323677063,0.016744548364435372,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.2337920069694519,0.2881599962711334,0.26074880361557007,0.264384001493454,0.016469850143940968,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.23369599878787994,0.28591999411582947,0.25465920120477675,0.25385600328445435,0.01593204652619194,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.2295999974012375,0.26556798815727234,0.24674240052700042,0.2497600018978119,0.010345732066199003,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.21721599996089935,0.29020801186561584,0.2394208014011383,0.2346400022506714,0.018747330357768585,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2717759907245636,0.305184006690979,0.28813759982585907,0.28809599578380585,0.01183422600545854,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.3917759954929352,0.43772798776626587,0.41130879521369934,0.4131519943475723,0.012992473640805227,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.6176639795303345,0.7009919881820679,0.642767995595932,0.6330719888210297,0.024074084919938756,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.0820800065994263,1.1674879789352417,1.1034304022789,1.0977439880371094,0.0233067292981566,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.9809919595718384,2.0415360927581787,2.003715181350708,1.992751955986023,0.022066076434645737,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,3.790112018585205,3.8651199340820312,3.829139161109924,3.8230879306793213,0.025177464160110564,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.212351992726326,0.24383999407291412,0.22760000079870224,0.22723200172185898,0.01050568575837594,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.47494399547576904,0.5184000134468079,0.4920704007148743,0.49169600009918213,0.011991064701471855,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.3984375,0.0,1.3919410907075054,6.0,5.570159765557392,0.667236328125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.6360960006713867,0.7004479765892029,0.6608384013175964,0.6572319865226746,0.020416877242438597,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.625,0.0,1.0307764064044151,4.0,6.138251855282827,0.5382080078125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.780896008014679,0.8301439881324768,0.80346559882164,0.8030399978160858,0.016801230312128875,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.859375,0.0,0.6903350635742038,3.0,6.5943747091218174,0.38067626953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.8607040047645569,0.9195200204849243,0.8783008038997651,0.8751039803028107,0.01719059253115595,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.9765625,0.0,0.49410588440130926,2.75,6.814452474347134,0.271270751953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.833952009677887,0.894752025604248,0.8619967997074127,0.863215982913971,0.018716378851797198,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,1.0,0.25,0.33693529145074724,2.125,6.9186075263155535,0.1867218017578125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.834879994392395,0.8871039748191833,0.8651552021503448,0.8638879954814911,0.015262430894400969,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,1.0,0.375,0.25567294018677456,1.8125,6.952441049154937,0.14349365234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.8518080115318298,0.9097599983215332,0.8810272097587586,0.8751040101051331,0.017005819633271906,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,1.0,0.625,0.1747801353218523,1.53125,6.978069554482723,0.09820938110351562,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.8470079898834229,0.9010239839553833,0.8694015920162201,0.8716959953308105,0.016138881171190216,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,1.0,0.65625,0.1158122428154187,1.3125,6.9901908183358845,0.06445503234863281,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.1698240041732788,1.2311359643936157,1.1888479948043824,1.1890720129013062,0.017978797794492758,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,1.0,0.78125,0.08347181893108634,1.1796875,6.994921772573154,0.046871185302734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.7702720165252686,1.8097599744796753,1.7919103980064393,1.7956640124320984,0.012641295676021557,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,1.0,0.78125,0.06866734477822484,1.20703125,6.996602562938728,0.03801727294921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.968672037124634,3.0278079509735107,2.9899007797241213,2.9824799299240112,0.01816414122631667,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,1.0,0.8984375,0.04399546833977376,1.126953125,6.998607314922362,0.024699926376342773,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.19574399292469025,0.2512960135936737,0.21939200013875962,0.2199999988079071,0.017532156418212565,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.20483200252056122,0.24006399512290955,0.22215040028095245,0.22433599829673767,0.00969639786132892,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.20559999346733093,0.24726399779319763,0.22126719802618028,0.22207999974489212,0.01328093478485499,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.20003199577331543,0.2301120012998581,0.21453119963407516,0.21598400175571442,0.010402239855151332,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.19551999866962433,0.22972799837589264,0.21238719969987868,0.21488000452518463,0.010706095835489097,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.19420799612998962,0.2903999984264374,0.2211231991648674,0.21476799994707108,0.025955008886196004,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2290560007095337,0.289247989654541,0.2543327987194061,0.24939200282096863,0.01663236753503115,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.30831998586654663,0.36953601241111755,0.3324000000953674,0.3288639932870865,0.018617999572156707,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.462911993265152,0.5497919917106628,0.4893856018781662,0.4816960096359253,0.023348152887178286,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,0.7662079930305481,0.8717759847640991,0.788454395532608,0.7744799852371216,0.03174746482604812,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.363935947418213,1.4143040180206299,1.3812703967094422,1.3798720240592957,0.014770450075530007,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize -0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.5507519245147705,2.680704116821289,2.579859209060669,2.566223978996277,0.03673283558365955,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +time_stats.moe_gating_linear.min,time_stats.moe_gating_linear.max,time_stats.moe_gating_linear.mean,time_stats.moe_gating_linear.median,time_stats.moe_gating_linear.std,time_stats.moe_gating_routing_topk.min,time_stats.moe_gating_routing_topk.max,time_stats.moe_gating_routing_topk.mean,time_stats.moe_gating_routing_topk.median,time_stats.moe_gating_routing_topk.std,time_stats.moe_shuffling.min,time_stats.moe_shuffling.max,time_stats.moe_shuffling.mean,time_stats.moe_shuffling.median,time_stats.moe_shuffling.std,time_stats.moe_grouped_gemm.min,time_stats.moe_grouped_gemm.max,time_stats.moe_grouped_gemm.mean,time_stats.moe_grouped_gemm.median,time_stats.moe_grouped_gemm.std,num_tokens,num_experts,num_experts_per_device,expert_parallel_size,routing_runtime_path,routing_assignment_policy,routing_weight_policy,routing_uses_router_logits,gating_runtime_context,gating_runtime_context_impl,router_topk,hidden_dim,expert_hidden_dim,use_gated,num_tensor_parallel_workers,total_routed_tokens,model_expansion_ratio,tokens_per_expert_avg,tokens_to_experts_ratio,expert_utilization,min_load_ratio,load_imbalance_cv,max_load_ratio,load_entropy,load_gini_coefficient,load_distribution,seed,moe_grouped_gemm_backend,measurement_type,profiling_precision,model_arch,quant_signature,router_median_nonadditivity_ratio,projection_policy +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.33926400542259216,0.405023992061615,0.36780479848384856,0.36507199704647064,0.01690507644474779,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,1.1943039894104004,1.286784052848816,1.228384006023407,1.2273280024528503,0.02832547242381263,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.3984375,0.0,1.346291201783626,4.0,5.59375,0.661865234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,1.679744005203247,1.766144037246704,1.7095808148384095,1.7015680074691772,0.02438921262998535,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.625,0.0,1.015504800579495,5.0,6.15516433212955,0.529052734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,2.1576640605926514,2.2921600341796875,2.2097824096679686,2.188944101333618,0.045572321842012986,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.875,0.0,0.6343057228182637,2.5,6.64370748444639,0.35369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,2.422368049621582,2.5130879878997803,2.4516672134399413,2.434159994125366,0.03278900287381846,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.984375,0.0,0.4921254921257382,2.25,6.817190042344769,0.272369384765625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,2.2217600345611572,2.289599895477295,2.2571327924728393,2.263375997543335,0.021660416089449488,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,1.0,0.125,0.3486861500690843,1.875,6.908192310183997,0.197662353515625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,2.18668794631958,2.318079948425293,2.2218016147613526,2.211087942123413,0.035380897213135316,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,1.0,0.4375,0.2525504668006971,1.875,6.953347743053017,0.1410369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,2.2291839122772217,2.3929600715637207,2.2908096313476562,2.2804640531539917,0.04479348924786221,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,1.0,0.65625,0.15765965680164504,1.5625,6.98229848728205,0.08779525756835938,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,2.0678720474243164,2.1297600269317627,2.0837119817733765,2.0779199600219727,0.017880044357986735,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,1.0,0.625,0.12169081635504074,1.3125,6.9892029662356325,0.06879425048828125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,2.916032075881958,3.0819520950317383,2.9805248022079467,2.9656319618225098,0.05482195799019572,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,1.0,0.796875,0.07935434147688751,1.1796875,6.9954297964750305,0.044734954833984375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,4.421599864959717,4.535359859466553,4.486294317245483,4.497056007385254,0.036990243787549344,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,1.0,0.8203125,0.060849326483103046,1.17578125,6.9973188375859685,0.033740997314453125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,7.302591800689697,7.402751922607422,7.354758310317993,7.3464319705963135,0.032142662400335566,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,1.0,0.890625,0.0412323087266341,1.08984375,6.998772433185578,0.02334284782409668,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.35280001163482666,0.39692801237106323,0.37662720382213594,0.37196800112724304,0.013401318050665304,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.4692479968070984,0.5523840188980103,0.5134752035140991,0.5100640058517456,0.02291433464135784,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.34652799367904663,0.4119040071964264,0.3789471983909607,0.38550400733947754,0.02073945105803335,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.31462401151657104,0.7456960082054138,0.38617280423641204,0.34545600414276123,0.12230201266253077,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.32521599531173706,0.419871985912323,0.36325119733810424,0.34968000650405884,0.03161798848223672,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.289792001247406,0.4663360118865967,0.4091839998960495,0.41655999422073364,0.0446001986506615,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.3761279881000519,0.4416320025920868,0.40686399936676027,0.40540799498558044,0.02257778769899645,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.7172480225563049,0.8663039803504944,0.7723807990550995,0.7591840028762817,0.04164772379451242,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.0195519924163818,1.2216639518737793,1.1253888130187988,1.1453600525856018,0.06548005322243594,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7490559816360474,1.9644800424575806,1.8529024004936219,1.814303994178772,0.08042565617327288,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,3.2479360103607178,3.385279893875122,3.296070408821106,3.2800960540771484,0.04525529026442046,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,6.344799995422363,6.517856121063232,6.464438438415527,6.478623867034912,0.05116674443145098,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.2648000121116638,0.325439989566803,0.28852800130844114,0.28390398621559143,0.01933778377635077,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.7347840070724487,0.862496018409729,0.7769344031810761,0.769216001033783,0.03290485328796285,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.421875,0.0,1.346291201783626,6.0,5.652114648336087,0.636962890625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.9198399782180786,0.9646080136299133,0.9412063956260681,0.9411839842796326,0.014939365085478117,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.5703125,0.0,1.118033988749895,5.0,6.008641773518898,0.580810546875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,1.2796800136566162,1.3484159708023071,1.3006976008415223,1.2929120063781738,0.020807998177176254,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.8828125,0.0,0.6959705453537527,3.0,6.60872850615583,0.38055419921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,1.3630399703979492,1.4430400133132935,1.3909215927124023,1.3892319798469543,0.022335744492366926,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.984375,0.0,0.5201036555341637,3.0,6.798826509158851,0.28302001953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,1.27948796749115,1.3904000520706177,1.3176063895225525,1.309440016746521,0.038060887827312775,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,1.0,0.25,0.3511282039725661,1.875,6.91002266305238,0.1970977783203125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,1.264415979385376,1.3145920038223267,1.2791999936103822,1.2753440141677856,0.014130605249568332,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,1.0,0.375,0.24692938483248605,1.6875,6.955481130775285,0.13909912109375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,1.3081920146942139,1.347648024559021,1.3292255997657776,1.329967975616455,0.014558863679016933,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,1.0,0.625,0.17143053326165383,1.5625,6.9786675275754035,0.09520339965820312,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.242751955986023,1.3112000226974487,1.2747935891151427,1.266207993030548,0.021093073517695057,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,1.0,0.78125,0.11000099875256815,1.296875,6.991308871213679,0.062183380126953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7388160228729248,1.8077759742736816,1.772764801979065,1.772704005241394,0.021056644077284283,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,1.0,0.78125,0.0864630150197678,1.1796875,6.994552526394139,0.048796653747558594,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,2.6563520431518555,2.7063679695129395,2.6785055875778196,2.6791679859161377,0.01639963463052944,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,1.0,0.8671875,0.06127686514721937,1.16015625,6.997291583027146,0.03497934341430664,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,4.386879920959473,4.452256202697754,4.4108480453491214,4.406303882598877,0.019768161791937636,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,1.0,0.884765625,0.041723768525324195,1.1171875,6.998746434318934,0.02298593521118164,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.24208000302314758,0.4028480052947998,0.3041536003351212,0.277103990316391,0.05660721484881584,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.2447039932012558,0.30502399802207947,0.26446720361709597,0.26265600323677063,0.016744548364435372,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.2337920069694519,0.2881599962711334,0.26074880361557007,0.264384001493454,0.016469850143940968,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.23369599878787994,0.28591999411582947,0.25465920120477675,0.25385600328445435,0.01593204652619194,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.2295999974012375,0.26556798815727234,0.24674240052700042,0.2497600018978119,0.010345732066199003,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.21721599996089935,0.29020801186561584,0.2394208014011383,0.2346400022506714,0.018747330357768585,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2717759907245636,0.305184006690979,0.28813759982585907,0.28809599578380585,0.01183422600545854,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.3917759954929352,0.43772798776626587,0.41130879521369934,0.4131519943475723,0.012992473640805227,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.6176639795303345,0.7009919881820679,0.642767995595932,0.6330719888210297,0.024074084919938756,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.0820800065994263,1.1674879789352417,1.1034304022789,1.0977439880371094,0.0233067292981566,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.9809919595718384,2.0415360927581787,2.003715181350708,1.992751955986023,0.022066076434645737,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,3.790112018585205,3.8651199340820312,3.829139161109924,3.8230879306793213,0.025177464160110564,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.212351992726326,0.24383999407291412,0.22760000079870224,0.22723200172185898,0.01050568575837594,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.47494399547576904,0.5184000134468079,0.4920704007148743,0.49169600009918213,0.011991064701471855,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.3984375,0.0,1.3919410907075054,6.0,5.570159765557392,0.667236328125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.6360960006713867,0.7004479765892029,0.6608384013175964,0.6572319865226746,0.020416877242438597,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.625,0.0,1.0307764064044151,4.0,6.138251855282827,0.5382080078125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.780896008014679,0.8301439881324768,0.80346559882164,0.8030399978160858,0.016801230312128875,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.859375,0.0,0.6903350635742038,3.0,6.5943747091218174,0.38067626953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.8607040047645569,0.9195200204849243,0.8783008038997651,0.8751039803028107,0.01719059253115595,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.9765625,0.0,0.49410588440130926,2.75,6.814452474347134,0.271270751953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.833952009677887,0.894752025604248,0.8619967997074127,0.863215982913971,0.018716378851797198,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,1.0,0.25,0.33693529145074724,2.125,6.9186075263155535,0.1867218017578125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.834879994392395,0.8871039748191833,0.8651552021503448,0.8638879954814911,0.015262430894400969,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,1.0,0.375,0.25567294018677456,1.8125,6.952441049154937,0.14349365234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.8518080115318298,0.9097599983215332,0.8810272097587586,0.8751040101051331,0.017005819633271906,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,1.0,0.625,0.1747801353218523,1.53125,6.978069554482723,0.09820938110351562,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.8470079898834229,0.9010239839553833,0.8694015920162201,0.8716959953308105,0.016138881171190216,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,1.0,0.65625,0.1158122428154187,1.3125,6.9901908183358845,0.06445503234863281,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.1698240041732788,1.2311359643936157,1.1888479948043824,1.1890720129013062,0.017978797794492758,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,1.0,0.78125,0.08347181893108634,1.1796875,6.994921772573154,0.046871185302734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.7702720165252686,1.8097599744796753,1.7919103980064393,1.7956640124320984,0.012641295676021557,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,1.0,0.78125,0.06866734477822484,1.20703125,6.996602562938728,0.03801727294921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.968672037124634,3.0278079509735107,2.9899007797241213,2.9824799299240112,0.01816414122631667,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,1.0,0.8984375,0.04399546833977376,1.126953125,6.998607314922362,0.024699926376342773,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.19574399292469025,0.2512960135936737,0.21939200013875962,0.2199999988079071,0.017532156418212565,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.20483200252056122,0.24006399512290955,0.22215040028095245,0.22433599829673767,0.00969639786132892,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.20559999346733093,0.24726399779319763,0.22126719802618028,0.22207999974489212,0.01328093478485499,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.20003199577331543,0.2301120012998581,0.21453119963407516,0.21598400175571442,0.010402239855151332,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.19551999866962433,0.22972799837589264,0.21238719969987868,0.21488000452518463,0.010706095835489097,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.19420799612998962,0.2903999984264374,0.2211231991648674,0.21476799994707108,0.025955008886196004,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2290560007095337,0.289247989654541,0.2543327987194061,0.24939200282096863,0.01663236753503115,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.30831998586654663,0.36953601241111755,0.3324000000953674,0.3288639932870865,0.018617999572156707,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.462911993265152,0.5497919917106628,0.4893856018781662,0.4816960096359253,0.023348152887178286,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,0.7662079930305481,0.8717759847640991,0.788454395532608,0.7744799852371216,0.03174746482604812,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.363935947418213,1.4143040180206299,1.3812703967094422,1.3798720240592957,0.014770450075530007,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.5507519245147705,2.680704116821289,2.579859209060669,2.566223978996277,0.03673283558365955,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize diff --git a/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py b/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py index da2a16d..ee2d8a4 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py +++ b/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py @@ -185,7 +185,12 @@ def parse_batch_spec(spec: str) -> list[tuple[int, int]]: def write_csv(path: Path, fieldnames: list[str], rows: list[dict[str, Any]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=fieldnames, extrasaction="raise") + writer = csv.DictWriter( + handle, + fieldnames=fieldnames, + extrasaction="raise", + lineterminator="\n", + ) writer.writeheader() writer.writerows(rows) From 6e8704d5256a6f22eb396a765f0f1d5aefac060b Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 10:13:25 +0800 Subject: [PATCH 099/148] Add fixed and exact-trace Frontier surfaces --- runs/frontier-fidelity-envelope-v1/.gitignore | 1 + .../prepare_fixed_frontier_traces.py | 122 + .../results/a2/capacity.csv | 13 + .../results/a2/comparison.json | 5638 +++++++++++++++++ .../results/a2/frontier_surface_frozen.json | 1732 +++++ .../results/a2/qwen30-prefill-ranking.png | Bin 0 -> 107863 bytes ...run_frontier_qwen30_exact_trace_surface.py | 424 ++ .../test_fidelity_envelope.py | 118 + .../workloads/f1-isl512-osl1/manifest.json | 50 + .../workloads/f1-isl512-osl1/r128.csv | 65 + .../workloads/f1-isl512-osl1/r16.csv | 65 + .../workloads/f1-isl512-osl1/r256.csv | 65 + .../workloads/f1-isl512-osl1/r32.csv | 65 + .../workloads/f1-isl512-osl1/r512.csv | 65 + .../workloads/f1-isl512-osl1/r64.csv | 65 + .../workloads/f2-isl2048-osl128/manifest.json | 50 + .../workloads/f2-isl2048-osl128/r0p5.csv | 65 + .../workloads/f2-isl2048-osl128/r1.csv | 65 + .../workloads/f2-isl2048-osl128/r16.csv | 65 + .../workloads/f2-isl2048-osl128/r2.csv | 65 + .../workloads/f2-isl2048-osl128/r4.csv | 65 + .../workloads/f2-isl2048-osl128/r8.csv | 65 + .../analyze_qwen30_prefill_fidelity.py | 26 +- .../run_frontier_qwen30_prefill_surface.py | 1 + 24 files changed, 8949 insertions(+), 6 deletions(-) create mode 100644 runs/frontier-fidelity-envelope-v1/prepare_fixed_frontier_traces.py create mode 100644 runs/frontier-fidelity-envelope-v1/results/a2/capacity.csv create mode 100644 runs/frontier-fidelity-envelope-v1/results/a2/comparison.json create mode 100644 runs/frontier-fidelity-envelope-v1/results/a2/frontier_surface_frozen.json create mode 100644 runs/frontier-fidelity-envelope-v1/results/a2/qwen30-prefill-ranking.png create mode 100644 runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py create mode 100644 runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/manifest.json create mode 100644 runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r128.csv create mode 100644 runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r16.csv create mode 100644 runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r256.csv create mode 100644 runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r32.csv create mode 100644 runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r512.csv create mode 100644 runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r64.csv create mode 100644 runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/manifest.json create mode 100644 runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r0p5.csv create mode 100644 runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r1.csv create mode 100644 runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r16.csv create mode 100644 runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r2.csv create mode 100644 runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r4.csv create mode 100644 runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r8.csv diff --git a/runs/frontier-fidelity-envelope-v1/.gitignore b/runs/frontier-fidelity-envelope-v1/.gitignore index 29cb6f8..71cf748 100644 --- a/runs/frontier-fidelity-envelope-v1/.gitignore +++ b/runs/frontier-fidelity-envelope-v1/.gitignore @@ -7,3 +7,4 @@ simulator-a3/ fleet-state/ fleet-artifacts/ profiles/profile-v3-batch/ +trace-exact-v1-public/ diff --git a/runs/frontier-fidelity-envelope-v1/prepare_fixed_frontier_traces.py b/runs/frontier-fidelity-envelope-v1/prepare_fixed_frontier_traces.py new file mode 100644 index 0000000..606e5dd --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/prepare_fixed_frontier_traces.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Materialize deterministic, prefix-disjoint fixed-shape Frontier traces.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +from pathlib import Path + + +FIELDS = ( + "arrived_at", + "num_prefill_tokens", + "num_decode_tokens", + "session_id", + "block_hash_ids", +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument("--input-tokens", type=int, required=True) + parser.add_argument("--output-tokens", type=int, required=True) + parser.add_argument("--requests", type=int, default=64) + parser.add_argument("--rate", type=float, action="append", required=True) + parser.add_argument("--block-size", type=int, default=16) + return parser.parse_args() + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def rate_key(rate: float) -> str: + return f"r{rate:g}".replace(".", "p") + + +def materialize( + root: Path, + *, + input_tokens: int, + output_tokens: int, + requests: int, + rates: list[float], + block_size: int, +) -> dict[str, object]: + values = [input_tokens, output_tokens, requests, block_size] + if any(value <= 0 for value in values): + raise ValueError("token counts, requests, and block size must be positive") + if input_tokens + output_tokens > 40960: + raise ValueError("shape exceeds the frozen max model length") + if not rates or any(rate <= 0 or not math.isfinite(rate) for rate in rates): + raise ValueError("rates must be positive and finite") + if len(set(rates)) != len(rates): + raise ValueError("rates must be unique") + root.mkdir(parents=True, exist_ok=True) + anchors = [] + blocks = math.ceil(input_tokens / block_size) + for rate in rates: + path = root / f"{rate_key(rate)}.csv" + with path.open("w", newline="") as output: + writer = csv.DictWriter(output, fieldnames=FIELDS, lineterminator="\n") + writer.writeheader() + for request_id in range(requests): + first_block = request_id * blocks + 1 + writer.writerow( + { + "arrived_at": f"{request_id / rate:.12f}", + "num_prefill_tokens": input_tokens, + "num_decode_tokens": output_tokens, + "session_id": request_id, + "block_hash_ids": "|".join( + str(first_block + offset) for offset in range(blocks) + ), + } + ) + anchors.append( + { + "label": rate_key(rate), + "rate": rate, + "path": path.name, + "sha256": sha256(path), + } + ) + manifest = { + "schema": "qwen30-fixed-frontier-traces-v1", + "contract": { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "requests": requests, + "block_size": block_size, + "arrival": "open_loop_uniform", + "prefix_caching": False, + "prefix_relation": "all request block identities are disjoint", + }, + "anchors": anchors, + } + (root / "manifest.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n" + ) + return manifest + + +def main() -> None: + args = parse_args() + manifest = materialize( + args.output_root.resolve(), + input_tokens=args.input_tokens, + output_tokens=args.output_tokens, + requests=args.requests, + rates=args.rate, + block_size=args.block_size, + ) + print(json.dumps(manifest, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-fidelity-envelope-v1/results/a2/capacity.csv b/runs/frontier-fidelity-envelope-v1/results/a2/capacity.csv new file mode 100644 index 0000000..da354c9 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/results/a2/capacity.csv @@ -0,0 +1,13 @@ +config,tp,mns,real,simulator +tp1_mns8,1,8,7.0,8.0 +tp1_mns16,1,16,7.0,8.0 +tp1_mns32,1,32,7.0,8.0 +tp1_mns64,1,64,7.0,8.0 +tp2_mns8,2,8,7.0,8.0 +tp2_mns16,2,16,7.0,8.0 +tp2_mns32,2,32,7.0,8.0 +tp2_mns64,2,64,7.0,8.0 +tp4_mns8,4,8,8.0,8.0 +tp4_mns16,4,16,8.0,8.0 +tp4_mns32,4,32,8.0,8.0 +tp4_mns64,4,64,8.0,8.0 diff --git a/runs/frontier-fidelity-envelope-v1/results/a2/comparison.json b/runs/frontier-fidelity-envelope-v1/results/a2/comparison.json new file mode 100644 index 0000000..4aa1522 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/results/a2/comparison.json @@ -0,0 +1,5638 @@ +{ + "contract": { + "input_tokens": 2048, + "model": "Qwen3-30B-A3B", + "output_tokens": 1, + "prefix_caching": false, + "real_anchor_merge": "both_fresh_server_rounds_must_pass", + "target_pass_rate": 0.95, + "ttft_slo_ms": 1256.0 + }, + "metrics": { + "anchor_confusion": { + "real_fail_sim_fail": 24, + "real_fail_sim_pass": 8, + "real_pass_sim_fail": 0, + "real_pass_sim_pass": 28 + }, + "anchor_grid_coverage": { + "real_only": 36, + "shared": 60, + "simulator_only": 0 + }, + "config_order": [ + "tp1_mns8", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp2_mns8", + "tp2_mns16", + "tp2_mns32", + "tp2_mns64", + "tp4_mns8", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "kendall": { + "both_ties": 34, + "concordant": 0, + "discordant": 0, + "kendall_tau_b": null, + "real_only_ties": 0, + "simulator_only_ties": 32 + }, + "pairwise_non_tied": { + "all": { + "accuracy": 0.0, + "comparable": 32, + "correct": 0 + }, + "within_tp": {} + }, + "real_best_capacity_per_gpu": 8.0, + "real_top_set": [ + "tp4_mns8", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "simulator_best_capacity_per_gpu": 8.0, + "simulator_top_set": [ + "tp1_mns8", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp2_mns8", + "tp2_mns16", + "tp2_mns32", + "tp2_mns64", + "tp4_mns8", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "top1_regret_best": 0.0, + "top1_regret_worst": 0.125, + "top_set_exact_match": false, + "top_set_overlap": [ + "tp4_mns16", + "tp4_mns32", + "tp4_mns64", + "tp4_mns8" + ] + }, + "objective": "maximum_tested_slo_feasible_offered_request_rate_per_gpu", + "real": { + "tp1_mns16": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 153.15452707000077, + 152.64549700077623 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r4p00.json", + "sha256": "d82e386ec3effdf57c148171311928298405da0e4f896584bf05db2485bb6bd4", + "summary": { + "admission_lag_max_ms": 0.11588609777390957, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 158.6234939750284, + "ttft_p50_ms": 151.05855697765946, + "ttft_p95_ms": 153.15452707000077 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r4p00.json", + "sha256": "1565be89131e3c8a9c352c6f29694ef2793734b0623a2db1cbea7e0b0e8a2bf6", + "summary": { + "admission_lag_max_ms": 0.11451996397227049, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 153.18561100866646, + "ttft_p50_ms": 151.18564700242132, + "ttft_p95_ms": 152.64549700077623 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 5.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 152.8308589477092, + 152.43642206769437 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r5p00.json", + "sha256": "e4447d94a326c541412b40157cfe283a6350680e3285b5d10fed6b8de5c0cd93", + "summary": { + "admission_lag_max_ms": 0.17109804321080446, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 154.35138496104628, + "ttft_p50_ms": 150.91424097772688, + "ttft_p95_ms": 152.8308589477092 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r5p00.json", + "sha256": "572c17e309c6c9f23c490979e7acb885475f142bc530ecfee88916b048259ea2", + "summary": { + "admission_lag_max_ms": 0.1321269664913416, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 157.11930498946458, + "ttft_p50_ms": 150.50161199178547, + "ttft_p95_ms": 152.43642206769437 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 6.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 153.56597595382482, + 152.16859406791627 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r6p00.json", + "sha256": "68c8b39154d46f1ceda17f47e8cbd84a8c8a9f088c5e82d8f0ac25a7180dc14c", + "summary": { + "admission_lag_max_ms": 0.11641765013337135, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 156.50876995641738, + "ttft_p50_ms": 150.41159105021507, + "ttft_p95_ms": 153.56597595382482 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r6p00.json", + "sha256": "a970a2cdf7034026a56af3ea4da59856fd588da35d66ef4f9e15481a892edcc9", + "summary": { + "admission_lag_max_ms": 0.11551892384886742, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 168.65782206878066, + "ttft_p50_ms": 150.03317291848361, + "ttft_p95_ms": 152.16859406791627 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 7.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 300.90478900820017, + 293.0445579113439 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r7p00.json", + "sha256": "1aa74c18adbff53b36ae8a6f07b5b0865f6b68fb3e7862a70b36f67c18f0bba0", + "summary": { + "admission_lag_max_ms": 0.10505830869078636, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 304.15459000505507, + "ttft_p50_ms": 263.44375393819064, + "ttft_p95_ms": 300.90478900820017 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r7p00.json", + "sha256": "e14bc9dba35d453862839eeed77262a1ed9b8394404ca90c60ec8cb26a1aa73b", + "summary": { + "admission_lag_max_ms": 0.10963028762489557, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 295.1545650139451, + "ttft_p50_ms": 243.97043196950108, + "ttft_p95_ms": 293.0445579113439 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 8.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1323.4397460473701, + 1315.5151740647852 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r8p00.json", + "sha256": "9aeee7453ffe5098d203e02b84f5009ed347c478b736e89297bd3695dd81d477", + "summary": { + "admission_lag_max_ms": 0.1251589274033904, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.90625, + "passed": 58, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1477.699184906669, + "ttft_p50_ms": 776.2576789828017, + "ttft_p95_ms": 1323.4397460473701 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r8p00.json", + "sha256": "7630f4901cf4ec4ea955738b119a1c32aba53f9ca1edb8819556fc60ae1fda09", + "summary": { + "admission_lag_max_ms": 0.11316593736410141, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.921875, + "passed": 59, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1404.566578916274, + "ttft_p50_ms": 761.7035870207474, + "ttft_p95_ms": 1315.5151740647852 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4892.253074911423, + 4893.758205929771 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r16p00.json", + "sha256": "d3a2a85cf06a4f080edc9a3809f05493ac55b0aa1d890806ffbc551c8eca14f8", + "summary": { + "admission_lag_max_ms": 0.39830803871154785, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4971.215195022523, + "ttft_p50_ms": 2742.7029330283403, + "ttft_p95_ms": 4892.253074911423 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r16p00.json", + "sha256": "67110284c38a9a60228f00b1d2b0de9477d6dec660f1c91eb4f39e1b79e272a5", + "summary": { + "admission_lag_max_ms": 0.11306500528007746, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4972.818040987477, + "ttft_p50_ms": 2742.512951954268, + "ttft_p95_ms": 4893.758205929771 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 6650.756464921869, + 6751.845097984187 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r32p00.json", + "sha256": "affc62b27382400497bb82e9da31ed9891b60155318fc4f250dca0704ccc5e02", + "summary": { + "admission_lag_max_ms": 0.11010200250893831, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6920.621327939443, + "ttft_p50_ms": 3650.839229929261, + "ttft_p95_ms": 6650.756464921869 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r32p00.json", + "sha256": "df0c8d703b023692814be41f47da4db4ce1b7eb0c3423a742c10884b6447583f", + "summary": { + "admission_lag_max_ms": 0.1288279891014099, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.15625, + "passed": 10, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6897.979240980931, + "ttft_p50_ms": 3730.34580796957, + "ttft_p95_ms": 6751.845097984187 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 7732.2855240199715, + 7744.056030060165 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r64p00.json", + "sha256": "fbfad78129c843c85905c3f94f278f468a0a019c0cdf0b62140d0fea8f0aaffc", + "summary": { + "admission_lag_max_ms": 0.11090096086263657, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7836.245508980937, + "ttft_p50_ms": 4328.587644966319, + "ttft_p95_ms": 7732.2855240199715 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r64p00.json", + "sha256": "0f005c642327708e1ceb058f71f806b4d46ea666f45695605552e8f234521608", + "summary": { + "admission_lag_max_ms": 0.10862003546208143, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7847.871358972043, + "ttft_p50_ms": 4332.9262170009315, + "ttft_p95_ms": 7744.056030060165 + } + } + ] + } + ], + "capacity": 7.0, + "capacity_per_gpu": 7.0, + "mns": 16, + "name": "tp1_mns16", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z", + "tp": 1 + }, + "tp1_mns32": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 153.44141807872802, + 152.2580359596759 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r4p00.json", + "sha256": "211fa276e4a99a061126882085b9850549b453520ac3414dd2d332c89e52b431", + "summary": { + "admission_lag_max_ms": 0.09491504170000553, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 169.56262104213238, + "ttft_p50_ms": 150.8563900133595, + "ttft_p95_ms": 153.44141807872802 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r4p00.json", + "sha256": "17428902a5b20e9084f20792963f0620a1af06cdc1afb76ee99fd15dd905b89e", + "summary": { + "admission_lag_max_ms": 0.12033001985400915, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 153.99700601119548, + "ttft_p50_ms": 150.5822929320857, + "ttft_p95_ms": 152.2580359596759 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 5.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 152.05012692604214, + 152.00478909537196 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r5p00.json", + "sha256": "b0eb19ff7ab8a26e34e91f213a983e58689c63071285e27b3719bda0a765ff32", + "summary": { + "admission_lag_max_ms": 0.11265103239566088, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 153.48999004345387, + "ttft_p50_ms": 150.21017810795456, + "ttft_p95_ms": 152.05012692604214 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r5p00.json", + "sha256": "3b857dff42dd16ebee95356ceacd6b58ff7bd0a9b91f3d6208d489b84985697f", + "summary": { + "admission_lag_max_ms": 0.11149293277412653, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 153.33813393954188, + "ttft_p50_ms": 150.45341395307332, + "ttft_p95_ms": 152.00478909537196 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 6.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 152.74227899499238, + 152.82704099081457 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r6p00.json", + "sha256": "7615fa664a99f02f9f6d4f1fb0f2d6cba11842772c086634b939bc08cd0f2f4d", + "summary": { + "admission_lag_max_ms": 0.1096969936043024, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 154.2669020127505, + "ttft_p50_ms": 150.6725640501827, + "ttft_p95_ms": 152.74227899499238 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r6p00.json", + "sha256": "345e3cd3f8cd4d7c7a5e9343f937dd9b1b5928500133830f3046b743da6df64e", + "summary": { + "admission_lag_max_ms": 0.12192933354526758, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 153.12304301187396, + "ttft_p50_ms": 150.68741200957447, + "ttft_p95_ms": 152.82704099081457 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 7.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 302.5522669777274, + 294.5978350471705 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r7p00.json", + "sha256": "3f54a3e3f2279a5325a173ff714d600fb1a2cdfcfba1e8e5c980f7b21b99be59", + "summary": { + "admission_lag_max_ms": 0.11122168507426977, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 304.24335307907313, + "ttft_p50_ms": 262.4604320153594, + "ttft_p95_ms": 302.5522669777274 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r7p00.json", + "sha256": "0832169cdf1bef49c5006464c6cbcd50f4567e0ab90bec98d722f64f7b476763", + "summary": { + "admission_lag_max_ms": 0.1059951027855277, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 297.1431679325178, + "ttft_p50_ms": 249.51713997870684, + "ttft_p95_ms": 294.5978350471705 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 8.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1321.9368209829554, + 1318.7565059633926 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r8p00.json", + "sha256": "a0f5900ac9d2c47dcc7d7ac08d3f3d23e1d6f529fdd0200989afa819dd3df055", + "summary": { + "admission_lag_max_ms": 0.17414195463061333, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.90625, + "passed": 58, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1475.8616799954325, + "ttft_p50_ms": 775.7647309917957, + "ttft_p95_ms": 1321.9368209829554 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r8p00.json", + "sha256": "c0fc3c85095ddacc66849e6f58be0851abab6d4022ee4eb7120c8cb8ea9e5d2c", + "summary": { + "admission_lag_max_ms": 2.749277977272868, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.921875, + "passed": 59, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1407.0930189918727, + "ttft_p50_ms": 763.8335369993001, + "ttft_p95_ms": 1318.7565059633926 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4893.3289990527555, + 4893.17379205022 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r16p00.json", + "sha256": "4e9ccaf5384f622b1da7c58d3de803c86099dd9deae91e363aabccc14aeabc05", + "summary": { + "admission_lag_max_ms": 0.11893699411302805, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4972.85822511185, + "ttft_p50_ms": 2742.8449960425496, + "ttft_p95_ms": 4893.3289990527555 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r16p00.json", + "sha256": "c207e415186aed1148e16085145bcdfd189314e7b92f74ea72fb2e4972ef4eae", + "summary": { + "admission_lag_max_ms": 0.09541702456772327, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4971.2590809213, + "ttft_p50_ms": 2740.232800017111, + "ttft_p95_ms": 4893.17379205022 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 6653.632597066462, + 6647.155530983582 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r32p00.json", + "sha256": "780f563a2739d892761905031f82ca8bd0ac4f4eb485c454390dcb2bcdfc1b81", + "summary": { + "admission_lag_max_ms": 0.105265062302351, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6922.717701992951, + "ttft_p50_ms": 3652.452490059659, + "ttft_p95_ms": 6653.632597066462 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r32p00.json", + "sha256": "2a5eb8fb37f03e759de149d24f9b23c812a26afe72ffaaffa030688904634e43", + "summary": { + "admission_lag_max_ms": 0.10375294368714094, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6917.111590038985, + "ttft_p50_ms": 3649.0042679943144, + "ttft_p95_ms": 6647.155530983582 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 7747.191094094887, + 7839.257381972857 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r64p00.json", + "sha256": "b94a39a91842ab39d545c473fc4f872ec79eb1888943c3e03ab03c18961a9105", + "summary": { + "admission_lag_max_ms": 0.6591080455109477, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7851.370444986969, + "ttft_p50_ms": 4335.578468977474, + "ttft_p95_ms": 7747.191094094887 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r64p00.json", + "sha256": "69ab47305c4bb14f3b4ed5bc0b18a37b8a5e7c1cdbee9232e6d2b9616ed10f84", + "summary": { + "admission_lag_max_ms": 0.10753795504570007, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.125, + "passed": 8, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7885.349316056818, + "ttft_p50_ms": 4009.1016669757664, + "ttft_p95_ms": 7839.257381972857 + } + } + ] + } + ], + "capacity": 7.0, + "capacity_per_gpu": 7.0, + "mns": 32, + "name": "tp1_mns32", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z", + "tp": 1 + }, + "tp1_mns64": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 156.76817402709275, + 152.9719429090619 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r4p00.json", + "sha256": "df798dd7a1cde3cfaf698286f44bbac2b0730a2c806f125268bdf7226e2bb47c", + "summary": { + "admission_lag_max_ms": 0.1147780567407608, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 162.9874319769442, + "ttft_p50_ms": 152.67050196416676, + "ttft_p95_ms": 156.76817402709275 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r4p00.json", + "sha256": "df80057224d69ce2a79f937aa45c2093d66d131765f3c7bfcc2289a9aaff3d6a", + "summary": { + "admission_lag_max_ms": 0.2131829969584942, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 153.917470946908, + "ttft_p50_ms": 150.96506499685347, + "ttft_p95_ms": 152.9719429090619 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 5.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 153.4973110537976, + 155.23677493911237 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r5p00.json", + "sha256": "e614ffaf879859407fc6420847f0140a81bda9a94fcc438fb6782c9019bf5378", + "summary": { + "admission_lag_max_ms": 0.1283780438825488, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 155.08639602921903, + "ttft_p50_ms": 152.1085740532726, + "ttft_p95_ms": 153.4973110537976 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r5p00.json", + "sha256": "13e825eb5ced3d0cb2b5dd68d9243fdf1cee8779b90e9aad77be04053be22be2", + "summary": { + "admission_lag_max_ms": 0.1683390000835061, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 155.6691420264542, + "ttft_p50_ms": 152.42570801638067, + "ttft_p95_ms": 155.23677493911237 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 6.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 152.53963600844145, + 152.85827894695103 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r6p00.json", + "sha256": "ac530f98f57ec452538ebe2311291d0a7c83bbd5554e82f53e78a2311802e411", + "summary": { + "admission_lag_max_ms": 0.13490929268300533, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 157.18225098680705, + "ttft_p50_ms": 150.29372205026448, + "ttft_p95_ms": 152.53963600844145 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r6p00.json", + "sha256": "2f1445ee0890b0972252d10180ef6f00cbad45ee73bd174f21b370cdf950b654", + "summary": { + "admission_lag_max_ms": 0.16514293383806944, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 158.88926689513028, + "ttft_p50_ms": 150.2534201135859, + "ttft_p95_ms": 152.85827894695103 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 7.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 346.96662798523903, + 295.1050230767578 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r7p00.json", + "sha256": "c4765794d442daf7eeb64a2f587792701ebe7659c7702a3bbbfc8900801fff0e", + "summary": { + "admission_lag_max_ms": 0.16206549480557442, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 354.881492909044, + "ttft_p50_ms": 310.15722593292594, + "ttft_p95_ms": 346.96662798523903 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r7p00.json", + "sha256": "30addb463839e23a7356a5cf6d4060ec1eebfa2b9564fa9500be5bcb7aecb537", + "summary": { + "admission_lag_max_ms": 0.10446517262607813, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 296.6241450048983, + "ttft_p50_ms": 246.06937903445214, + "ttft_p95_ms": 295.1050230767578 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 8.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1376.107804942876, + 1321.1636180058122 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r8p00.json", + "sha256": "89e2975f9c7319e477789ea77015541d5235efa61d90fb8bd5b0420a58f7f186", + "summary": { + "admission_lag_max_ms": 0.1416490413248539, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.890625, + "passed": 57, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1501.330335973762, + "ttft_p50_ms": 802.2130889585242, + "ttft_p95_ms": 1376.107804942876 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r8p00.json", + "sha256": "9ca6935f1320033c3ec4b72a35c5d9909ccd7b0fff2e4b2c92c6a26f52887df1", + "summary": { + "admission_lag_max_ms": 0.12422900181263685, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.921875, + "passed": 59, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1410.047525074333, + "ttft_p50_ms": 766.8001820566133, + "ttft_p95_ms": 1321.1636180058122 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4950.155022088438, + 4892.750842031091 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r16p00.json", + "sha256": "4fce2749435bf2e8c77999e11be0a788cd85f459a647760f3db34f1dbc0aee85", + "summary": { + "admission_lag_max_ms": 0.10520406067371368, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.171875, + "passed": 11, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 5031.207721913233, + "ttft_p50_ms": 2775.8131299633533, + "ttft_p95_ms": 4950.155022088438 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r16p00.json", + "sha256": "33ded0192d8986dbcf0a7f5ab6329a440994243d4e528d6c1b278bb6b92720fd", + "summary": { + "admission_lag_max_ms": 0.14005706179887056, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4971.664474112913, + "ttft_p50_ms": 2743.070787983015, + "ttft_p95_ms": 4892.750842031091 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 6750.775643973611, + 6754.7571890754625 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r32p00.json", + "sha256": "44030c4d3374cecebe67b886b428ba8014c87d941c00a178d6049bc57c809f46", + "summary": { + "admission_lag_max_ms": 0.7358170114457607, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.15625, + "passed": 10, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6899.517056066543, + "ttft_p50_ms": 3728.6347220651805, + "ttft_p95_ms": 6750.775643973611 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r32p00.json", + "sha256": "f397bad36552c6df5bf5bcee702cfa968ddf3e20b20e9976554ddfbe838ef3b2", + "summary": { + "admission_lag_max_ms": 0.7476879982277751, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.15625, + "passed": 10, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6901.606322033331, + "ttft_p50_ms": 3729.3128239689395, + "ttft_p95_ms": 6754.7571890754625 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 7800.616211956367, + 7748.703246936202 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r64p00.json", + "sha256": "61eadc2af60386d6972d69fe7b083874e705864162073d54d1bde2bc0e6de7e7", + "summary": { + "admission_lag_max_ms": 0.11387700214982033, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7905.77321103774, + "ttft_p50_ms": 4366.223473916762, + "ttft_p95_ms": 7800.616211956367 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r64p00.json", + "sha256": "db02e9a814154a9259cbe3fefdf15700923458f7e1208596e861b69b6a6ddd41", + "summary": { + "admission_lag_max_ms": 0.14900392852723598, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7852.748095989227, + "ttft_p50_ms": 4337.09563605953, + "ttft_p95_ms": 7748.703246936202 + } + } + ] + } + ], + "capacity": 7.0, + "capacity_per_gpu": 7.0, + "mns": 64, + "name": "tp1_mns64", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z", + "tp": 1 + }, + "tp1_mns8": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 154.07429495826364, + 153.62840006127954 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r4p00.json", + "sha256": "23d5ab491b70c02f51c2f06dec0407edc0627819a33b9bcef1349d974a4e20fe", + "summary": { + "admission_lag_max_ms": 0.14362391084432602, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 171.46877304185182, + "ttft_p50_ms": 151.62539994344115, + "ttft_p95_ms": 154.07429495826364 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r4p00.json", + "sha256": "563aa230e0f5e420cf453dfd9b829d7df493919192d638cbe76f85eb77b6d2e7", + "summary": { + "admission_lag_max_ms": 0.11323392391204834, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 155.62100906390697, + "ttft_p50_ms": 151.69239102397114, + "ttft_p95_ms": 153.62840006127954 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 5.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 154.20477103907615, + 153.81942794192582 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r5p00.json", + "sha256": "6f2a04a5c11eed5826e2a6e1ce6e6f8d98504b31d0fec30d9440df2cdb62e56d", + "summary": { + "admission_lag_max_ms": 0.10794296395033598, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 155.37187200970948, + "ttft_p50_ms": 152.59731002151966, + "ttft_p95_ms": 154.20477103907615 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r5p00.json", + "sha256": "27a0bfa65a66e5cc77e10a5cf39a846f0fd1dd266c10926dae9db7a80c581d6e", + "summary": { + "admission_lag_max_ms": 0.1204309519380331, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 155.40366806089878, + "ttft_p50_ms": 152.0398510619998, + "ttft_p95_ms": 153.81942794192582 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 6.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 153.00951700191945, + 153.5524509381503 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r6p00.json", + "sha256": "4562fae05d510cd50f2c9d7479916d056c8ac4d04851d2388b98f8410192c23a", + "summary": { + "admission_lag_max_ms": 0.10347133502364159, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 156.3970319693908, + "ttft_p50_ms": 150.414744974114, + "ttft_p95_ms": 153.00951700191945 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r6p00.json", + "sha256": "bd13283e2807150396b278f46ff630ae9820895e0c054a9efcfd8fc213947d47", + "summary": { + "admission_lag_max_ms": 0.13600767124444246, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 159.63539096992463, + "ttft_p50_ms": 151.83425601571798, + "ttft_p95_ms": 153.5524509381503 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 7.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 336.8800369789824, + 329.65395506471395 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r7p00.json", + "sha256": "fd5afe52238d6ddb72bdf7f7b142fddbeb3f62e160e4a1b3a639b308717964e6", + "summary": { + "admission_lag_max_ms": 0.14346837997436523, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 341.1952300230041, + "ttft_p50_ms": 294.81438896618783, + "ttft_p95_ms": 336.8800369789824 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r7p00.json", + "sha256": "b0fdc1ef617313261c490010adeb2f4ee93bbb053f1db10734ed5a63fd37fa76", + "summary": { + "admission_lag_max_ms": 0.11047185398638248, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 334.06115300022066, + "ttft_p50_ms": 281.1476809438318, + "ttft_p95_ms": 329.65395506471395 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 8.0, + "round_feasible": [ + false, + true + ], + "round_ttft_p95_ms": [ + 1289.2397560644895, + 1254.356418037787 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r8p00.json", + "sha256": "8877ed44b90f421afd22ec110fe6337581c67cd973b44378bcb7528fc8624de3", + "summary": { + "admission_lag_max_ms": 0.09899691212922335, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.9375, + "passed": 60, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1454.3529660440981, + "ttft_p50_ms": 813.0663179326802, + "ttft_p95_ms": 1289.2397560644895 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r8p00.json", + "sha256": "7ab24f5ab33bf166589af7e2f5c2a274ceef6a64f0029a96ec1879b42332123d", + "summary": { + "admission_lag_max_ms": 0.11694896966218948, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 0.953125, + "passed": 61, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1347.0055809011683, + "ttft_p50_ms": 744.0344559727237, + "ttft_p95_ms": 1254.356418037787 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4903.538785991259, + 4900.28300601989 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r16p00.json", + "sha256": "363d9b43164b5451bc3dd668caa0ce141d15114087cf444e7fd79d12b7563fd6", + "summary": { + "admission_lag_max_ms": 0.11420203372836113, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4982.48829995282, + "ttft_p50_ms": 2749.9175920384005, + "ttft_p95_ms": 4903.538785991259 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r16p00.json", + "sha256": "8aec70dfd6b69f4903bccc02c5d5474165ed07d9dd010c860ff9b447429da950", + "summary": { + "admission_lag_max_ms": 0.19414699636399746, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4979.007771937177, + "ttft_p50_ms": 2749.115635990165, + "ttft_p95_ms": 4900.28300601989 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 6760.187636013143, + 6534.733187989332 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r32p00.json", + "sha256": "7684ac0d2d6f851218b88fb37db3a5b54d877a6f1ead3bb604a6cb4d9a61cf3f", + "summary": { + "admission_lag_max_ms": 0.43534801807254553, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.15625, + "passed": 10, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6907.940509961918, + "ttft_p50_ms": 3735.540736000985, + "ttft_p95_ms": 6760.187636013143 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r32p00.json", + "sha256": "947974a9ae04e170b8c1aad32ddbda8c1000f3902a9b572c6338a7d0d4f23146", + "summary": { + "admission_lag_max_ms": 1.221106038428843, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6798.48151106853, + "ttft_p50_ms": 3595.78589303419, + "ttft_p95_ms": 6534.733187989332 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 7754.704045015387, + 7756.428897031583 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r64p00.json", + "sha256": "abfc2a158087f0ad75b560dd420604a9f81359ffdf39f0ce8e2ea25b451e6904", + "summary": { + "admission_lag_max_ms": 0.13654399663209915, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7857.916672946885, + "ttft_p50_ms": 4341.527819051407, + "ttft_p95_ms": 7754.704045015387 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r64p00.json", + "sha256": "192c0c850a52d2b64780bc6321cbfd10faddcc0ff10741ddad283b7d789b9def", + "summary": { + "admission_lag_max_ms": 0.11000398080796003, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7859.458995983005, + "ttft_p50_ms": 4341.894763987511, + "ttft_p95_ms": 7756.428897031583 + } + } + ] + } + ], + "capacity": 7.0, + "capacity_per_gpu": 7.0, + "mns": 8, + "name": "tp1_mns8", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z", + "tp": 1 + }, + "tp2_mns16": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 99.1211449727416, + 101.08128399588168 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r4p00.json", + "sha256": "f149882da390114f2a2a140a3bd2ca6f15b6f7761591ea7b49b5a961c9d18df5", + "summary": { + "admission_lag_max_ms": 0.10085199028253555, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 101.28417506348342, + "ttft_p50_ms": 95.90165293775499, + "ttft_p95_ms": 99.1211449727416 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r4p00.json", + "sha256": "154dfaf2729001a8ef5b7430290f4ca906df1d4758e867d5b350489410f36d5c", + "summary": { + "admission_lag_max_ms": 0.11863000690937042, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 105.76263803523034, + "ttft_p50_ms": 96.27402503974736, + "ttft_p95_ms": 101.08128399588168 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 97.1850419882685, + 97.03529300168157 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r8p00.json", + "sha256": "9338d5cb994b0c12a6420f030ae2a9335a04986b4a81e31f4781ac838aeef2d4", + "summary": { + "admission_lag_max_ms": 0.12352492194622755, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 102.26434096693993, + "ttft_p50_ms": 95.07436107378453, + "ttft_p95_ms": 97.1850419882685 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r8p00.json", + "sha256": "a35b67970948976482ee835e595ee5631fa921607276d5ea7928460514f0d06b", + "summary": { + "admission_lag_max_ms": 0.11170096695423126, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 101.72453604172915, + "ttft_p50_ms": 94.03157397173345, + "ttft_p95_ms": 97.03529300168157 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 10.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 96.10817302018404, + 94.37024802900851 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r10p00.json", + "sha256": "f67e0f4f6b41861fcbcbee04b4efbf0dee46e273239ec94ef22d650ba6afa22a", + "summary": { + "admission_lag_max_ms": 0.10012194979935884, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 102.40214702207595, + "ttft_p50_ms": 94.17163301259279, + "ttft_p95_ms": 96.10817302018404 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r10p00.json", + "sha256": "5a64b52f0298e9ecda2fddb8230cc27ab4b9c1db01575f68b5331816b324d1d7", + "summary": { + "admission_lag_max_ms": 0.10465399827808142, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 101.07159405015409, + "ttft_p50_ms": 93.24954205658287, + "ttft_p95_ms": 94.37024802900851 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 12.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 447.0475659472868, + 454.8899739747867 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r12p00.json", + "sha256": "c6e27b9078281a54feeb1fab09616385a70de819791ce14f46b001bf540a80ab", + "summary": { + "admission_lag_max_ms": 0.12051803059875965, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 455.56937297806144, + "ttft_p50_ms": 280.09015799034387, + "ttft_p95_ms": 447.0475659472868 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r12p00.json", + "sha256": "861f9c4186c33e9ee6365d19c53e98c10615c5ee529a84e69f84dc292f53ac4f", + "summary": { + "admission_lag_max_ms": 0.11322635691612959, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 457.4526828946546, + "ttft_p50_ms": 290.1579710887745, + "ttft_p95_ms": 454.8899739747867 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 14.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 1000.5082850111648, + 1004.6314439969137 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r14p00.json", + "sha256": "2dcaf1087edd1f12fbfe0618a6ab636b22484756a5729919bc0e51de153f8f4a", + "summary": { + "admission_lag_max_ms": 0.1881921198219061, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1089.2393708927557, + "ttft_p50_ms": 611.2175739835948, + "ttft_p95_ms": 1000.5082850111648 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r14p00.json", + "sha256": "16937b31f72298c7a50ec3c7b9c1b9c798f305d83469b3d3c058efc4d53135da", + "summary": { + "admission_lag_max_ms": 0.10659568943083286, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1096.6680150013417, + "ttft_p50_ms": 617.3943240428343, + "ttft_p95_ms": 1004.6314439969137 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1458.4544260287657, + 1471.3564389385283 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r16p00.json", + "sha256": "5576a7fd06977f51f1c43bb73c3c474cc21a93cd288b422ec3d09db671b3afe7", + "summary": { + "admission_lag_max_ms": 0.135264010168612, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.765625, + "passed": 49, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1583.2410319708288, + "ttft_p50_ms": 904.5663280412555, + "ttft_p95_ms": 1458.4544260287657 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r16p00.json", + "sha256": "2ccf16426046c412829442f72932268d5b27ce051a4312d135d32b240b48ec7c", + "summary": { + "admission_lag_max_ms": 0.11741300113499165, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.78125, + "passed": 50, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1537.9131329245865, + "ttft_p50_ms": 896.1611289996654, + "ttft_p95_ms": 1471.3564389385283 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 3348.027400090359, + 3348.575382027775 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r32p00.json", + "sha256": "d258760d15f1fdf2adefca72f7a3505c0b839036799dcb074450f7f526f76718", + "summary": { + "admission_lag_max_ms": 0.18610397819429636, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3442.3487790627405, + "ttft_p50_ms": 1801.6119640087709, + "ttft_p95_ms": 3348.027400090359 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r32p00.json", + "sha256": "edf5cccc1400639a94b5430897a04f20d2214bb1ffee7f4bfa62514150179a6a", + "summary": { + "admission_lag_max_ms": 0.1144439447671175, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3443.0311269825324, + "ttft_p50_ms": 1801.435066969134, + "ttft_p95_ms": 3348.575382027775 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4286.911493982188, + 4287.898648995906 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r64p00.json", + "sha256": "2a6303efb826637c3c6448d6f766f7de4fc4d7b63db163a48f72c20df68394f4", + "summary": { + "admission_lag_max_ms": 0.10997895151376724, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4340.311317006126, + "ttft_p50_ms": 2396.4726959820837, + "ttft_p95_ms": 4286.911493982188 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r64p00.json", + "sha256": "6fafe3bf206583b8689be1b2b57d6b92cbd1ba54f7fb230ca05f6c68b3ab9757", + "summary": { + "admission_lag_max_ms": 0.12669397983700037, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4340.538130956702, + "ttft_p50_ms": 2394.112486974336, + "ttft_p95_ms": 4287.898648995906 + } + } + ] + } + ], + "capacity": 14.0, + "capacity_per_gpu": 7.0, + "mns": 16, + "name": "tp2_mns16", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z", + "tp": 2 + }, + "tp2_mns32": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 97.2925309324637, + 96.13083000294864 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r4p00.json", + "sha256": "812f8a108a9390fb759b46e5bd41e3ddad66bc144d47bf4024d834e42b47c47e", + "summary": { + "admission_lag_max_ms": 0.11386198457330465, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 100.76773096807301, + "ttft_p50_ms": 95.72782705072314, + "ttft_p95_ms": 97.2925309324637 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r4p00.json", + "sha256": "70f922eb7651d46d81d7d394b8453c5b778f921069cf48fb2ae2677afa94a842", + "summary": { + "admission_lag_max_ms": 0.30313804745674133, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 129.06646099872887, + "ttft_p50_ms": 94.53632694203407, + "ttft_p95_ms": 96.13083000294864 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 95.42673197574914, + 98.94769196398556 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r8p00.json", + "sha256": "24097d38b0ffb45f6cdbd93f0aad04dce3b7da41ff45e948eb3a794902b6547f", + "summary": { + "admission_lag_max_ms": 0.11571904178708792, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 100.04934400785714, + "ttft_p50_ms": 94.34181300457567, + "ttft_p95_ms": 95.42673197574914 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r8p00.json", + "sha256": "7251f076d34ba9484b08fa3fc4b898d2ba8d1199cb1eaeaaffc19930f8c889b9", + "summary": { + "admission_lag_max_ms": 0.1177559606730938, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 100.93720292206854, + "ttft_p50_ms": 94.77824496570975, + "ttft_p95_ms": 98.94769196398556 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 10.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 96.28185909241438, + 95.9081610199064 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r10p00.json", + "sha256": "85796457715a25c15fc909f6def32776fbd49956eb7890df1c4abcef8c5afe78", + "summary": { + "admission_lag_max_ms": 0.14786410611122847, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 102.46025596279651, + "ttft_p50_ms": 93.54631893802434, + "ttft_p95_ms": 96.28185909241438 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r10p00.json", + "sha256": "0bf4d4d006bf0b5bd8b989ceeeb4bacb0e3ee1b8ea66f30b3deb120a14871cf0", + "summary": { + "admission_lag_max_ms": 0.11162098962813616, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 100.54603894241154, + "ttft_p50_ms": 93.77408598084003, + "ttft_p95_ms": 95.9081610199064 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 12.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 439.0145930228755, + 429.9719400005415 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r12p00.json", + "sha256": "68de6e36bd7c5624cab107fe5cb21bb817711691e545439a958c926a27e3fe29", + "summary": { + "admission_lag_max_ms": 0.10475562885403633, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 445.90794993564487, + "ttft_p50_ms": 270.3119309153408, + "ttft_p95_ms": 439.0145930228755 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r12p00.json", + "sha256": "d8d3a44a07b46f1673817ece2948fd97c30f41bd451d93a2fa02edf074536c63", + "summary": { + "admission_lag_max_ms": 0.4323029424995184, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 434.6071509644389, + "ttft_p50_ms": 265.2625450864434, + "ttft_p95_ms": 429.9719400005415 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 14.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 1014.2896320903674, + 999.8802930349484 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r14p00.json", + "sha256": "10196c2f9a9b69c2e47f64d06bf5ee6863af4cd2b612301622f0e304949a942a", + "summary": { + "admission_lag_max_ms": 0.09005970787256956, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1105.1907669752836, + "ttft_p50_ms": 621.6445650206879, + "ttft_p95_ms": 1014.2896320903674 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r14p00.json", + "sha256": "c10cab85588a3563bed36fc4480462635d34d240938beaa0e5034f13186578a6", + "summary": { + "admission_lag_max_ms": 0.11327012907713652, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1090.6395060010254, + "ttft_p50_ms": 623.5586369875818, + "ttft_p95_ms": 999.8802930349484 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1488.3801550604403, + 1484.6198010491207 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r16p00.json", + "sha256": "ef5046c49c32178855bed6a0488ea21e8da22f786ddaabc503bea2d1025bb2f3", + "summary": { + "admission_lag_max_ms": 0.7892940193414688, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.765625, + "passed": 49, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1613.616122980602, + "ttft_p50_ms": 905.2404230460525, + "ttft_p95_ms": 1488.3801550604403 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r16p00.json", + "sha256": "de02c1d49414cffe7e20023abf4d502a4efc6d2e01fcb3e49e928081aaef77b6", + "summary": { + "admission_lag_max_ms": 0.10548799764364958, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.765625, + "passed": 49, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1550.9842830942944, + "ttft_p50_ms": 904.9503420246765, + "ttft_p95_ms": 1484.6198010491207 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 3391.811708919704, + 3362.9186518955976 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r32p00.json", + "sha256": "876c5444d17c4ce81eca907ae17d5cf7062ffa67fa266b491580e52f712614ec", + "summary": { + "admission_lag_max_ms": 0.12557301670312881, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3486.074637970887, + "ttft_p50_ms": 1840.204154024832, + "ttft_p95_ms": 3391.811708919704 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r32p00.json", + "sha256": "02b4cd93ce1f9816e822022fb608ba8d4c89265f673f6d27c8fa3d4f1523ffe0", + "summary": { + "admission_lag_max_ms": 0.11014204937964678, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3456.965696066618, + "ttft_p50_ms": 1812.343467026949, + "ttft_p95_ms": 3362.9186518955976 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4306.351027917117, + 4302.602957002819 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r64p00.json", + "sha256": "13ba282f179b2d6ce13a0aa2fd04eb29f82e579b7995e3406cb0afd69a2d5ee2", + "summary": { + "admission_lag_max_ms": 0.3526840591803193, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4359.505696920678, + "ttft_p50_ms": 2411.1130589153618, + "ttft_p95_ms": 4306.351027917117 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r64p00.json", + "sha256": "767622d62f0757ef2d8c7651ea470bfa2e62edad0ed4e75f66f84a0b722e35da", + "summary": { + "admission_lag_max_ms": 0.10626309085637331, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4355.409953044727, + "ttft_p50_ms": 2409.5974090741947, + "ttft_p95_ms": 4302.602957002819 + } + } + ] + } + ], + "capacity": 14.0, + "capacity_per_gpu": 7.0, + "mns": 32, + "name": "tp2_mns32", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z", + "tp": 2 + }, + "tp2_mns64": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 96.54812794178724, + 95.08875198662281 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r4p00.json", + "sha256": "6e6a45d0c1010094c3395969815fd8cf9c35b919db202d772fb514744f981ebc", + "summary": { + "admission_lag_max_ms": 0.11306197848170996, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 99.27749610505998, + "ttft_p50_ms": 93.01548008807003, + "ttft_p95_ms": 96.54812794178724 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r4p00.json", + "sha256": "58aa71d694704cfea15c37cb29420dbac1dc505045c29a731e8be904846cca5f", + "summary": { + "admission_lag_max_ms": 0.11853000614792109, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 98.74943597242236, + "ttft_p50_ms": 93.57558900956064, + "ttft_p95_ms": 95.08875198662281 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 94.24941800534725, + 96.21043398510665 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r8p00.json", + "sha256": "311350af236cb0306aa635f05f8999b66bf5080bad7cc1f6703fffe0be31aee6", + "summary": { + "admission_lag_max_ms": 0.19771000370383263, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 100.52211803849787, + "ttft_p50_ms": 92.4453770276159, + "ttft_p95_ms": 94.24941800534725 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r8p00.json", + "sha256": "60fe989a36a526e76f6f5c98e217b9feaf081bc680b16b9d53650bf085047c06", + "summary": { + "admission_lag_max_ms": 0.3933720290660858, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 102.97230398282409, + "ttft_p50_ms": 93.26327103190124, + "ttft_p95_ms": 96.21043398510665 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 10.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 97.98661107197404, + 92.9473100695759 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r10p00.json", + "sha256": "d0a04444a0e41bf76f71ac6ea3815bfbe76f7a51f8ddf669976bc3a4896dd0c1", + "summary": { + "admission_lag_max_ms": 0.10270101483911276, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 103.49742788821459, + "ttft_p50_ms": 94.55623896792531, + "ttft_p95_ms": 97.98661107197404 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r10p00.json", + "sha256": "f330f26f45f6039c48150a0b6b146ca3dbb9ea8b2dff086a9efd5938ab981eb7", + "summary": { + "admission_lag_max_ms": 0.11239596642553806, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 97.4133670097217, + "ttft_p50_ms": 91.2688790122047, + "ttft_p95_ms": 92.9473100695759 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 12.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 260.68170997314155, + 319.04888805001974 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r12p00.json", + "sha256": "fed5eacf18018402b0c2675b8c5595f92693a9bddec30f354d989098f9fe4965", + "summary": { + "admission_lag_max_ms": 0.11784560047090054, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 345.19017208367586, + "ttft_p50_ms": 211.41935500781983, + "ttft_p95_ms": 260.68170997314155 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r12p00.json", + "sha256": "f2be27cbeff3e5c1a95050418f2dc3c712a4648cada8cdb3d36382cfffc9593a", + "summary": { + "admission_lag_max_ms": 0.12930459342896938, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 326.86293800361454, + "ttft_p50_ms": 229.8808820778504, + "ttft_p95_ms": 319.04888805001974 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 14.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 895.7287209341303, + 912.1733580250293 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r14p00.json", + "sha256": "c2e157808e0308cc443f3abdf5114898cef66c435688d24321f80bef2d966208", + "summary": { + "admission_lag_max_ms": 0.14902697876095772, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 962.3544370988384, + "ttft_p50_ms": 576.2168710352853, + "ttft_p95_ms": 895.7287209341303 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r14p00.json", + "sha256": "749ef7d31bf8782d42719b75ee1a2e4a443f097510af721d02cf53df191bc66e", + "summary": { + "admission_lag_max_ms": 0.42896345257759094, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 986.0872740391642, + "ttft_p50_ms": 569.7114520007744, + "ttft_p95_ms": 912.1733580250293 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1364.6171110449359, + 1366.6857269126922 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r16p00.json", + "sha256": "7545ea9210348019de88ede5d2cba7527913534274deee034fa6811c173a6e72", + "summary": { + "admission_lag_max_ms": 0.12612901628017426, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.859375, + "passed": 55, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1435.8308439841494, + "ttft_p50_ms": 848.9949750946835, + "ttft_p95_ms": 1364.6171110449359 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r16p00.json", + "sha256": "d75e27137f6a7811126d8c957bb5a081944fc1bb7643852cdd785111773f6010", + "summary": { + "admission_lag_max_ms": 0.1258660340681672, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.859375, + "passed": 55, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1436.2916270038113, + "ttft_p50_ms": 846.2129919789732, + "ttft_p95_ms": 1366.6857269126922 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 3243.9196659252048, + 3214.524411014281 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r32p00.json", + "sha256": "366ebddc167338ed096f373de5667740721422c13d2db1f887ed09186afafd19", + "summary": { + "admission_lag_max_ms": 0.09669095743447542, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3337.068362045102, + "ttft_p50_ms": 1751.3202880509198, + "ttft_p95_ms": 3243.9196659252048 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r32p00.json", + "sha256": "01b8f5081e36a64d5776285c8c5f423b53b1ad684c4b8f95a8caa5d997c6bf09", + "summary": { + "admission_lag_max_ms": 0.8310690755024552, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.328125, + "passed": 21, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3277.297410997562, + "ttft_p50_ms": 1790.5721209244803, + "ttft_p95_ms": 3214.524411014281 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4319.725521025248, + 4367.031275993213 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r64p00.json", + "sha256": "92202709fd6bbd558ff6f1986df213848254110c4274447a7bd5879153290d46", + "summary": { + "admission_lag_max_ms": 0.10392803233116865, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4360.145599930547, + "ttft_p50_ms": 2403.5812759539112, + "ttft_p95_ms": 4319.725521025248 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r64p00.json", + "sha256": "9660d15d31bc70c5550ed29477f563313d4d81baf0e561126aebd3b7d097cfa3", + "summary": { + "admission_lag_max_ms": 0.17276196740567684, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.25, + "passed": 16, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4414.442001027055, + "ttft_p50_ms": 2260.2788450894877, + "ttft_p95_ms": 4367.031275993213 + } + } + ] + } + ], + "capacity": 14.0, + "capacity_per_gpu": 7.0, + "mns": 64, + "name": "tp2_mns64", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z", + "tp": 2 + }, + "tp2_mns8": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 97.17482794076204, + 98.03147299680859 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r4p00.json", + "sha256": "0e315f6d9e0087ad553e874b0c16c2e6de79eb6eca22eb118dc5deca52e6efe3", + "summary": { + "admission_lag_max_ms": 0.11708191595971584, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 101.92840604577214, + "ttft_p50_ms": 95.13642196543515, + "ttft_p95_ms": 97.17482794076204 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r4p00.json", + "sha256": "baa394a24d2a3db40a3a7ae6dae0c5ce49ff05499a1e17eaa0bca4b479ae33bf", + "summary": { + "admission_lag_max_ms": 0.10946101974695921, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 100.52333807107061, + "ttft_p50_ms": 93.19468098692596, + "ttft_p95_ms": 98.03147299680859 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 97.67980605829507, + 93.80627400241792 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r8p00.json", + "sha256": "303df27c6d4604d5af65f9056acacb7c4373fb6eb7edb6e5d42689eeaa384aaa", + "summary": { + "admission_lag_max_ms": 0.11111306957900524, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 113.2329449756071, + "ttft_p50_ms": 93.05417106952518, + "ttft_p95_ms": 97.67980605829507 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r8p00.json", + "sha256": "cdbb1bca17b24b67e18ef3dd4997494d6bd2a887b3b5d1264fbf411e688c53bc", + "summary": { + "admission_lag_max_ms": 0.12463307939469814, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 101.12245299387723, + "ttft_p50_ms": 92.26030297577381, + "ttft_p95_ms": 93.80627400241792 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 10.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 94.45723297540098, + 93.57301203999668 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r10p00.json", + "sha256": "fe17ef2417319307d651bc10b01a618234d61bab797ad87a7332cab20d78ecb2", + "summary": { + "admission_lag_max_ms": 0.3267280990257859, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 99.28593400400132, + "ttft_p50_ms": 92.22474799025804, + "ttft_p95_ms": 94.45723297540098 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r10p00.json", + "sha256": "550f693708a34624a82d77afd4da0ab4bc01e09f42242a3a79e4532897681a51", + "summary": { + "admission_lag_max_ms": 0.11238199658691883, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 97.77756198309362, + "ttft_p50_ms": 92.07128698471934, + "ttft_p95_ms": 93.57301203999668 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 12.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 320.49866404850036, + 336.9630330707878 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r12p00.json", + "sha256": "edab8169453151de732923c076b8f7dca4082ee47ccd2597e65e7ba68516bc2e", + "summary": { + "admission_lag_max_ms": 0.09914662223309278, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 327.930542989634, + "ttft_p50_ms": 231.4880370395258, + "ttft_p95_ms": 320.49866404850036 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r12p00.json", + "sha256": "a6dde4363d362ca10254409e3dc734d0f0e640e63156426f3f4fd7115db49e16", + "summary": { + "admission_lag_max_ms": 0.17016706988215446, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 428.6657409975305, + "ttft_p50_ms": 237.09688894450665, + "ttft_p95_ms": 336.9630330707878 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 14.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 911.7195709841326, + 910.7652489328757 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r14p00.json", + "sha256": "7eba81ee5c85c457a056f9aef28abd94d375fdaa116929cfe0a0c282a0738bd2", + "summary": { + "admission_lag_max_ms": 0.13163150288164616, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 999.2953530745581, + "ttft_p50_ms": 580.023527960293, + "ttft_p95_ms": 911.7195709841326 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r14p00.json", + "sha256": "6a113d3d876ff01a9c1101e8b75efa2fc71f96683d6bb430e26d5976426b5d23", + "summary": { + "admission_lag_max_ms": 0.11634768452495337, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 988.8870939612389, + "ttft_p50_ms": 578.9327350212261, + "ttft_p95_ms": 910.7652489328757 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1389.8832038976252, + 1365.0342189939693 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r16p00.json", + "sha256": "d677251ee1a941c52469052a239c2e13be0a837285069aa8182a16e196445e48", + "summary": { + "admission_lag_max_ms": 0.11070503387600183, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.828125, + "passed": 53, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1460.5093500576913, + "ttft_p50_ms": 864.3530959961936, + "ttft_p95_ms": 1389.8832038976252 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r16p00.json", + "sha256": "f5177f53aa0f87a4e7291f470978d86ad8bc46bd1a8362b79f48cd128646dc10", + "summary": { + "admission_lag_max_ms": 0.3672370221465826, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.828125, + "passed": 53, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1491.050804965198, + "ttft_p50_ms": 841.7225609300658, + "ttft_p95_ms": 1365.0342189939693 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 3258.010295103304, + 3258.664960041642 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r32p00.json", + "sha256": "02a6ac2b9c2b9ab97854d6fcc13b64bbb36a079e4afdae84231e4cb187fac4ef", + "summary": { + "admission_lag_max_ms": 0.1296499976888299, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3352.278529899195, + "ttft_p50_ms": 1755.0864539807662, + "ttft_p95_ms": 3258.010295103304 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r32p00.json", + "sha256": "9ca2f44fc7e3cb3d38ada548af858903b31bbee5cd50f1723c29255ebd4224e9", + "summary": { + "admission_lag_max_ms": 1.1343059595674276, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3352.7101600775495, + "ttft_p50_ms": 1755.3453800501302, + "ttft_p95_ms": 3258.664960041642 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4190.255103982054, + 4189.1050330596045 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r64p00.json", + "sha256": "53908d3a254e19691839831b4337014b09591ced9189c2a9b29e0d92e80fcb56", + "summary": { + "admission_lag_max_ms": 0.11591101065278053, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4242.029857938178, + "ttft_p50_ms": 2340.5529640149325, + "ttft_p95_ms": 4190.255103982054 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r64p00.json", + "sha256": "1c8c85c873aa96c01dc0200f848474a53c1173ef72f3828e8c23aeae70e92b11", + "summary": { + "admission_lag_max_ms": 0.284243025816977, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4240.770195960067, + "ttft_p50_ms": 2339.3583690049127, + "ttft_p95_ms": 4189.1050330596045 + } + } + ] + } + ], + "capacity": 14.0, + "capacity_per_gpu": 7.0, + "mns": 8, + "name": "tp2_mns8", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z", + "tp": 2 + }, + "tp4_mns16": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 74.66826005838811, + 66.5705680148676 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r4p00.json", + "sha256": "fb354fc8a097caa21844d4ea5515b98b03459f400816e5e5590a6cda81cf4815", + "summary": { + "admission_lag_max_ms": 0.1035250024870038, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 93.38432096410543, + "ttft_p50_ms": 65.75745099689811, + "ttft_p95_ms": 74.66826005838811 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r4p00.json", + "sha256": "ac8e8412f79a8a32ebf085445a94aa548dd2d646fce0b9f56b4378081dcc8960", + "summary": { + "admission_lag_max_ms": 0.12108997907489538, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 79.89856600761414, + "ttft_p50_ms": 62.386377016082406, + "ttft_p95_ms": 66.5705680148676 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 70.44799195136875, + 72.29184894822538 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r8p00.json", + "sha256": "05ebbd68bbf24e9c8b25684474411c7db122b92701cc1617f9d786cf19592751", + "summary": { + "admission_lag_max_ms": 0.1825150102376938, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 96.92095802165568, + "ttft_p50_ms": 60.598687967285514, + "ttft_p95_ms": 70.44799195136875 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r8p00.json", + "sha256": "d24f7daaf41073cc8aa030b74c06203783684c3e556770366883c37b826406f2", + "summary": { + "admission_lag_max_ms": 0.10628998279571533, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 75.13270596973598, + "ttft_p50_ms": 60.161596979014575, + "ttft_p95_ms": 72.29184894822538 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 16.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 66.02151494007558, + 107.58775402791798 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r16p00.json", + "sha256": "a29b2128983af630e1da8f970cbda802849036bc522eb506821bf2fe1a34172f", + "summary": { + "admission_lag_max_ms": 0.10468997061252594, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 72.93136999942362, + "ttft_p50_ms": 59.5597909996286, + "ttft_p95_ms": 66.02151494007558 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r16p00.json", + "sha256": "275ae35651cbe2200d16c9b06a68568efda527564065255a10dee6fe93346010", + "summary": { + "admission_lag_max_ms": 0.3144120564684272, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 119.78317995090038, + "ttft_p50_ms": 59.12985489703715, + "ttft_p95_ms": 107.58775402791798 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 20.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 193.8455159543082, + 187.52167106140405 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r20p00.json", + "sha256": "4cb485f30b81c004bab379e6522d94b811c0bdc0be6d8e42c9d8ae842d9600dc", + "summary": { + "admission_lag_max_ms": 0.10407401714473963, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 197.7908629924059, + "ttft_p50_ms": 142.73560303263366, + "ttft_p95_ms": 193.8455159543082 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r20p00.json", + "sha256": "63aa9dc03a08b781c5d6e8805f2b61698c75e71673608ecd38bd1132c62e6987", + "summary": { + "admission_lag_max_ms": 0.09386602323502302, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 190.29632199089974, + "ttft_p50_ms": 136.1723990412429, + "ttft_p95_ms": 187.52167106140405 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 24.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 568.2198900030926, + 573.4539540717378 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r24p00.json", + "sha256": "5e4d5b44850090d07fe1d3abc44eddea3bfdf2b682f59b12a45a866621e38f98", + "summary": { + "admission_lag_max_ms": 0.7511250441893935, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 613.2891150191426, + "ttft_p50_ms": 375.27791305910796, + "ttft_p95_ms": 568.2198900030926 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r24p00.json", + "sha256": "239e7789aed2e9b2bcf1d127215a6b122071eb856be2db64ccc184fda153aa48", + "summary": { + "admission_lag_max_ms": 0.11180061846971512, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 614.9205249967054, + "ttft_p50_ms": 375.5133739905432, + "ttft_p95_ms": 573.4539540717378 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 28.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 860.0242879474536, + 886.1482660286129 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r28p00.json", + "sha256": "8b249b52779e5b3cd9028aaee30ee73ef34b0b0a64dcd75b9aad57266c29fbc2", + "summary": { + "admission_lag_max_ms": 0.11425174307078123, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 930.8854410191998, + "ttft_p50_ms": 534.7980660153553, + "ttft_p95_ms": 860.0242879474536 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r28p00.json", + "sha256": "c9b28c8f3187542a343ecf1067783569ec2abfa64631d6827a94ba6876aa7b63", + "summary": { + "admission_lag_max_ms": 0.312068616040051, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 922.4315020255744, + "ttft_p50_ms": 549.7413279954344, + "ttft_p95_ms": 886.1482660286129 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 32.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 1147.1082780044526, + 1137.236590962857 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r32p00.json", + "sha256": "d27ff9f940a6cd7fba339d9a176de39cf4b5bd164dcc500f2af36bcb07cdef30", + "summary": { + "admission_lag_max_ms": 0.16146607231348753, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1201.626160996966, + "ttft_p50_ms": 696.4708579471335, + "ttft_p95_ms": 1147.1082780044526 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r32p00.json", + "sha256": "093044e87fffdaa222e5bad3807cbf4900aa64891d5b4ba27f187c7a411252d7", + "summary": { + "admission_lag_max_ms": 0.10430300608277321, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1201.045956928283, + "ttft_p50_ms": 684.3201500596479, + "ttft_p95_ms": 1137.236590962857 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 2064.763988018967, + 2039.6780440350994 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r64p00.json", + "sha256": "fad7bdd80d5ca603790a4e5d19dadd4f494540b106ce583e3e8c1c05d2374a58", + "summary": { + "admission_lag_max_ms": 6.394341005943716, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.53125, + "passed": 34, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2144.80376499705, + "ttft_p50_ms": 1158.5425049997866, + "ttft_p95_ms": 2064.763988018967 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r64p00.json", + "sha256": "24fdb945706dbbdd29916e0f08e0eff807334db212fded94c729da2e1b46efd3", + "summary": { + "admission_lag_max_ms": 0.10294502135366201, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.546875, + "passed": 35, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2125.4275270039216, + "ttft_p50_ms": 1131.1213460285217, + "ttft_p95_ms": 2039.6780440350994 + } + } + ] + } + ], + "capacity": 32.0, + "capacity_per_gpu": 8.0, + "mns": 16, + "name": "tp4_mns16", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z", + "tp": 4 + }, + "tp4_mns32": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 67.80304794665426, + 64.59337309934199 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r4p00.json", + "sha256": "2e82a30f02c2702668e59661930431cf82323746a111d42bf38b7fdc0d083b3d", + "summary": { + "admission_lag_max_ms": 0.1332589890807867, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 72.74050498381257, + "ttft_p50_ms": 64.10286005120724, + "ttft_p95_ms": 67.80304794665426 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r4p00.json", + "sha256": "8294196c7dc52765a6820f5e1d8439c480490daed9505263f707810f73c95783", + "summary": { + "admission_lag_max_ms": 0.1283179735764861, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 70.6796320155263, + "ttft_p50_ms": 60.962425894103944, + "ttft_p95_ms": 64.59337309934199 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 70.46496798284352, + 68.22343391831964 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r8p00.json", + "sha256": "38bbfeab47e5eb6296fcccf20f04ef25dc24db658be92300c492734bbfe333cf", + "summary": { + "admission_lag_max_ms": 0.10555307380855083, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 87.7126749837771, + "ttft_p50_ms": 62.65173899009824, + "ttft_p95_ms": 70.46496798284352 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r8p00.json", + "sha256": "716f90177d6f203717c80a8e9b28697e53141f204af470493c299bc73a1ea3ee", + "summary": { + "admission_lag_max_ms": 0.24518708232790232, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 80.40577603969723, + "ttft_p50_ms": 60.29119901359081, + "ttft_p95_ms": 68.22343391831964 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 16.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 89.53388908412308, + 76.93439698778093 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r16p00.json", + "sha256": "063016efe822c31c2425e7d963eda4c054f3f7c0341df0a3cd73ab86a942e873", + "summary": { + "admission_lag_max_ms": 0.2996840048581362, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 97.41822991054505, + "ttft_p50_ms": 60.97789097111672, + "ttft_p95_ms": 89.53388908412308 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r16p00.json", + "sha256": "eecc6bdeff90a8bf195dfc5387c6bc2f4db20ed7c9aeb9b4ebe050efccc6051a", + "summary": { + "admission_lag_max_ms": 0.36267400719225407, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 79.42048599943519, + "ttft_p50_ms": 60.29245804529637, + "ttft_p95_ms": 76.93439698778093 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 20.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 206.2494510319084, + 201.13301498349756 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r20p00.json", + "sha256": "22a62729358fea7c38ca166a1a49f1235e98c7f07ecfe4948322110cda4b7178", + "summary": { + "admission_lag_max_ms": 0.11367700062692165, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 255.14698098413646, + "ttft_p50_ms": 155.46583198010921, + "ttft_p95_ms": 206.2494510319084 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r20p00.json", + "sha256": "be103a759100bd2ab6dd31ee4ce41b6f0da40d232a60ebe137e3d933194a487c", + "summary": { + "admission_lag_max_ms": 0.17853605095297098, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 209.22932599205524, + "ttft_p50_ms": 155.12292610947043, + "ttft_p95_ms": 201.13301498349756 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 24.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 580.5047769099474, + 577.3971510352567 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r24p00.json", + "sha256": "890a733a97f399deb4c5efcca661d9a442ed76080cd5d8b6a2c09368aef701b7", + "summary": { + "admission_lag_max_ms": 0.1084543764591217, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 621.5670789824799, + "ttft_p50_ms": 374.6346529806033, + "ttft_p95_ms": 580.5047769099474 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r24p00.json", + "sha256": "87e8543652b1743b1b9b8c3de0fc680401b0bf9a33d61fa6132120c944b2a6eb", + "summary": { + "admission_lag_max_ms": 0.11801638174802065, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 623.2662099646404, + "ttft_p50_ms": 382.11427396163344, + "ttft_p95_ms": 577.3971510352567 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 28.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 873.0683700414374, + 889.4930009264499 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r28p00.json", + "sha256": "f877781730a5437b4cbdb76948915f1c871574b19a6df27076d21b65bb18e678", + "summary": { + "admission_lag_max_ms": 0.6807943573221564, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 943.9258739585057, + "ttft_p50_ms": 542.9852310335264, + "ttft_p95_ms": 873.0683700414374 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r28p00.json", + "sha256": "82151dc539b35302489c17976cfe89b8f4c64944298954607fad8f5c9cb9e7d8", + "summary": { + "admission_lag_max_ms": 6.750226486474276, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 936.5667289821431, + "ttft_p50_ms": 546.0105559322983, + "ttft_p95_ms": 889.4930009264499 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 32.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 1135.3467140579596, + 1146.743991994299 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r32p00.json", + "sha256": "082cc4c47b2eec6e49f0edd14cff5a51b880bd998d98da8b9830bbf7615483ef", + "summary": { + "admission_lag_max_ms": 0.10443700011819601, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1199.8225760180503, + "ttft_p50_ms": 680.4794340860099, + "ttft_p95_ms": 1135.3467140579596 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r32p00.json", + "sha256": "517d3cbc1104b5ea1bdb0e381c033cf978181a93449b6498d30b52cd3dd3e770", + "summary": { + "admission_lag_max_ms": 0.4415180301293731, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1210.0602959981188, + "ttft_p50_ms": 693.0603379150853, + "ttft_p95_ms": 1146.743991994299 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 2084.9693010095507, + 2041.9401369290426 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r64p00.json", + "sha256": "7e11771ed598826fb3c5594c7e05d0989e01030be2d8b36fa21813aff462496d", + "summary": { + "admission_lag_max_ms": 0.10768393985927105, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.5625, + "passed": 36, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2132.128676981665, + "ttft_p50_ms": 1113.0280290963128, + "ttft_p95_ms": 2084.9693010095507 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r64p00.json", + "sha256": "cf550423832a92cba942b7e1dc14a42000e8c9507403de8cd49aa56af9db6706", + "summary": { + "admission_lag_max_ms": 0.11262902989983559, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.546875, + "passed": 35, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2122.8132860269397, + "ttft_p50_ms": 1131.2310450011864, + "ttft_p95_ms": 2041.9401369290426 + } + } + ] + } + ], + "capacity": 32.0, + "capacity_per_gpu": 8.0, + "mns": 32, + "name": "tp4_mns32", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z", + "tp": 4 + }, + "tp4_mns64": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 66.89354998525232, + 83.23234610725194 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r4p00.json", + "sha256": "266cba4088f8a9483ceb41fdc02d04a2181cb7ffb6b52372b6b0687e62f2863e", + "summary": { + "admission_lag_max_ms": 0.10568101424723864, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 72.09669100120664, + "ttft_p50_ms": 62.014322029426694, + "ttft_p95_ms": 66.89354998525232 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r4p00.json", + "sha256": "26f3926c40504b96a37cb7b9dae1af5040c085c1a730e20feaf1d3e9ddbf8e54", + "summary": { + "admission_lag_max_ms": 0.108166947029531, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 96.42452490516007, + "ttft_p50_ms": 61.261708033271134, + "ttft_p95_ms": 83.23234610725194 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 67.26283801253885, + 67.34374002553523 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r8p00.json", + "sha256": "e84e6ba2c8822e012aceea7cc6f376ea1b6265fe4a3bb68b7badfc36b8813545", + "summary": { + "admission_lag_max_ms": 0.15694997273385525, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 86.54619695153087, + "ttft_p50_ms": 59.63437003083527, + "ttft_p95_ms": 67.26283801253885 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r8p00.json", + "sha256": "dae50073c9f79919181e8dcf274ca8929ba41c512f64363e40187de68839b87e", + "summary": { + "admission_lag_max_ms": 0.09962497279047966, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 69.00161597877741, + "ttft_p50_ms": 60.2562150452286, + "ttft_p95_ms": 67.34374002553523 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 16.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 71.37343694921583, + 64.9534990079701 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r16p00.json", + "sha256": "776c9956f65acf7c9b6c8df517a4f8c4d462289893a19e625a2a0ad7dcbf5506", + "summary": { + "admission_lag_max_ms": 0.1066719414666295, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 82.25136005785316, + "ttft_p50_ms": 58.52357891853899, + "ttft_p95_ms": 71.37343694921583 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r16p00.json", + "sha256": "90057204a5f6343ee4ee9cdd08323d87d83660d5bf8f02480882eb479d2e587e", + "summary": { + "admission_lag_max_ms": 0.10102998930960894, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 65.3582609957084, + "ttft_p50_ms": 58.98014397826046, + "ttft_p95_ms": 64.9534990079701 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 20.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 192.98715889453888, + 258.1990590551868 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r20p00.json", + "sha256": "6ce029388b206d0663acd41f4da1d67a6562389e1c84cf1826ce77c05d176d9a", + "summary": { + "admission_lag_max_ms": 0.12691703159362078, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 198.67739104665816, + "ttft_p50_ms": 142.75032398290932, + "ttft_p95_ms": 192.98715889453888 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r20p00.json", + "sha256": "0dbfb3f012574988ddebfb0af1532c735ac62d6d88863d24cd820aed50a92a59", + "summary": { + "admission_lag_max_ms": 0.3961039474233985, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 259.55891504418105, + "ttft_p50_ms": 162.6282810466364, + "ttft_p95_ms": 258.1990590551868 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 24.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 583.3089610096067, + 571.3486529421061 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r24p00.json", + "sha256": "0774ba2f285d81447c63343b32551d6fc33369b54010629381efa9a37df23a9c", + "summary": { + "admission_lag_max_ms": 0.09822729043662548, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 628.714332007803, + "ttft_p50_ms": 366.9840869260952, + "ttft_p95_ms": 583.3089610096067 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r24p00.json", + "sha256": "1986a05c366ebe73af0f89d2cd0e4efd3e577c800cf1e80997cd26ea0aaaec09", + "summary": { + "admission_lag_max_ms": 0.11414836626499891, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 617.9165309295058, + "ttft_p50_ms": 376.65714998729527, + "ttft_p95_ms": 571.3486529421061 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 28.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 869.708820944652, + 866.4831441128626 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r28p00.json", + "sha256": "ab987729fc833a59817eabaa18c320b0d0a2ecd402fd02bdb534b6c773c029d8", + "summary": { + "admission_lag_max_ms": 0.578614417463541, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 941.3277570856735, + "ttft_p50_ms": 540.4732850147411, + "ttft_p95_ms": 869.708820944652 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r28p00.json", + "sha256": "38f9c82c9ee30927cc0ba857e5e3fce2e79b8a24de03efaec77ce27258633a7e", + "summary": { + "admission_lag_max_ms": 0.12532551772892475, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 938.2033230504021, + "ttft_p50_ms": 541.139594046399, + "ttft_p95_ms": 866.4831441128626 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 32.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 1132.6635719742626, + 1144.196804962121 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r32p00.json", + "sha256": "358f5b8e1b72bff5752c076f2966b966d98f6f5767ce52d5a255b7f68f061e54", + "summary": { + "admission_lag_max_ms": 0.32093306072056293, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1197.7118930080906, + "ttft_p50_ms": 678.4568330040202, + "ttft_p95_ms": 1132.6635719742626 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r32p00.json", + "sha256": "3f6dbb760582fdd7470422d8edf6b62949d158ad0d437f04015441a0a4509003", + "summary": { + "admission_lag_max_ms": 1.1304179206490517, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1193.1529619032517, + "ttft_p50_ms": 692.5535890040919, + "ttft_p95_ms": 1144.196804962121 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 2042.27216495201, + 2043.943466967903 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r64p00.json", + "sha256": "a24eae7db6eb4cd105ed903917988e8fe6eef5d4cc2549d4e05e9a03dfafb107", + "summary": { + "admission_lag_max_ms": 0.134881935082376, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.546875, + "passed": 35, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2132.23939598538, + "ttft_p50_ms": 1137.3268909519538, + "ttft_p95_ms": 2042.27216495201 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r64p00.json", + "sha256": "e85efaada0b64bb9dd686f59282171ccb256a1d8483c6f037589a68e3e1a104a", + "summary": { + "admission_lag_max_ms": 0.11892011389136314, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.546875, + "passed": 35, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2125.3299129894003, + "ttft_p50_ms": 1134.724210947752, + "ttft_p95_ms": 2043.943466967903 + } + } + ] + } + ], + "capacity": 32.0, + "capacity_per_gpu": 8.0, + "mns": 64, + "name": "tp4_mns64", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z", + "tp": 4 + }, + "tp4_mns8": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 82.33969006687403, + 69.57727600820363 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r4p00.json", + "sha256": "e69b89ad2e07e6560ebd870773532b33e4415e9ca1515e585cbe465616678569", + "summary": { + "admission_lag_max_ms": 0.14186999760568142, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 86.54833806212991, + "ttft_p50_ms": 62.749656033702195, + "ttft_p95_ms": 82.33969006687403 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r4p00.json", + "sha256": "2c76dcf112a828a6720e079d29bb54a6a65d37d26abaaafdc6d5580751dbad65", + "summary": { + "admission_lag_max_ms": 0.11478399392217398, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 81.3983449479565, + "ttft_p50_ms": 62.220132909715176, + "ttft_p95_ms": 69.57727600820363 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 63.71171900536865, + 68.14328604377806 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r8p00.json", + "sha256": "0e5d687dfcdb4b6a0d1d060ceefe8a74faab94a5b241580d45b53c452f893b23", + "summary": { + "admission_lag_max_ms": 0.09818794205784798, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 69.63772000744939, + "ttft_p50_ms": 60.89008797425777, + "ttft_p95_ms": 63.71171900536865 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r8p00.json", + "sha256": "1bf016b603473d46fda4bc6d4daeaee5e1091ff4cda1cab7224e0c5edea512f8", + "summary": { + "admission_lag_max_ms": 0.11399004142731428, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 76.76690397784114, + "ttft_p50_ms": 60.9520060243085, + "ttft_p95_ms": 68.14328604377806 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 16.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 133.86176899075508, + 61.95738597307354 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r16p00.json", + "sha256": "197808b1b5fa8082e603a255f358826b5f8f48707223c7fd63abfe029674b817", + "summary": { + "admission_lag_max_ms": 0.10530801955610514, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 191.68931001331657, + "ttft_p50_ms": 61.92394997924566, + "ttft_p95_ms": 133.86176899075508 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r16p00.json", + "sha256": "53e809470daf9efcdabb254e85e8cc3b45d0ab8593c6c2f3aa3396e60d1656da", + "summary": { + "admission_lag_max_ms": 0.1288399798795581, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 70.40882098954171, + "ttft_p50_ms": 59.417470009066164, + "ttft_p95_ms": 61.95738597307354 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 20.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 201.29013701807708, + 200.0734859611839 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r20p00.json", + "sha256": "05b977824fb301517cdbe2d077b360b6053d57fee1c20acad3537321827d027a", + "summary": { + "admission_lag_max_ms": 0.3848400665447116, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 204.5107699232176, + "ttft_p50_ms": 150.0862929970026, + "ttft_p95_ms": 201.29013701807708 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r20p00.json", + "sha256": "baf422b7a3c95b0d3d376a31b0a30e075f69216c794ebc2ef51afafe1ec55229", + "summary": { + "admission_lag_max_ms": 0.24754798505455256, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 204.8192109214142, + "ttft_p50_ms": 148.99871195666492, + "ttft_p95_ms": 200.0734859611839 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 24.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 564.2021730309352, + 560.4996709153056 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r24p00.json", + "sha256": "029b2a9695322d536539c3dd7665725f933b21a8bb48fa94c0448c9f7f8bf0eb", + "summary": { + "admission_lag_max_ms": 0.14956865925341845, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 610.4218009859324, + "ttft_p50_ms": 368.3909500250593, + "ttft_p95_ms": 564.2021730309352 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r24p00.json", + "sha256": "94b46583af3a7a4bbb9f7256c4c4a5ce804b3092459abe3984ea0ce67f591f3c", + "summary": { + "admission_lag_max_ms": 0.11110503692179918, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 606.6705230623484, + "ttft_p50_ms": 367.50321893487126, + "ttft_p95_ms": 560.4996709153056 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 28.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 870.7952990662307, + 894.6108610834926 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r28p00.json", + "sha256": "07bfe263107bb3d2acaa356257343f90045e4295c799178bc0a0b5f1c1994284", + "summary": { + "admission_lag_max_ms": 0.6798935355618596, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 942.792383953929, + "ttft_p50_ms": 543.340316042304, + "ttft_p95_ms": 870.7952990662307 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r28p00.json", + "sha256": "861dd55076ef40a78c34824345c5279dff86262b604db120e9e38efa27b5be55", + "summary": { + "admission_lag_max_ms": 0.09912776295095682, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 930.2741150604561, + "ttft_p50_ms": 549.9696529004723, + "ttft_p95_ms": 894.6108610834926 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 32.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 1136.603599996306, + 1134.0074760373682 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r32p00.json", + "sha256": "332b805322b571087b018ee459dd91c443dfe0b2bb539b5bdfb4de902bba6da2", + "summary": { + "admission_lag_max_ms": 0.19067595712840557, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1201.8779760692269, + "ttft_p50_ms": 683.3873559953645, + "ttft_p95_ms": 1136.603599996306 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r32p00.json", + "sha256": "996a12a5ed962218dfbe6f30be688119030f10f6539647b6e4a9f25f032b22cb", + "summary": { + "admission_lag_max_ms": 0.11958007235080004, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1198.4891929896548, + "ttft_p50_ms": 682.7383439522237, + "ttft_p95_ms": 1134.0074760373682 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 2040.8174369949847, + 2082.3236129945144 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r64p00.json", + "sha256": "c5f17f625071e905b5f1bf8045cd233b2fea0fe6ca00044935dee9c663d1526b", + "summary": { + "admission_lag_max_ms": 0.7995429914444685, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.546875, + "passed": 35, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2131.160691031255, + "ttft_p50_ms": 1134.8516750149429, + "ttft_p95_ms": 2040.8174369949847 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r64p00.json", + "sha256": "775db5b941347d9fd5d6df32d6ee037f8168bd47b9eae9a7bdb3dbd8c40aa990", + "summary": { + "admission_lag_max_ms": 0.17620902508497238, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.5625, + "passed": 36, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2129.518053960055, + "ttft_p50_ms": 1112.3740070033818, + "ttft_p95_ms": 2082.3236129945144 + } + } + ] + } + ], + "capacity": 32.0, + "capacity_per_gpu": 8.0, + "mns": 8, + "name": "tp4_mns8", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z", + "tp": 4 + } + }, + "real_campaign_resources": { + "fresh_server_anchors": 192, + "gpu_hours": 12.074444444444445, + "measured_requests": 12288, + "runs": [ + { + "duration_seconds": 908.0, + "gpu_hours": 0.25222222222222224, + "run": "qwen30-prefill-real-tp1-mns16-20260717-v2-exclusive-20260716T164730291172Z", + "tp": 1 + }, + { + "duration_seconds": 540.0, + "gpu_hours": 0.15, + "run": "qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z", + "tp": 1 + }, + { + "duration_seconds": 937.0, + "gpu_hours": 0.2602777777777778, + "run": "qwen30-prefill-real-tp1-mns32-20260717-v2-exclusive-20260716T164731376588Z", + "tp": 1 + }, + { + "duration_seconds": 558.0, + "gpu_hours": 0.155, + "run": "qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z", + "tp": 1 + }, + { + "duration_seconds": 972.0, + "gpu_hours": 0.27, + "run": "qwen30-prefill-real-tp1-mns64-20260717-v2-exclusive-20260716T164732482666Z", + "tp": 1 + }, + { + "duration_seconds": 572.0, + "gpu_hours": 0.15888888888888889, + "run": "qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z", + "tp": 1 + }, + { + "duration_seconds": 886.0, + "gpu_hours": 0.2461111111111111, + "run": "qwen30-prefill-real-tp1-mns8-20260717-v2-exclusive-20260716T164729267199Z", + "tp": 1 + }, + { + "duration_seconds": 525.0, + "gpu_hours": 0.14583333333333334, + "run": "qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z", + "tp": 1 + }, + { + "duration_seconds": 983.0, + "gpu_hours": 0.5461111111111111, + "run": "qwen30-prefill-real-tp2-mns16-20260717-v2-exclusive-20260716T164734745039Z", + "tp": 2 + }, + { + "duration_seconds": 569.0, + "gpu_hours": 0.3161111111111111, + "run": "qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z", + "tp": 2 + }, + { + "duration_seconds": 1033.0, + "gpu_hours": 0.5738888888888889, + "run": "qwen30-prefill-real-tp2-mns32-20260717-v2-exclusive-20260716T170909930153Z", + "tp": 2 + }, + { + "duration_seconds": 584.0, + "gpu_hours": 0.3244444444444444, + "run": "qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z", + "tp": 2 + }, + { + "duration_seconds": 1063.0, + "gpu_hours": 0.5905555555555555, + "run": "qwen30-prefill-real-tp2-mns64-20260717-v2-exclusive-20260716T170910954319Z", + "tp": 2 + }, + { + "duration_seconds": 597.0, + "gpu_hours": 0.33166666666666667, + "run": "qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z", + "tp": 2 + }, + { + "duration_seconds": 957.0, + "gpu_hours": 0.5316666666666666, + "run": "qwen30-prefill-real-tp2-mns8-20260717-v2-exclusive-20260716T164733581881Z", + "tp": 2 + }, + { + "duration_seconds": 563.0, + "gpu_hours": 0.31277777777777777, + "run": "qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z", + "tp": 2 + }, + { + "duration_seconds": 1008.0, + "gpu_hours": 1.12, + "run": "qwen30-prefill-real-tp4-mns16-20260717-v2-exclusive-20260716T172922998830Z", + "tp": 4 + }, + { + "duration_seconds": 559.0, + "gpu_hours": 0.6211111111111111, + "run": "qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z", + "tp": 4 + }, + { + "duration_seconds": 1024.0, + "gpu_hours": 1.1377777777777778, + "run": "qwen30-prefill-real-tp4-mns32-20260717-v2-exclusive-20260716T172923955862Z", + "tp": 4 + }, + { + "duration_seconds": 556.0, + "gpu_hours": 0.6177777777777778, + "run": "qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z", + "tp": 4 + }, + { + "duration_seconds": 959.0, + "gpu_hours": 1.0655555555555556, + "run": "qwen30-prefill-real-tp4-mns64-20260717-v2b-exclusive-20260716T175248149074Z", + "tp": 4 + }, + { + "duration_seconds": 567.0, + "gpu_hours": 0.63, + "run": "qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z", + "tp": 4 + }, + { + "duration_seconds": 995.0, + "gpu_hours": 1.1055555555555556, + "run": "qwen30-prefill-real-tp4-mns8-20260717-v2-exclusive-20260716T170911977980Z", + "tp": 4 + }, + { + "duration_seconds": 550.0, + "gpu_hours": 0.6111111111111112, + "run": "qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z", + "tp": 4 + } + ], + "successful_fleet_jobs": 24, + "warmup_requests": 4512 + }, + "schema": "qwen30-prefill-fidelity-comparison-v1", + "simulator": { + "tp1_mns16": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 171.58529929215405 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 516.4107605377559 + }, + { + "feasible": false, + "pass_rate": 0.3125, + "rate": 16.0, + "ttft_p95_ms": 3193.8404181099845 + }, + { + "feasible": false, + "pass_rate": 0.203125, + "rate": 32.0, + "ttft_p95_ms": 4962.713638565687 + }, + { + "feasible": false, + "pass_rate": 0.140625, + "rate": 64.0, + "ttft_p95_ms": 5853.338638565687 + } + ], + "capacity": 8.0, + "capacity_per_gpu": 8.0, + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "tp1_mns32": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 171.58529929215405 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 516.4107605377559 + }, + { + "feasible": false, + "pass_rate": 0.3125, + "rate": 16.0, + "ttft_p95_ms": 3193.8404181099845 + }, + { + "feasible": false, + "pass_rate": 0.203125, + "rate": 32.0, + "ttft_p95_ms": 4962.713638565687 + }, + { + "feasible": false, + "pass_rate": 0.140625, + "rate": 64.0, + "ttft_p95_ms": 5853.338638565687 + } + ], + "capacity": 8.0, + "capacity_per_gpu": 8.0, + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "tp1_mns64": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 171.58529929215405 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 516.4107605377559 + }, + { + "feasible": false, + "pass_rate": 0.3125, + "rate": 16.0, + "ttft_p95_ms": 3193.8404181099845 + }, + { + "feasible": false, + "pass_rate": 0.203125, + "rate": 32.0, + "ttft_p95_ms": 4962.713638565687 + }, + { + "feasible": false, + "pass_rate": 0.140625, + "rate": 64.0, + "ttft_p95_ms": 5853.338638565687 + } + ], + "capacity": 8.0, + "capacity_per_gpu": 8.0, + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "tp1_mns8": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 171.58529929215405 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 516.4107605377559 + }, + { + "feasible": false, + "pass_rate": 0.3125, + "rate": 16.0, + "ttft_p95_ms": 3193.8404181099845 + }, + { + "feasible": false, + "pass_rate": 0.203125, + "rate": 32.0, + "ttft_p95_ms": 4962.713638565687 + }, + { + "feasible": false, + "pass_rate": 0.140625, + "rate": 64.0, + "ttft_p95_ms": 5853.338638565687 + } + ], + "capacity": 8.0, + "capacity_per_gpu": 8.0, + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "tp2_mns16": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 120.88989089722979 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 120.88989089722979 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 636.4131651102598 + }, + { + "feasible": false, + "pass_rate": 0.453125, + "rate": 32.0, + "ttft_p95_ms": 2395.158730843253 + }, + { + "feasible": false, + "pass_rate": 0.328125, + "rate": 64.0, + "ttft_p95_ms": 3272.153523650326 + } + ], + "capacity": 16.0, + "capacity_per_gpu": 8.0, + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "tp2_mns32": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 120.88989089722979 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 120.88989089722979 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 636.4131651102598 + }, + { + "feasible": false, + "pass_rate": 0.453125, + "rate": 32.0, + "ttft_p95_ms": 2395.158730843253 + }, + { + "feasible": false, + "pass_rate": 0.328125, + "rate": 64.0, + "ttft_p95_ms": 3272.153523650326 + } + ], + "capacity": 16.0, + "capacity_per_gpu": 8.0, + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "tp2_mns64": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 120.88989089722979 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 120.88989089722979 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 636.4131651102598 + }, + { + "feasible": false, + "pass_rate": 0.453125, + "rate": 32.0, + "ttft_p95_ms": 2395.158730843253 + }, + { + "feasible": false, + "pass_rate": 0.328125, + "rate": 64.0, + "ttft_p95_ms": 3272.153523650326 + } + ], + "capacity": 16.0, + "capacity_per_gpu": 8.0, + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "tp2_mns8": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 120.88989089722979 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 120.88989089722979 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 636.4131651102598 + }, + { + "feasible": false, + "pass_rate": 0.453125, + "rate": 32.0, + "ttft_p95_ms": 2395.158730843253 + }, + { + "feasible": false, + "pass_rate": 0.328125, + "rate": 64.0, + "ttft_p95_ms": 3272.153523650326 + } + ], + "capacity": 16.0, + "capacity_per_gpu": 8.0, + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "tp4_mns16": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 87.63426896254956 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 87.63426896254956 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 247.38770725677827 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 32.0, + "ttft_p95_ms": 1141.943009455039 + }, + { + "feasible": false, + "pass_rate": 0.546875, + "rate": 64.0, + "ttft_p95_ms": 2030.0951199619265 + } + ], + "capacity": 32.0, + "capacity_per_gpu": 8.0, + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "tp4_mns32": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 87.63426896254956 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 87.63426896254956 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 247.38770725677827 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 32.0, + "ttft_p95_ms": 1141.943009455039 + }, + { + "feasible": false, + "pass_rate": 0.546875, + "rate": 64.0, + "ttft_p95_ms": 2030.0951199619265 + } + ], + "capacity": 32.0, + "capacity_per_gpu": 8.0, + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "tp4_mns64": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 87.63426896254956 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 87.63426896254956 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 247.38770725677827 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 32.0, + "ttft_p95_ms": 1141.943009455039 + }, + { + "feasible": false, + "pass_rate": 0.546875, + "rate": 64.0, + "ttft_p95_ms": 2030.0951199619265 + } + ], + "capacity": 32.0, + "capacity_per_gpu": 8.0, + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "tp4_mns8": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 87.63426896254956 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 87.63426896254956 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 247.38770725677827 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 32.0, + "ttft_p95_ms": 1141.943009455039 + }, + { + "feasible": false, + "pass_rate": 0.546875, + "rate": 64.0, + "ttft_p95_ms": 2030.0951199619265 + } + ], + "capacity": 32.0, + "capacity_per_gpu": 8.0, + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + } + }, + "simulator_sources": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/simulator-a2/frontier_surface_frozen.json", + "sha256": "1aa84bcabe810467c0a5c6b7f848d94bb4f14c725ab214b50e0ae8c8efe6656f" + } + ] +} diff --git a/runs/frontier-fidelity-envelope-v1/results/a2/frontier_surface_frozen.json b/runs/frontier-fidelity-envelope-v1/results/a2/frontier_surface_frozen.json new file mode 100644 index 0000000..741a63b --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/results/a2/frontier_surface_frozen.json @@ -0,0 +1,1732 @@ +{ + "capacity": [ + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 8.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 8.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 8.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 8.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 16.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 16.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 16.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 16.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 32.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 32.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 32.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 32.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + } + ], + "collective": { + "allreduce_csv": "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv", + "allreduce_csv_sha256": "9d693fd406616b599e57bcde66c980c7fc2831b3acf37d3eb633cec80ea0070d", + "backend": "vidur" + }, + "config_results": [ + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "loads": [ + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 159.88838052749634, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "f0cfb072afcc2fc82a92c59ddb4e8cfc1e3201e15848f37785eeb9fdd975e779", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.0197002243768605, + "ttft_max_ms": 171.58529929215405, + "ttft_p50_ms": 171.58529929215405, + "ttft_p95_ms": 171.58529929215405 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 9.092494249343872, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "cfebfbd8aa5d9603158d8ff4e786119433b4c0d705f733ba242da46a6a6fdfb0", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 7.742175153637853, + "ttft_max_ms": 570.3133207139617, + "ttft_p50_ms": 391.4107605377559, + "ttft_p95_ms": 516.4107605377559 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 9.190775394439697, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "03578ba406b3299e70ba7975e6b9872d381a5f4bd4fbc3c8f6a8ba49e9b53f2e", + "score": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "throughput_requests_per_second": 8.916224898348922, + "ttft_max_ms": 3318.8404181099845, + "ttft_p50_ms": 1814.2305264490028, + "ttft_p95_ms": 3193.8404181099845 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 8.99299168586731, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "0b761b5f42e65b694eb34735433d43b473ed67415fb4e17b89894fc840b645fc", + "score": { + "feasible": false, + "pass_rate": 0.203125, + "passed": 13, + "throughput_requests_per_second": 9.07297385348011, + "ttft_max_ms": 5147.666503402199, + "ttft_p50_ms": 2739.3537469047046, + "ttft_p95_ms": 4962.713638565687 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 8.941024541854858, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "078209f2f56f21243384bdc70add105bf4942ee44ec8cc9573cad399a4a10e03", + "score": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "throughput_requests_per_second": 9.07297385348011, + "ttft_max_ms": 6100.791503402199, + "ttft_p50_ms": 3208.1037469047046, + "ttft_p95_ms": 5853.338638565687 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "loads": [ + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 8.843815326690674, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "f0cfb072afcc2fc82a92c59ddb4e8cfc1e3201e15848f37785eeb9fdd975e779", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.0197002243768605, + "ttft_max_ms": 171.58529929215405, + "ttft_p50_ms": 171.58529929215405, + "ttft_p95_ms": 171.58529929215405 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 8.990388631820679, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "cfebfbd8aa5d9603158d8ff4e786119433b4c0d705f733ba242da46a6a6fdfb0", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 7.742175153637853, + "ttft_max_ms": 570.3133207139617, + "ttft_p50_ms": 391.4107605377559, + "ttft_p95_ms": 516.4107605377559 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 8.942370891571045, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "03578ba406b3299e70ba7975e6b9872d381a5f4bd4fbc3c8f6a8ba49e9b53f2e", + "score": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "throughput_requests_per_second": 8.916224898348922, + "ttft_max_ms": 3318.8404181099845, + "ttft_p50_ms": 1814.2305264490028, + "ttft_p95_ms": 3193.8404181099845 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 8.891057252883911, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "0b761b5f42e65b694eb34735433d43b473ed67415fb4e17b89894fc840b645fc", + "score": { + "feasible": false, + "pass_rate": 0.203125, + "passed": 13, + "throughput_requests_per_second": 9.07297385348011, + "ttft_max_ms": 5147.666503402199, + "ttft_p50_ms": 2739.3537469047046, + "ttft_p95_ms": 4962.713638565687 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 8.989603757858276, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "078209f2f56f21243384bdc70add105bf4942ee44ec8cc9573cad399a4a10e03", + "score": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "throughput_requests_per_second": 9.07297385348011, + "ttft_max_ms": 6100.791503402199, + "ttft_p50_ms": 3208.1037469047046, + "ttft_p95_ms": 5853.338638565687 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "loads": [ + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 8.939709424972534, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "f0cfb072afcc2fc82a92c59ddb4e8cfc1e3201e15848f37785eeb9fdd975e779", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.0197002243768605, + "ttft_max_ms": 171.58529929215405, + "ttft_p50_ms": 171.58529929215405, + "ttft_p95_ms": 171.58529929215405 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 8.94373345375061, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "cfebfbd8aa5d9603158d8ff4e786119433b4c0d705f733ba242da46a6a6fdfb0", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 7.742175153637853, + "ttft_max_ms": 570.3133207139617, + "ttft_p50_ms": 391.4107605377559, + "ttft_p95_ms": 516.4107605377559 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 8.992903709411621, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "03578ba406b3299e70ba7975e6b9872d381a5f4bd4fbc3c8f6a8ba49e9b53f2e", + "score": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "throughput_requests_per_second": 8.916224898348922, + "ttft_max_ms": 3318.8404181099845, + "ttft_p50_ms": 1814.2305264490028, + "ttft_p95_ms": 3193.8404181099845 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 9.04556393623352, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "0b761b5f42e65b694eb34735433d43b473ed67415fb4e17b89894fc840b645fc", + "score": { + "feasible": false, + "pass_rate": 0.203125, + "passed": 13, + "throughput_requests_per_second": 9.07297385348011, + "ttft_max_ms": 5147.666503402199, + "ttft_p50_ms": 2739.3537469047046, + "ttft_p95_ms": 4962.713638565687 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 8.937517881393433, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "078209f2f56f21243384bdc70add105bf4942ee44ec8cc9573cad399a4a10e03", + "score": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "throughput_requests_per_second": 9.07297385348011, + "ttft_max_ms": 6100.791503402199, + "ttft_p50_ms": 3208.1037469047046, + "ttft_p95_ms": 5853.338638565687 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "loads": [ + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 8.944840908050537, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "f0cfb072afcc2fc82a92c59ddb4e8cfc1e3201e15848f37785eeb9fdd975e779", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.0197002243768605, + "ttft_max_ms": 171.58529929215405, + "ttft_p50_ms": 171.58529929215405, + "ttft_p95_ms": 171.58529929215405 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 8.943378686904907, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "cfebfbd8aa5d9603158d8ff4e786119433b4c0d705f733ba242da46a6a6fdfb0", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 7.742175153637853, + "ttft_max_ms": 570.3133207139617, + "ttft_p50_ms": 391.4107605377559, + "ttft_p95_ms": 516.4107605377559 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 8.993504762649536, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "03578ba406b3299e70ba7975e6b9872d381a5f4bd4fbc3c8f6a8ba49e9b53f2e", + "score": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "throughput_requests_per_second": 8.916224898348922, + "ttft_max_ms": 3318.8404181099845, + "ttft_p50_ms": 1814.2305264490028, + "ttft_p95_ms": 3193.8404181099845 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 8.916944980621338, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "0b761b5f42e65b694eb34735433d43b473ed67415fb4e17b89894fc840b645fc", + "score": { + "feasible": false, + "pass_rate": 0.203125, + "passed": 13, + "throughput_requests_per_second": 9.07297385348011, + "ttft_max_ms": 5147.666503402199, + "ttft_p50_ms": 2739.3537469047046, + "ttft_p95_ms": 4962.713638565687 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 8.940248250961304, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "078209f2f56f21243384bdc70add105bf4942ee44ec8cc9573cad399a4a10e03", + "score": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "throughput_requests_per_second": 9.07297385348011, + "ttft_max_ms": 6100.791503402199, + "ttft_p50_ms": 3208.1037469047046, + "ttft_p95_ms": 5853.338638565687 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "loads": [ + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 156.6941306591034, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "e1c3e7924869b2b30d00d9ed7c8f5261fea58b8bae09af2222f15b909a6e29b0", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.032540105813934, + "ttft_max_ms": 120.88989089722979, + "ttft_p50_ms": 120.88989089722979, + "ttft_p95_ms": 120.88989089722979 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 8.991374015808105, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "14089aa0fe7d62dd18d486b0ab2bf02fc8bc11de772da9dd42b09ebca2e484c6", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.00411222181281, + "ttft_max_ms": 120.88989089722979, + "ttft_p50_ms": 120.88989089722979, + "ttft_p95_ms": 120.88989089722979 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 9.145017147064209, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "fd82e6bd7c47563866485bb023b11fdec286c8688c09c6f00fbf75aecb846897", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 14.2457389654175, + "ttft_max_ms": 694.7908916608797, + "ttft_p50_ms": 461.27998545840177, + "ttft_p95_ms": 636.4131651102598 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 9.096367359161377, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "2bdd946e6b94ecda68aefa93b07566db206b6b48cf1d26b251e01ff3b6b20354", + "score": { + "feasible": false, + "pass_rate": 0.453125, + "passed": 29, + "throughput_requests_per_second": 14.66575126735821, + "ttft_max_ms": 2488.908730843253, + "ttft_p50_ms": 1333.2347933749338, + "ttft_p95_ms": 2395.158730843253 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 9.091314792633057, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "bf39a299a3079ec0bafe4ce77b632d793000ce77f34a65f8766c04a080e0a328", + "score": { + "feasible": false, + "pass_rate": 0.328125, + "passed": 21, + "throughput_requests_per_second": 14.665751267358214, + "ttft_max_ms": 3410.783730843252, + "ttft_p50_ms": 1807.813828365548, + "ttft_p95_ms": 3272.153523650326 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "loads": [ + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 8.994110584259033, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "e1c3e7924869b2b30d00d9ed7c8f5261fea58b8bae09af2222f15b909a6e29b0", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.032540105813934, + "ttft_max_ms": 120.88989089722979, + "ttft_p50_ms": 120.88989089722979, + "ttft_p95_ms": 120.88989089722979 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 9.044063806533813, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "14089aa0fe7d62dd18d486b0ab2bf02fc8bc11de772da9dd42b09ebca2e484c6", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.00411222181281, + "ttft_max_ms": 120.88989089722979, + "ttft_p50_ms": 120.88989089722979, + "ttft_p95_ms": 120.88989089722979 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 9.093687534332275, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "fd82e6bd7c47563866485bb023b11fdec286c8688c09c6f00fbf75aecb846897", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 14.2457389654175, + "ttft_max_ms": 694.7908916608797, + "ttft_p50_ms": 461.27998545840177, + "ttft_p95_ms": 636.4131651102598 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 9.091583013534546, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "2bdd946e6b94ecda68aefa93b07566db206b6b48cf1d26b251e01ff3b6b20354", + "score": { + "feasible": false, + "pass_rate": 0.453125, + "passed": 29, + "throughput_requests_per_second": 14.66575126735821, + "ttft_max_ms": 2488.908730843253, + "ttft_p50_ms": 1333.2347933749338, + "ttft_p95_ms": 2395.158730843253 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 9.093712568283081, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "bf39a299a3079ec0bafe4ce77b632d793000ce77f34a65f8766c04a080e0a328", + "score": { + "feasible": false, + "pass_rate": 0.328125, + "passed": 21, + "throughput_requests_per_second": 14.665751267358214, + "ttft_max_ms": 3410.783730843252, + "ttft_p50_ms": 1807.813828365548, + "ttft_p95_ms": 3272.153523650326 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "loads": [ + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 9.043614625930786, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "e1c3e7924869b2b30d00d9ed7c8f5261fea58b8bae09af2222f15b909a6e29b0", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.032540105813934, + "ttft_max_ms": 120.88989089722979, + "ttft_p50_ms": 120.88989089722979, + "ttft_p95_ms": 120.88989089722979 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 8.999051809310913, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "14089aa0fe7d62dd18d486b0ab2bf02fc8bc11de772da9dd42b09ebca2e484c6", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.00411222181281, + "ttft_max_ms": 120.88989089722979, + "ttft_p50_ms": 120.88989089722979, + "ttft_p95_ms": 120.88989089722979 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 9.092182636260986, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "fd82e6bd7c47563866485bb023b11fdec286c8688c09c6f00fbf75aecb846897", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 14.2457389654175, + "ttft_max_ms": 694.7908916608797, + "ttft_p50_ms": 461.27998545840177, + "ttft_p95_ms": 636.4131651102598 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 9.042077779769897, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "2bdd946e6b94ecda68aefa93b07566db206b6b48cf1d26b251e01ff3b6b20354", + "score": { + "feasible": false, + "pass_rate": 0.453125, + "passed": 29, + "throughput_requests_per_second": 14.66575126735821, + "ttft_max_ms": 2488.908730843253, + "ttft_p50_ms": 1333.2347933749338, + "ttft_p95_ms": 2395.158730843253 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 9.09327220916748, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "bf39a299a3079ec0bafe4ce77b632d793000ce77f34a65f8766c04a080e0a328", + "score": { + "feasible": false, + "pass_rate": 0.328125, + "passed": 21, + "throughput_requests_per_second": 14.665751267358214, + "ttft_max_ms": 3410.783730843252, + "ttft_p50_ms": 1807.813828365548, + "ttft_p95_ms": 3272.153523650326 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "loads": [ + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 8.941684007644653, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "e1c3e7924869b2b30d00d9ed7c8f5261fea58b8bae09af2222f15b909a6e29b0", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.032540105813934, + "ttft_max_ms": 120.88989089722979, + "ttft_p50_ms": 120.88989089722979, + "ttft_p95_ms": 120.88989089722979 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 8.992353916168213, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "14089aa0fe7d62dd18d486b0ab2bf02fc8bc11de772da9dd42b09ebca2e484c6", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.00411222181281, + "ttft_max_ms": 120.88989089722979, + "ttft_p50_ms": 120.88989089722979, + "ttft_p95_ms": 120.88989089722979 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 9.146737098693848, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "fd82e6bd7c47563866485bb023b11fdec286c8688c09c6f00fbf75aecb846897", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 14.2457389654175, + "ttft_max_ms": 694.7908916608797, + "ttft_p50_ms": 461.27998545840177, + "ttft_p95_ms": 636.4131651102598 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 9.090755462646484, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "2bdd946e6b94ecda68aefa93b07566db206b6b48cf1d26b251e01ff3b6b20354", + "score": { + "feasible": false, + "pass_rate": 0.453125, + "passed": 29, + "throughput_requests_per_second": 14.66575126735821, + "ttft_max_ms": 2488.908730843253, + "ttft_p50_ms": 1333.2347933749338, + "ttft_p95_ms": 2395.158730843253 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 9.190985918045044, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "bf39a299a3079ec0bafe4ce77b632d793000ce77f34a65f8766c04a080e0a328", + "score": { + "feasible": false, + "pass_rate": 0.328125, + "passed": 21, + "throughput_requests_per_second": 14.665751267358214, + "ttft_max_ms": 3410.783730843252, + "ttft_p50_ms": 1807.813828365548, + "ttft_p95_ms": 3272.153523650326 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "loads": [ + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 141.65436458587646, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 1.0, + "request_count": 64, + "request_metrics_sha256": "10a656450567caf0b8b628600b6e6acf8eeeb3f37c5d82713f12966b5aa9e533", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.041007571782521, + "ttft_max_ms": 87.63426896254956, + "ttft_p50_ms": 87.63426896254956, + "ttft_p95_ms": 87.63426896254956 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 9.093120813369751, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "519248e66e8451d0f7bd61566a67453e862caf3b7cbfe466f83d78388f4997c4", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.037541074750196, + "ttft_max_ms": 87.63426896254956, + "ttft_p50_ms": 87.63426896254956, + "ttft_p95_ms": 87.63426896254956 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 9.287490606307983, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "29cfcc459ed774299f781ade2234d11aa9c62cf5f5bb952df4b7e9c7afe7f2b4", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 15.584885703455418, + "ttft_max_ms": 274.25967286812414, + "ttft_p50_ms": 187.36059674989036, + "ttft_p95_ms": 247.38770725677827 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 9.1891348361969, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "b51aecf0f91ce9d2c55d9d6b854c8a6c09c290133a65733e5cb436abd375555d", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 20.409287348343973, + "ttft_max_ms": 1204.443009455039, + "ttft_p50_ms": 693.4638889220377, + "ttft_p95_ms": 1141.943009455039 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 9.092704772949219, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "174484e0f73c517ada2adce7dea0c4737f89b6252eab700d80c0c4055493418c", + "score": { + "feasible": false, + "pass_rate": 0.546875, + "passed": 35, + "throughput_requests_per_second": 20.862900609687326, + "ttft_max_ms": 2114.521306587048, + "ttft_p50_ms": 1129.8633894955506, + "ttft_p95_ms": 2030.0951199619265 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "loads": [ + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 8.94224739074707, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 1.0, + "request_count": 64, + "request_metrics_sha256": "10a656450567caf0b8b628600b6e6acf8eeeb3f37c5d82713f12966b5aa9e533", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.041007571782521, + "ttft_max_ms": 87.63426896254956, + "ttft_p50_ms": 87.63426896254956, + "ttft_p95_ms": 87.63426896254956 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 8.993168115615845, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "519248e66e8451d0f7bd61566a67453e862caf3b7cbfe466f83d78388f4997c4", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.037541074750196, + "ttft_max_ms": 87.63426896254956, + "ttft_p50_ms": 87.63426896254956, + "ttft_p95_ms": 87.63426896254956 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 9.14280891418457, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "29cfcc459ed774299f781ade2234d11aa9c62cf5f5bb952df4b7e9c7afe7f2b4", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 15.584885703455418, + "ttft_max_ms": 274.25967286812414, + "ttft_p50_ms": 187.36059674989036, + "ttft_p95_ms": 247.38770725677827 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 9.090214967727661, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "b51aecf0f91ce9d2c55d9d6b854c8a6c09c290133a65733e5cb436abd375555d", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 20.409287348343973, + "ttft_max_ms": 1204.443009455039, + "ttft_p50_ms": 693.4638889220377, + "ttft_p95_ms": 1141.943009455039 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 9.094237804412842, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "174484e0f73c517ada2adce7dea0c4737f89b6252eab700d80c0c4055493418c", + "score": { + "feasible": false, + "pass_rate": 0.546875, + "passed": 35, + "throughput_requests_per_second": 20.862900609687326, + "ttft_max_ms": 2114.521306587048, + "ttft_p50_ms": 1129.8633894955506, + "ttft_p95_ms": 2030.0951199619265 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "loads": [ + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.140523672103882, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 1.0, + "request_count": 64, + "request_metrics_sha256": "10a656450567caf0b8b628600b6e6acf8eeeb3f37c5d82713f12966b5aa9e533", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.041007571782521, + "ttft_max_ms": 87.63426896254956, + "ttft_p50_ms": 87.63426896254956, + "ttft_p95_ms": 87.63426896254956 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.091987609863281, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "519248e66e8451d0f7bd61566a67453e862caf3b7cbfe466f83d78388f4997c4", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.037541074750196, + "ttft_max_ms": 87.63426896254956, + "ttft_p50_ms": 87.63426896254956, + "ttft_p95_ms": 87.63426896254956 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.141303777694702, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "29cfcc459ed774299f781ade2234d11aa9c62cf5f5bb952df4b7e9c7afe7f2b4", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 15.584885703455418, + "ttft_max_ms": 274.25967286812414, + "ttft_p50_ms": 187.36059674989036, + "ttft_p95_ms": 247.38770725677827 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.240424156188965, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "b51aecf0f91ce9d2c55d9d6b854c8a6c09c290133a65733e5cb436abd375555d", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 20.409287348343973, + "ttft_max_ms": 1204.443009455039, + "ttft_p50_ms": 693.4638889220377, + "ttft_p95_ms": 1141.943009455039 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.140634536743164, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "174484e0f73c517ada2adce7dea0c4737f89b6252eab700d80c0c4055493418c", + "score": { + "feasible": false, + "pass_rate": 0.546875, + "passed": 35, + "throughput_requests_per_second": 20.862900609687326, + "ttft_max_ms": 2114.521306587048, + "ttft_p50_ms": 1129.8633894955506, + "ttft_p95_ms": 2030.0951199619265 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "loads": [ + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.143561601638794, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 1.0, + "request_count": 64, + "request_metrics_sha256": "10a656450567caf0b8b628600b6e6acf8eeeb3f37c5d82713f12966b5aa9e533", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.041007571782521, + "ttft_max_ms": 87.63426896254956, + "ttft_p50_ms": 87.63426896254956, + "ttft_p95_ms": 87.63426896254956 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.091549396514893, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "519248e66e8451d0f7bd61566a67453e862caf3b7cbfe466f83d78388f4997c4", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.037541074750196, + "ttft_max_ms": 87.63426896254956, + "ttft_p50_ms": 87.63426896254956, + "ttft_p95_ms": 87.63426896254956 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.191416263580322, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "29cfcc459ed774299f781ade2234d11aa9c62cf5f5bb952df4b7e9c7afe7f2b4", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 15.584885703455418, + "ttft_max_ms": 274.25967286812414, + "ttft_p50_ms": 187.36059674989036, + "ttft_p95_ms": 247.38770725677827 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.043817520141602, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "b51aecf0f91ce9d2c55d9d6b854c8a6c09c290133a65733e5cb436abd375555d", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 20.409287348343973, + "ttft_max_ms": 1204.443009455039, + "ttft_p50_ms": 693.4638889220377, + "ttft_p95_ms": 1141.943009455039 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.046180486679077, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "174484e0f73c517ada2adce7dea0c4737f89b6252eab700d80c0c4055493418c", + "score": { + "feasible": false, + "pass_rate": 0.546875, + "passed": 35, + "throughput_requests_per_second": 20.862900609687326, + "ttft_max_ms": 2114.521306587048, + "ttft_p50_ms": 1129.8633894955506, + "ttft_p95_ms": 2030.0951199619265 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + } + ], + "contract": { + "arrival": "open_loop_uniform", + "input_tokens": 2048, + "output_tokens": 1, + "prefix_caching": false, + "rates": [ + 4.0, + 8.0, + 16.0, + 32.0, + 64.0 + ], + "requests_per_anchor": 64, + "target_pass_rate": 0.95, + "ttft_slo_ms": 1256.0 + }, + "frontier": { + "git_head": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "git_status_short": " M frontier/cc_backend/backends/vidur_cc_backend.py\n M frontier/config/config.py\n M frontier/entities/request.py\n M frontier/events/cluster_schedule_event.py\n M frontier/execution_time_predictor/sklearn_execution_time_predictor.py\n M frontier/metrics/constants.py\n M frontier/metrics/metrics_store.py\n M frontier/profiling/common/layers/rotary_embedding.py\n M frontier/profiling/moe/moe_impl.py\n M frontier/profiling/moe/moe_vllm_kernel.py\n M frontier/scheduler/cluster_scheduler/__init__.py\n M frontier/scheduler/cluster_scheduler/base_cluster_scheduler.py\n M frontier/scheduler/cluster_scheduler/cluster_scheduler_registry.py\n M frontier/scheduler/cluster_scheduler/sticky_lor_cluster_scheduler.py\n M frontier/scheduler/replica_scheduler/base_replica_scheduler.py\n M frontier/scheduler/replica_scheduler/vllm_v1_engine_replica_scheduler.py\n M frontier/scheduler/replica_stage_scheduler/replica_stage_schduler.py\n M frontier/simulator.py\n M frontier/types/cluster_scheduler_type.py\n?? data/profiling/compute/h20/\n?? frontier/scheduler/cluster_scheduler/prefix_lor_cluster_scheduler.py\n?? runs/\n?? tests/unit/test_attn_prefill_prediction_fallback.py\n?? tests/unit/test_vidur_cc_large_payload.py\n", + "source": "/tmp/replayserve-frontier-rs1b" + }, + "profiles": { + "coverage": { + "attention": { + "1": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + }, + "2": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + }, + "4": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + } + }, + "manifest": { + "attention_tp_coverage": [ + 1, + 2, + 4 + ], + "environment_contract": { + "dtype": "bfloat16", + "frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "hardware": "NVIDIA H20", + "model": "Qwen3-30B-A3B", + "tensor_parallel_sizes": [ + 1, + 2, + 4 + ], + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + }, + "inputs": { + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931" + }, + "outputs": { + "allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3", + "attention.csv": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d", + "attention_true_mixed_fused.csv": "43ef4be90bddc9aeac6dbbe339feec24162cd1f2129a08fbd959e6ee4eaf5f60", + "linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "moe.csv": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0" + }, + "profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-fused-mixed-total-conserving", + "projection_contract": { + "allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity", + "attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target; true mixed rows use a total-conserving compatibility projection", + "attention_true_mixed": "The directly measured fused total is preserved in diagnostics. Frontier's two targets are projected by the same-TP pure prefill/decode reference ratio, with projected prefill + decode exactly equal to the fused total; the split is a schema compatibility attribution, not an observation", + "linear": "Frontier profiler using vLLM 0.20 CUDA operators", + "moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero" + }, + "row_counts": { + "allreduce": 24, + "attention_frontier_compatible": 132, + "attention_true_mixed_fused_diagnostic": 30, + "linear": 36, + "moe": 72 + }, + "schema_version": "frontier_qwen30_vllm020_frozen_profile.v2" + } + }, + "root": "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/frozen/profile-v2", + "sha256": { + "attention": "76dcb767cebb4ec1c4e24bd04d93ddd48b5d271986ebfb51a197ab33e1b3d87d", + "linear": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "manifest": "af40545e75aff55c6333cd2d5379ccf042a5a0b7d7fc7df4f745ce256cb290eb", + "moe": "0e4dcba72918a1c4cf4e96ced31ee3829248a19ad54553cebef14417725808b0" + } + }, + "schema": "frontier-qwen30-prefill-surface-v1", + "status": "frozen_before_real" +} diff --git a/runs/frontier-fidelity-envelope-v1/results/a2/qwen30-prefill-ranking.png b/runs/frontier-fidelity-envelope-v1/results/a2/qwen30-prefill-ranking.png new file mode 100644 index 0000000000000000000000000000000000000000..ed8a22189147be87e8e5b641ec4062cc0de410b1 GIT binary patch literal 107863 zcmeFZhd-BX|37|eqam|IMo~yoDiI>1h?1F+m5S_yG80*$LaCH8LNc>wiHx?CSu)CA z+57uEx~}{4`#rw*U+{T6z8>AzeO>Z?d!Ofd9Iw}Ny^eqrYD#P9*y$*WT66S>{7H(U zwWO$}?yFYdJCsiCbNo}nQ9;*H!}hGBvx$Q_rE216XJzYXWns$gWbSa$!q#TbPSIUE z5A5JR@91cEQ9?w-`hR|6r>(;|5q`xlr|>Scc1KQKq^M147=BG!%7IURKj3 zs{gyQrlwB)is5tJs)r8IsK2tYYG9#PsIckhWKrqo|Gbr7RPpoq8*lEHZ)R~#3s}A> z(Rir1mRTWw(xX(yrORW_uHfB|e^00EykfCK zn%1iW&8ivSJsUHz5jXxdA&FC!o2_WC2SVM`d{yR>yD^;(0{${ zk++eF3jg)KapuSB{_9Wns@& z#(=4lU4MSPjXZdH_kA&oB^NGS=bilQJ;t;mu={iDN~fV8Yh)P$nJ*@&J?sog!eVaVJUgDI>%<6jqi~H<^!q;b7>Q9et?C$A#RaUm-Qg5B?$4644&VvTamM!Dnvq!^(tE8l) zJ!4{GV&_bV<>v?cZHGU~EWWI)T=~7w*{`^`c!TTF1+UJ50b%E0)(O7(k>o^?#?jv$ z9U4CeI@;T_7*&oQJ#ysGp+nT29jClsU0WG@_Wk`&pFgiRqg%7)YhGHr+gaAm$b!N` zw)4@~yuEi_{&S`-S!bQLw)U=l`>w~vZfVN0xVCZ6S?*oC(htqwGPSU<&@H~QSHgL4 zwWOpZxn&0V4(C6|Eccin*O6CJN*>rECYE(*@@8mgXwAZA8IK!y27x2u5)!8EIlaAYf^V^6I@?8I1WM_(U37Z+D{eY>B{^q*hDBO|Zys8uAH|5@lWBJ2Nd zj)g6hzleEzd%I7G`)p0N*7)rsTjfrkJo%(j$Xd#Ef*FgXXvnaOxZSf9VMvompEvdouW4(m*;l33=56jt$PEOwvWlj#xjQ4qmg@x607?ikOZYvL95}c)_r4{%j zCFVL|74L4CXD7Q0>mu(Vp8fDxk48jBl#J);n@6^K;~E(_I1D0R%PrF2&D$@Hi-t*G zQ~5JX!y@wNb#3LvL!_K&+FjxPdt<45t;Z6VdVPd+QYmU|e0*fOD>@dJmY$pI$Gyt` z#*Ok6{am}5F^eLPd3JAZ=6nfFivrKZk`10O&c6TI)^_#&-V0?|-D#(b%NFKm*E2Kw zjEyB&G-%>U$S-SK|a&of0mG5}(S0Tr~Z)==~ek`ZD zySrb0|IYbP;*wlNAj_?L_X5~NPX8Qkd!6Sxxzp6tbl0U`O@rbqELa-ePo6v}^GGwu z_YUEdFMc*PM&0xJABxF`j0u;p;A(ujg5_Ztwz>0n@USc zqX`ue~>!rA!_?mqvciK}PXV3CqU&|R77q=A!z_hn6DN!R~?atFh{`l+m_Yd|D zPfRGDIB~OfpfZ$eoh-wy3*UBVCY@BUF8WVD(>xS^cZ=fR<(SE>p(S_A{Oz59zK2m! zK1!jR1qOt+ZBtscdNp%VQIQX}pj_}IPojQq>Mo1gt!*P6mCDUK3=1?{)=5c8wN;00 zQ*K7>sW)x@@KBOIDJjXJFE%#z*E4H;&uDp|^Gh$9CDar3SVixVuC8|Y zs<6+h;c>d5on1PfFL#cl%dqiPQ=uE3ooAk4g|$a!TYhCsHOP<2n#Vfu*==6qKcE!M z9-o!+;>C-G56#Vzt@$@2A|tn3HDw*0`gPZcIdxH2&~o|m<+Mg_BfjZp-?O)&3CLZ! zQoy}x(No0Z$YQNg&h6VzzefGJol>T8(>*LDG5*=^<3~{&7@3%SvASqW7Z(ccCi+kD z?A)owuxvVT*ZoJ=+C9#)o?{7&isE?n^{W=e-QC+u=eaPQaP#5Alq`1_7nfU=b0QZ$ z$N0KDD#qP<_wJpW^~6B)HY|DnkIo9#%VZZ81Zwt7{CsuIRK|6p|CauK+aCw*zYjF$ zve<}NYowi);@Pq#_LKdmgix03?!&Jx>dv^V{abM9)Dm23i~s(sv`2HUE!$n4w9_eB zrps^Y--^G~_pPDelF+eZ$CP;wrsUbREm@f96RCTqrDE|QNh2Y~wzY)vdj5Pze#h26 zEGOM;OBSz|UP<(?>$h&*GT4L;YSoyz99^H*EY5krVHrN))sT?2WX;=WCkL^K z@Cpdf9U0i!qV=tnceiKUEB77k3`yxWC^vHESLh z+Sb*zZF7C_rv0YqrPEDPF2j`bU<(bU`An0x?_0`S=bdNJ`Cm0RZ`yP2!;YH4@B*xI z6V#&(`)xSx@3W>`y7XG@+5qJ$d~o~G->SZSxUK0IyVTrf2XcpphrN7#6`oWId^mU3 zUcb;u_(QbJI!d?5MU?;eqt%?8zZRPlP@wo2>l3FN1D(dEg z6LD68xhNe^3=5qoud1peQIgJT15@ZLk>d8el<}X=>hn!mNs9JM^J&kV+-q5PqA>?B zp*{2I7eQ8SUEK}XEEKnM!E5a8*WbQr4>;}IxieDAHD_w_`KcE-!@{=ID^BP++1t0R z8yu@XdhA&F)uk(kE7*)415epGIlZ~HMJY?##$I|)P#E_r-OMwz)MLGDOUr~0LTi8g z_<`nyKdaWPSx)u0mU0;uILi9cu^3xf-M|*=C>$zbN|+803=ORuDqRfWSH1rjD+fLB zF!~TS+I_c=FN$hwYtt>i3c6gmQugD==Iht5eZ)D3~Se} zC4k<|-TmvrweX@2)_Ch5qqJ0^(||XAdw+kw@%i(s0M+sOxAVE)g6it(QzMmJ>IrI` zk@xSH)zr|vzrXi&dqq%$nqNv)UwxV=nps|6-b?(=%*@Po{hS?^b%|GVteZ`7k$itb z&-_y`H^dv({v~yZafjY(`n-AbrhO(oeOF58r09VwV~H){*8|4}nsuyR*tSYX%+D{( z4w*I&mCRyScDxB@qE@j8X)`l2GA@&4@FA#DY`m@KOG1ScK8(hJ4@REzd^v_CZuw1l zf$d~}RQ6)0T+LBcRb@dH5VG;?ZxGENFLWMat-tKz!hGOcsCn;U^SBJpBN~}@y|2CT zH;>ZKf98Gl{ktI+=Uvw~tY?kr_KZJyDt7rSo!p2jo8w^10sEq8&xQOW?iR`u8|UVx zFPe0JdNTbXpnvarc6Py4-rNU0=5!OaQ|{O1%+F4;IXE~lhfL$go}khw=SH}rpsHM$ zl?I@;j{*fsw%WwRv?b>&+L*YZgT1OdX`6JE57vB_{@sTUg*R;2u!`~l5~xmTE^#lU z4y9*jvzVHh(My#GW`5-K^z_U(i=2!HCz+a>+CI=%5ya-5nwr{l;bmQ&W<#lGX{|@* zix&8q4 zlR{wDRTQBRJOTm&turX9uPQ1O%nh(8%0Mmx-8jyCJm=_`_35eVCf|>?r%-{I@Q@NF z8iF@V>Gd!_tGKj|UT(xaM7h%O;zcH#mOLKxCjp~_m!m&k`1btP?2wW|8puHMTnLY( zjjXAr{Dj-nr%&mMuZ*8$zx(8gNy(x|RUa0ncW`iUt#t3V6d|AM*PltPTwItMN`Fq^^)z%Ms6S>g(emrFHZ%#!OJ;G736R58KxwS)=4Lj^&e2h``RUOx ze!!n+K;lIIgV7F1>r%xw# z=%!PYv9&eoUfPmijT4~GjCak2QqPxFRjpdOV)aX0eML!uV-Ig?8Wv3+2M7b8GJ$3a zHzE_y@bU4ry}P$Z++$9h;865gH*3`5PbKcIjss0BC@edCx3hfdyeA+eA;Hcm;rJ5m zc|9B3N)$Sv%jKIlZ{8s3^t?qX#ow{!Fm?V%(E%*U^vk1Xr)Fl_!F-P>D|@G;2&3`% zq94VWRfhPs9$4-%)wZ&~Id|O}b}_HJciBMX)^FSxFy2?+ft$37p8hpxg8kI630e#8 z-x5+S{nm1x0B3J0y37RfB=6v`dt&f=5L$xiyGT(7GgLvmv}1a-i}Jd@W>X9)hkDt9 zpjhiSZ3+Uwebw5^m22C&^V|Ge6$T$UYb^d?zja*Cp3_}!D_bsh;oEB9mcy2of~e%` z0BDZ=4eJT2tVxmte4(dStzG-3qcZf@Q#9Yq8901q8nw`BDz$MCd_Qs9XXU?6A z#rinkS|UYSk;AVVo^9J!wzaj1yU&PycqA1Be7u@nY?Z96?0KjtB_0cNqsQmwCtE2> zMMcHs@@4Wg2}@E|rXwurR}IV5>}*$a&epWb46{m8Yy!$bW)al2t-XDTp`oEcsR!q# zeO51XTRk4w7N{tk)X*?7H(!4I_;E4II$Y;kR3{~Q`DHdgiUP0#polQhv+g(qA~gbd z|JGUt*il|V!Q}V5d+l}a!F}HCSNqZP+Nj9oD)>9EpdcMZHlX9s56+N~kg5+K-T)T< z?CT2xEuZ`IRs>3jx?aRvl)>lS*;viKS|N3(a&0(qn|3+%>lS$~ZU#GJD!ou+*^(#f z)zbAWAb=Ks`EzI}6yIbk4II|A$5M;{u_4PGXH8m%7zo+C4nppdX zgeYPmjSwb`4TN%h4Il)IxKmIo#jsF2s)?1lGS+<+eCJhdE#vWM>58YS4;UC3yYKL1>^!H6 zRaIBNLVvRB`oI|_afxnZWTff9hQGv-E2j26(-u<31P|PtW-APQ7cFCys)}p$1tn5? z{!_E1BPlE@Y6r(bmvr6J`)yjbYp(X@UInl(H_((_;eNoO^GL})fDi9Sj~=0_IgwA% zcn&16!3cyeJVyl$)biVN0gBX!7R9 z#xfvjU!adAp;{a)*_)-VXj;{e&Q1=|i{*$pY1h?^^XSxFnHqkQ?lNNf1UjMVcyv)3 zDi^Irf*K3f4LJ`zr@&Jt~uyUO6 z|5p<<5HCxpFKujIbxZCH~?wFm^ z+$Jfhf4(Iz!)sA%1X^8anG=|)X@xKzA8C-5mD-c@#~*rYn)QBqa*a9!M)B5pb!(}o zh;(V(*Yv2Us6XTOSfrv8kViZ9Wjy9w?A@Ut`Cy;ASxK;RbQT}hHX0j`M}MTnCDhIEIiTBb=;|c5Gv0c+)e&(RzMKgB2n zxyjt7nev* zMox})bHdXBPxR7}$}-3V)hR+pV(xHvE;LLxo{JWcQpAHR16c9x?mnOG&hqt3!to0G z;2SsSzE;Mo+%1=eC>Uw$ZfyLxtVb8RpD*w<9a-u`$|x@{PvlG%XUzVyaKtG5ZfU9K z)i>JXOHW!%pJ2=uyKKQ|WMXy{8y89}@3F{TvOVDjfZK9+cN+Qt;%K0Ye9N=FlogQ} zU%f}81_eLy#1rpbmUY|PASyCI>KBF>uSMwWiBb>u;OLIbXv5l0Z8WsDX4_Jh9CT_R z2)bCs;M)&&{~7%#BU0;8BYmy!YjT3vG*ASm!|an@KRxuU3_}-}yq1E8lFosBYdOSE z#5Dvd@4Niw)hpSZBLZnaR@DQG!0r)g!D90 zXC4KBPU<%(dHCt$-rWAQO}_bT%Dq>@RmU2#FWjwlT3A?c=qlJwd+glDM=Z0A1(%Y` z_ANhJ*97d`FoBzKqHzNIsXg3^iZTtN6FUP8F*uW@k-GSbSaEwM$Qyd)?bpv~rptbJ$G%oZ4(pwL@PCP$h&%mMp z!$x{>?&9;)#q3a|+h?_swJYL|hS`mEtHWjKz;hs8*fRj%E3YgMeL45E4qImU&o33L zhIAT@r^jA-(JcM>>(_D^D=o#YY{}XwjJd5vm-k>br=~|km$#qB-A+7RECG1o2f2dt z@b%TN{MT=RnYCisiWNlJr|deb*7yG?F4UcHZOOC8z0t@^1%_$QOikShAcbcFJI(*? zIxgvO{d1ns?V+Ob9Y9V3=+IY9 zaQWz?t3h=90Lo1P>{ijz-g=Pf44Dl7roN$hoy?pUczxFc-VC6Fh8O47tysN*qR_g_ z(aX43E%8{GKBt+Y$8u<}tGeFEa}8h<@ul#?8xA;Zg@Qq6=cCzImlP(h8q16Ax7YE~ zC7SJzw2y0~=w&nBg@1A1ws`FRm5Kg)HA#ic0s(!Z!oq8y&@H>^x-c_2l$MnOhml?E z{D~bMHJfwM=$Z3N;XN#&-o1Z60+lP#O8>=KRWaKi98|gc;)%S%!l1W-PY*0}NLUy| z>kyAO`Qb!E1Alwz(xv(5won{WSD1TxdMGb2`wutdh8)4#0ixUA}Z zv7^7bBE}9XZ)J2FeZOCDRuD2nfztpTx3lHx9uJmqY#jZk$L_^A%tL5{K=UD4f)zT` zUp4O#gAcFNM>-x4)`Nv-CR<{&28^lL>Ea0m-2 zlOquj`eQv87l1yOP;)bVX?FJZ0=F`tm5W{(JAcO9{0VfgNHL3(w`TCrImy$;+J6k| zP&eO!4@G1x5OR5CWyjp~Xr!d`&cT-aYY_W3qq`adH0EyLazScb&AELv6;!ScxZ@6rt(k;B-usmOhn6?XF! zy2*M5hR3^KDQmzd(8>RScK{;T2&xf^qRcBTeY)+^=+9S6t$q}31Q&$1x3RsW1Muld z`bTvCU#=g&d|`r^|0-4~6e{#eBAi1!x@NFDe>b1K)Q|j&O2@b_Ub>_;`>-6w93Hb) z^AK=jy7^nysDqaoQBmZrte&?_E;$t#qAv^6lZG<3wGG8SNwgMGtp()=Kej9xP)Aa6|8yP&2@N`W2aA>X1YRxY?0?MPR+aB<$cU}c z#fm2s5%w4dYP`$TFb`OZQQZ86=A4w8@!HYz78XhBwc%n1gkbjJPEGZ{`1mJVpjYbo z8gu}`u(H--*KPc%5|Wb1^C5M>a-vrY#TmKwe$6yjiRk0x;6PP=>lU3&*$p)9wLkTJ zRPXy0^z9w`IcPTt$GPUFIzov+RW`K!FUwgJh}Zx0k9631=&W*(_Z67tWEq&0`Otc! z(;66E?D`-)F`}&huY2`o5jY~*%BV$yYo|fpi72_>fK7C>pB#H%!=ken+=aMAyhcJi z)FbHFf(Z$NJNZP%pX@$$>J&lBKBO?K?pe{PCOQch)~%Vjvd%C5D7$2W^gBMu<&9{e z^n0D`DJq&kbD6@3c`7L)p%PMba`ApoG znex@B+iTOJYhVO5HDVqDPk;t@FAOZ)XWu}TiH*-~nhshNqH#U5+BW-Keayh<#o1u) zTSq!mZ{eQNL+WBgYJvBl+f@3A*^{~ew@grkm`@i{%TP*9!0F_mC{#=$>Y?8~awd8f zI<}yr3?33O!|_HwFJB%SnMY?gpt#8v`f||E&yPNseJ>-m3R_~wSY+nMhk(n8)~(4p zA{p~URr&Ud0Oy3`Y>tkO;hSfRL=;-T!ngOKV-c*%-h+IBUapPsv1n;Sar<_kn^^K< ziq=MT`knb*0R?w6d%lQ0oY^HRs@{>oTnu1RH|VU+ z?ATNU!HAAp29?`hA`kMR=xG@l8Ciw}Pr8$hL+3KTRL`THP0!pZ{>|df_O=gux@NzP zZ1tl`5P=Bml#~>D_svLbbj}E-0!9UT_P!=)$jW(k!a8FBF4)%z>}a7JXM1~f2^;(4 z?unM{Z;Px)9h{t4>yxgmP(G!n_w3+Gbp2{)=?isFxiT}uY97DWi@{MARii1OW2dVBxw-DS1$ zDix)RxOCp&}u=Tq29z*tB2m`2=@L?@jn03PG$a z;;b=c5L3#~F&LoX5v2hY7nX>xizk9NCRi(f0|=UlUMQh$DDiS&98F!vjtacn^MV@~ zu~X0tqPDovrpB)GfQXtC=PWG)LAz|kB#yyHr9>Q2{xE1TDpU}sc5@=J=$+c4rXt|x0 z+aa^CNq7h}CZ&hpQsZ1%bv$i#r*^V_*7^0g1&@=HgP;TDo# zCY!NbWZm2jQjhIB-b6~fOOTKiOvQjoRi~yGF=|QMIFr-EwR8l9QGY2ftkevN+gbPT zWZk}FRfxtAprP_s?b>!j5fvvvyGvb+Rr7->=bOt;>Ksko-nDz^@(wKJO?8}pwbIA$ z3)Xa{jNMCyIYe?g17A7)ATzW972HX#n(A-dLwXO|iz2MTU1zEoiY|{n{UpjnEK+wX z>5%EA|FB%{Z{NN(B{h##O+BJ8I$D6V5}O0Pf4b1aDjkYVB4=+Oe@Q z;%580SE&sFj4b+V#1QcoVh>wACqy0tf(L8niA#qjW@RXa3uR+0L;g4ii1RgUsGt=1N z{@G6!F%h6G=(8K~vxIO*!B+!7^NWt=iiv^#!XHRLRO!Nu5Ndc^Pfq~BLFg$+{)mj^ z(a*lyu^k=5{s5Xq>kL+Z8#(~>*uhd(pmCpqYTR}Uq>FF|4?8-Fq2G7`AGWpRU(9nF zV8_z48);X_viPzB7v@t~Sop0YBRjhSe@!xN@z$HA-L~)C4}as6Sx*~ETnr4N z6lo7;f{6#C!N2_WjTPXKxwRCbf@QK^^1QOxxW97j?d^#zOfU;siEw^=f2^d~`68?N z1*-Wa2)=sdy_|ALxG^?4sdGOn@u;^f(TTAEO?ztN$(AE>B{~9JU{;7jE^`paN?^`!( zAA4!}qbT3y14K09opOtct_Q`2e|bIflll4ciFSG|Bsuc-e$>*&Wr~f~*%j~Jt;Ti0 z9nEZ+obnbNbj^#4_;$HQvLa*O(V=E6&@*SxK9gG48*)va_Z5sw0-u+yT)Ab}E(S!F z${|yYVCh4qH|zTF5K?hDT$s+MviyYb&S?G{3rJ)j87t7(K$(&f*tHFnxKzUuE*1I< z<^JnoylOS_@7_%c_G?rT8Vag2i9sSjr)WkT>*J3O(oi6Lhs|TY!3g=46ULXK$Fs_N z>Trkz`W)~sILAUB_k%<}SDQLM!=qanBOg2nl6kgu+qMgQ>hu)YeT94q3L#N?*=--7 zxze^2Zjf-?PSz375{IPI zYAjq-+x;YDpxxY5=qy4R<7tA^TLx6<)}1>moA^v_6$ST(;qKm}MYpYEkDAu|RW&uS z13k!r+ffuM0x4Ba{S8;&y*oyrs6+JL_+QYaO=CoFhcnBNcme6U0&--+uHg(~~d3Mm4o> z3LV)UfFPKGfx*xAc5mB~=@lC{ZX{YcNfSUTM-pT+__^%-%s7ipT5umBwe1c-a!nJ@ zn$Da#^Al-9$_qlgplaeZ_Wn#s8mHFhy>t7hT}8WNhhIlE&4fHyoRI3z?n@=)qA5?IZ))FLF1vID_D zr&^I~^Sum8@W|{?so6+-g+gF7DZbriFO${FFdZS2$Dl(4apozma2nhAE{^RTL(CAYK&YsekH}g>WhuQQ#Ma z%}|aHX>6H?QSToRKyy?7sM16O+nWSO;V%jB@>dG$Wxb3#aM5nOS4&<|(e&*d;SF#E z4C0e+KVPlI@$r~yJR*ZYnC-2B9z{d7@j-cR9dFhEp`z8<_x1G&JzP6s)`8p~3U|;^ zDHsV;+)w#_WLbpe9fV`TJAR2k`Dn=gbOx$@m0X_kXU=TJq8CLU^0f}vvZ;}rH&#EA zTq?i>90JFWrmFadc^GYLiM!BFon&oQbD7?fphQz+V=p3lfYY_Rf=0318gbOef;302 zzPK=VIh*BD9LoddfRGT@gdpLN4*uEa8e+n6J1myx{QPMge0xm!wxdklz_oBO8u}_# zak{l@SrcMx>Qf&Y=v%aFLqLMD!^i((`~3+9uNIlz;?V1A#9^?lhtJNPs~q-B{(d`e z3P;qPJKAvqCscYjf4hTw@*3ITLpBk|8y9U($md98_8+)&o4@7RtRG`-(moY$tEk>b z^$IEF&4^eK*>q0-is3&^xA)(?`j||n3`ZgpJ9{9get>5^(qIi8_6`m(QY!wO)KV4V z=U)N$mLd7tI83bO9P4YTsWw;J2filXjU0Y~lvR=Iq?$<~IDXLLm(~v-cD9D!l9rZc zANT^^Ty71;X={l{3M((4eg-m?hJuqOiwpBmgO*WsX+~U#{@fpM;~`*QrU@9wcxt%K z(zBn&!87sR-@p=D(WusNKv0mztjMT2LTB;@-`-@eKJRgQ@8yj9-#qp69exqz8-ebn z=sU0BxQSl;eJopMiuG>$m9v(Xs6M8Qdf*TzP+{DxVHj5=WudPn?tXFxK(GB`Dy%by zFUjux*qkRaOch3KEliayF3(Mg61SAfmw6MqanI9F?zGhnS6OUIqaNPiTGq+r#3 zN&4Tj&SkdN11iMl4Bg)PJ=FWyRHQ`zEN;ley1O}7#H+>i3i_~eGao;H;1?dD z``WBii$mW{OiTom+sPJI@Xs?o(=kV4L_@1D(|SMtJSz7v=HR)%r9)c#-+_mjXnaHy zxY(8nPcM)6vOq)Hwr~hG@<~(LB^AQb(wjLYotQma`WjzsLxzb@`--4HV4Vq4=ahqI z_c$V{dV2Q_9-z#lb{RZ3*oCK>vY01^+?M@iIHwIc)(^^@AVox|{6PvNQoCb`1Jf>O zCG?cbpHAAAmKM#^p3pR8q4-xk~<*|8yj(%`5XpVyY^sDPYZ@=QO3|KrVzHSDD|A_*%oiz_5Q)OozTC8qGATI8XVk7 zhI%<|zNU3+2`3TSPDONMW5SJP^=j#fK(&a?c7Yt9@MrEjA_%x_`72i>C}Z5@?K-Ij zB~C|=9*sP3@o9}k3Sx1)5Vp@KxPg}R^muft{d16h5_3d{4d>shoV<)?nrR8*ecb$+ zN0PU&C}?Z5LM=-i9_>N)XMIBPZc))!KD11o-}YNKX&OM-E?v*acsnf=V*C?iJOzbL z6ki@)(dsdqQZ^6|Kuz)%Berm;?emEp*JA)2cmsR6>jEkQovp2H#JsMq?ql#TAhl>| z-29S#_`E~_Y3hxoIt52?O&aEi^v)2?kaA#}22i~aQQZksoj{id21Xolv+JR!i^!^P zY&>J%Gmb@Y+g9Q=-ZS9q&$xjb!NH$n6zF~D@1SerF)g|Iz)l9brRYYLZ};|`BhQ`6 zul6FJS<#+N3zlyAvUnoJ1ANeRgw@p4l;Jt1JN#14pA72Tyi*nJ`f5goC}oVdMb5~x zWFMv=hX4Fw6fz%2hO20$d9^kx5&c=TogeJIzzWgS(4Z~(SAYKv%pOpv%}BPHmXsfq zWf(?Q;;q)4<{2oQe+%+O0oAi%A^+&GMkj_B!;buwWCLods$L>gK-756w9}kGyXPCy zd5|Yv344<0bRY#~7_BJh`VA)0xzLrlyOv9{&H}Klq<4K&UAzjL{r%_AER3se@`xjG zR$~pJtDJ7#<@b-IHf>XjdI{=aKR0DBX4A3(>4U4+t}T5e;~4@x87Mfdbln>W4Cu1%0PZC}^>;ll@*5}9QSNZX0RYNztQ zyiB}rd4v-h2jL(fiWShChzFB({__&x&%@YD>L`&wy}aV$8xS%m1OF_yUb%)n?oX2| zwmBfrbr=<~upQ1fX7WALNm~nJjzs=Zn2{CAYgxAgbi^{@H~io`Z|X+@npNENviVf* zaPQlv-Bv){*bE({&wHfbY75fCT$p8B)I8{Y?HU3pA%LIAqggOk(_t7_!WpWDgD!gN3! zcR@ZTRS{7#CET^Y8CXPOh*<%_Is@`%#_l(4Ms7U&&D;G30nQ@#oLdVee+fyvAov5q zhY188G6R7X`4ie{dua8Zv+qtc9Jl{ok01`AbqG3#K;er;nS&pw2<$;*dT`;u45CZ* z$!h<*PLh{t0$Ys5Lj+LyQ9{crmTp_~xIoogW zKwwq5v*XomK9x_;PwgSfG*O*FQrB~Fg`$ucW4V}?nE^BWy4@JqcSSP+SQ5hq*VEEO zz--BjkmHO6ZAY%>6{t&weTO1~C?t_e+ze`a)b&fKpHryGG@bm%*KkTOTQ_CN-rH?% zhY1Oi6T^!HLmyZf)cTZ=+%RlO#NwF4h9D;GJRK_`A%OxQ18@(0>?$&v#t5IFcF9CN ze29j|qwP=#nhDkA8iG=!^3zp1IXhPZ(jxV+O7m{qQyZB$ErnU!6`YH@Z`s$cED(S~8$OV1VfHs95!=yUU1r7vFzw(6 ztZ2~@qq_+S48}4zA z3llv=2!z#1Qgo1LDzIox%*?J058L%OWz$1TuY4?fO?m5ObxRE8oUe_e#T5?2V(5Sn zvq9QT9E-&V`X>QgXkW63#goWVyvBa~E>?s-u0MRpNd}cr4vCe4kLNN!vlni^H?{~> zm2wQNDT}dLiE!YUlYD=b%XuY=!6*=kB08AnsSls&!0hz}T(l5-^fZkGYAuF>L=?dl zBHXY0_^bk&JK3m&^5cW39B+j*g4r3RfN@Dy4{qP)MZC!(%M*&096~L%B?N)yT!MmB zD}AJ*vXY)8tO51v-st35oiYJDlv7g+sa+t6PWadt+|>9O1Abya586}G_b%_Ex%v5^ zUsKFJv9V@XO*JpAHE;Iy@i{O2p=^Sakuhe(UEl)rx^^dA`OdnA3YNE+c%i>!EpR2N ziLJ5A_o;|N;OM=c{{H8mC$1pwj^*GBTAN+D2XPisew#jK3TQlCg+6~jiZ}jDXi~ui zUHHTOFZhQpey^-litUKUcA1-KwzV`9=yi7omC4Qs7tm0I4YinU_ z=iEc~0MvB+{QTbXyz3$MJkGR6#Nh;)Q!)c1qeamW{?w3(@DAu_84T+1!323;^~*9! z3JSDX**XQq$lVRU6)9bXA5e~72Q!*Bm7Lq0~FM@i&j1^w@;{`5D4wxu>BDQ<3*kfJ~QN?1#N>v>l9rD7mjPETi z;nLga1R=L4o_g`LRy=&XHT!~2z7vVQG}z$jI1G49|6^X>9#ocSA*Xk;;O4W{oO_Q_ zvHbAi!}0D28XCsVW^#9Fgyn$I{mtg){mth3`X(J*32U6)a2paz-qF#~O?m$yT$bnm z4Em1Apv0A<&_I+t3Jbm+?h1w?phX>ACVL1q>L+G;;Hbv}dv!x{c=EJxrVhKI4XB_! zYwxUFa7Xn5iIwiK(wQC?`onV#xtya*$vjR72xWk7bcH)oe6E zY8^VuRhZ*1k#4;!V)Od`r< z;!uz7nu?7HG~rWXnkJMeiufrq=?0AvP}SJn+!rO_CE5@gagEMrg^hqga{$pqEdqS? z^75iGYJAagfYV762CUJ2Q23NZW`2wb69e|Y-)$rlevr0aL9QhufMnPKkP$iKmyx1o z;o1)bKg`bHb6q35V}2e2_Zo-?@H#g`LPpG;on$5db?nV$AlMEgG=lJw9TFWyE+b12 ziG&z&6$^Z-u`x+w8>+^uy1J}G_Go;*5d9b_G)IP>I^^8hN~e28FaSEry{8 z%pyDf8(o0s{5TuT{y^ArKk>M67qE=hH04+arIpMO2TwUSJ16H2mQ4|a^+SrY4K#>M50QL0I31fD9qwc{TrySUpD?u$>Ni= zj@j+xg&eTz)uxhyV7;kwn6PV%8EkXdB9gwH~>c#Mcfv6UbOPsY)(0}f+! z9Q)to%$a52WMqN{H?LVpPrJ%78FiK(n8pZms z2u5O9tsN>IPvpFzklL(M`cNhbYR|?P+#s< z;z;XL#y}g#YG-^NO1raDBr*Rk>cETeSe}^nm$Ub!wa88Z44|Qa45QE7W~DHdwiZvB z@>)*My6fm)3*6sdTKPXG9@Kt<(>5~rT@u6cCO?WU$8)Z=xFz<)3;~OZeHDD`BX<6% z9}^{ma~S!LcXUU%7%5(a3|CV`n^hLfV%%2jhXzsJ>q1BtzPL6O*RJ*Ydj;KQq()FL z!5`G|oKA*Z&*DRT9H>_oiqP=;{jK3n;_{lSU5vg|_M-ZdY>|PX;pZG{eKMtl8jD9m zHf6@SkDHN7p(rx$1qgk|p}c9!{ATpq(g=sGe}ApKyl>@yj)zzSn6U=Bk zL9#TOOc>Nf+uc>UUceE6TcI80acaG@volHU7ZeomZ8%k306=Hqcj%wD(>(K^dqf_< z3^m^Uc_~QRYe;W|Y#|`e1JMm%n5cnKDom1QST?a)hxy10@U`X$CXJLOSN-!}|G$66 z9KWY#vGU*F^Y4cz|2LNO|BkZ2ZTjDR@&<}7`TBq7nvlEjzu*{_&nvg$M1s=!iB04X z66he9@;=0S3x5c~rU=O>VPj`Uo5DPneQ|DXZkc`E_=o^>GnUEE_4T<~uctp2LTe_| zB_KbTCk^QO4dHdgm%&+KP4c}8bZ%&^aVmS zeSCd^;ptHLNL4{v;uCN-W*XTbJI8bOt`vgB2+&;#Mahm}DS3Hr{5Svr8O;SWAr9HE zU%ud zm;@mT!o!CTYf1=EM8q(@4$f%@jJjZpt7NO@(5}}O!2|g z7$BbsJp+sb)F2#WxsFTkxm|3`P70V?TdWdjMQJ;?K)i zk%KN^|FNV71qB_(c?={52^9DkB%Y)LA@oUC+2cQgahuxF3>}76!W)KgMnZePAJ`5< zo@J95#6SRIf4qo_ldDJK)fcI5GswfAJ_{_wyYZYPW?QJRnB6XRa zNGZDns6x*EfHWoQx!|_rROT`~0%6DJkYCAh9j8m&0-!XtBWPQJh42ZKPi}Hr08(vm zWSkaB4?f=^plPCJQNv?n^04bn&YrzaazyY4j;N`<0d+Dtf8PJ{=Yh5w3)fPih-4Vc6rLpqon@q==hG%yOHAPYp&*{%!$muNZG&HD&50qn%$MPTts zlcnL}a%f{qDMUqqNT0w{rfZ&_nQ;)5nK^Gz=(GmwekFBdvlKfyJEF`9CoN4gCilE9 zJGYK(aPp=8`N*;bqZ3%YGv!2I2E-$-CPf0Ws2>W*O8?1zkJ&7;o=AH1FH(buz6RlO z_kcHZPw3aI@x{ievD6}<31SKZ3(Il}@LK1VP&jU{H}_+l>ymeQl%FE!7lG}3sK1+> zECi16I5zegXjv>&|LK|A4`)t%7rO>^kmS)3S6N-+I=L3ptk0Z_C?xStz;D5fC2l{7 z@(^GzsFQl*=1qp++rKvBa1#(kRK)GpFN|L;fnT(gB2(yK>j;JazOYSl{0Zjuu+v#4 zFln$3sEUMPs9o+ej`3f=ET~{T`Xb{>lqTCBMFNg8fT@Wm5tAZVQHI@C37ZZW{3ALgel0o41r-pBt8M-y!a$m}IAA%*y z`~NwBhZKs%#P(=sf}s<`gOR^GR_kDI{|b6H3%>=87l!*v=KiQaMN>I;^4@`q{J`?s zrXwD!F?&>}CeV%H1^z$OONb9LiT0tGe&M&s29?LL2qzorF@ua-H^LIf8?hT_x+sB~ zP!ysFc~Eq3&4&Mi%dP!*Bh)(-xn&fXVI{Ww-riW;RzXr7h^vhPNOh`7Ze(WG^hu;a z=$3eUbNcsN#*OYidbF0J3;XFWtzFy4{xkcpgYkCe>XCRQC4#yc5fPDAOXLpF6Z|13Px-oIEE|*c%2o+hO+`~vE=n+>f7#WQ9OZS$rd9`q{8mS@VPQ@l2N#!7=RzD{u@Emr zrzbG8aSTRep7c<-zk*gvSFU(y5Du6?XhxSs@W6qMfYox?>tuRWLqmhaXo&QH!-X^_ zA|`)SfA~N|YGee!yeS-96aO_qElQjQh8P6grRFu^S20)$Lbw9of%YTV@}|CCheHIE zkYZqER%ug!6m7|!A zAR}1T*tik82RXR{ToZ8PDv7c{pHN&Z*O_JK1O-C^}(C{t5C3DFRpmWB)KvoZ#3ZJoT1fKUf$WF>N5 zA-I8MI1~;n8d)|a@LVEb#l^=baY%A=AL7t&@&vp88`MlS+=P)dXjk$WjUh*eU`T=G zicCC)tFEF1VAUN$kOashOh0X~F*YR<@|r*P#w(=L0B~?t2suy4g{w+h>A>DCq=1qy z_fN~nu0DPQk%Z;c5{kIb%bWILdIG~UIMo4SG_-pR2jf@JQa-rU?JWyvh3yTL480z$GI*y7olK0t6Lxjzm1jA=8T*>C-!p&^k}=ZhCLHbo~SBs46bov{yu z@xwh5o{;pIb3s&EdiEH_O$I^{Q>)l_8nvW+1b#+rdT{@xuA!l{%u?h;Gpx~Uf|wuR zpe5=M@khurG<)1_j_C(<8&wOV%vl_%B8(g$zY!OfpF=YVb(}{wrN))jYZpRoICSW+ zO*4_$zpwQo=Lpz8fVSp8{lfP=@}dRHn0Jx9Aa zR04)`*-f6qVg>y8;4BlOc(E){-LD+zvrrtfYQ@hCCE@1LBf_TLq=OkHUyNA@4IKWe zGI6@Lvw+2?T`gB;z!*Z*JCgQu_nAS4br>_CRq51CVZ zKsdb=QjbW2u0(qO@9AKeV#k_IYRSdsZiDGVtQ|5{137vNkUlvvs$=ZWpE4jdWK&~D zM8n40E9i;P-rnjC2TN0?X#*19jKE+N1vZjBi>$JSJzWLj0LwIl=;k2oPEXIE)F_Q2Tr{z$L)}*hJcm(LnT&BrzO#11j`uoVbH892J+^ zfKIjo0;~%(Ca)F&G2-I^f*eMrz-sYBfyeGAmOY9|wU(O&*1K-oc^F#+q~Nyr;rUiq z)B@d^9#Cv@z$haDw73R}1kz1_7?2Fz&AVj_(U73o;#@*@Qnvx$+Xe?yGe6?>Ca7vC z*Iv1~`-uXGHA1X)kV0?T3Cy5u*`d5HhrQ@Zdo^ocaJ}DG)3mBt%cnxIA%!0pBLW10Z#n-(P~D(td)} zNEQdDNuacPUA=lWZvd9^&D*!Vu}j+#o?doS|MQ6ngCvRHftUbcz>Q!_K8wStRw1vN zh#cmR689i{bIbPas|bO*L#L5RJb66jE$GMimOPjap^B&`;^N}|1IWJd+&Tiv!h{LT z7&4AO4NUVI7XJtiQY0Eaj-Gl2l7O&&k^}5aThdr?7$VWsL02*C?~nUuhWrj815np? zc`nS!sW$f^l8Jguj?^Pq2Tn7H%gLV}fs()f=hTa#mFVmUBOHP#PCQY#nZpCQt>knN zg}{iv5+s3v<>6=VY{1S|wVo%qWGiqTD9>K+-w2;VI6lkR!8Y`!ZRwhf4z`(JZE$ zgmK`Y)fa?neC?BxGhZMh68vl!D@Lv;;g7bb(1$=h;sVc-;TjB_hUZ9K{C$G~<2RE( zP+}{@aWIoFuJBj$P6-J;81PBrAF!2@(-(^-!(-@fq5{Z~7}7p+bV9@3r(Z+F@@>qc zl@R#Tbb0uz_dCo(pOZH*y zPN!;V|IgvVG3)+0?6^CO%+HM|6#$d4Jqrb~33MmXcN;^TPnHLd+uorMYD)RCWSAMW zIFdwX%IrMGyv2R%(Go_3Wz1`sX%-*UbfF{*ntjK7(T0!cP@^e#RrjgyTN8`fb!>Mr z%Wc{4RP(vy&5iILY#4G$$_XvoUXzEu#@dV*=nUxgHg(PO=zUc|r?zu;W?Mj%ig&>C zG&+UW^p;W#cIBP^TBmopn%}VVdnl$~Xwh3W^WA@J-KHf1E5`&NN)Ghv>ZFWFBUi7X zfx#a>i=$*H4s6)88O=Gl#Sf>;B_69SukDX z)p)XJgl*Y$k^=T|v~97~?O?zX1}3K1^k_ViCrBM4au)Rf1UW6ZVA&JcC&VnKxU+1& z>sh=*nulSwvCuiKrJ!B8j*gz5iAe_s{rKX|*z!VQ#sy`N(?u-EnM%X}I(D$;!7$&d zKa(H|vEdUqB3K}MAW%rX4^O0h2J^kNpeM?+w~IrA_NZUL=#KB8kO=M(P1VCC>bHub z_Y&v3i98Lh9F04P**KbV#5Dkk6R@hK5u$8j>dD(h-%1NH^Hm9L3j6=gvL564LJp7m zyp5NiU(rB&zwgc6na$M`D?8PKw?Cv49TM}y{gs!KTY^wxR-6X=Z)<0SmOE#8 ztS4C%8~`QXy|9oFgP8Bg^stIU>^NUWX>IJe74WGl5?!%1eJ7A%g6JAupE&vV?WU%D zsHWzA@%kWcTQ7l=?m{<&U1aOqYu6cYu2dD>SZV3fiMv+lkbr~uWE8-e&Y&(fi~XWk z&_k@;x^?e?QD0(&J#GCdOzWL-n$P|Fw)e-piS-UVME4$vf2sJROEZq_4v8Onc}~$+mt}3Q!!HAyS_vm7s2v_7Ad$PBPj>ar-Dc`A zVo7^}div5(;tFKHu!!0=H~R0c&QlC)(LBcSA#?I9&^C0Vc?es*V^b%-MB3>$u(WnJ)UGT1aTZ&zBtJMP z=pv4?A3$D7jswcHH+tglaWa{$u-IKP{ypu;7!YusY~4T$1_QpOu;Xk2e_1yxo#lVG zqOx-Ck|kHD>gw6ZH>I;c1x|)8E=e{GkdOmKgtr_A)!cP@{)d7C1k2VbAk!5RY)rvD z21z2&$OCH)%D3r+{m*i98{6&^tqx7&J)%ayd>ZidrP^R*@W4-XvcG@~PEMW-Sn_bf zMB(P0aDWb=a-?PQfrSC;6Ti54#+!C|ZHh&~-vqa`2M%q>-b7YhD3%xH$iW2+DL)aZ zmdnVHTpB!dy|<`uCxg?XJSy_qmHfhx@IOPgJBfir=?M}bovWq`6ugGPSo<)v9+4@K zJ<2^x&hm3^9F7q!>tIqcTE)h0ekmvT`WmzO`Y+J+N6X zfUEr{0wa0cflYGD!L(0YQc?lQQ90xkGZ(C9Af}uE>s8vR<2207E_r#EqJ~zq%6N)% z-WSCQu`r0m0hrnzLf@c}#M?OuKc#y+m_3pkT zDjZLgWafAT-;uGld>F=JFKUI)BDeKSwLF&rW#hG~lOy|=Fy!&TtFwb`S3a;c0RSPg zI%d*@$1+aW*%AV-g(AwDffQv83=MflMo+KkikP7Vsnvq@`%c#@AION&-5eigbgsF9=ci(K1WO2Tg|#A4Wmq>n*L&(0&Z|AQ&T2hR zd7GP2C~Zx6bFJE22s{Nu*vlR&@xN7P)?c&HQUcoxNd`~$YOh1{uNmncMlwVuj!5st zj{YeRFZin<)y(D}8a6f7ZRYk9>_U}iEunDOB%?CZ+F@B=#FPWdT}v!ipt541H4$Il z{va{YDqYYz(RG;;FvB^(%%@+wApxLij)cwIKhULtOJfWl3~NI$KS!GDh4-8WEev<4+GVfU%HtCUz0!Yq?&s03m^n za+)IT<;@wz9_&R-S<&NZ>!zNL()05>VhX~@?FT&aPOI5+&ejqwA-3OneFx{Rkrj)$t;GskFlf(l* z*;lp5oU&|}=c;fF2n>up>hd8^B7fQCN;J2Mn4-{DP^pS_QGC4FZ3bM1$qWz;B2qhQ zCThjb^PR(>c^G2bz5_UeznXukMScbUjT5?W+NE)p|NJWV{A+AUe+e#-NR;1mGP|zD zfmX9}{2nI*b+!Im^-v2QG-GkRiOr?Z>Y$y(Y5=Ke7D^(sEeuXB35$3gcKGCO^Od9L zJergR5p_kl^fM(>(M>f-7m|@Nw$)8R5qSB<02sYm#_~sW{W1=`@b-A2-bn_meIYGs zm)PNhclgCOY)D6$x!rbkY}bPMK{f&c&Xs>yQ|8$daF%>ogR9k=6R!3bxD9;{SM*xR zM-w9gYh!IBMfs(?c%G{T=;+6){xOyA49=EyT~5J8G*AN_45XJSwnvYy2U3RnJ*nkc zU5=e;$}@f=o<5>&Hkr80KHYd;)7heddq8%TaEAT#88hHjyLhqD?dnVkfzhpI(CvWp z7*>(3z^WJJ1h@y5^#Py2pkZj&Ng%;Rk zyD#WTJzIdz`$l`s!Oh>7htJT3p}NT07fr?9-1^mpXI;=_zGsXMUThDtyQ{&Bs zY9fK$K5*A<^qAOi6r7LXq`G?)Z4$ptKJC~%u-Q2T+wkRjRWMvD|= zY#x7<7^2_tVJH_lA=#)aH%y;8l{`rWcozK?N&r^*{pUE|41a$`5Dg|Vu%^*M*O8v3 zqpm(1LHq9Ju_pFv@C9?7^2I#BO2a zRzJ$Nsf#kxaEvz^MN=wsWJ)V4v~)OEXUR392w8-#=DZ0^nUQgF;+$~4d#|{sJlZql zJQ>fD{9?GC%+HY9!N+aydn|R%rG-_9oMLD;=57L?g0ni~EXCjbi2(gU#|zU*&Y;!w_!yYc!Ee<^BE*sPJ;keiMU{hced$WYD-Uth{hVL} z2$V!oNFk|6#U>zp81`Bsqid2tU_CW4HUz83jf6}fI6nrskCYw*Sva8xBd9AqV*vv| z&Hd+c+vgA6!+Z*4ALqbQfc#)Rm?Bhd0J2>#DOm)dsuwB2<&xjY#h1#cjjt0DR-s5# z`n+}9AAtWZ1D@E2M*`EYX$-PAKoxsIdKt;(kx^iZz}2fqot>*sG%Vk7aUY~oqA&j9M9Kj_9xY<3*k>-2S&*HbO$OKKKlTC&V#uSqV_XAEL_Xap2qa)lb?qlvqfJO4h15u&s%5Vb2Nh>1>fAFAzZwX*~fL@93e(L zA>tTn9GZ|oU_Le#$pCjga?sf;_GyD)MJRwUgo1N?-et9gRvAHX@Bo@ehL28N zRU8!l1760GneYD>#acke|8NE-;*nAf4=W{aydHX~Q-~zKdU7l1NEgvG`*_2y;0$y) z4TT^@H1=DS&WlA~2rP@IrLt&C?m&>y8GMKIl_=LJv=opZfo!%?uw&Kbd}h)uG$-GK zA56N@ij6>{D7Xc!-D_yWv7l)45dcuc=SMeUD$TmU{-=$TyAhiOt3=ztUd0c^tEGgS z38EIF#p~@FuB(hV4x_)dXm7k0hz3yFP60IpnCUsVZ{5T_bFf7NB`~dC&jf&dG0rFA zDEmAc!w#>`F0ZaGLXt~_G@lWjU4Zz&#!$c-p`ySTj+>@8zzq3_PmKPVL;4jM)Lezn z3-L@)geihahtpd~O4q^!n_yK?x*U@T?C6`mL98M8gplG*)p1}tz$N5X#{GR(VjwA^ z!5?zx@c#XNUrfpcQ5_QefRFnWytiwBeQ@lWH{)ct-0w^50_F)&Kqbm@G0~XvmlHn*SUVN** z!tWu10XBHWJv;EGa;L62;AMz9AE4ox5X)FAv+cAEspE+J7z^?dcJSBnF;}dnN8s0B zqbZO;0(c2c<|0I?*hCZLe?si&e1Lf9PXZ|bfFRmZw++QY6#j;&Ei`ZntO>U_AF+BS zp=J0GxC9yVlRXZ)m=xX+rpZ-n{GO=+)*{BL5Hpp~6_AeTmSZ-!U5+LYSgtwGYeONI zK|6nUvYsgVi9`W|?Wv33$jcXC9l<85h?sI@S`UII&QVZCs6wNrAI6F#85B!MNfpx} zMsW)_j;smMw1d=j4%Qkvr9{QT@t^>#=lAeCL$@434MW;~2agq8kOkzCh?NbM)^4~^ zk;6UOMFGi^2X1x3WGW;<(y2oe0|)~;zT?AF_Dk+ZxGECekbsh`_V@3nDQ;1ye;2?Y zrNZI~3TW!>8r(fNi&U(@anX&Q0u$&RBpZOe32tNj&Y|Q@7%M{0$^c`J#)sSa;8Z~> zFRV%udy)VP9FmpjmNUG^-1XyunG?K3LQ!aEK4Od08I36P2EOmAVVmXx+IogxR*_ii zY~i0pPEc430WKLq62pVgp#0Pq!5j1Z`Ke$p(C3BHQJe%8XjghS1oZ!7G5_YpKj!R5 zwV2#NoT=Y&dyz;T1F4FXMX^hG=JH$waSLra($l?p^QL09r>q%ZMdGNTtG^40puzn~ zyKjLntP!R@0PP~pQBdBzq9X2}k{H3mf2Y6YGgeQ?a8TkiF&DS#G2+l92JS6Ixu!x7 z)6(5Yk*!WnjOTB1GepF=9y2ia=hU172XxnUR{jC26m-Pr%O8O z(~ym2BgVqmjH184es}ju1j&tHg3Tj;gNAHTBEX@;Ob*8g=oHcr2p9~s2zG)@_{E4S zLHZiG*%|oP+}7AvSaxzhzV5K{g#EL?CuF1xHao;Lx`G4t_MZJp|E;n|af#mnSq7#` zZQS?_z9G2(9B`_AjDyl1Orcm?ut5nZZp!&@^7`n%>Xxx~#PeI|IgmICz90tm@2_z} zf&K&q4V~@y6@FykNMV1@0;;*~1by4JfB)yd56J5O7fPJ#&;ooF#_Y}5rIY{r(XhU~xTfC9 z-0_pyIsfYw82yQn{xfKR<)<=jYERN%r{v-S5BN4a3Tk)d{mbkN;j$ z+WM;%{nsO>fB#>#&Ch%Bzu^?&@)?iO#KEBqm0|2s&be;Tm&~e_PMTYh7u{M5vmG9b_mO16-S^JUqmR6pcYqooI#|65(ojOHEa;Ls5 zUa;UHeBXcw??Nk93>tn=*bWO^^aZp=;hx5=!Qvt4?XjPXaO8Xs0rOgo;$Jiw5Di-f zLrO{t^W7|Ensz-Zj>=rQYME?reWzZDTgoN=-weHO-75k-4}61h8jZe1&@{&`n&Y-0 z#_K*v#(W3nE&lac@u$SWNTn(i3)~$V`!jw4JJjZ%6=Oh z)C)~GfB|l3;{@)r5O@OQ6ZjM=T7<>EZGI+PXzl(}B8X))GS#g@n~(GCX`JHGka$pr zfb`=QT#KOV1!an8WB4`96f(5(R5pa8s&D};meK#X2*i3a1Cml_0y$%sy!W^v!Lc8y zzeocs*;&l$3}blIu)JHlGVi1#iGI;=S=*!Ot&3x)_j|87+HG+V&-`42J=+Mktvj!NV%O@kg#470#brh=(J+%C1x%kI!ZLcSQP7g$L%h@vCo3$}T$J~M z{5BiMDKX4JPm+hpV1<2*InAvV6Pt_EdnYV-_S@LZ1nYqEM|7G?a>CIqhFR*0@x8cX z>Fcm7k?Py;m6t#Ly+9mPD}a_fVDo{|`ST3J*$9Oc8SyCg7DA?hRj>na<$g0Wtv^R! zpheowYBp>iryb913W+XBMlsYsV*t*F|&IY zN`4w&m1~=Q`kd>E1VzUNj z*!#~OZ8{VNa!{k>t&UARt@j6@_s2cHe5&ccT_-wl687~&z-$Yz?OBY6;J&-K3?J7; zp-D0r6vwMqk9lI^-CDr+?M5CzB>(_SK{h{`*pNgy$DbeLF%rl@vmt;8nHSii_`Hhl z3F9w2bMKxg zT)c20=7{p4n^H_5Dtd%AFc?F*qbvOkk%xvRl9~!=FWhsssjAMrx@@aAhOoSc6WCAF zvq3C$%y^p1C1q=41B>C+IVMy>(BNgTIT`#?9K|T}Km;uUL69Eh6+XFTNILcE=?yV{ zj32@xy~c^N;HjoIj#-(RO$JPc4ilmcjd}vIAE+W273 z6hJf+n?w8ePr{rP5;`$R35eN)m%3jBt`?F~v^UN{@FpV6t+Flu3g|qILVI_s7>afy z5+Hvp&5=cA_|vOb0!jzwj%clx_VyY4Qn$8cC(VNC81)Cy9_PdsATuHw(}@Cx-V<|a zHo*NKE%Ym=c=j^L*8&ngfN?bS4aRYPfEVD85RJ4GYx5zv>fX*CF<$zpTjgOc^LQih z6`DH-NsaFKQPT$w4 z5R2aDCYhsuGEm71j+2=Tq-oykth8dK@m#R*Hqg>Q2%)G0@hVUc@*f4@wjWRx#Ett9 z9F#||V@3;hElKe0Cn9A|M(v)7mG#q)fQE$Qf%NgcUTBP;so}zT4== zwz5cH)b3Pp;thT>D4e^dGQNL)@C7p<&S9sJ>E0z@-(BFV(a8encg2=1i-}-4Vq6u3 z0bKh~Dbb8FI^qCJ;pm~=M}#RXDlv#7a&2Lka0baj?-#(4&-Zd=)T-F;&B4LGrvIeS z-v=4}H|p_l3|AKXUK#v3hR?yGB_P569oh=(A(FvEipDuj4z^YoSOQECX=qKO7@5>E z9&!Z6JGBSl?o95)7iTU!i$fq9H{UFfEb-0pFbrVv4a!|^Up*wDm_mPs8VX+)E?96A zTX_>2hv;I_xKp>D~oV4zlW_FZVg$Cp=M(^ ztd>y2XtA)#C(w)LI`)tfkf13 z0)>`%It>e&5 zb7c_#N{T2N;Lv0+kTWJrJi|$`AA5oTm1VHgmbxm9`uHiVWM~cuw)cw^BoEv}xA=>T zQ5)k_ECQ-ShFj6_v~lPLxb&CP1zLv@X?9jw*E2ISC;J#x^~8+UfxYoERPuKa=bs== z+lMWR*lY}70XLDbC}^kAC}y-Qi`VFB$`I~hc(YiIwH1l)xldxfV&&1PULXbYKY)H- zwZ6A;)J}OmcKZ&PMhhmMdU1(CXSoEUHZVefQ*JdHxL6$ic3|eX!Ww@CgpO zk$D(GRjXd<7=H& z8J&N4J83Dc`!+sz!@SBTs+_0f4!`Pqd9t+bP{am~;|H3@WGcN3+5K2g$rdGSAfLfd zJ165KI~pO&MFVsVPPSI;-YL=NB>`Nmd-M$}rV5xtYe&a|_8&;0UzIF@7Uq)N`lsuf z5*N!kX+}J@pHM0sez{U~OP_pk>Wog4edo`v{9yOM`EY|xiMeB=e@@x6A46|fl(-(s z6JvFTd?`4lY?w2mKTS7x&3xAzAs3f*XQH31prBBK(NM79R;teHTnKbya}L^JMszElb6dRPwGnTlq=uaMPdL zUY4B)YdSonN_jx}sm6)K&qEJd@`WCCw@Avok!1O1yRBTP_oLzK`cKUtdGceH%;so* z7<+oE`DlZjF;+@2_czzI*(4FBpl%9k$Jjqas0ysd$Y ztuu)P+mRnW>Tfb@!vmH)KtI2-qeoE2v zn#=8B?ftJRuMVgSUAM>=sF8VR5hXc;!C+jI+=^v^t<;6yKBiv8A&zxmTkE06Bf!Fm zUJWQR3oekhZ|`hFUZjf>MN1;BgnMF6s)qMiw0){^nR57&7k6t8aXlCa?EfNtT~+WM zh7}^#mt!$yf%95@dyh6?@5>toOOmzAC=xt@NDybeH{w_rtgufSqn6JC%>vHr+rQ)wMZnpf zWw~(E(VMv%HjjhC!}pH3Bl&w9f7*jkXEl_*n$J3#B%thn4n(H>24Y-ZTLPupd zr_7_Kv3jsMa%lfhns9LZ-n?ASoy|I_#T@!04&p`384aF|x~atzf_t~l9+x_^Vb@oI z)(nkJ<)tfInj?jBBQEun_N@BW3m)^lC8MTjopQJn7uCw-0pLw5Ojd#^&}HKP!EZoB z@UutKduIqv&;S#+TfX?kt5<4TFU!tNm~1(g4-7xT(m0(1zt^gqsbMos&DC{ulbc!2 z?kh_p8;2TKKCS)XXrk=Wb9SYw^_a??Y}2vhZl7#pmZ(p!8>{Tue~C4>oV~0WAC^OZ z`{)-*^LOq`el9c(y@NE#y=UIx@gF7DhhMwMh2PF@Vk^}C zu<{5tov9>KCwI5B_4}!`;Yzbtl`O$Mjx}4tE!YD=2H%|gy=snBHPDGPjWbge<^5%fB z6U|ab@28a_hWXMe!>Rm|v4kvP(-1?9r8Zu3sE?FygS3Q&J=jwVm!8$#xwUsk3zC4|Y%MjP_^~UXV?`l*vqX0T#&gNY znUj%OTB0U4d~hsCR+wUkRl_I&+i%uFai+RM5GaIXZv_3Y{e_&VO^Zi%+V+4wH zcBBWT2AYY~*gTjQdt;JV?Vfu_Z!zg{Bw@sAb?R6q>^aECI99{lUKe;4Jy+;^Zb8&aLo<ftcl95z$f z^+=?pN&F(bv3Cr=90r{P3v`-mrDuyw6Po`&r3PCPJa z2&jI28 zBdr$aqtpb)!2MyzAW_=Wb3dp8QJ1BKV-o4f2Ipz(E!;p(ZR#&ei`-8P81^_^GG|`xFvTsE;=>x> z`bc)vF0SX0vqRyG)Dea54^N(Q9W&><{4|2Yb@7hUhz3z3W|58P9vpNQQPcl>AFi21 z)a?|F7CTU_h?L%3_L!sNU2E0nOOO!Z%obu*9Ex;{yL2gpa}}rvZAt3X+Dl8bV}FyG z=`|Yck(w+1Ni*b$QyOeoT<(qZo9#b0WpK7@AlU7dESt)(gs2_gzmlxk0ngMFb7 z>LLl_sxb)+n^*@Lh}LT(@e2rYC<(dt~2c^PcB!c#ZF_6%wOI?>&w-8-L?I zYM&|lc0m%$^+{l5$$g{vq9YzB`i^8O+WN45gSW)>K9FNRy#gu==gz%_BiI@{*?T-bpuA+uZtHQ!JE`BVx9nT#<5Bl0 zs`ho8<$C?6p~u&3_bX4-s2RC)HD9DnOhiIL2lT59qJeEa@%3TjX9(N@J0;-2PBJYl zC^GY?K;lf}b5vA1@$Z&Y<_?LN^hOA|l!9yi;Vd`&2BUTu{B zT+XXX->)lu&GotJMWf(ox(%wW6)2uC*GpDy=-0D)C~dPrnsFF&9ke24-TKkg?n1)g z0$rD+RRuE;7z-P%N?|%VaAE{9^C!kfGt$!=v-O@gis&TV1oO-iy~EJ;`_E!rqcGk9 z5nBVEFT0^^b%Kx#Ei*O6=-sHIv<>#1KD%N(Jet*euk3}V^ODRDHZkM(`BK~qJ*^+! z*Ua6bG=7WC(kkpa#_1E##^-x2E1=9g+Iskkbl>sJ+3&oEpSUUKY&FiX@B%A|Z4n6< znoa9*Y&0jPPaYPu*dFG%1s(N44Qs9n76B;hf>(J zYc)Ur!^Rxsn7qEGY`{BNu%))exL>W8UraEqbgN6hQk)0739ihT@-|W`Qmc_nq;PZ@ zE8##;Wp%;gP(auxvzm9WObi?9_RMOZAtYDsamAxy%P9&rgt9JH2jE?AgzX!~G6^0nZ(~J656e z`WCun)25$^BhJFgK^j&1pRYUKhq9xBxV1Z@(Wzv|Ksk~`MDSzBf{>fRcIP&l5h`ry zF`juvoaRkm`=`lKD3jb0t; zNjNaUih15<`KHNat;@tTSDse~ula1tkX%%CC&7oe@O=!Q;g;AJZqB3eNkPoq2M>}_ z&UQNN&k}Li*)5WOKt4* zGio2#F40Z9;}g~sQ^;2=!zDHOgWh-Vp~m`pku_^nk#BZkPy$^yFy#2oWq&^=2E)Mj z5w}s_hlNyU!O*P>RX`0O0cQ+;S~zn$IGVcvWody*B3dBe|7hQO)5T@i(w8kbQ$qkMD%Hv=`DGHU8dhzdk`IREv+79zaK36ii1;~v61*c!yc zR!kj6G1Lvz!@4rg2ZTI9#8{OJQ~&jQ*UuKs%L4om5oZs|-EJVTuz=nyYH&v$?Sgb2 zrGNeKHDLpr?kAZ^Mmdn}g);q1VnCQW=(OoyKeW$&VMKxe3n0(6NBiCY@!25F{jVRm zb5GST0dm|_w9z^d92PG9_oWyZ$0=2@V=i6#t**ZQpc95{|M$z{5+Kdz`_|*14xmp9 zm#W9T@9D7}HwJvXK|0K$=vn=p{X3Fhnj(5)gY$?QR*8I@awh_B=nV76t?oYz0A;@F z(ERT!GXBiXM6(S1U-)O~XreSBMbHWbZz9nx+Uk;t=&U7ChPFjD5@O(FJJHD~^%wF5 z8kdX{TzJ*0=cUIdG346Hp3&7`+0NsDiZM%_{mWI(J zS$JCrXidPIRB?(xZU?*E9YB)39LG^W(j%c#aN&l7R2c#pWXIaU1X*?64z4Uf-~Dg` zwnX^$WBbCrV~kNHv`xFQ&UXtNwmZL~9Em$tuX^*i7Wg3MhHiGd#3-?f}F- zEKAdJJ~FLX>~hO0OpwcT?$(2io;7+wXyXMSLA*A&n8Dz`%-X~wJqw>F?1MhQ2lAK9 zA8K;N7{Iikb*5`^S5+=d{`HnhXG<(9P|*mma8kNpic+OaEeL{e{t#nn0iqc|(ZPIM zh64Oc-nXm6W1yE-oKj;!;ZqI8L_COF3AphN&dz}_@`KVg8qdla&$@4DNW+E zcR&JeH#-K8<}f%CRBwY@-O|<;hg`f96Hzq5756rSGfx!+YqL^k)(30x(_|a?s7v(ij78 zLkkjI*d$ar%+2mt!);}&HZK1HMB4>>fCRQklV0F+HZ*Ke=w-G5_pe1(DS`nrc!eaG zAGvg=lO&L?{Wac@8blm%~=+Kv~3I4B*kh+9DQe4~7LjN=E`2 z_-LOdBoO}jo(Pl4Xda1}Hxpyc6D&5wy|hhZJ44Y?)g%#-Dmxb&AHD&JP-a+T?2BQpLoDI!Q77jtm5xCs z*ojim{xu(_xDv36r9+FBAor5ZO{c!8!8%CTqyY)C6XG3(7c7?=WNBRCLhyXSK~se% z!PowS6lW;BDYk@SARQda!#kD$WiCIQme0HHy)e=|jDe^^H*MhE@n)Lmv}m(~MbtML zD`*=i`$G)Us*)3i(60;XZeS#MQ9!34ic5;_$=GRns`mnxu7A`pmU9dapDy$V?O#s} zxfenXgf{fL@{{27O5#4dVX}iazAAbD_+y{@GB67g(KJYaxIj4X;{q^9D=3!645pa znMM>ojwJ+Yz4d@)VzGugQPdiyn_WQ8J%DHs|Ks49HBS-Per@N|x0fG#v4d>#gHSX{ zg8dJt#%?f>T_9seWuA;!kpM8&8pTGGBcRsOQ<(8Ez)Uj&bp<)VDvTIGc=ao>U_aWY zUCmA07LGKcYWI9|WU(T_Igb^j0oa^NXOZ-SAVXuJne zS%9KeAjD!Y1|oL@lK7$B0Q(B5z*W1qLoi264*m@L)LZDFhYEnp?i7Fw6PHc_Ld@7; zF36&0Rv~N?(H>CL98uE(V{jCJvIfvC0*{nwUVrzHFPhK%kgyJXerJUD3`G9{)r*jZ zMj5kMum$T%Qpl>Vp@B{zOw@|Fq8JVB)lF>mE{}=PNIc8nPsRv4+E99MFIln%kS-Q~ zGE9C&G;f~90z~+b1sI`8tp_YgV2v+zqarL&dpuigXNh{6&BVK7RjSh+qezPJA>lBx z$8M~^9b@k;p12PNM;J2&>c~VK=cSQyg}4H$SC-1T8+-id-Ic-fejXPOPb^TCQp~Mk z!}4(>T|N>qZ4Ba}>J1LB7d$DP2O57Tyx3F*>#Y>BDB$YSV&KE?DBaWTJTg zeit$#3Jutk^V<(`eO;4b7^R2X2qGfAGJ>OFxA;F6pAHNHt_{D%crXOL^+L{1aZCgv zg8+s=uK`Sb7^UD8s%_^D zaE+cVmcws-tA{nIM&ZQx14Q(fJ?Ljb` zpn-s6+d{l(R6VW=o~FP?tby(qy3M%+V?#w^y zNOOU$?C09FNUSnM- zJ%CRHLt+G2kja!B;o5^C8e>49IbIVZ1!RAg_k#h8+t|j}xy-?0M~2q7iDj4$QAZq^ z%4oej=7C$RE-uKzw`#P7tXPyEy(&fusBveJ5k83|BBVw82u8^`gV4zmGuj}S&9mQY zp1{jU6-`aeJP|F>H){~9(RBE(yoR+}4|`A-*JSb0RqInwG`s)xmM_fuvVH3l;NF}V zYQ#oBmxT2aOU%m;@=M=fomDIZSL70RUw$e4{^7DkyL$J~N2J^-*-7e?LL1!f@T z@1^2!eh%q9GEQ_FsvzkUb!Pp#$?}V!$S6MfrKczSPu}y`>J{)-#lr;RpIjWY2$^s# zT0=VM)U>|5u8r;rila7tG;UwH*)`ttYcYmv<24^2wWuB>a_jlzT>4k3H(-~);#fts zALOf*4qNQ=ur^rP#m~&euYb@L+;a5K)5G_0n|zfS?-sS#&dU z6G=W?KsP#>h8blCtz!=MOe`*gsDrC#NTM0I?Hv5Niq^})>uLFep(N&u5;^|xuHbv= z%T`}2HRM=qq#cPw)x@6Pm;;)F8qPgJ2wHo3Qt-}(T^Yab;AtK2l$%Tzl0I$ZKwLcy zm}~@UmJZHtX95(7se%$Tf3j@PJ!JEh4wy54P^`p^;`kJ1f^PK>|rTeLBkFkyiCrsH{uG z;$|J06B3Yl+ikn~M7LGMF5!cR-=I7va64>$`-cxvD8IKMI8fH016##~KD_T1%zo~) zX7D`Urla!H&uwdV8`>Mj6cvv9@g8#nkS=_%u=_{(RzZ#Hmn3b}$Vh{!rE>6EQ5N#k z3Veqvyc^G^#AFo52gg-4A3X8v{?z0oR!Jv1K(+9U$TBP99K)cU5ws&Dkbr&*iOKHw z@7Lj2p&@u?>-Bqs5)pzCLOB6W_V%po34bx6tfj5~MsK(J2<5yu<*@qUQ>ODA9L-i& zKNj+Y-%h{EF%0=B0^}0(Gn!6)#@+-%T7Z;21#d=Sf+eUf@XYY=L?(w0mN?~tl+Jx~ zPsU;Er{@Z)bjbo3Hr`u3#*TMu@_o zuNyHTwk7Ou7LgNz&jLhrl|dQ~jyb_Hp5R%y&?;5774vT6IMpCAtVXyJU_k{|h4hZ= zo%PoeW&N6#&!3OTx{GrNN6YLD>;& z3#4I_T0#0uRXHVPum`|ELxPDY`=U|lKFhY&C#5qM7Mby6LEI4t5$(>qtIS{g85>h2 z@N1Jibb;ZLg{(SDAxdcUhg3SD?QI_sCxkYa;nuzkOhM8hbnel$s1uR!pxlU~fmMM5 zj7zP89fN8z)@3|%6%y286KwYzYim`N3r-EcEQaZt5}vkOUwfHdU`RxNizJkgAvO~+ zBQn=N4Ykx3JtHX#sj1>nA+c<GOi$NYhJ^av3_EE!IFmvwSrof@ohd6Gy{#w=#9plgkY5Um1P6i)|Kt2{51vRn- z-gt}$X>0q4SR}By7F+36F-lY=`#cR7hxtP5)}=8=1?T(5g8RJ#fy1APt*DH{h85f` z7>5L>?IT#`D@zJs-HGSr$F04a5GOq!H8%3MHGmvh6<8Ji?}A`7oZ^*@mw54`ZUQUX zK6{;N3M!i$zO}da__v*mP((<7+2Eoqo~EW8i#XWOUXtEfg3K4(@;8g^@|Lr}E2+fJ z+rZ>~Sg%y|>*7wgtNN`_&_QdSRe>B6h+f3b(bqSmG!WR>F$mfvMMVPYuE1Vi@tmgl zk+@;ZO&X}VC{g=1iuyE8_!wCA-+ICEp4WqH$~t_L{R*{OrOT9EvbmQoO`+D(yVM)k zaLpx;9&KYwpb~rswRCv_!f*cO+(c|O^cK@yPNCFtX${n87BQQM0^l_W!_gfX)hVdz z)(Rh7T!d;w{FOauORj290}zP(_iK&$kMF&0AAy^8 zpm^#Xfs@C2^@?)Tm|S6pk!ZJlgx@m3*6OaX90!BOLKd57bd!-_oFYz=SH&1Rjf2?t z!$4i3MJc(V=hOv;hin-ImA# zJmH2}@oXlD?zrtQt;NP_;f81_>kb|jFtPL#ZuCU3&xN+Y7~z~wjM7|I1ts9fqgI17 zT#i^cJ_7HjhocgUU3wYlafkaf4gxV~)tAxh;8(=DFD5R4RDO}pakLzSv>niNr z32W-ThIjUz$qlX9@+FV!(2J)+R1@I2R)Ba;5eW&Y8qg_i0FDFjz985g-YnOc)Py#5 z8_*5?2tIo@YU5OdpKgRQ5s(r_hQ6fX_g6ESJE=8`KMJty@&|%|w+6Hwj=cz#2}}}_ zKrp4AK2E_3sKk<>U#ESUf!l%5?~DLC2D~5%hGfV|4`!X90ta2*E*vTKxsLG&aq-A- z5^)bX(N93}i@a6K@$sAZNcYgLn49st0fNqgKM@#{RahSV7{Z+N|ElAu1O<_)647g3 z_zMgVhR{R{B?dRFnNUqpdzR<|I;qB+kspa*I0B^J1UzS&7tcmXU0i3jz6MU_EdVr7 zICKCKNHWYyGI8yZg!Q^6b^#UK2ruzC47!00*B7`w*vwiZfJ7+{vWDj%gyL`XVeg8q zln62>(jFC}6(U-ql32D_+Q~9Wu3?Vc4^2ewKwn>9nlgZ+2e6q5jamdoML_+lNI6sR z9oS|C4OG$%IBS(+K4IjkJ}=p^er*J6ee`iK)iIU;SeONO0s0z2bngH9eg{jexdCUk zk?1o1mFWJg&JN)JwTAc)lRCfx#`Hk?pQF^KI_XX`^4OnZj&?ffXHGvy)jq@)9mp_z zYm4DQ4W5z?P~3_oj-Mon+y;BIZd}tMsUyZ=vS?9+ikV-$qG*0!z?vmivWwtF6vPD1 z5kUB;Y#aNE;^iZ!d3B(SXsvwG-sqpS)iP$m@dQ`=cR($9GB5L=Q?b6~$hZ`a|RAvdei;B}En!vZDf z8bB7W32<)!`?wo%DHQsJv+1Nrq*`)7QfuX4M08XXBqwXJ@K%=>V7GaHo5?6!bXmX} zOQB3ld(TemBu834$Y*lQBz`@MYr_kC6bX$aXj%f&pPskrJCM4vnLMjBvEYy-Vc z?5vEeEM3(3RhTm{Yr%SXI#qLWhO|vrXPKOMKaI*r2$gEF-JKDdK>0C6(u(;Agt=qr zdMeht(N;0&?E&Y`4{P)xX<;8v7iozk|z3?`jI_c5J!->Ce>@cEv`Ty1QU8dND1_J|ok z$p!F4G#aX*utK00)peAkvme{C8lj9Zww}7XyKuVr!O@hHd(};HbViY$gfJDF=BGu^ z2PXz-6oLqF8blPt0w&xL=Q^OBx5m4BVuMXGTx8@G#Y0pDwH0UVpM|^1%7(#C$cr{` zgk(xo>#Mxg=D8}$)wkCg_+GinKk#hikkiyXi-EK4M-bR!`%3gq6SmVeZvTYw#uDpp z(cYyVhW`t5y6gDKa=-2f>7f2&)#)lkoxa}T`Z_e;Up1rc_HpSFX18d1Tc;pb%E^OI zn-#GDPq!c?*E{G4u#?$8RS6FowKC#))*xz2DN0IgHj zt)eyb*G|BnB21B1bT#EafIce`mHN}e-_V}|v_5a?7EHZZy4m3=`#n=6J2c4Cb64K^ ze?XS_kZG_NDJk!R9Feyo2rxYjE*~gQ|Isi2v}TI*0JVnJi>YZ3lovK>>rd zb7oE7;XOiRx!11f7vL<8=?r~7)g&aed~QsWoYAQjJ{b=~8>UU9*yy`Y3_bO5|I8m~ z{KCR%YGQFH(UaSFjlWEcZ)wfFR$XKCVas@?!MAn|JtN1O(N5x_h*y8JgauZO$m`tF15Ups&R-T$J~!k5z3)Wmji?Pn zhep~yxh-6}JxMhEZ_>gU(Pku^aH&LFWZ^id349X#^%B90-r z!t4%SyEj8;69=`2(GFNA+*6&&EU^$5s9Sd9683%SruMP1F*tiHk1k&YF}#H7#Gd{0 z=LVK9kMTbDNDLgCtqm`^9QBOsgdNjPDjCE??U-_ak0-&{cVZ;xy`0=}@rojbIWE9D zcgciuf%ZW#dIHzv{O6tuWEp&MdVKsx;+-}PPf3j#w}-T$G7{~5@I!d(m&0_Yo!Bh! zd;V9I!5Y!6-m?XQ?Y_y}!5u})(enPipXDe{N*qifmJdLnfIMOikNR1&WGgC;P<>~2 zWwAeiABrMo<^%VG_Cov;`1FB!7kA+6c==kmBPpyONt99>O5^mle^7~a(Fupe8 z*-_UKt%iBFXt_DDKb|{#7KgQY;?WNrbLP~-K4cAxG8h75lOY-Z$JcY~CCRhXzlYu0 zRhacXY)@cO*YbA2Qt19*q~Nxs;wHp()J&zwZAX?l1_|l>_ zJ7Qxn)3&)_YqfcfZ3FM2*@rK^8@z*)g)}bJssD-q7Od&#T20sAHHgv!)(s{WWfD5R z;Tvs3kx3v!u)(K?jJFy|9g~Gl75j|mKi&@vDd~)?s+|N~!fvo1Q~<<_-6Kfwv?Pw; zPxYt+AZta|HwVKyi{S?qgTxA44a1&Azg|b2cN$vfWsChwEb2IB@g@(SpK9=_@)fRD z(RZ0Bs3SniJ2Acs%~a2MJs!9vFp4rnGJ(wd4b`+uHambma2GF)V_V}VQ+E|wDZGWh z=*#qOOseK%Lb?nn(__B?r9#lZQ;_%UyV;XeSL6c_M{V^H`VP)^k!BIr<6uP0+or; z8@dUa+c3U57XAj#(5gCW>D_*taTrq^h&J*QA@qqbgi{-ezJo$h3yEl!HJevl2!mUEpmQy#O+DbW>UvW;SG>gF zH(|qp(Th&GO*rzha6}T11ywW~a~1Hc(ZTU+xkq^;8P%g6Yi(^60LL}=-n|5*T-4Fc zvv?KYlA^oeq{}9OktBl*%d;pIPbCb^7|^bZnx-RRcPz`QNomG-)}wMju0jYJ~j-N=Oq9V=S7m%D`^qq7$rqdfAn=v8b-Pg z2i`y4nQ$z{$Z*UBgqL!ZXV<#yk*@h8|7Mmvd9s7xSR^Qvi$nQMtt7)tvn&z)l%TkK zr2(tg9X~#FqWy0B^MzlG(Sp05X_p-BIY(Ikvy6yNOv3EX6WUAb^P-w3-rn*KOn%yF zhb_?eWGFz$&SchN)`x$PL;l?Stj1ICK-NxC#vWgI)X_Z$2%d-_jo#Ab0-$&n#c1Jn zAeQI>NY(|yu+T@WXykRITpD#^k=kgbrP`IY|7njlNUU@!hkt?AyODLV318Y~bvyNc zT`p-|Z`@S4#vfFUpf!fLv4hq6{<5`s3NJqNo8}!Yn7#CE8lSr%|9t&go#KBk?Q8Qu z8C>6u-aihA1MVKcU$}inVel5X;n0sRJ%IB+>$-UY${lS3-Sz1MbFysSJu~YSw3v`H zS~AU+~9;x;iDJULy9w!pZ#JjFy!M7tv~2A zwM;_xooXp?d8AqZxbFi;3(E$V_CK_%iSaKJAAD>6<)E2>(4h^xVrU6ZyqrPfeV`}&IE*3S1a z+Co=vj1G?uTUE42ChmS`C=wwT?A`hzF>c`S;hHOU+{>0lbo^Lco1)vm>!T~ywvX;IFD}>lMGbo2>9J;U2o1rm3=Z##5Uen)po5n~d_C9dmj9Yoo0}#gopkV`NT#d$u`JDlLBUXku%beWDmXtKNM9?EF=U}QMqXXgm<@WsR9^vM^o)NJoFpyZJUrE8A)W|o2<7|C zL?yo!eSLj|iS-kW0{|FOcYo9&LHlHA>mkrG(U_vnKU#Zj*+?iYK|*Gmu^j+=r%>oX zB}+8{RVLIR#pn&91yEK9|IR>ReG|B|w4HQlFY!+*>v+(qE$RNUxb*t!9BPp<@ z?k>i1DDqyliMexh_L7>WG0)7C-Rfg~m8uKJ>%L?i%o54Sxm$ZdB{)#%_}ajJk%JM* zX1U^tBe%s3fdMj0I=$KjC3%1JM;A;SyZ?Cmj=rRfCJyb!8>uiGw?g&+v9}NMs#~+UVy$c!9REkM92f?C(43oX=psi0e zFLq(od6sG7k+I;4AH%Al>(xzFucv;_Z}uy<)Bykpyn_dA^JU>%I@){|chyJdDEcmp z|FqdbiG9R)sQPr%yR5Q9If5SJ*T?M|r+zynbaiR<`E)U!(r0dBEVq$4VAAPeB$MEk zF*%<_qYCWa#(Tp#c^-R`Og$o^Ya8-busAWNFeN4BNONv-?fw3mm2z@p&6kEG9Xj?o zVz5u;q3Xn49-_m}Std!NQPcZ7*Rl)?_GRbI7u7AB#FucOQn7D*zsuR$cXv+S5L6wO zet*;E6PHg8+T|nYxcKJ@sn8(G*O9)7F#~2j_e=WK-*yGsCqGFsxNo3cxFjw2_uvHO ztS&MZNYnDjKmRdKBpj z(whzSP!%|IL$k#a_XV3$Tqm$yyq5_&223hPCe)j7WA z^kVH$;oRnXte_@!NU)g7zoS!}<^oQ}v$Z*=&TZUnbRm3*WoqP>PzSpoPohqD)!D4s zO8$ut>fI-kALLnqnmX@rh^9*^tFNEcS%Mh_A>`3()6Q+C#VHHZIyh)45tF+_6T9zZ z(bCRdbie6tn9AUGIc#>9J47T%MFZoI-?c!vY%Vg}tgKS;?$9;$lvdNx)SPfGz>}<+oNFaH$CDKz)bi z$L72HA9Zq335Y^5#E}oWja=P=c(S~9-T`~pg`hfmR8IbsY+4JSjw>x*L;IN5o`IL? z`?|+OvrT;|HhoR|o>>&;6$P5ew&YVn0Jc8;rHrf*6)a~G&V=mz3+~Nu5#WP2^zPymcB=KLr=IXPBFdPUf-e^{&_X&#f7x_ zwU?D^-@L>jOq`|BdI-*ubk4X$Kx{oEq!I?afdp%#T5Qg`lz5(*Q|&B{gVpB=o5f;w zk<2XJET)633GC_9xGx$Q9ph+sB%*OjRAr?6uS<~IFXK=BGT+D-yPv8fH-8)lf!~F2 znQc4}IuAiN$D3D*cUWT)@YGH_V^`DWrr5KM+-|vhP7jX47RPyXnKzN0EAG zI&Z#RShaCKRnWXe<=KxjJ_T^a!tggjKq4ClXruCzGxs}JMW4+cY~=5}evxHy@q(#V zj*5=UluHPs4iKaHRBv>S)3}G=WTt|I;@P*lasl(c6vNpEw`U#uDg34*9vV3=asv%1 z^@2KbP$1nY`n9^@5+;0q|LbZ`a7o4$vg->co4!KU(J?nJZORBEhe1Fcxig`cPoMt$K{w~-98Y9*9Eg(xnGA^n zu9>zLsM8;CsB)vRB8wJ*fAy7_cGKG>TRsl6QbjE0uVh%=`0Foe`n7AjgBiJbx%=NY zRMzna+J|fw8}1o4muSsjI-7CWeSWi&$|TMeqv6k zvnO;gD2&1>Q#vRI67`Gcla39qrCx zr?Mn=G;wHW-&!U+_)Y1;zzMKsS3XC)GbX{Bd#TdYkuP7~f|qp!y|hpX7Y%T- z>Va>PitrxBJcCt&j<4)Qfgw8A>~3z7pGE#qj@}fX=leE5&FiP1y6$sn7D)UFo~fw+ zrW6IrgB5ML6as%#`UI>G&M|!cAa%3Z<8%dUEl@HhP!VVx*-9noG#WBI;3kH<&Qj57 z$ty|yotkZH+uafO`nBHNX(=hg&YQy=Zh!tbd*S6l>bsJ$p3dRn*ai`6Ue?Vl_fGL& zsXf_hb-AzEU4~sJ`!TJ3SuVlE*mg(%T;beMzDDNddvN_uAbJ3>fh_Ip8aPr5BA{RF z^;JDjRoe7+ECt{!w0psXu>=CGEE%{5K(X>OK}N~RgoBEZACWQC`w%q%l?)gJ?{HJ| z;up|aEoNif!HgyojvpOgL4aw}+X22dj-~O6S}ScbASuJ9ot?B{K3AcqDvW9o;deUQ zkgs6Z0H2QL?E24V1!`F&p#SIi*N*@VUcr+EJ=;0C` z#snH-%sl6{fWll(xTJ~QLmFtLDOELtdO|Kl=p0=cn3Rwys!D&#=ds4n1mkGshAo99{ z7#`X+FG$QJw&G*WpaoEjU&Qbsw!YW^_(*2pA&-kzu!l z8iJtK>!ToX4s=oI&P9oi6o10M`t@FMHi5(Ce^>ILn>!awB1e7MH>ZOpcLUt?;=$O7 z1EC6{p8(75ig;pUgo&;I`)knwg=0@Z!h7@-V?lZ`LZ5yVgH+IR+`__=fL=E{+Rvy3 zXE8^+<8B|8Cusu>$0*eO)z)y(uC)Nr0n6yKL;ObXIB5+Ex1iJA%B2&XBl5T=k3aFT z>t`e8DfUMujhWzW9aZJgQAE*z8ghq$Rx~?}-?2i-WwPw_to_TIvzPnzm=CI*#h>vj zE)GB7Id|rTva3t`^PE4crWPPE-*&}y>+&+>PP$PPztXPk|M91DDRbj^g)Uc4P>DQ?+Gpv8@DfAh;y}ZE5E`bPeCe-CnuQLFe>_U2kpLj76;aqwWd_u&Xqq)hT#(*!3snJlRG{53B;; z#$6mv1die8HvRhNx$)i_ReW-ro2d3SMz|8{wK}&iPW`QOyQp4@J9(DveD&STX-VuY zo3g_63=Cq?`%Hi^AqZ}@WV7QP5TgLlg`zM*mj+nc3geilO3T0xs0U zPz9m~4?%oLyRi-imX=L|nqr9bF21Q>{;N>_RB}73bo5KpCGRa0R?w0c9~(|+duXS` z?{-HeH9;y-wIg&&eJqm_f92D0i>hs?TTrO51&N;a9Ax3^KXGsS#6^z|wof$K@@*CJ z7E7@}`l`2x%PjdeTkWOV{djz{m|JgW?#~Q=ZY34nS8n(F)xO@2_M8{xL#k$~M1OrE zLuq-(ynq~>;4Mhtj73mxtU7^^`=dG1qD?LTl(&; z^W3!TB2U-6pM9Ias~xe``~HT-T!lUB`dK%6nXqdVJc)>mu=zBvvu6EzUtkxP-NdO5 zW1UsKhn8CVCgS5Hj>>sLKvqvkl94Zd zd{_6}!HPodC<=%w`swCNLn}Kw)!X#w2ZE^vr*14m3Yacjo;aBd>N2`er%`LXgED9f zBw?Vb1b#rrlT0cB7Xb>1#->kVd}zpyBd{6)qa$fGQtc@82K24fj^^dtf4WAAzt|Bo z(sbS-D;fVcb-cOktsbp^sp77pUw4uSxCk$hWZF=_Qp&!BudqSxj&vL!BMc6!D#-g8 zm(MV6fQ(O2KE!Pr%9{yvY~p}wZQr%k`}gA~j#H?fp7Pswu3q+VjLyWBK(CANrKNkypn8Nabq!Q>Q z5?LrwumBn$gdWOp&tPCp(0qw~Vbh>KjZI0*-7+_^K=oo_%ZSbA=4^hOJ@PEFiXJ8AsA@=xc%IB~65fkjrl>Bs^&vv;bfeprx&4gz13tyCwZh3Km^mgT`kJ zjFfZyJbD<>mu4&z3>TV$_&b8_879c>$dVr@YM@ZF*nJPD*3qRPh4^Rn%ZH>p*$&gO z70w&i;gwP)7pAYX)FnqpIAkVjjU4Z9OAK~s1s(FnmHr&bIF%Qwo#)aXhm3AfozYm& zfBCuIij@-b*3IFoHY9cpn6qg<;w5)#6I77Eoam}BX|*h({HFcuT|c*H6-LZiI0a&$ z#37Wddi=ZI5f-8_!KkfQ17t=s@wY*(lBpuZxPkReScczZQy{_-Nyqys@O8c#>F5-5 z1Nb6ZA8b(YGu{$OHQ0^B2?norl}q zBm*M_CS zgFaD3xk%8=XsUIgsnD*tC~!RRukqmp<*5sDuq`HRJ(&DHMCw`&CwtJX-vTBlEK9;P z8+ql7*>NybNu>~?x-w7}!b~Lq%AR$5e0#J0k1hr8e|0GW!W-FXi5zZ=KXy+fXk`U~ z_D`Yaj&)K;#5W9mkSXTYjEE@lNCcXkXTKPtvXG~RTA;d2JQy82PoagSBA3&ZIs*|t zv5YrBncUYhd*8{-tkNk+ve+DJ*wwVYsmKi8G^e+q9R4D3)qIY zdJd_Am&Af|#2Z>1JVep0JMz%Z&SiGwAirA|jhS1KQWvf+ZQa+Tt(|APoJ_+|&}i56 aLkDkhsSf*ggoYB$Yp{AZpO&`f6*(E`< zYZSw(YdVyL{PU7kOZ)2>Wserz*quB>H1ZvSjf(!qCv9pfIk_v08>K^Zts z3+7g!=cnYgDB>`i)RblR$eh(n71-x~DzaJS;b(h{Ci3K1#y?%*nh4Y1Z8uk?J+X88 zScP4b4v$>mm$Krv`+a;zEGv5AJ(dFmiHCmW6}h%$1XUk;>@$TB?~=6vOJ`#BfKaRg zk{*$|Ai%N;-9+C9XvpTmQzq%Z%4BG`q@%y_wZuwLre>$&><6L^tSO0E-cJzbyMzJ= zq=`razUn5(W5bpuPV~g5i8M<{w-Fcd2eF4}!o9t{?K|KaOlS_|Kcl#jhcLjK0=9>Q z{1V|c0%IVCOhpicS!HnnF2$BefGp(pt68J`hK~J)RMML_>@3yYvVD7eT`{+ewX@3E z6j`d=+-QwZc=?jVuSqJE!3TMH6_tuZnPZFHjy@!vj=I$Ar#;oh(0Q=Df?25D6fUac z=d6a>7`Yvg7pg0G$jyU)8!k+hp_6K?WiY(6d+7d0o2^S?I{rM`%0As({4)c?6;utn zGTI>o${su% zj%ZRmA!nQ**POK?17}(_diKm79wa=jM664DWTz!{> zv(R}bWEj{+$@V=uXe3iTRA)E^)D6CTe6~6TF^p=iZ9g}x88lOQWMZI^_F2a?UZ8SE zyxiESpoIlHIy(Cb6k&OQ5X$;!nkVImhs-lMt6 zsjAvw-?Q0qD%f-zRyFe{kM*zFG5RDQGMG{ibnI9 zezU!+(X3DnZfLz_SEO+-h#n=Qno=TMY-uV>zvRUo+HShUM&;(NGMXKg@$mGlPZ6K1 z>=B$#*H)|uVx8XA7wk-xT9%ysk1n|RgekRh-LDUUrw22(!G43739?>Uz@WxFq_3yk zw|muPOK?If1Y9x-a5X!|f>S5a33!M>G0LDPQRc)Mt&pn`lTM&N(E0_E0v~4_>w992 zM~?zpX9$RV^`tPALKNZ-G14Uc;Nbm_0q<>`=cg?~#=w^Li4tOd687M|19jH4L~;Et z-m||ydv<%$Vf@R*2vhZAjqmoOOOqm7G4sK02rPW`aLoF)t9V3p!_V{6HKbFd5~c6o zT-2keziH7kT-*vS&~-0)j*JBxuAa=bhSNpgux5_pR_8Ta*i<;5jpbUFw(6V>8Dmvg z+E1OeSlr8{lO^yhy8X##vKx@*z`Wn+YTbbV|#$-FLy!S=PD zRPDlqsg>h2VA!)8b|y`R=EYq}Z@09u@~OV^fKj!}s4XUX@7{g%La*_9oXZO?bMgga z62Yoy9I)DM zr8QQ0?X2N%+rifFG`xu8aCKngsqxNU0`iry1OSAja$YG$ZW(^p`^p}>OOY@#Qfw)2;f5T2%{5yTltv=DYc_K0( z`%J5I1@lXn2yH{K>m4fW3}2QCG4bC0{nPUU>Xpy}Rg{wKHjM!ZDWGHsv^)s ztU-pt^+Ccyj2UoT8jieA3R)=$ejic(ll~J9S75>HpmIVpN3;Tj*#Yqp9GOUJX#Cfx zY#E+8k>cPv_9A8(L5Lc1GKAPPp$Vml&#CKx%Yi$^@YUVztFzr;m9#p{&f0^G&yMF$ z4^3CE-G+TUl9~sf*OrCtyGda?^t5g7SY=2@=*OAi%S#lhQR|b7A!F+Y8mNa>bLKYG zDXnVPeo@YG;22`%v?JIw|M+IyTs`Ruv%HTSlwLPptXgAu@ql6_Z&iES$ zB)4AqAWc6q5E~BmOGvu~SJ06Ml*;L9QFvt-EGx;buG2rHvB)f%^~}4od=s)%Vwb!C z`vyyEYuQ5v`(d2K!gu*D={&uCdsf_ens}2CClrU$rzBim-zbrGMphxiu`mPV`vKI@ z5OjnQgDYbBi33v!LV9A;OpavI4YU1~5kjClEI3khBVCGx^;++z7XisX&Y=Q_^ntA(0+!-t>;4qZt4}6wKlu1$n^A)= z#eor+yt&jfh|yMHq1_^yqx#2N-8(P+HQ+hsLqU7ux>UEQ%Vfb{F5`wF41LtJy;Juz zNv8YgdNMj=@Ul8X$5)DiiajT&jzkuSVj7z?Y^Q1<@j63yo&L0nC>*=TkS93ixb0an zJ-1&d#h4+%K174F*ylRYmbQ15J4V7AB?V01&Rw5XKhj!j)R5=iUzZYn$3Hc$Ylrzj z>X-$`O_Vl9b1CBe?~KX$SkW^#HFN-VVW^mk%AUJBH&2OrUOr$19u2Wb3Q11~dL;^Q11cSo5)vCAfhA}SJqm=m19_2Z8qwvi=okvw{W6J2 z?4dn6#A*hbRn+PKFhDBjAtezJAwYLhhbd!7`H6i2azRx@jS0Zw2_WFHqpAYWAsOx2 zG8nK!WLJ;(&b8?@WWhvb1b*oitN_wp0V|TY>yUCDpt=6F4vlQH$RM7AgXmUg+2@A! zHzY;8nmB(pSz6zwdz{T>zR#Y#sussdKfsc%>b=Q8`4W*f%Z#B&eqY0T2JwXyt1GzvQ z{S_nYcRjW*Nf6RB=pL&`%x#mqH{)3`Kc)U!HN)dCxB0-Ld28#Ht<1|}+R~w4IhST{ zmgz~4PCSM-=yJ+Kc+4%@K(rRXSN}?k57NV zSy-;|Nh}k7N0I37oIt|_XXwpZSj?afjwd265X*{Y`#I2L(}ZhU9fZh)Q(hN$?gV)D zHGrbHww?wGgQA)nmO5JZ@81^z(+Cz!gq#N>gCoxl6CqZHW%BHLmc0z;8S9BR`YpwF zs-IhXA6AM^CYa24Y`w$7k2*Il+u_a&ROOFJU>`@Ly8$O?}>!#hIWm&EW zEw)zox_(VL)s`z+TSRAgqhsl{`=vb}KR(3zBC;sFL^MhY;=X+OgNSYj?Gxo2r)wp~ z=#-=QCSV~B$chhEDPR)upSM3;ZBc@+T4GR4YHg!h$755&Ej8);ZrUsjY12|Pln*_vv_gH6cJ%dTj^(D7DH-%B`se{HCGFMV;@8$6TIUw^ay2r(7H=pFhI&~}Jl zvtd{wo<6xBN*|V`-+vet7cx#tYy1c|6#P0CDO*gO8@#Wr;T(eyp|LN-nmOOR+fA|w z$Sw>T&%pH#EZ8@<-tzj2{_uzhXE4FzP4E9GGPrb3oxb1)jnA8AW7od=6DwvU>E$ig zM@5cIw*8g9xP5Dl8}qJQZiq&*5Qp@MkhoZ#^Pt!PO# zHe*Y2faD8v4~YJ&b(3+J8EB5qeGuqOcsJyniX>rvRTr|Q4lvky-m!eqvG5TV_tU49xBss z*TvFl7gY{Hv}LuQOJ=_zn(b6D!u+==*}tJD1;zQY96NSQrSY$2`UfVarxg znBEr~C1q{zLLQ{LQ6@=Tpt1bQkpwi2{Z>!z>C5~@kCz-tIBGdiy8M@u=ND(*U%8et zuHhI6@Og}_leh&XI6>V(1n0tGCj+&w!0_TWi0g@4H3VR_uqHY{Mbj`G(4%Sf6E2l= z#%KeP_D_fv%6n)cRU>o(yF%*o*lEfPNk51iZHk|k-!YT@D-75)LRCb+lyII<3II6C zK3+mo@(7;7*c>p-e$2z7lh_ZPA{(+l((aQW6i&f5XWfzaTo6uydI^vj9)dRdWCDEh)f~g1)Ta0mQ8;H70WNh z^xF$Hu5-qSio{X^m|ikK;WPA=8+JnQ#)=guLqFm;eucOr0pos4-bY7=Zt@?zzOXvY zwtui?>sLA48{HEpiM%uW+4fz3s(%;6LA>*&7FAH{6Y)Eaw~VZXsdr$k zqwC|;o%0hWPP*Y~#LF&@q~bf%RtkGoBFLsrtonBSJ1N5dT$#!6B(A(Ynco5)c7)dg zvc?!GJ0L#kLxap>Rssxp6e_4Ia#a3pGVFa3&l9{ZHpKgW6lcj1V9INQfB=uz{qk*I zFgBzs=UE#XyDwv9Vy&{RKIjS12ja>Ovzf{s zl4RlR9Y6{27U`um*07Z(uo+@enfNjTvxSH?8*2Ca6rzMd9trclTD(=VY!EZ$al7en zCYHS#rmGcI7asoX$B(ze0S`Zv66XuX5ZqKCqa&hN(i8;AgRtFzh3NSTfzS~N)+rmf zJA|nSYp-ZTRD?Lf`yq;H{Hp8Ir;EM=|9Wg!vGeJQFnuFN9xBE1uq#B35reHjAp17h zj)oBP6ynngHrQjw^k6GOJlTc6h`FXz7^ z>Tj8RzpmTr_DM%2F^;&Wp=E_BE$RcC%KjF%z{&$9S4G;tmxWkclFHXdfigCye!Prq zZX|IODvk|*m?lD$PuzUUu#{1a)xnUK9A)}ycbC7rqnD523zID+!&4XqQX0I6o@w}> zA^uOWs^z8r;m;n#=HlR-vJ>j4q=}KUKJkQ@_mKP z#cq;v>5@FsPfi;^9&FiGBuj!&VDI({=+V==74GmWuwqzOnWeGWvkBKGx7f7Sz8k~S znBEn>!8mM#+Hfqr3$kXr870To{~ztL+WHk~NpH}7L~AVpbUb4y2#L7Fb?&oZ3#{^5 z*j0hslnRGojw-a=Ja|V`Md0IEL>!r-rA{ZX6bCOaulA*XEvGNe=^9d8;H->|K%-s< zw;8=sXv6R9m%(8i=!IYv52M?X0bO)CabBZ^MSmQ(Pg)h`hPPm@)R$du>`5~< zm}B4|LMOPPcI9I92$&gi<`(cZIs4KZC(PmUnIyUF}E9>h6`Oz**7_8FYg zv;0;4&*gvC{``GWU2TGELd%;KBcG1a$`F|YiU*Sj7jOY|CAcB9bG)$}?qbpt!I7+5 z|KW)QGnrgMB`ODOZRS*Q5JO29ZA&KY67k666I=daOpMzGBIgp}k%*+EkP@#DE-?$S z1RTBChygVX0g2dzQi-*m-VuqKJ^#uByoIhapFfn@RDUzj9XpJydi1(TKMaN#-lza! zGo&}fhXx?rHpPF6Ttc1v4{AQ?Jo58Jn>R&g3gh_<` z-yusKfG{rcz+n$op^4{ExYMht4XN z2O&ix-lb^1y%OFAUp{EGafLHd6P&$wdH2;D7@q z+jMFnX5StA8FBv|MrpzMk3t0}4^B(Im~o3<*0N{T_j=3rgo8^Q9c&Hr178QPnbUj zZiZ)0YOEZR)Z+gBIu3{{&ViX!`=}c2z2>GPJRs3LqdXHxiL1MBXlYqA(H6j zEKU_K5t29L;N-_}=8}Fh&LEJ;RI*f-O=h@iy~)-nEUMQ{;a3yN^>ZHoQM7)^EVWAO z+yl*vjm*ZsU$5q_QUgr;dWR@%rp7mS=q?tRuG#3#CU*3b7x(J}cKVIXasC_Zei9Rf zEGnj3j-|h;E|E^bk&Y~~@Aan>B9_zMev27r1_aG%$RYRy&g%V#9WT3DWAiXPXy_z& z66%4yK_uGvP@x`0ec1quhT0+*Cma#RXg4n35wDNnXgdf(517T5tq)#hmMZ@kayom( zj@LzlA1srabv%7%%Zf}?8#@o~sM+^c^_K>_mpiskeBnU{wEBh2q<$M+x62iOP+?L( z?{)8a&X~5s4v*3Nkn@5+>p5VD8MO`3wDTQlVKbwKEw3cUpGuzD_|vtl3)P!&!t}jQ zkL<5o_OI|~zh>iI zK7rY4nD9MoJz*;qT-NLTxoyO=QC~GFcirR=J?dcb9hIWZ9~(!;uFZfz5k)Z^d8bns zXR(CyOT8Uyji!*Oj=puNzrPGClPE^~VPT;m;_#KeA1XJ)faRtz55De<#bh)Oh%P zpByvSR;6^a>gOx^Sa$HP+LV=4ZgiW_$j~+R2e1(S0g@in^h9F3{hkHSXUv>WKKp$y zez}_uj>rr%%q#)l^@ZBC-lw1_J2PA~XSVY6e}3D2{qtt?hz@}TC9d)T2ZZn^YR*fyNLq@M#o zKMQ*Iv0RV~@OH@z5Q|wDs+FVOL7d52R)yGmZUH|5tSA*|H7R65LJeBtd_wF@3J4iB z&L)3UY>%xxpt^!eXKU8^MOBbPq++1C`)64V}!RpI`6&ooU!Rnz$L82fo z--gY`j$rY3kLvrQv)_<)g#3&S^4yPq+(2(n<549DW`#P~}JdVQ-L6?UaYYV#nc<_ZRF@Oy{qgv<> zKDQN1R*!&jS53ynMM$kBtUNMz!p4S%dYrh_m&0zbY=B2QPZ=?sP_f`d8jB2y$b}UtlT9ARWjoHOv_~ zc;LhQ1j1nTqW>0G+xy487>d0k>T8w;wn<4A?FX!&r;~7foxtRFi?v7n#xQ@aqI70# z8y@@Ok4xDLE(0nN3rtaA?l~((nV6z-s3(spn~rL~yWIQs5%FRpjWP6|&(N`6$+iUH z_*Gv%Qu&`bO#e9z6<*+M@q!Ood)6K(u`o?70Xyz@D#FZtWJv@IAw4q?4v4M>brG|< z2|7H4tVSi<6FN^!gV{$^T*$d$4wggg&(Vz4>}n-vH`qu0XhWjP@yDMgtT&ts#(=Pi zfj0g%)xHBspcgDGFAa| zuWQjP2Rh6_rsp&ppk+roS~G?C-neIS|DaR$qQX84Afpyi#&Q>-*R}Xhd!2L6+ zs*o)%yppl%`k$NB>h+LL;lb3PKZZNu@nuswg(-9q7$xu!tNT?S$$JP}2-Hj1qhu^g z$DUeY__`eZ(qgY#l3KcusoT;*4)IGvYoixAZUZVoLia^Qhw9}tR8LE+C_>;0s{vw) zhoBwQ`b7DFAVUgdlpJRvq6Y~h0ss>sR&-xmS^ovy_I?}~goR8fCd0$2kj;=`6(EJE zz~#&u*}i=T0E7rg^?Csu5$OjI2Ij3>WAKkebX5`dREa%343-T-0FB|&tjKENvEC2l zj+obYkwoMeXv=tr|#4SX2-He$9Zr_-GBD`&4@jr zO4Bo49*b_1L!z>Y{mnbR42M3l+w#M+l=PRaXKF9(ufG!UN_Um08!8{ zf8d6As=_ES*$=)Xowr%T4tDAVA|vd9{Y9ni2UAr$^W?1hH$QC@^Tslf z1OjO*4@OBd*3U$1{$NJ=<-kPZR)nJv@|hx&$JM7Cfc#j&>9hOlPaEQPjMXYB?583K z_$tW@N4L%aH?YnIKh~spVRB+f65lXBIk^%9!7x-3Pg*lk7xp-{&9(lx(AqniJ{>GE zT2`@hOqw1=%^8j^d)F5B`N{UkFzL@vmZEA4P6W9aPd!K|nyYUa&xG=2*ZE82PAhAo#%42jsfG!2@s$>YQR0Uln-%U&=6R416qQ$# z7w)w*c>O^vBe(bG>E?`b+r&9BSm6tM?%*FSHz>Cq5Gb{HdX3|8WW6c@rM3&0s2Ld< zWpw#vO6val_ev`A6lOk$(IcX;ySIT(_uTLzT=2Zn)1Z`bYCr7Od!Kd@lLutb1#7X> zF)Nzs7F(Jhn*lUq@Zpi(%Ui#U@h@F#!usf$Ny^R^z%0!&fe$81!!wdgGYl=ag$al5 zKm*=OFK;z&J-1*tMAjL6lV2AvPNa?_N4~k+@k(}!?FV(w2lELeXya_1K^J~Bbl92o z^lk?|vR?S>)jEuA1`qm|Te+mHeH~#u`_oA`T;fm4q9YE%TcG)$j4;2{y{lwJHc3BGJe_2^s+xd-@ zQ}X%~tl=Jru*Db#(MtW57Yl#F&=^@24FhX+xSl6qSpyC7*S2fn&uOvM#6+kdh?Sy& zdy*#$q)$*{Z3i>-!SFYNz2syCICHRbLZNW}@H{`^z~qWnl)JwbQENu)1+_ z78&X?nEkED%(>Lq}!xgIKwsv9#?z#q(M%g9EE>nBM|q zqMuorbA~Q9`u2L}E+uaNvDf=xACNu3@8({?sV*I@Si)1JeC1I+8QG+dmBPRtR-39E zadWe($HzU!aC$GX(KAC&Iz0Dw{QVr$ZgBWYtJ^l}*W{SDHQO|3v-wI3cRf%VMMSIH zF;y2ihoQMz!v^YBdD*Rw19^5SAWt+~mUG2p18p}5xew22Q7o5A^e*^tcHiACE+V-& z;^kA3zgGE#W9R+TmY>|W9iT4a2op>#7=6Q;t04601|^e0`lSX~*SY-(i5ckF8z4?G zASDU;V3yA%Ia}?OQKzD*|NO@4iPk=xuH=0If&=m^F;kyY;j0cI6oK<_{mBPD-ZPUB7(Y{Mz$1Sz*{c^X=P;&j72o zggHHOzQdW1uPr?PVb#trONqRD*x3IU5N_ly{LgJ>ijwnW!354!F^^ zVpHKu^F5TZi#K=qWmbuBBqnETBqa1N^_t>P(czn!57e}%Q4I&So;x(QE!Xb-#hY5L zGhONip*D)z+Ssk_hJB#WEK!w%})D| zf98(7+A&tIX|9h5k^Ax5+IQ$!AhyB&){l zh7~?{DijTd2s+5d6g%m>33pjE$SUI^7Tn=r{#2O5eEOeM ziQz2{cyv7o7Gadei>09%mDwNoFDtW9tzNueOvVy(167@p&0F~Md*}=S4=cme!uTH2w#jbbsCLUcz5iLHn+b)pM(HUVoIJ-72381SaRDBU3luu zZ_taJO&$wfHsH|j#j|y(N37L;^o^CPsPHP9o-t?RlVG85-?KijWBNd-V3O+Vx~LD` zYzn)=n{K~Q61A|8DHIlCTUd&CKI$*!Puo@G+t0mY+lb87{fcCXlxk1u-5fUexj0Ja|q|Pv>p)$~A`Sf0UuiGKO2|)5SZ#k!r zoIoQ5V=c>Yic2&I!|^%*ds)b?)xm&lg#P~iBs8g^ICQbBI9wW~p7~}^qNbYq2*lNa z6ye$4$R6QrQXqksRhaqo?`;3V#biXuM#Lr`N>7NDe2hYPSm)RV1xVd>=$+Tra=*rc z>JedJCl9!^0P3ccAv-woVTAl)e1j)v8+*EtlDvk`!sT2^7=UOwkCfouhY(}LJcF?} zE_F23u9hXY&N{bXR2*7kB;ru}ZMM#d45#VExmjNp4m|9TqUz8+?l|m^!?TZF@=T4d z?HO9WuTPArL^vxoD4R-~qt#edx{%lioZN=kuX^yU$-+6j^ipBO}1YofDdN3NR?a#&<}3l5Qqt56h7##A(%oG6Q)HIz zcG5#aBrhjdY0|a_$1%1@VTet=x&wxBBdqelu?#lNLad?IibZoR!`&EOl59Wpe6a)% z*6STWIt=PIEP92eml(2mj_>RDtmIrHLy_Hg6oIJ=QsdNow^oN9GP(^3wk%HM%x34Y z$_bA|2}~kucIXLhbeU?&1UOJ1Y*o6;PL{5P10uIO$hhCNicBU8`Q!QOTnxC%#5|F0 z5dWh$KiO=>XtpTb6?jy7>PUiV5Il#F_*%Fp#D};vQF&3p zg<#zXcL(Tp2h8CHs+Q&%OP!cOL^4!mJ2wz)zgrF(N9 zKWm8ff5vOKxMc~mYA4{6GVlTUu)EiRl0>o;I-|I)??cY8NgHx)?J+!O{YHKN%6(p9 zbaJ=+{QNTri6MAJ-8xINsvCqpUiIkVPC85S$$0!_S3eaZ8+3#X&D3&587`Xg{|@~u zdW4V`O3q^{U1yNsDBfaat@fy%#W?9c8qD3`?uPZe=PAl?Wf|LE*mPsFT<6kObR)%nNA6+`^aw+Fq6qz!!yscp{pTC8ix%^A!k0YnsT zzqQZu-_8QxsldVu`dTkC%nL?dHQRUG%XAR{XMcr;u{=VqKb++y(Y07~F0MhM zk-OMh+1Hqmm=dJxDSV(5*^qN8_(M56!B)uRV)=<_t2m9eBI~Kq1(YIjbMtjIOsEP+ zJAD)mQ1KmO8;dmAeZQ`8tfvZ4yR@pcP$r6COAta{>WU7JOi=;K#d8N;m)Oi zdSn5d$`_8k;y6OTCudAdhqm1*uf91pS{E^m=#eutR*aD#dHa$Zu=jQw9Bs)=RY~?O@3O`SSGR=$of_=ki1xozY5e_O)Df?Q zWMLag?X!q5i30-QQ!|p%U$NLQ@Qr9wxqwKlLOK%3uY9BZG!phUt{I5)rR*?CJBFc^ z(K}I}l78x&F3UnPvRSzI;dNg7FCQI?*}i?o(HHE#(#*vyil(NfB+9|bR*=<$1xGSsZEoy2~2IjVVaEzictp{)9Kw+!P$W4Rfu!>TO zuY9}yg%QB8#!%R?p@$YV;f~HPeh_0h#c(QTiUD8wFowWzfC{gC?MIKpIg;G#pm3FF z=m8kd!0ru32opd_i+M5*q#XTrHZd_>;AZ(4_1r@Uby>g1V-lB1W-zv31~dH8;I{b# zD6FKvW(L^3jb{0u6_N3vE5e8J*PgD2@b-paN7c>l%toeJbt+>!Gf8Mb8CZ^_#D|7|D$P|MI$C>qC-#i zAt;;amVRGiTaS^m>P^?^aYhZ21Cs0sBj&Pu8!d8fa@W9hQ8H}DJ?ozS_A^(_qkd$$ zFv5J#b^Ic|0P73s0~ITRMfCd&_@gEKj;E6m^6;+5QR=+es%cXF-s0OT{kS7ODq7{y z8jjkp(9Ene2-^-AIZd(&99u*@T0Gyy^DPYH$12=Kk}ol<#Gx?)c{qf_TgXO(IYeXHGLF7eSZPonE@T99W94Wo`(X z-68}HJ@@?)Ud{bu{35vwhQyrm6l%YPCpPK;Q~^njzayhTP4}Kj{^xAr$@jV>*C1!h zcBm*zJY{9+I0p{HultKwsrcyYeFQ@&`Dd?AC!nx*ok-Ln2tzLNFax%Cvk4#w_aG&! zyc}%mhXYa&b*M{8CXg%rKz+Z!4lq)KRh{L&-J@E_l{i`wuCp88h@^3CGgWro!jqzH zGx7u#!o5a$bC&ZIJ0m_89`F9{?-AgSA0jot5kuaknuhxNZUlm5+wah}{ajgVXtdgw zAm~;h1t$1%rPUzGz=DE;a(65I=i7Hkd%~V{))+fnI#;Xe(aqgKu=+Ok>}UnlwbSc2t3i5;oah!Zz z{uiEJMQL`y&jP`&oGV9b2CzVy;A!lx1V$gyV$yf79h1ppRvS0L=QWGGi~9b4O6n&kn_UIHrS$OFS}$yU zB)*F2Y6RI%E5TjEx`5M58Q_Q=Mr95fII{W|HwKd?`o7q2;hD)#*?pT4NSze&63KiP z^g8u?X)5tka3`0PLjh)mIB^-&dZ~ZTfyji+<%s${6v6HQHDmM*hv56XZ_n|mpbWz0dE`_i=8(7JDL(lg zsKJw=fM-kffGH8-k)Hj;tXZtZ%@~^j))PXBPe^#P(e=x9kL2%x`cAN7{OC3v?BW5s z40jpc?5I!pJjGwzR$ZX!N~E%?D#LaVr?)@F0YEBy_9CPK??~x^?8X7_Y+y|espl&A z7+=qMz}vnQRLJIJSug*X9jEtfO-zg2h}}B^m?XrpKYncI2jt@fqaj$D6d&m+I4sWe zzy}gzEReJDA)7=^lx0@&Y}KBSHq>s++TVlz$3zm{fxVNVe_ei>W<{^yZOx8kCiVR+ z;mi-aPUVY%Jm&!8jWINPd&p}6?~6Q1a#}HdzqymnmHj^AlJU1p>}k97Pf6qeghXvj z7QG%U!7$Sb;?e>(F2U;YIN?l95H73n>eH$w^50&_)zrv@xZA3zOJ&0)#F4MoigG7T)0Y)GLT0_WntlD2}CFgu!U5~_a%Sj2(Gc83YF_9dEaE2 zV)61ps*lA+kavPS;2Z#+6-rtXs0`KbpZorVau1TqVOr5AVw;*>iw)l&AD5n?U_sNppFPL;nZ zrGhL2fVE0OV&Yh4_(D?L{_mf^#zOh)Sq}N45^OR8fyr&n`cAfcdg&2fAWqzcyE_EA zF|Uq#PUhQ>GBGQDdxR6nBhd6g4lLuL@on8OT}VIszWOH83DRY z>0J5uds+Kv3s;(lGBPM21;G@8ka8DIw`-yLQb(X6wK~~zlc9?751`c8%KY8O>ft#o z0+N|I>M%lt`G_rOE8slgA|=a1-*57W2Ya;W*GGjn#A#&o?G7G3Kcdd=cbx0nudiOi zD*v8gaY`JiWmlo;eqV4*YVZ2~`Oi_(GReo6{rc>oU0L5RgrLd!&o7zQ#6*72`Ml|V z8t#h!Tt{d&?SiMo_a}ZZ!1H@Tcr@X0ExYn3cMU*jxj2wV-=5g0kETVSn1dyWfq7r6 z6sU*8@mz4c;epQZ)DF`JB)JDgBv@YyxhBbKkR0OW#>V~gq4#3a zjwcg*8X4X(yvlr1X3g{|xCO!AtIBqez@iifY}4iGHFBSX=@S1nqO)I+lF8xX?%U1> zBqZcETu(-oZiuXn-0Z*hZgfh*S(F!!h_yAC$PNI1<=9R@6YoGkMEBs)Ki}!;riU>x z2R4d*Ia-Cn2xsqZ(!w4Pu?HR3JY{$ie=idGbD>9aDIRFDh*4dAeGHoQV|CAyw73mW z@sP#1Joi=8@?>{LJ2ZpDirSr|^Z@|KU)m!Vq+FDgfCoUh2?Qo3TmvMp44XlKFyZk^ zv6n#;_}*ASL3g7U5IH$e@&;~fmVXVdT+iGB$r->TVdws6f~B2>-0kbd&`$G4=hG%d zmmxBcgu{Y>S3Lhzhk#M2Cz46y7y&uiZuoJHym8c#u~ckdFXWTtbOGS(fcR`6+6~~Q z0z9hlRlDtfgO09T!FAMU+4GgT3TW%t^!#UvVZuPC}~G0|xQgs3250ZS|s1QF>{j8YV^fru!@#Au>KjFYYc7LX1C zB7z7eib}UoL8&v>4@4N&Qk zmMV>?jKw&H!pQuxl%rs~0^k}kTDBFx7PevnunTjm;IMZWHH!ERkw9^>Pfz@L^&!o` zK$Z?$s3swKaefdfBnQoshBaaDJLBRl95{iHPK+8aa}$L4#(g~2u{eI6X{M#h*4I`4 z1|z_R4)@60Vg&I(F>l4EBW+Mx>eDOcdRMSX`)f&}03TKlu4{ixnQGTvq#OV0$6Zs# zd37Xi^HRb!;@^9BT$^Bzn|$^0#j}C;5@#g)F9zIGXPVn(%m#k=>IErhoHSMrjD>#{ zm){1!bvOOrePAIB@sk%`^U-&a)FK0{x6 zk_pMdeKBtC`7>hE$g>q-rmPNahAtO`US5N&Oo!O;NUCmhul33}<7ygd5D(x{Vg6ae zfe)mX9XGhV@7ui!)5{)!W>)&3O5XlAnb@pSz{@Vn8k1I@t^gCII8)Y2-Ch~=MoW)f zYca2|JZ=NhRw%;s;eZh_{#lo?LYVApPpyYAi&+k9-LVa)2%*Sms*X_GY*;-X7`pad z43p$)M$g>vU3CxWc?V$DL}kNq{jMON#5Z1?k(%qWc?0r%-;7MR6BcF81U}|C-&bxY zLfBZD+v`?ciA9?awkD)b>}NrV4=6u>0VIX1prxDb-QB^aMQCVj9LTjFebPlIZ?*=` zC_q9%5Lz{!J*uT6qO#zGbE=T-%)!Ibbs$&t`v#-vw)6x5@oeNg%pA2 z>+8dnCfI8kD`s!2N3$?NS&z~nq+8G=Gt|gjNIpGXcNVze3=r)2a zfh5>MS5kl}IvhIKC_vY4r4&@*A4oQk957S~ZmyAa0b@+wPr*3Q0k5KBS-Rv;o?cw< ze#J~hv%@`V?fp_P2uRuO@;0GL8}i z>ri5g(9_?=$&u*?OP(Y0Cjv0Zrb&;Ds~XoA`pip9nRHb{C%R{=^NY_G9T>4oZGv!c zcTY9+T`YHRT55g5H{uX&;4&d!`}b!sK3#vo>+9VwxA_#}&>`0eZdWU9d?hHoaTt%d zaQPJ<)LqvYS6z&DO9Nt}`K$gp{~ zbSdxCjQMXi_0IoCYh)Zb(BquntrUpX<*r83S&8l?%1E)?O8O12Hm;ap()C1F;dzYP zt#4qiBUaJfWSSf2zHKqzI_9^)U)0vIKPkak_eX=Q0}BZ|Ji@UW90wp2^NmN;n+B)n zTryP&ZtUzG_4KfE8TCYbS~ys2!OIllRI$1>Px^z;&vlt=mBMF&s4V;-Wso^(kJcG= zzjDtl#Mv5%PDdLGh~ont;i=z^63$k1KXOurVX&A?9d}$Ns9#qoV>EUNq~F^t(rCsK zz0jcVP$DhL#$){KL|abA)D|e!<`=Ra>8eK2mo_nOI92Ugy#Rm#dafS8C)Xb{D!>0x z(DoySG3uBhNM;dQV-=dv{rFQ&OixNVB}m$#crF@aS0;S%N^`bt)_RzIWc2KF8=O=Q znK+p~1Qu{B#NKJnuAe^1_^fBN<)gRrG58mTYy8*V91Q*olOKDlzG$ER_VN!M^K%Sd zbpDajD1X&kH6U+m?(n`K)yQpK?-7KN#};PP5)lEUpb@fcQ|U)?`|{R)e_*>0&NfhN zVGQ;NazNSI2I=0VM>s{u=kox!jqrLKyZaz0R(q^BP+2x)C=Zv)V(N7d_mA0UD_CX( zRoGfh)f3b<8Bh%+$Om$J42>fnR3=V9lJX5dYj=OYtfG$sEnMNOO_GMwg^6v6DmvUu zt-do)O~67bX1x+MXb&Hme9+^63>h(sO;9xYzOVTuoI$HvJ!V#zwzbr13td1;gYf3PNa5L&f5~9O-y*LrNrdrZzNmEff*UnCKJ%Dh~b4YSMtV;8%;lXEs@($A}+0|MFhxlKe~WDar|fYcR4% z*FIBoWtdjS*Whkf!YGu6pI=$sf(DfH0OZ*CzQaK13tiB!kfTnsM5EzdeP8ue7zQ>4 zb6K|I<6MPZQ`TcJo{cEEva+&B_=$ufvUWldDy!$kmTvd&Wr)P>Z{v1e#bX+uPBLGR zk7>iiJsA!#qM*>!HXRwS?d=J5_HSs@Ol)6535}zt^z}TTC(Sp!ceG#ziNEzg`b)|6 zC7$=Q;A&$5Y@DjyDdg$TIUB#+dZr=ubw>2XYP^NeT3_VZysq|xjvMD3KwQb7$pMBq zD^bNHpa)ny?2*ynrR^-qV!!opv+O?B^+E%)(IA7zs={}x??l#@1RP*LGI7uO6efD zZ<%@ElEfwd!3LZj7%KPWEuC)Rs1UmtS~%cFI7x=HF#<%EhTo;i4-&#mP!MA$$|A}a z?VoJQF=Y{@x*PGD+;Cx^p1y!D1x{zCd!kD*Q% z%H1?jIrb39c>oe7AO-EsbW~H_HL?~(Zj7<1=j)J5T=;T<#FZCH;Ds}vfREHCu?G{=xd$7U$IT#2uP7I6}h z(JSxBfx`+r3*F=g2ff;Ha1AC}eAxG`8NS4nWD*C1 zS2Y~Tv4N&i<3`2@s-0Psx=@HlxdeNPf$y*{Ge$8`(ff7Olx2F^Bxqo3NqaN_jN(N+ z{zm}yNhaJw$$x0NY>oem5pPr22mfGoF0>HAlrCmZ|26H zH(xX}@K}}VQaeI#%YlWx| zQ=A@Fov)vCf5~+R$8Cx=j*Oc)wJ*2tU9N-(0VoQcMRQVe6Y3%@8eoeA?vHk?I3^>% zn2bI^Wg z&hs+UuNp;AB956y1fZF?jD@*{sN!~Zx-C#g$;%kRszG+2VYjSEO4mV0mT>3 zuAz)gL($g5(bwSQ2~LdT=HH+DJVyr9BZZ=EWsRS4D0>557+jAHQK6oDhZRsMBpL73fu9%^y+@zc8T{3HMbJ@LdJ8v(hnumD( z#Iv%q$92tnz|*a)YY89>S6ma=cvvav7Ht9ipfGR}X`TbU4hFoqXR?!B?nvCk0~X{& zf-D?kEivr5wevn8z$|o`;ZSxNA_10CK5C%&`ta4Xzubc8{}>L+seq2+jp_sn zia*7JL&2(Y$i#u7G;^W!i5Z+X#CwXLMxhev5-4Fc!7&#q>z!-h4AS}gFz??0bqk!5 zGVQ~CdOKGk_ZwZKIfXM4i5H|DE`az=7jUSbkzQuVPsD19Qk?VUT3Hd0kGYf@SHO&N z;5X}9YQV3~v1ze}a%D8`UFTNL?RmLPqQw$O5iiP&IcmF7qDD~~XnpIHM+7+_*`ONF z9$tcCoT&a-WC9^X_a~utSG{cssZ-K-vfh9F=SEgmJ?XVQXM#4f<%PQ?j`AwtcGjFM z$(1Igm2|<%l!5sr4BZ=B`c^(pFi;7Xwb7svzZWkAjsF2OeB@Nz)w6>~eZonI%SFJ( z!hSvuyg8g6zk-^aN2ss01)FBF|;dvdZL@r%gnj?U!qd-9fQtCF8xaIM|}h2Mz%H(^ekqxu7F{ zRspHa5X&A{I9TJ}J;FnwvZsp29dIXXqjM9RO^t13$oIZw_pyqawZKR@`aONGA4)ik z&9n{N!7@D=3}keF!Ex&u0SBLjq{k?MNY}@8>!Da&_6HP9Ha&}y&bRwCLn>eYPm{GtjZvj$g&wsnaryl?f|u|CdCV zfNp_y2N&GfU5`O~H83_j+~!D>cF4k)V!-X}HI=uHaf4!P&Kr^R0bgAZptJ1l)wl#c zr_(hpAjqk;A~Y2Ex!Q!kHw#8NKaa2}|N4Lh9Z2(cIg4y#qkYo8dr?|Tl9azwbGIyGEqDX#D_ zcOqyZ?7@QvIQ6rdDgFj#6AIW~mFA;!t3nN*%YM9^@2`iy@|p{kJ_Rytp3!cs#b^)R z{4|cT5az=nVY!szV4C|dg*BiAx^sZ`@}!aJC<1boD;j8MMt-g^zia!g-@eI&3XC77Iq9+r z3Sad&-wPt!s6dkfxEdpHqQDkUmB{E>k%qEAy$=tDJgNdDCAGOe__SQvONVOO=G4vy zR&kF|OQGEdW6Na2Jcx7N6W+wZhIGGzbly}$2J+ngE6$LF&~YQjGl>3+L>;Mu0(oRd zHIHi+Z)9&M4zw#~a(;@+CVBH5(Z)J}t!bLisZ3q1Uek^6| zz+#E$1oo?LvFeof8RF91sKW}5U~4RdA{58*n z6A8>og7+v)70!Ycg;+ur2C`Iq2z%%_$tuu=;6QRC8M>*?0$G(m|E(ejdyLd@_~Z%! z`~uq+1B;wb4IUK|<~MS%BWX@ZtFUmY2Slk0=8X&vPgcVnA^`!OyO>L<6Ze2L09D7I zXbe6f$KTdMo8DPY2`S6Ny}O}d%yMqrc{7C>=$Z8CJZoQR9x|OUJSE|M+c z9aXR)qfy~e%Yh!PJa3Kv^0_(c@IsrpZ?iwT-*O=J{!8~KwXFx<=+%l3*p-gGH8Hgj zfTQxncVYL%gVH{G35mk_-CZ|EI5N2aRxHWe1&0CuB*sZLwdUU+P#B{*Ui#eCk_b+; z9hqLX2bHZ!%LjZW%a%Carbm$rMisJ;lPCn@g`o7po5ia0w&I09ce3j_*HQXY9DajE z^U`;pO_`*8Y1y;V$4@TS9$IK)_BQ*6`(l1vk5Ah7exJKKrB?oW?dt>h?fe#QlY6`O zwDPWZVI3Ks-+0|?m;BzTzr4uo+4{BbEj)zg^vmNbZg(nn%y2NwR-BF5|B<`)tHdpJEf}LXOF`b zJ&kQIemQS8eRWvC=(D|np%bozKUn>9wB_t6(<{%FWp9a_mlQBXY0I%G)f(2PG^4NV z@Hv<3vZA^&u|4bb-4yem0?tI492=S2r*UF1FNx1-ztS>UNT_SMe*eThWGe_xeYk|+ zd(WwO2bBR^2G)Zp0LuXA@E3(bqvkza{OR}rhz#}*MFxF%N}W=0VW(cMe{0&c-l=`N zpM`tA`TZAqiH#KP-u&{XM|W>-U8I(Ff$^x5HTTzipJM3R@;Y=*$FNY%?$Cg#emVv9 zkw#&yb*pE)mgUZBi>-Ai&{h?C+$ifjp1kYHr=h(Msy~UgU-MpGQu_>S>vK2*MRjP_ zS5KPye&?J@AzqfW=y|*;rrmp(jXAGSB|&oLhM&a_Qo-c>DFDJR(r+jhq1?>br53b) zSZ!P5BcFAMfiHdOUM%mkcS`%v(LsBE{-Ut0T|2P&V)aa6Yw(GrlHo^dYD3n1|JpD- zG%m_Sn0937s=_dpmCXy2h0P0-bPB_K!lo#-uM!oF>(M!vv;AInv|^=wlh|jaV|29R zb7Noqp|zjYb}UIm7o{PjPi?vW`^|Xu%wyjrQ4q)xM36YU0f{sc7QwA?;eakKq)aj? zf?)O-APiNZ3*OZ^)CNeJN>EdY->SaphCsuaQ!2TPAqz+82CZnxoFTlswZ%+w`&&U7yY?9`eK8BW{LV!?W5=3`u9cdInZ)0t8m`38}|P)RD7(RQx<7Bd<*&* zn|IX{d$oswYB#SO z&vu``ANMM|!ZJd`Jk!BGd;T<`;k)O1V;$4qINvPye!Jt`mDb=$^Al>d>uXo_m2M35 z8(g~*4GY3(4bX~1fc|mr{EQ88$<36y9QA&4cAQ(;nogX{5oqNyw}$aXNCZaQf~)8R zI8UD1Q%`PRlGo3Xo$pwODkU*GQT9PWFwXNCMA5)a-n)Niq=lZIo=Ev$xg$ra7OG@y zSpG%N>Kd=7=|w=Pf;J?Mz8BpXreUooO2~QJb>v_F3R)S}lswL9T~gT7OR@X6t+LnG z)%B`5Ya`6WE^d#bkOuuCDZvq&8z?%d@CghR)rwfnEUWRC(NsQHP^I924Gl!JmN5ea?xO zkB~*a(>16O5i=X;Q2Q02myom=g8}I9#vtH|2BV&8 z5AU~W5yh>qC^n_WPe|)Gl>qF5tYO>D?s($M2SgDd1uhDm=~zEgC`e?l3Y4H<^&Y#o z27_5vDG2%PgKlua?6 zIl|QL)?%P@{pSf!ehyj3!iqya3Bd)m>ie>*Dh( zT5>d>W@toSyBp!|dt#oQ%Q^kd?$SF{eMU8;dCu9qWaILmCNIylJP})J<=+~xLuhYhPFsRYVr2@!A%V0&LVR?JqL zjVO@OT&VO<)uhoBLER_1Xz{&>UN!DDR<1o8xm(ODwp1jTQ-BAM|64{HP;NCfeD|5( z=bM6Uw@Z6OGY=mT&lWbw*3gd9zNh(kUsd1Z3B3;k^3^jtM%}D$8CDZkURm{ZwDtC7 zChMH41^GU`Jvv)2y~tj$r}U*qZ^Y61AGIGHGp;bZd~Eo&CtDu)`VBA++SSKImB$bU^nA6GsNYHj{p6d^h`TcJtQSl*A&ha_Tp zsbC^*2}aov$^6O5rvd*Iqu~wPPA7ix0D)#(+1kdr+t9rG3s?#Xhjx}=G&9HVKyp%2r&kFYPUW|g?ydOQMS670kyrB|6 zP-y)XjLk)r-GOhRS({C}3?*8uO!&D~${WWhjrENzPLMM6!AgY-OZ1a$qw(V@&MVXsNKAWHGm%-?^ExnCQ?Dvwjh)dXI!Y}CWF z1ra!FXiWpAqe{#@4eNU0@F}ZqabnWx*H-UKuJ?_wgK&l9F7ljK!~A{$*o8LmiLLtc zo=5I^x`O^q5AXSQUg@6^8vAtI_C>T3XYGX%jwURMLk0N=1D_4drE26#g_kfO7vpJ~ zwS&!dyW&g+1b|(|*Hyjm8h`E6Gfm99_Dg$Be$nWNQC>UOc}HpbHRn80gz53S)nB7SbZ5yUeFW%)wniDuF;WN9 zIjHaxsQV4Dck>^oN1qhIoSsl~3E*gjS2*eudYJ{ui5WFnuflM9j25tVU1+{tZ;W=R zBkzkkOwyhYlx-wF%~OF`#VGCqT~SFZn|%;!CZ@Aj^b&&9O##lzIdM8Ry&+xKhSfwo zy|h}pGv<|Re8rZ*#sAbj9;@1uX?@HjO?{VR-FJ%&fA^EHVkqRhAlI$Z%Kh`;Sm2$l zS6aMOA+AffoQwBnkCdN6bXgI^-K(HrRFsmniW|`1mM}O55PM_DM!$S`v`(Vj3xuFr zns4Bu3}v7VH$E<=R##K#f*_W9z_Z6+hG4kyDXtsR=<7^_Ikm>0H!n zqgR#u!~Kg}CE8lwmg^zgWe=Ddx0{nLZmImHFfrR=<97%0?qXkC_FcM`xs%yciK|aE zi#I3tsNb{6oiDt*Cwc5hErG9c&xOOrDSj3t#Np5(0B9c0kCTrEMdSP1` z-XPB{DNJ_i4ACI86OBO9O5yEAz{5Q~jl}tbsCc$|?1AWSvELE^5`Q{JtVR+7ViR6e zuFU?cAuthIh`Zl^(Gca~8&tKUw8vl2=h0JtvN!ir+h^;N=?wzZ>Cs+X*;+WMy|}q= z4|KVETP8IPp7cZJiMOp=Y({QQ%q{g)*!+=vG-?rxBS~z`KwSu*Z;H64Co6nnSE6j5D-WlXs`wZ7T*V%k95^0=Y$J1ZBUj zCy8Fs3)Pf^07@@O2H5eJ9n=Vdc1(DoYOhtS^pGqwmU_lFlTv<~`{Me&*)I1lkM6#{ z(arRpeX+mbUbJ4~*oPhUkZzW7M(3=}=4addrk=Au(?>SB^oAG|r$FG3~ZX`P)v8C1SpnP(!=ZE{-AxP9@tO`p+eG4G_^|uH9GGc z^$uVI1QtghYnusKP^?!tncar!c8_Z~5RFV^TeK3F1VG3H3(gu;f`jpeY`YleQ`7j2 z`vyF@U>KbarH8hp^1yVl>gS>3$=a1wRM-Qp$|vOjS!>4cX(=5Ls#LlBY?W3&^zxit zBg`fmrN0WO*jsII>rrt<@0#O@J!h9R*xxa{`~EejL(?)<*wx-*)6$DRy%o$dqVwLq z(0u3)lr4Dgp`Ow3`Beq}7B1 zl6q)DBs!&SInWc=#YTr0Kz$GqXksxP!Yxo+LZ)s}2E>S#F>fl{jj^AKDPbeHiBo$I zj(XZ7fHT&yoMP%d`|w|K%8TDK<#>H$#OXxS#C4}M_Sms7@wEmq%!I--|Z^V-k`3Bn%qI8jbK#0D-q+knbQc$N9~41rDghp8O>^FeD$EH{G>>PBGqymJq7R zJ|sed;i>@9O||jsCZ; zpSd09*MFccAF*nM%ssS7H}S{+ogXLwYS@4QV~^Y?_twY&^Of*zEFnRUSccf^YJjS! z+fo{Ea(~-bwpk$_0s_3jjqd>qQZy zw<0V9FWd}9Xnxlp)q^;M&)x8Y8bE=DH(sJ_<_hkF-}jw38lO9jiIEMI-~ec(ORsbn z({Tf5I~5P zy1wA%#6MA1@qxr6=xdt1mjhJ484uOhk?|?vu=iZScumPPEXVUpQ>d5$o5`Z~H~q`q zC*=lCDQmGF!Ryia22V{`BQ!xbsbod4&FMwej32Mzz=@Rw;I0YQ5B5h z1I0KO3@;<2)2L=ggecR$4qJ6N(RNPGg=j6x4?8J3jZG{zM>(hDjg+_u5#L~k!Ke^z z2)_oYrQbqG>+^xeYw}9KRU8;0P-I3L(VBf9{Ufz_&O;EjL-Qfk8#8~yLfRhliHlfb zpk&w=I54UdXmSL4xs(4JKY%5aY90#vY)42}+jj zF^X3i9QKoPEON>p4aBJ$qvN5`b?LB$0YD%pajPU^S%#xhCF|K4f@aKUMMH6d{gQm@+w2j0-h3ndbr&tA zVfrpkR6#!2nTNMF=ahZ@e)wgp-yc8P3UTx4_ZldyY3TQuqqXIi0qwZY^z)}jPQPT} zgjGn=ycIqcb>OT;I#G9##DbRJn>8{UsDU{upYN|SwVU$qe>Y9L99L!905FR*m|N

wi}y)U1=0k&m%F&P?{b%?6o@7#%>sA z8$t9@c_G4RI-8>#tl-)YQ=l1g5Y+)4`rEggrtp_d6x)09H;|@;u{liI&Uub9JH`$@ z0u&+?Vf3*-qFxcR`(QFn)7voUfi$5>6e_QWmN&_hlSIT}w?=yYvoQUI`?4FsEUp$& zgaZExu^rY0- z3WSu&dieEJw|L{aLr_oyFYdki^XG9cE3tMNi}fpv;(kw)kXa}piSc{?YsI(^pWIg3 zPyNqKsexE575Gd8^XWJzKpw3MqzutO;AOdZNjXxYfG&!zyymR!vb`rCADIfXCW{5s z;U{?_UETK)MU&01CIDv_iclo##IM#-0yAKY=BHgTVTuqN*#+CRk*kK$N(qQdWGEVO zM&iYsL>u*1niVE%&+NqL`W z9y$dqQvd)3IBKj~Y9O}aHZ2ub<0<0QVDb6jkFcpaZGz@I^$!S2y%qvDB7}^cbK>cI z)Gj(P*-5c!g;Z#?yWtN6ncIgV_wRzCuJU;b_qe%9_Pf|s|1x^ zP1t(%AHz7f962s?OPZy^mFi^8wM)4!O9%esNk$bb*gRzf4*1*Ig*|Ni87hp8Cn6*V z)MSDy%BdYJK*_sh>&X(<;qHHemS-deJ_K4m z2t#RV;AF~%1^D+GcORUwdJv5sw&H z8`Z}e*L|jp*wt$P&HtPvVK99eAKU2a9a{$frOY8}3T?YyFMx`4Kp#Z_~;*DwNY&zlvi5X>5 zkf4r&Y6-@n|HzOVVkL8beLelWK{KOINf%{^FHGk9D_7Jb7ZC{hSDS^kZX7RJ}ebP zW5rkvbj}h%^A?0O_@p}tEq`lSbxhs`AxI*d&(8*L+US-XdSVmPfG3@sE650 zS^1i&?*;UI6T>b@zj*x|HCQ4~AFw#`GGMJfG{9NtdR_kbe5#tuMCsR-G&++LP8*1x; z9~6VAYI=o<@A4}#HrsKO36|2%ceQ=tkQEr&R%0#iiO|M@oasj!^)gx1*1o(o`G5X= zKoR-(-1sDIJE4GP3|oNxy$Olvz^gW$J<^Iss1g4^`Tm3cs5VaQ z*ENQ*SSo&6&iEE|p5dgz7>R;Fst;DCSg8*f&s`E}f%a1aJjJWeu4m%tTnF27TK<$1 z|IrjeG>GNrjl&_@QY9MbjO)U?e*xq#peBh@K%6rl%%_sv7>Q97`!b9i36BR?9GLVG z2t~|rXD~cv67&X)hE0wFGJBY+KeV)IM18E%q=qnW6r=_aXr;VCf!~e>p}iNum{rgS zVUFpTH%^>0O2x%A@BiObQ8dhS!c<1ukzqn35XhcNVuJwjMnetYY$F!>4@WnybR2Uz zw-Hi%5?hR_A?D|yp;Sa-*D5?##?XwsvOxNi7M;aN)f|rq^dQ{Xrd~H7@AaIv`3K3% zV3IMTc8c3Oa4u;8l(zbC#b8R9fIz0Rj2GZ-6rEr7F>VJA=)XNv`8{v`d<2y7T%^w! z=?_;M95p|WCYeuW;&AjtgUw8jflxuYm`hNh<0hRYfJ{tVYG7m#<3ZvrB5?+76&pDK zmbbv3qyhLd9+>^#>p!V|W;E5FcI-ZXbPJnvYEA$yJ!-#2K2c5I*k<7cl|hd9a=;`+$2vgicm}SzJ*<&F}3)0RH4Y4Hw;!b zu^;G*PeQkvjyA{o_i|=M5Zr_ZIiXDE@>sN3=Z<0gN`-}8f5oG3HYzKVKBh$nB-QDN zOl&8&UTA7IfN!mTeH!V3I|tP4A4b*e3$pkS(>nDbNH%?Qtnst~$Ze5M(}ltH&z4Gm z?)nru2)Wis>;c;IM@WYQaLrDxgL5sD{GmPpu}R8yql!jGnuFJ(2?MUs{}d2D<>@ z_!hoI^gted<6&7O#!HFWKrOiX<$Ane((+=^HvgX)*U+G?#x4P)a126sB?e5e6NKT% z>*7;DLrLQpj0lxFPAu8fwf=JrZ9Re6&On`r_8lolZ*T9uvT>xZP&bi?Is^1Xl8|so zm1M(;4?^wv3sj5Fhqb*yNJp}VB+IGjk{e8kZfHHCfv|Z%Q=f|8ryhetTstE#iQl(E z!ZR1OvKV-G==*L>`cAHX`55_@IwGbUyDhV@Ne3fk%ODdG#p8?9ikb1i=su=MG6kBG zubAHt(4Q?t?;_Z_)R~LX@2wrT=h7(XQyxXIN01MSSM8n6ir4^l z$ih~L;Y=4?tHcpwBBB*TA{zt7-QTFPrrl%27rrS$J@mF0CHXR#4+kd6eR+>)PDW9f z(wRMHvwqI*IcWTT9Y6o#;?!bY?@5n z48g1VdV3n*X^fUW?GF1BAjGR1p&2UP@yIm7t1avXt_(3jm%url7#LMFU@ambYSbVg z#Z`sm$4L|*Lp^0~`-SwItsjbB@2pMjKQVI4Bl3Y)rC_YdhE7Qmtw^i_$!aQe>5H*s z<^WeMB^{dp`X>`x@bF<7wxIkGo96aRcxw6$J8I`}##JB>j3IK2I8Fl8A!%a}!BH(n zDVv9-;>^o|JOT7d!q|b^$Y4--fHY#y4pvsgy53QSGbXFlb$(t;d}&UX{zk%=K(+{7 ziP_(z$T>k$RZQ>OB)_>6(MNC$``}c_d(Xc3sEVDK1w zlb*6Uq*5)&Z=Ljc$=TqtvUtLy(E0-ox=0-JD=b$*K02S;^;Hc~XjGJvE=?pb*7BMz zJ_3~=G!(k~{@-$it4~!mFc&L%BYFp?7cx`e)rkoZ#BmL?V+Op!oCBc9wo^yl;psDW*u=odk(F&^isNmF;m~x_j;SCL`^8FKPcg<%{ZrlPAW1t>|cwu*T@0 zL&hxq^_yKkn4O7FIX%cfW>eAflAVsN3Q{Rdv*WAJ)5qG2vrHp=S4 zIbHMVM|9jCGwugaiI*f{8^7#o?7g+hBk#d+InL|N9ZLNPd~QeyfxBWVRh}4f*2TH> zFOQYH1t;2I8Xa&>StB0zP5b#BtG~n*h;u3$zE6A*&8NT4cJz7#g5aFNH08F(5ekX> zT?|vFI$HH7LZ(5eYYRy|Yc{2ZXiXa+^VJs>g)f|sn6@31Er5xem<#8Ru5tgHUbKIW z@<_=&VIWV6A!Ff?8i{^{fj11|qP>K;4LMhM6ce7ioIayg_fAZJhz>r15O5NCh;#I( zx{p79F2LEK8O_b=(+0PEx^Mbm+HeegNXsbaB}C~4V_KX7?nVdCN!PglY+Y3K4%30e zKSD1;d@!=Q1sM9E7ju^cWRW9fn@f=q4m!&XeZ1)a#xyQ(Q`h?2f2qHDCq@BVE#gT) zdOdjlFvH>a9(2rxjUU!@RwVCu1^*sT6Tea8mxUmFGRiLU+AK2dR0a)_2JGehftPfG zMQbzi9x1K>{;YkAx@pv6JKV&|032m)FC64Z()*7zuwv@ZfwM)n~yGZU>#KL1(zF_`wG_q49?hg=S-eqA#a5kG57$$ zxSg?X2r<&Z8zpw;P;!zNb1-)szdztp?T4ZO!R^=Nvu*5S9sUrlycopIK0>+91qKLxgWx$O0+)6=Vlr_*-LYee|{ zDs$eZOS*}GeE+G9*a=oDmmdGvv<1=ip}0ZO4z)zZSQ?NikA7$^(=7t)?lW*0o98(1 zd_d!b#mGPdF@b)j-!i5Se?AI@o9H4RFuDh_q2y+X`_++XK)!^d9OZN%Qpj7fk z2j-!XabajJHJEFsjcCPBy=+w9UpfP>!08;@wV5&YC16~1UBV%EYrrEch7fo5-nTEB zk}I+1vth04r2K?uJlPq1mRb0(MaR{ZBq2i>;PG@PT{|cr|GLu#M=u{Lt{Qa2?-axs zID#HkrQShHPX9(m5JjoR@ExWgE#eAnEcvaNkZoTY9!AX}Q6xS*Vix%47Q=V^s!s4k zX$&=l93)g@n>BkZG&9OQJZU9YOk0t-C4mpt z8tXO_H~~FY96s7ofdE|IQxzOs+3?j*ItHnW6M13X*F=+st(Ts6fNpP^jh=Qy3zo^8+AC=_O37^Q)8F3vg!62JytTf~z_AQ=TFzkLT_ zwH1|mO6QZ|r%C%axly16@fB}0U}@HU(T-_isu24y>jg?q`OyGBRzsfFag@`e?tC{vfTv<@!l0nM^ z*Yudm^Vy$z@k6pZ=Bs3_3-t52z4+6~LTWN_3$2OjHgq{-)VrQkHeyHi>P$$6#3DX> znEbk^M-l4j*?a#>aasPL^dNK|B))H9<8bDQX9c^XpXq956v2UqWG>a`^?}$+Em|B< z=R4Jjmj?@5u^m*?y)lqWuDimskOXi-F>5$&1pZ>I{8S;ZDt|0rqZaa>yctR8a`mH6 zB{!Gx$rhc^Xi;h=zZfdu#S`2;Jh!1(Es3-FR(?4SMkPak{JwdP6Qt0>skeAXCpb%J zJlejL%{GS@f?*8m5J-YmN$OzFS=DdE%GncIctUM+s?Nir2yLIh(lA~(^I9jd4@)h` zObZ@grZQ#X%}OG<5sbli!zqR}kElc2#lH5KC{EA%wTVKR+%_Ta-D`m;DNo}&&^MK#OV{3BElvPrE zaQg8?p@M)B*-X1uwzx2jkR6RyjHZMs_5PRdE;v<6?5R}oG2)8M54dN4-k`z9au1nB zBsy$nRIctZ`Mlj>OA9K$o=J9o0b9JLy1CE`CHOz>uk1!~T=$rBaaSN_ezR-k30VpY z4>`W>f|xRQ4-XR}$I)n1snNes6zu}je4EnIhfwU%NLz)gao?MNNxn&fY4Ky&-8Z0i z$M*jFavp`yOzRbj;HPBNlq5zNwXP0rhc~uC#AGcOzXFq2?TxKvc?&<2w+fS&M0p=r za@yqPv=M{N&#zxR-nd@?VGxQY-?*m@<6`rIg7hG!iN0ZGH#$43N(hiQA)Bd&5;dwahTl0pe zP+dwW2|Y?e;nK=m#NM$Ox9bJ>6JZ8Z={q9I4a3MkNC?^CYgZwDNF)$9mNE{OKY9wK z(jrbN@=lg0pK z1k+C(GD@Y08of^KA6kIX`ot8yoDPLXZo( zCf=sX9+Q$NVsQ9IcG$&*UXj&Jw}Erexm50h=5x1rMc4E29qCv6PVwLHg`uM~);=AN|HC9VW@Z?6hr~GG__YjwP=o5$^nRo)Wy{Jj z6MbbU@GO2G)=y^_ZKK2J#vynkITD?xgfwlD$o2j{XRP>Or#B% zKugJke==mGL>EcF0i+Ac?URPOQ)h%`h$$Ol9F+|ehf`RFp_l=>bpKJisQUV!$&}!s zZ`{Th3(5mIa*zwfUgZ^pO!XviM~jZF_N1m@Zm{Tck`31L3}UuN-*T))pp)c2Jl@Vn z#Rq#=lyH4eSCU&Y&)28m|hkCcL!-V$~TBQ#On6HemOa_vrTW#4l}i-Ko7Ah zJ<67sz6Fx()=(OeJ8!j(!{7sd=xk|6K1<*wQENqJ@fmfiKNLG-s)C%delj*aYQ4Wc z$bF*tfvD4QqS#{k<)Y?qTORZ-Tvii*Xm7TOt@Zp_Hu7~=N_N5SXM!sYqa3W)&9c#J z5;nA2CKG zP6Ts2i;edylsZB>c^hWFe3((n!<>nhP^LsJ@XAMjnQWQcacCVCJWbQ0oqNRoBA-_K zFpG@N=Kq7g!lI>f(Af|gV8Ncerg6UoX%O0MdAk!K=oGV6IKWAY+uuI5n6xV6kh8R; z1BTq9E5;7{EqE~odg^1?2x^dG>7D zVpHFe(+0uP=~7OvYzTiCIE9Mx)5!D~J)1?csb$m*ivR)IOMS+tHr2D=f|k)_ub1>Y zFS=Iw_|tR>LbJh6>(0Q$Qphx4Qy*>MrN55TbWt~viF%9P>FNf0p@AltVus#p?85YoOzw?6YH)vXkzKlt`O^=FF{*}ee%5sJ`2DIrHhC{lAQ zo5mn7fC;s@Iugn-{9yYd<+7wDHD`HmsKse#<>?UHMhK_(Wo_>aA%&~S6*8^NzJfdJ zKTYSr4Dd=($SdRRwgNU|+bR%3%W~y`V}CM>oSdTWesCUW0SPb)@|h&_=;g>{T;c7A zXx1Ey|NKhS_qbd*e2xTPt`4sB@|Ym~#r9{iKNskbiwcmHFYoLr2N{M$now4b+C;v{-*DjaBRQ9{nW&NZaE5ip?}NtL;?Y>oNSC!6flkkA7vsKO;1QxaDD5R_0#OyUP-F4R50L<< z_6x^>>%d~h;T1b!fY8J7$KYs$bPI$~G6&EG#V8OFrKRAK+K($MNFV&BtAjD?T{`DG zEeb7KG!X+XW>u{(cVHUelqj-gD+?9ZU5pOuW|P~jP8+Oj&AVx5I2w(wv5X#}1?EBvZ>B%i#PTy;OD{EfT7COqF60a&ifq<$Djw z{-d1J@&CCq>CR(?!3T`HsO%NwBa1B9q7s7^CU8Xq|FU!I zseajqR^VtV&`7Xzw#@00H!avOD|rF`0-qC;IR%IOv~&>7+hgt6Z}Qq0j)TDM*KM$Z zzcRQZ2jfe5A+In2SE+FwOkoowGc%@4*9SA4`H(3QA(3;z_D&A+!PJsRQ#k$w z+QO!KW_`W}jx@alMe&d;K(pKt!B-VU9(Kpv>yLTV$2J#%d^634g7GjRuMH3J2}k-M zz-om52quhHwAq~($yTP$YxfM*vB>eBDSS-{<4oZWY{KMgO|hSc@~wRpiVdAGZi3fm z@iEZhyx(>Ja1yO+AmpMJ0dQR2am6EF9+Q3obCU*Y!KqDzUD^B47JU1FAu5-?7S2My zSZ*=ZZK>=9d-%Aod%@rxsKtYb+Cm&?Rb9X4)n~2!O9RUHZ~wlx#&A z=Vt@F6N;wjc{=XS6iI%6d<=8z8np1BG9^Z}a%0oPLwBeBYv9)j{Nu9d{9(`}fjBZI z3z>YQZYdwXzeve9n{TBw&fXjEW3rI9-HR%gZI;zQPvaV{FkUkRo+}!^CN3)MIzH{# zyeVR?kx8y{a;HT9F8fiTBLc>1n)M`*jwFT;RX!CQX(Lh$tkUwizRzxMFVVXlhK4K zZ3l=`2Tc!%N^sF&6${p^7$1lUIT8=m>C!wM{DEbXCm($-l!ibmCNP{u;AEcxJgu`y zG1THW*+UG?Zp42hd%&j-89o}1frQu2WDj0+3udVFoVC-OAsBu;L%ZN&2Yzkh!@xyQP!EX1Y( zNason1=!0@SpuDv{fkJx7Th{i0cj+MTzfd}flDGskJ}oo6Vum>8CXkW6xrYZ)pr6+ zGB+=_PyXAHCr3K9^Pj7haxMXsHD@6xX`>@FIuVh)w5{| zeJDR}?rqgKW91~pTv8U~#({>R^rR9cd98K|+3jvxGKJo!YLO52-v%|2NIya_*RUsK#5|D>| zD{*+S)1>hQ{(HlBymgFjR>G$|lmO5WfT0i}@xicDKPcX`oPg+OWJAn_&mQTw3I14T zfao_yCACAOCApLhL|jH!+ifIof&-telPG@dhDx)ZA!D+Sa6?&_{j_K;->OEXsM)E)FyDVu z>TB931@rGYA=YS30%q^GQ2UG#EI6!zm0c=oW5dak2ZSu)FEfapwvJS>ymYs*mqs7>)le=LY&<8pz z7Dx1XMwQn77Pu~n7Gbh8)TvR*Zql@XkT=U2;4fb6$E%TDV1AO2`M@h_jSfu8{AOB4 zkq?^vI^usQ9DzOvMJc75gHMx=1cnWgCe(HIK@-|Q@L9X}^-*$-ykzq&(mZ$@2cuhE z@%)+zho9buQF4)LhEGeOBF~8G!l9E zwn=}iKK;c9Y?l#p50wE^Cp@Z?#BA9Ho}6J>8Fn32D`$G zW@Uv=XQd0hvQYQzBkLi1It%zqEc42~STNJ|O7<@P9hGp?XA9Q&4th$07G1a)0jir1 z)sy{z4o8fiG6{5fo9S0i(ekavXhI1}y_UthLW0^@=2O|P-vydzw^j1hXg1?}y>X`S zdfbcmH~0e8cRREt1Mv`)C-OomwtfoXxnBn8CJCA65e3>CIQ{FvrN;#D6czKTZSE#+ z;xXzLUE2AmQ&zDX)dlqdc&Np}nJ=lJ7T9zb-cx;7;-2G=is2Odl0Sp?d!Yytwxshl z2P03mYQe@%#z?{rAXeG%oo9K^9{S#}c|Zl!&1!KgXykLta><`_r?7d6W_{w($|EiX zma3|WE*>+U)x)9YqW9)?mZ5%VbG_Tyld~#u=j*L7^KgAk3^Gc}+b_7!Z=04DGP)rSZv6KWc}a7`s> zv=F@M8SW7{S(pJu)E4pwMt75FZ_0w`hOY2yMmHjpTX0?`#)igy{6G93UH*ickr72BhYQ;?2r9G)aiOdKN) zp0RAT_RLMn?d__7b+Tg&!BGb>bO$rXS&deV9rqy_@N@m4RS~<@3A(__J^Xo;KJcHCm z3`*Z*z~w{^fzZ525`jf6%vryAir9}zZoh(stEXc0MOB(BtZP2?7a`W-dZ7l6BC0|t z8zni=WY?Imk7G$0IZye9?|9i4XGz|lg$_uQHV}r7QA`OnR{=Kn#imiU`vP^7;+sIU zV6uPuC+*y*^A6y_=%h=c`iws~d0E%?Uuif7xjpDUM~2~Li4$#*k; z{mi4%bK(jnR~knhRkeA4jB8?9V_23yuz1FplMI!@{@KM#$0)32J_2i!&dp7rX6bJs zaDaVe9_w`x1)Lj(O&l+3XhcI-2)Hu{i+L*hq_$nW`G=eucRqze7m;AZDUbV|>y}S} zQ4eU-F6*dk>}*?Dd3+PBT%t$}1z2&#Up7l5$LP$=;aDLUYY`T08Mv_l3MhR}J8Wzq zlD|cjAhx)3=0i#tJHh>hLXI+8_@|D5FhlYi)+?@hC29 zw{%)Eq0jMs9cWwl{*0dXh2x#aa z>}~MX{^~5%D3s{4k!S1{v(t)MmN57lC&iwQZSiPN$=THo5V+aB9i|Egm28@DM~f}EGV5EY$}F#B=1v&ZMnDfM6lRS5fb-f#yq|Bag}H+beRzcAU_q0 zzU7w%qB*iZa~oD#-ZR?o7^OVHlPvIW&u3jm{_v5q{+2`FP~cj@Z9&Y zPkVq7d$6|Uf$ST>1!jnxw_-a8&Rn4#*|{^ z#GCRz)vKFExUzA{!-w7DvAGOOOV+rQp=aC)`>eiS8wRiitRcOa$W+p|g6p6+6k^Cm zTR?)PU@`m&Pw_J0jmWrV;2cA1Xd-IzVs0dF$i{_uw_pSdz^Ed?xw4r{1V~t4K0F&P zh_a-<*3HmosbPexo{0T@m;5QHQzi>ZfAChjJtx3&oA_MmSksJ0Z8~(}o7>}1Lm7Y6 ziVC<8()9I|v{Q-%jhdTFaea)IgBGN8j28t5vJTvcB)A9?MpK5T@~Wu*gCzAkZ~8Tu zZUhE^tqZeEg5#pt#K6jHHMP=mD_)DRH7%k?8J-9m$qqoprs|H@vT^Jo{LkbSI(zZE zM8%h)aYQ|Hy@&y?5`VmFHy`rwE1qs~_)k}|8=72`4ry5rZqvj7qK3C{NYdqeK>&@Z zN4)T%&{J%Wu-^ire5}Qc&mrz)iV5}x#uKM(|!Msqx#b#p99;MTEBcB?h5Lv-UDJZF+Lr~| zzzLq~dHpGZ^;gB=rPIU3YWIA4@lAHpZ#>%lYJ{|)xU#A;XGaaT-G@yGywpe!i9kV literal 0 HcmV?d00001 diff --git a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py new file mode 100644 index 0000000..3d8aebc --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python3 +"""Freeze Frontier's Qwen30 exact production-trace response surface.""" + +from __future__ import annotations + +import argparse +import csv +import importlib.util +import json +import math +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + + +TARGET_PASS_RATE = 0.95 +TPOT_SLOS_MS = (50.0, 100.0, 150.0, 180.0) +WINDOW_SECONDS = 600.0 +BASE_RUNNER = ( + Path(__file__).resolve().parents[1] + / "frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py" +) + + +def load_base(): + spec = importlib.util.spec_from_file_location("qwen30_prefill_surface_base", BASE_RUNNER) + if spec is None or spec.loader is None: + raise ImportError(BASE_RUNNER) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +BASE = load_base() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--frontier-source", type=Path, required=True) + parser.add_argument("--replayserve-root", type=Path, required=True) + parser.add_argument("--profile-root", type=Path, required=True) + parser.add_argument("--python-deps", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument( + "--trace", + action="append", + required=True, + help="Frozen trace anchor as LABEL=PATH; repeat in increasing load order.", + ) + parser.add_argument("--config", action="append") + parser.add_argument( + "--rate-contract", + choices=("trace-window", "uniform-spacing"), + default="trace-window", + ) + parser.add_argument( + "--prefix-caching", + action=argparse.BooleanOptionalAction, + default=True, + ) + parser.add_argument( + "--cc-backend", choices=("analytical", "vidur"), default="vidur" + ) + parser.add_argument("--allreduce-csv", type=Path) + parser.add_argument("--timeout-seconds", type=float, default=1800.0) + parser.add_argument("--resume", action="store_true") + return parser.parse_args() + + +def ttft_limit_ms(input_tokens: int) -> float: + return 1000.0 + 1000.0 * input_tokens / 8000.0 + + +def percentile(values: list[float], fraction: float) -> float | None: + if not values: + return None + ordered = sorted(values) + return ordered[math.ceil(fraction * len(ordered)) - 1] + + +def parse_trace( + specification: str, *, rate_contract: str = "trace-window" +) -> dict[str, Any]: + if "=" not in specification: + raise ValueError(f"trace must be LABEL=PATH: {specification}") + label, raw_path = specification.split("=", 1) + if not label or "/" in label: + raise ValueError(f"invalid trace label: {label!r}") + path = Path(raw_path).resolve() + if not path.is_file(): + raise FileNotFoundError(path) + with path.open(newline="") as source: + rows = list(csv.DictReader(source)) + if not rows: + raise ValueError(f"empty trace: {path}") + required = { + "arrived_at", + "num_prefill_tokens", + "num_decode_tokens", + "session_id", + "block_hash_ids", + } + if not required.issubset(rows[0]): + raise ValueError(f"trace columns missing: {required - set(rows[0])}") + arrivals = [float(row["arrived_at"]) for row in rows] + if any(not math.isfinite(value) or value < 0 for value in arrivals): + raise ValueError(f"invalid arrival in {path}") + if any(right < left for left, right in zip(arrivals, arrivals[1:])): + raise ValueError(f"arrival order drift in {path}") + if rate_contract == "trace-window": + offered_request_rate = len(rows) / WINDOW_SECONDS + elif rate_contract == "uniform-spacing": + if len(rows) < 2 or arrivals[-1] <= arrivals[0]: + raise ValueError("uniform-spacing traces require at least two arrivals") + intervals = [right - left for left, right in zip(arrivals, arrivals[1:])] + expected_interval = (arrivals[-1] - arrivals[0]) / (len(arrivals) - 1) + if any(abs(value - expected_interval) > 1e-9 for value in intervals): + raise ValueError(f"non-uniform fixed trace: {path}") + offered_request_rate = 1.0 / expected_interval + else: + raise ValueError(f"unknown rate contract: {rate_contract}") + shapes = [ + (int(row["num_prefill_tokens"]), int(row["num_decode_tokens"])) + for row in rows + ] + if any(prefill <= 0 or decode <= 0 or prefill + decode > 40960 for prefill, decode in shapes): + raise ValueError(f"out-of-contract shape in {path}") + return { + "label": label, + "path": path, + "sha256": BASE.sha256(path), + "requests": len(rows), + "offered_request_rate": offered_request_rate, + "first_arrival_s": arrivals[0], + "last_arrival_s": arrivals[-1], + "shapes": shapes, + } + + +def find_request_metrics(run_dir: Path) -> Path: + matches = list((run_dir / "metrics").rglob("request_metrics.csv")) + if len(matches) != 1: + raise RuntimeError(f"expected one request_metrics.csv, got {matches}") + return matches[0] + + +def score(path: Path, expected_shapes: list[tuple[int, int]]) -> dict[str, Any]: + with path.open(newline="") as source: + rows = list(csv.DictReader(source)) + if len(rows) != len(expected_shapes): + raise ValueError(f"request count mismatch: {len(rows)} != {len(expected_shapes)}") + rows.sort(key=lambda row: int(row["Request Id"])) + request_metrics = [] + for index, (row, expected) in enumerate(zip(rows, expected_shapes, strict=True)): + if int(row["Request Id"]) != index: + raise ValueError("Frontier request ID/order drift") + shape = ( + int(float(row["request_num_prefill_tokens"])), + int(float(row["request_num_decode_tokens"])), + ) + if shape != expected: + raise ValueError(f"request shape drift at {index}: {shape} != {expected}") + ttft = float(row["ttft"]) + e2e = float(row["request_e2e_time"]) + tpot = (e2e - ttft) / (shape[1] - 1) if shape[1] > 1 else None + values = [ttft, e2e] + ([] if tpot is None else [tpot]) + if not all(math.isfinite(value) and value >= 0 for value in values): + raise ValueError(f"invalid latency at request {index}") + request_metrics.append( + { + "request_id": index, + "input_tokens": shape[0], + "output_tokens": shape[1], + "ttft_ms": ttft, + "ttft_limit_ms": ttft_limit_ms(shape[0]), + "tpot_ms": tpot, + "e2e_ms": e2e, + } + ) + slos = {} + for limit in TPOT_SLOS_MS: + passed = sum( + row["ttft_ms"] <= row["ttft_limit_ms"] + and (row["tpot_ms"] is None or row["tpot_ms"] <= limit) + for row in request_metrics + ) + pass_rate = passed / len(request_metrics) + slos[f"tpot_{int(limit)}ms"] = { + "passed": passed, + "pass_rate": pass_rate, + "feasible": pass_rate >= TARGET_PASS_RATE, + } + ttfts = [float(row["ttft_ms"]) for row in request_metrics] + tpots = [float(row["tpot_ms"]) for row in request_metrics if row["tpot_ms"] is not None] + return { + "ttft_p50_ms": percentile(ttfts, 0.50), + "ttft_p95_ms": percentile(ttfts, 0.95), + "tpot_p50_ms": percentile(tpots, 0.50), + "tpot_p95_ms": percentile(tpots, 0.95), + "slos": slos, + } + + +def main() -> None: + args = parse_args() + for name in ( + "frontier_source", + "replayserve_root", + "profile_root", + "python_deps", + "output_root", + ): + setattr(args, name, getattr(args, name).resolve()) + if args.allreduce_csv is not None: + args.allreduce_csv = args.allreduce_csv.resolve() + traces = [ + parse_trace(specification, rate_contract=args.rate_contract) + for specification in args.trace + ] + if len({trace["label"] for trace in traces}) != len(traces): + raise ValueError("trace labels must be unique") + if any( + right["offered_request_rate"] <= left["offered_request_rate"] + for left, right in zip(traces, traces[1:]) + ): + raise ValueError("trace anchors must be supplied in increasing load order") + + selected = list(BASE.GRID) + if args.config: + wanted = set(args.config) + selected = [config for config in BASE.GRID if config.name in wanted] + if {config.name for config in selected} != wanted: + raise ValueError(f"unknown configs: {wanted - {config.name for config in selected}}") + paths = BASE.profile_paths(args.profile_root) + coverage = BASE.validate_profile(paths) + builder = BASE.load_module( + "qwen30_exact_trace_frontier_builder", + args.replayserve_root / "tools/run_frontier_sweep.py", + ) + frontier_head = subprocess.run( + ["git", "-C", str(args.frontier_source), "rev-parse", "HEAD"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + environment = os.environ.copy() + pythonpath = [str(args.python_deps), str(args.frontier_source)] + if environment.get("PYTHONPATH"): + pythonpath.append(environment["PYTHONPATH"]) + environment.update( + { + "PYTHONPATH": ":".join(pythonpath), + "CUDA_VISIBLE_DEVICES": "", + "NVIDIA_VISIBLE_DEVICES": "void", + "WANDB_DISABLED": "true", + "VIDUR_DISABLE_WANDB": "1", + "FRONTIER_LOG_LEVEL": "WARNING", + "PYTHONDONTWRITEBYTECODE": "1", + } + ) + + config_results = [] + for config in selected: + loads = [] + config_knobs = BASE.knobs(config, paths, args.output_root / "cache") + config_knobs["enable_prefix_caching"] = args.prefix_caching + for trace in traces: + run_dir = args.output_root / "runs" / config.name / trace["label"] + result_path = run_dir / "result.json" + if args.resume and result_path.is_file(): + loads.append(json.loads(result_path.read_text())) + continue + run_dir.mkdir(parents=True, exist_ok=True) + command = builder.build_frontier_command( + python_bin="/usr/bin/python3", + trace_file=trace["path"], + metrics_root=run_dir / "metrics", + run_id=f"qwen30_trace_{config.name}_{trace['label']}", + knobs=config_knobs, + ) + command = BASE.configure_cc_command( + command, + backend=args.cc_backend, + allreduce_csv=args.allreduce_csv, + cache=args.output_root / "cc-cache", + ) + BASE.write_json(run_dir / "command.json", command) + started = time.time() + with (run_dir / "stdout.log").open("w") as stdout, ( + run_dir / "stderr.log" + ).open("w") as stderr: + completed = subprocess.run( + command, + cwd=args.frontier_source, + env=environment, + stdout=stdout, + stderr=stderr, + timeout=args.timeout_seconds, + check=False, + ) + if completed.returncode != 0: + raise RuntimeError( + f"Frontier failed for {config.name}/{trace['label']}: {completed.returncode}" + ) + metrics = find_request_metrics(run_dir) + result = { + "status": "completed", + "config": {"tp": config.tp, "mns": config.mns, "name": config.name}, + "trace_label": trace["label"], + "offered_request_rate": trace["offered_request_rate"], + "offered_request_rate_per_gpu": trace["offered_request_rate"] / config.tp, + "request_count": trace["requests"], + "elapsed_seconds": time.time() - started, + "trace_sha256": trace["sha256"], + "request_metrics_sha256": BASE.sha256(metrics), + "score": score(metrics, trace["shapes"]), + } + BASE.write_json(result_path, result) + loads.append(result) + print( + json.dumps( + { + "config": config.name, + "trace": trace["label"], + "rate": trace["offered_request_rate"], + "primary": result["score"]["slos"]["tpot_150ms"], + }, + sort_keys=True, + ), + flush=True, + ) + config_results.append( + { + "config": {"tp": config.tp, "mns": config.mns, "name": config.name}, + "loads": loads, + } + ) + + rankings = {} + for slo in (f"tpot_{int(value)}ms" for value in TPOT_SLOS_MS): + records = [] + for item in config_results: + feasible = [ + load["offered_request_rate"] + for load in item["loads"] + if load["score"]["slos"][slo]["feasible"] + ] + capacity = max(feasible, default=None) + records.append( + { + "config": item["config"], + "maximum_tested_feasible_request_rate": capacity, + "maximum_tested_feasible_request_rate_per_gpu": ( + capacity / item["config"]["tp"] if capacity is not None else None + ), + "lower_censored": capacity is None, + "upper_censored": capacity == traces[-1]["offered_request_rate"], + } + ) + records.sort( + key=lambda row: ( + -( + row["maximum_tested_feasible_request_rate_per_gpu"] + if row["maximum_tested_feasible_request_rate_per_gpu"] is not None + else -1 + ), + row["config"]["name"], + ) + ) + rankings[slo] = records + + manifest = { + "schema": "frontier-qwen30-exact-trace-surface-v1", + "status": "frozen_before_real" if selected == list(BASE.GRID) else "partial_not_decision_bearing", + "contract": { + "window_seconds": ( + WINDOW_SECONDS if args.rate_contract == "trace-window" else None + ), + "rate_contract": args.rate_contract, + "prefix_caching": args.prefix_caching, + "arrival": "original_trace_timestamp_and_order", + "input_output": "exact_source_values", + "ttft_slo": "1000ms + 1000ms * input_tokens / 8000", + "tpot_slos_ms": TPOT_SLOS_MS, + "primary_tpot_slo_ms": 150.0, + "target_pass_rate": TARGET_PASS_RATE, + }, + "frontier": { + "source": str(args.frontier_source), + "git_head": frontier_head, + "git_status_short": subprocess.run( + ["git", "-C", str(args.frontier_source), "status", "--short"], + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout, + }, + "profiles": { + "root": str(args.profile_root), + "coverage": coverage, + "sha256": {name: BASE.sha256(path) for name, path in paths.items()}, + }, + "collective": { + "backend": args.cc_backend, + "allreduce_csv": str(args.allreduce_csv) if args.allreduce_csv else None, + "allreduce_csv_sha256": BASE.sha256(args.allreduce_csv) if args.allreduce_csv else None, + }, + "traces": [ + {key: value for key, value in trace.items() if key != "shapes"} + for trace in traces + ], + "config_results": config_results, + "rankings": rankings, + } + BASE.write_json(args.output_root / "frontier_trace_surface_frozen.json", manifest) + print(args.output_root / "frontier_trace_surface_frozen.json") + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py b/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py index 2326466..3f9813f 100644 --- a/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py +++ b/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py @@ -12,6 +12,7 @@ from pathlib import Path ROOT = Path(__file__).parent +REPO_ROOT = ROOT.parents[1] def load(name: str): @@ -85,6 +86,123 @@ class FidelityEnvelopeTest(unittest.TestCase): str(4 + 8192 / 1000), ) + def test_batch_profile_retains_one_single_request_anchor_per_tp(self) -> None: + runner = REPO_ROOT / "runs/frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py" + spec = importlib.util.spec_from_file_location("qwen30_surface_runner", runner) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + profile = ROOT / "profiles/profile-v3-batch-final" + coverage = module.validate_profile(module.profile_paths(profile)) + self.assertEqual( + coverage["attention"], + { + "1": {"exact_prefill_2048_rows": 1, "profile_batch_size": 1}, + "2": {"exact_prefill_2048_rows": 1, "profile_batch_size": 1}, + "4": {"exact_prefill_2048_rows": 1, "profile_batch_size": 1}, + }, + ) + + def test_exact_trace_parser_and_joint_slo_score(self) -> None: + module = load("run_frontier_qwen30_exact_trace_surface.py") + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + trace = root / "trace.csv" + with trace.open("w", newline="") as handle: + writer = csv.DictWriter( + handle, + fieldnames=[ + "arrived_at", + "num_prefill_tokens", + "num_decode_tokens", + "session_id", + "block_hash_ids", + ], + lineterminator="\n", + ) + writer.writeheader() + writer.writerows( + [ + { + "arrived_at": 1.0, + "num_prefill_tokens": 512, + "num_decode_tokens": 1, + "session_id": 1, + "block_hash_ids": "10|11", + }, + { + "arrived_at": 2.0, + "num_prefill_tokens": 2048, + "num_decode_tokens": 2, + "session_id": 2, + "block_hash_ids": "20|21", + }, + ] + ) + parsed = module.parse_trace(f"u0p01={trace}") + self.assertEqual(parsed["requests"], 2) + self.assertEqual(parsed["shapes"], [(512, 1), (2048, 2)]) + uniform = module.parse_trace( + f"r1={trace}", rate_contract="uniform-spacing" + ) + self.assertEqual(uniform["offered_request_rate"], 1.0) + + metrics = root / "request_metrics.csv" + with metrics.open("w", newline="") as handle: + writer = csv.DictWriter( + handle, + fieldnames=[ + "Request Id", + "request_num_prefill_tokens", + "request_num_decode_tokens", + "ttft", + "request_e2e_time", + ], + lineterminator="\n", + ) + writer.writeheader() + writer.writerows( + [ + { + "Request Id": 1, + "request_num_prefill_tokens": 2048, + "request_num_decode_tokens": 2, + "ttft": 1200, + "request_e2e_time": 1300, + }, + { + "Request Id": 0, + "request_num_prefill_tokens": 512, + "request_num_decode_tokens": 1, + "ttft": 1000, + "request_e2e_time": 1000, + }, + ] + ) + scored = module.score(metrics, parsed["shapes"]) + self.assertEqual(scored["slos"]["tpot_150ms"]["passed"], 2) + self.assertTrue(scored["slos"]["tpot_150ms"]["feasible"]) + + def test_fixed_trace_materializer_uses_disjoint_prefixes(self) -> None: + module = load("prepare_fixed_frontier_traces.py") + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + manifest = module.materialize( + root, + input_tokens=32, + output_tokens=4, + requests=2, + rates=[2.0], + block_size=16, + ) + with (root / "r2.csv").open(newline="") as handle: + rows = list(csv.DictReader(handle)) + self.assertEqual(rows[0]["block_hash_ids"], "1|2") + self.assertEqual(rows[1]["block_hash_ids"], "3|4") + self.assertEqual(rows[1]["arrived_at"], "0.500000000000") + self.assertEqual(manifest["contract"]["prefix_caching"], False) + if __name__ == "__main__": unittest.main() diff --git a/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/manifest.json b/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/manifest.json new file mode 100644 index 0000000..f765bce --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/manifest.json @@ -0,0 +1,50 @@ +{ + "anchors": [ + { + "label": "r16", + "path": "r16.csv", + "rate": 16.0, + "sha256": "dd00640d57184b1607fb466d2735bd4edf42e585ac6d6f2d6c6363c76b34aac8" + }, + { + "label": "r32", + "path": "r32.csv", + "rate": 32.0, + "sha256": "de448fa237c367c4910570586f8574a5020c3186a2adc1db5daa26acf0d2b9e1" + }, + { + "label": "r64", + "path": "r64.csv", + "rate": 64.0, + "sha256": "c796965d7afa1a9df4b8ba523412cef2e4dc7ddfc3cef6bce7e91265b20aaf2d" + }, + { + "label": "r128", + "path": "r128.csv", + "rate": 128.0, + "sha256": "30706d59e020ad0dcb670d367201de77ca7337b5fb41da0efd5af8c1915b2f04" + }, + { + "label": "r256", + "path": "r256.csv", + "rate": 256.0, + "sha256": "f93c2390f084c4c60e7ca0d3096c7bdf024b7b286fd7ac72f4dc21d3d2af8cbf" + }, + { + "label": "r512", + "path": "r512.csv", + "rate": 512.0, + "sha256": "e2612362defdb10f876eb478a642ef78a9f6101833b7cc4ff1f0b02b8a453767" + } + ], + "contract": { + "arrival": "open_loop_uniform", + "block_size": 16, + "input_tokens": 512, + "output_tokens": 1, + "prefix_caching": false, + "prefix_relation": "all request block identities are disjoint", + "requests": 64 + }, + "schema": "qwen30-fixed-frontier-traces-v1" +} diff --git a/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r128.csv b/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r128.csv new file mode 100644 index 0000000..aff1039 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r128.csv @@ -0,0 +1,65 @@ +arrived_at,num_prefill_tokens,num_decode_tokens,session_id,block_hash_ids +0.000000000000,512,1,0,1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32 +0.007812500000,512,1,1,33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58|59|60|61|62|63|64 +0.015625000000,512,1,2,65|66|67|68|69|70|71|72|73|74|75|76|77|78|79|80|81|82|83|84|85|86|87|88|89|90|91|92|93|94|95|96 +0.023437500000,512,1,3,97|98|99|100|101|102|103|104|105|106|107|108|109|110|111|112|113|114|115|116|117|118|119|120|121|122|123|124|125|126|127|128 +0.031250000000,512,1,4,129|130|131|132|133|134|135|136|137|138|139|140|141|142|143|144|145|146|147|148|149|150|151|152|153|154|155|156|157|158|159|160 +0.039062500000,512,1,5,161|162|163|164|165|166|167|168|169|170|171|172|173|174|175|176|177|178|179|180|181|182|183|184|185|186|187|188|189|190|191|192 +0.046875000000,512,1,6,193|194|195|196|197|198|199|200|201|202|203|204|205|206|207|208|209|210|211|212|213|214|215|216|217|218|219|220|221|222|223|224 +0.054687500000,512,1,7,225|226|227|228|229|230|231|232|233|234|235|236|237|238|239|240|241|242|243|244|245|246|247|248|249|250|251|252|253|254|255|256 +0.062500000000,512,1,8,257|258|259|260|261|262|263|264|265|266|267|268|269|270|271|272|273|274|275|276|277|278|279|280|281|282|283|284|285|286|287|288 +0.070312500000,512,1,9,289|290|291|292|293|294|295|296|297|298|299|300|301|302|303|304|305|306|307|308|309|310|311|312|313|314|315|316|317|318|319|320 +0.078125000000,512,1,10,321|322|323|324|325|326|327|328|329|330|331|332|333|334|335|336|337|338|339|340|341|342|343|344|345|346|347|348|349|350|351|352 +0.085937500000,512,1,11,353|354|355|356|357|358|359|360|361|362|363|364|365|366|367|368|369|370|371|372|373|374|375|376|377|378|379|380|381|382|383|384 +0.093750000000,512,1,12,385|386|387|388|389|390|391|392|393|394|395|396|397|398|399|400|401|402|403|404|405|406|407|408|409|410|411|412|413|414|415|416 +0.101562500000,512,1,13,417|418|419|420|421|422|423|424|425|426|427|428|429|430|431|432|433|434|435|436|437|438|439|440|441|442|443|444|445|446|447|448 +0.109375000000,512,1,14,449|450|451|452|453|454|455|456|457|458|459|460|461|462|463|464|465|466|467|468|469|470|471|472|473|474|475|476|477|478|479|480 +0.117187500000,512,1,15,481|482|483|484|485|486|487|488|489|490|491|492|493|494|495|496|497|498|499|500|501|502|503|504|505|506|507|508|509|510|511|512 +0.125000000000,512,1,16,513|514|515|516|517|518|519|520|521|522|523|524|525|526|527|528|529|530|531|532|533|534|535|536|537|538|539|540|541|542|543|544 +0.132812500000,512,1,17,545|546|547|548|549|550|551|552|553|554|555|556|557|558|559|560|561|562|563|564|565|566|567|568|569|570|571|572|573|574|575|576 +0.140625000000,512,1,18,577|578|579|580|581|582|583|584|585|586|587|588|589|590|591|592|593|594|595|596|597|598|599|600|601|602|603|604|605|606|607|608 +0.148437500000,512,1,19,609|610|611|612|613|614|615|616|617|618|619|620|621|622|623|624|625|626|627|628|629|630|631|632|633|634|635|636|637|638|639|640 +0.156250000000,512,1,20,641|642|643|644|645|646|647|648|649|650|651|652|653|654|655|656|657|658|659|660|661|662|663|664|665|666|667|668|669|670|671|672 +0.164062500000,512,1,21,673|674|675|676|677|678|679|680|681|682|683|684|685|686|687|688|689|690|691|692|693|694|695|696|697|698|699|700|701|702|703|704 +0.171875000000,512,1,22,705|706|707|708|709|710|711|712|713|714|715|716|717|718|719|720|721|722|723|724|725|726|727|728|729|730|731|732|733|734|735|736 +0.179687500000,512,1,23,737|738|739|740|741|742|743|744|745|746|747|748|749|750|751|752|753|754|755|756|757|758|759|760|761|762|763|764|765|766|767|768 +0.187500000000,512,1,24,769|770|771|772|773|774|775|776|777|778|779|780|781|782|783|784|785|786|787|788|789|790|791|792|793|794|795|796|797|798|799|800 +0.195312500000,512,1,25,801|802|803|804|805|806|807|808|809|810|811|812|813|814|815|816|817|818|819|820|821|822|823|824|825|826|827|828|829|830|831|832 +0.203125000000,512,1,26,833|834|835|836|837|838|839|840|841|842|843|844|845|846|847|848|849|850|851|852|853|854|855|856|857|858|859|860|861|862|863|864 +0.210937500000,512,1,27,865|866|867|868|869|870|871|872|873|874|875|876|877|878|879|880|881|882|883|884|885|886|887|888|889|890|891|892|893|894|895|896 +0.218750000000,512,1,28,897|898|899|900|901|902|903|904|905|906|907|908|909|910|911|912|913|914|915|916|917|918|919|920|921|922|923|924|925|926|927|928 +0.226562500000,512,1,29,929|930|931|932|933|934|935|936|937|938|939|940|941|942|943|944|945|946|947|948|949|950|951|952|953|954|955|956|957|958|959|960 +0.234375000000,512,1,30,961|962|963|964|965|966|967|968|969|970|971|972|973|974|975|976|977|978|979|980|981|982|983|984|985|986|987|988|989|990|991|992 +0.242187500000,512,1,31,993|994|995|996|997|998|999|1000|1001|1002|1003|1004|1005|1006|1007|1008|1009|1010|1011|1012|1013|1014|1015|1016|1017|1018|1019|1020|1021|1022|1023|1024 +0.250000000000,512,1,32,1025|1026|1027|1028|1029|1030|1031|1032|1033|1034|1035|1036|1037|1038|1039|1040|1041|1042|1043|1044|1045|1046|1047|1048|1049|1050|1051|1052|1053|1054|1055|1056 +0.257812500000,512,1,33,1057|1058|1059|1060|1061|1062|1063|1064|1065|1066|1067|1068|1069|1070|1071|1072|1073|1074|1075|1076|1077|1078|1079|1080|1081|1082|1083|1084|1085|1086|1087|1088 +0.265625000000,512,1,34,1089|1090|1091|1092|1093|1094|1095|1096|1097|1098|1099|1100|1101|1102|1103|1104|1105|1106|1107|1108|1109|1110|1111|1112|1113|1114|1115|1116|1117|1118|1119|1120 +0.273437500000,512,1,35,1121|1122|1123|1124|1125|1126|1127|1128|1129|1130|1131|1132|1133|1134|1135|1136|1137|1138|1139|1140|1141|1142|1143|1144|1145|1146|1147|1148|1149|1150|1151|1152 +0.281250000000,512,1,36,1153|1154|1155|1156|1157|1158|1159|1160|1161|1162|1163|1164|1165|1166|1167|1168|1169|1170|1171|1172|1173|1174|1175|1176|1177|1178|1179|1180|1181|1182|1183|1184 +0.289062500000,512,1,37,1185|1186|1187|1188|1189|1190|1191|1192|1193|1194|1195|1196|1197|1198|1199|1200|1201|1202|1203|1204|1205|1206|1207|1208|1209|1210|1211|1212|1213|1214|1215|1216 +0.296875000000,512,1,38,1217|1218|1219|1220|1221|1222|1223|1224|1225|1226|1227|1228|1229|1230|1231|1232|1233|1234|1235|1236|1237|1238|1239|1240|1241|1242|1243|1244|1245|1246|1247|1248 +0.304687500000,512,1,39,1249|1250|1251|1252|1253|1254|1255|1256|1257|1258|1259|1260|1261|1262|1263|1264|1265|1266|1267|1268|1269|1270|1271|1272|1273|1274|1275|1276|1277|1278|1279|1280 +0.312500000000,512,1,40,1281|1282|1283|1284|1285|1286|1287|1288|1289|1290|1291|1292|1293|1294|1295|1296|1297|1298|1299|1300|1301|1302|1303|1304|1305|1306|1307|1308|1309|1310|1311|1312 +0.320312500000,512,1,41,1313|1314|1315|1316|1317|1318|1319|1320|1321|1322|1323|1324|1325|1326|1327|1328|1329|1330|1331|1332|1333|1334|1335|1336|1337|1338|1339|1340|1341|1342|1343|1344 +0.328125000000,512,1,42,1345|1346|1347|1348|1349|1350|1351|1352|1353|1354|1355|1356|1357|1358|1359|1360|1361|1362|1363|1364|1365|1366|1367|1368|1369|1370|1371|1372|1373|1374|1375|1376 +0.335937500000,512,1,43,1377|1378|1379|1380|1381|1382|1383|1384|1385|1386|1387|1388|1389|1390|1391|1392|1393|1394|1395|1396|1397|1398|1399|1400|1401|1402|1403|1404|1405|1406|1407|1408 +0.343750000000,512,1,44,1409|1410|1411|1412|1413|1414|1415|1416|1417|1418|1419|1420|1421|1422|1423|1424|1425|1426|1427|1428|1429|1430|1431|1432|1433|1434|1435|1436|1437|1438|1439|1440 +0.351562500000,512,1,45,1441|1442|1443|1444|1445|1446|1447|1448|1449|1450|1451|1452|1453|1454|1455|1456|1457|1458|1459|1460|1461|1462|1463|1464|1465|1466|1467|1468|1469|1470|1471|1472 +0.359375000000,512,1,46,1473|1474|1475|1476|1477|1478|1479|1480|1481|1482|1483|1484|1485|1486|1487|1488|1489|1490|1491|1492|1493|1494|1495|1496|1497|1498|1499|1500|1501|1502|1503|1504 +0.367187500000,512,1,47,1505|1506|1507|1508|1509|1510|1511|1512|1513|1514|1515|1516|1517|1518|1519|1520|1521|1522|1523|1524|1525|1526|1527|1528|1529|1530|1531|1532|1533|1534|1535|1536 +0.375000000000,512,1,48,1537|1538|1539|1540|1541|1542|1543|1544|1545|1546|1547|1548|1549|1550|1551|1552|1553|1554|1555|1556|1557|1558|1559|1560|1561|1562|1563|1564|1565|1566|1567|1568 +0.382812500000,512,1,49,1569|1570|1571|1572|1573|1574|1575|1576|1577|1578|1579|1580|1581|1582|1583|1584|1585|1586|1587|1588|1589|1590|1591|1592|1593|1594|1595|1596|1597|1598|1599|1600 +0.390625000000,512,1,50,1601|1602|1603|1604|1605|1606|1607|1608|1609|1610|1611|1612|1613|1614|1615|1616|1617|1618|1619|1620|1621|1622|1623|1624|1625|1626|1627|1628|1629|1630|1631|1632 +0.398437500000,512,1,51,1633|1634|1635|1636|1637|1638|1639|1640|1641|1642|1643|1644|1645|1646|1647|1648|1649|1650|1651|1652|1653|1654|1655|1656|1657|1658|1659|1660|1661|1662|1663|1664 +0.406250000000,512,1,52,1665|1666|1667|1668|1669|1670|1671|1672|1673|1674|1675|1676|1677|1678|1679|1680|1681|1682|1683|1684|1685|1686|1687|1688|1689|1690|1691|1692|1693|1694|1695|1696 +0.414062500000,512,1,53,1697|1698|1699|1700|1701|1702|1703|1704|1705|1706|1707|1708|1709|1710|1711|1712|1713|1714|1715|1716|1717|1718|1719|1720|1721|1722|1723|1724|1725|1726|1727|1728 +0.421875000000,512,1,54,1729|1730|1731|1732|1733|1734|1735|1736|1737|1738|1739|1740|1741|1742|1743|1744|1745|1746|1747|1748|1749|1750|1751|1752|1753|1754|1755|1756|1757|1758|1759|1760 +0.429687500000,512,1,55,1761|1762|1763|1764|1765|1766|1767|1768|1769|1770|1771|1772|1773|1774|1775|1776|1777|1778|1779|1780|1781|1782|1783|1784|1785|1786|1787|1788|1789|1790|1791|1792 +0.437500000000,512,1,56,1793|1794|1795|1796|1797|1798|1799|1800|1801|1802|1803|1804|1805|1806|1807|1808|1809|1810|1811|1812|1813|1814|1815|1816|1817|1818|1819|1820|1821|1822|1823|1824 +0.445312500000,512,1,57,1825|1826|1827|1828|1829|1830|1831|1832|1833|1834|1835|1836|1837|1838|1839|1840|1841|1842|1843|1844|1845|1846|1847|1848|1849|1850|1851|1852|1853|1854|1855|1856 +0.453125000000,512,1,58,1857|1858|1859|1860|1861|1862|1863|1864|1865|1866|1867|1868|1869|1870|1871|1872|1873|1874|1875|1876|1877|1878|1879|1880|1881|1882|1883|1884|1885|1886|1887|1888 +0.460937500000,512,1,59,1889|1890|1891|1892|1893|1894|1895|1896|1897|1898|1899|1900|1901|1902|1903|1904|1905|1906|1907|1908|1909|1910|1911|1912|1913|1914|1915|1916|1917|1918|1919|1920 +0.468750000000,512,1,60,1921|1922|1923|1924|1925|1926|1927|1928|1929|1930|1931|1932|1933|1934|1935|1936|1937|1938|1939|1940|1941|1942|1943|1944|1945|1946|1947|1948|1949|1950|1951|1952 +0.476562500000,512,1,61,1953|1954|1955|1956|1957|1958|1959|1960|1961|1962|1963|1964|1965|1966|1967|1968|1969|1970|1971|1972|1973|1974|1975|1976|1977|1978|1979|1980|1981|1982|1983|1984 +0.484375000000,512,1,62,1985|1986|1987|1988|1989|1990|1991|1992|1993|1994|1995|1996|1997|1998|1999|2000|2001|2002|2003|2004|2005|2006|2007|2008|2009|2010|2011|2012|2013|2014|2015|2016 +0.492187500000,512,1,63,2017|2018|2019|2020|2021|2022|2023|2024|2025|2026|2027|2028|2029|2030|2031|2032|2033|2034|2035|2036|2037|2038|2039|2040|2041|2042|2043|2044|2045|2046|2047|2048 diff --git a/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r16.csv b/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r16.csv new file mode 100644 index 0000000..6123705 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r16.csv @@ -0,0 +1,65 @@ +arrived_at,num_prefill_tokens,num_decode_tokens,session_id,block_hash_ids +0.000000000000,512,1,0,1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32 +0.062500000000,512,1,1,33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58|59|60|61|62|63|64 +0.125000000000,512,1,2,65|66|67|68|69|70|71|72|73|74|75|76|77|78|79|80|81|82|83|84|85|86|87|88|89|90|91|92|93|94|95|96 +0.187500000000,512,1,3,97|98|99|100|101|102|103|104|105|106|107|108|109|110|111|112|113|114|115|116|117|118|119|120|121|122|123|124|125|126|127|128 +0.250000000000,512,1,4,129|130|131|132|133|134|135|136|137|138|139|140|141|142|143|144|145|146|147|148|149|150|151|152|153|154|155|156|157|158|159|160 +0.312500000000,512,1,5,161|162|163|164|165|166|167|168|169|170|171|172|173|174|175|176|177|178|179|180|181|182|183|184|185|186|187|188|189|190|191|192 +0.375000000000,512,1,6,193|194|195|196|197|198|199|200|201|202|203|204|205|206|207|208|209|210|211|212|213|214|215|216|217|218|219|220|221|222|223|224 +0.437500000000,512,1,7,225|226|227|228|229|230|231|232|233|234|235|236|237|238|239|240|241|242|243|244|245|246|247|248|249|250|251|252|253|254|255|256 +0.500000000000,512,1,8,257|258|259|260|261|262|263|264|265|266|267|268|269|270|271|272|273|274|275|276|277|278|279|280|281|282|283|284|285|286|287|288 +0.562500000000,512,1,9,289|290|291|292|293|294|295|296|297|298|299|300|301|302|303|304|305|306|307|308|309|310|311|312|313|314|315|316|317|318|319|320 +0.625000000000,512,1,10,321|322|323|324|325|326|327|328|329|330|331|332|333|334|335|336|337|338|339|340|341|342|343|344|345|346|347|348|349|350|351|352 +0.687500000000,512,1,11,353|354|355|356|357|358|359|360|361|362|363|364|365|366|367|368|369|370|371|372|373|374|375|376|377|378|379|380|381|382|383|384 +0.750000000000,512,1,12,385|386|387|388|389|390|391|392|393|394|395|396|397|398|399|400|401|402|403|404|405|406|407|408|409|410|411|412|413|414|415|416 +0.812500000000,512,1,13,417|418|419|420|421|422|423|424|425|426|427|428|429|430|431|432|433|434|435|436|437|438|439|440|441|442|443|444|445|446|447|448 +0.875000000000,512,1,14,449|450|451|452|453|454|455|456|457|458|459|460|461|462|463|464|465|466|467|468|469|470|471|472|473|474|475|476|477|478|479|480 +0.937500000000,512,1,15,481|482|483|484|485|486|487|488|489|490|491|492|493|494|495|496|497|498|499|500|501|502|503|504|505|506|507|508|509|510|511|512 +1.000000000000,512,1,16,513|514|515|516|517|518|519|520|521|522|523|524|525|526|527|528|529|530|531|532|533|534|535|536|537|538|539|540|541|542|543|544 +1.062500000000,512,1,17,545|546|547|548|549|550|551|552|553|554|555|556|557|558|559|560|561|562|563|564|565|566|567|568|569|570|571|572|573|574|575|576 +1.125000000000,512,1,18,577|578|579|580|581|582|583|584|585|586|587|588|589|590|591|592|593|594|595|596|597|598|599|600|601|602|603|604|605|606|607|608 +1.187500000000,512,1,19,609|610|611|612|613|614|615|616|617|618|619|620|621|622|623|624|625|626|627|628|629|630|631|632|633|634|635|636|637|638|639|640 +1.250000000000,512,1,20,641|642|643|644|645|646|647|648|649|650|651|652|653|654|655|656|657|658|659|660|661|662|663|664|665|666|667|668|669|670|671|672 +1.312500000000,512,1,21,673|674|675|676|677|678|679|680|681|682|683|684|685|686|687|688|689|690|691|692|693|694|695|696|697|698|699|700|701|702|703|704 +1.375000000000,512,1,22,705|706|707|708|709|710|711|712|713|714|715|716|717|718|719|720|721|722|723|724|725|726|727|728|729|730|731|732|733|734|735|736 +1.437500000000,512,1,23,737|738|739|740|741|742|743|744|745|746|747|748|749|750|751|752|753|754|755|756|757|758|759|760|761|762|763|764|765|766|767|768 +1.500000000000,512,1,24,769|770|771|772|773|774|775|776|777|778|779|780|781|782|783|784|785|786|787|788|789|790|791|792|793|794|795|796|797|798|799|800 +1.562500000000,512,1,25,801|802|803|804|805|806|807|808|809|810|811|812|813|814|815|816|817|818|819|820|821|822|823|824|825|826|827|828|829|830|831|832 +1.625000000000,512,1,26,833|834|835|836|837|838|839|840|841|842|843|844|845|846|847|848|849|850|851|852|853|854|855|856|857|858|859|860|861|862|863|864 +1.687500000000,512,1,27,865|866|867|868|869|870|871|872|873|874|875|876|877|878|879|880|881|882|883|884|885|886|887|888|889|890|891|892|893|894|895|896 +1.750000000000,512,1,28,897|898|899|900|901|902|903|904|905|906|907|908|909|910|911|912|913|914|915|916|917|918|919|920|921|922|923|924|925|926|927|928 +1.812500000000,512,1,29,929|930|931|932|933|934|935|936|937|938|939|940|941|942|943|944|945|946|947|948|949|950|951|952|953|954|955|956|957|958|959|960 +1.875000000000,512,1,30,961|962|963|964|965|966|967|968|969|970|971|972|973|974|975|976|977|978|979|980|981|982|983|984|985|986|987|988|989|990|991|992 +1.937500000000,512,1,31,993|994|995|996|997|998|999|1000|1001|1002|1003|1004|1005|1006|1007|1008|1009|1010|1011|1012|1013|1014|1015|1016|1017|1018|1019|1020|1021|1022|1023|1024 +2.000000000000,512,1,32,1025|1026|1027|1028|1029|1030|1031|1032|1033|1034|1035|1036|1037|1038|1039|1040|1041|1042|1043|1044|1045|1046|1047|1048|1049|1050|1051|1052|1053|1054|1055|1056 +2.062500000000,512,1,33,1057|1058|1059|1060|1061|1062|1063|1064|1065|1066|1067|1068|1069|1070|1071|1072|1073|1074|1075|1076|1077|1078|1079|1080|1081|1082|1083|1084|1085|1086|1087|1088 +2.125000000000,512,1,34,1089|1090|1091|1092|1093|1094|1095|1096|1097|1098|1099|1100|1101|1102|1103|1104|1105|1106|1107|1108|1109|1110|1111|1112|1113|1114|1115|1116|1117|1118|1119|1120 +2.187500000000,512,1,35,1121|1122|1123|1124|1125|1126|1127|1128|1129|1130|1131|1132|1133|1134|1135|1136|1137|1138|1139|1140|1141|1142|1143|1144|1145|1146|1147|1148|1149|1150|1151|1152 +2.250000000000,512,1,36,1153|1154|1155|1156|1157|1158|1159|1160|1161|1162|1163|1164|1165|1166|1167|1168|1169|1170|1171|1172|1173|1174|1175|1176|1177|1178|1179|1180|1181|1182|1183|1184 +2.312500000000,512,1,37,1185|1186|1187|1188|1189|1190|1191|1192|1193|1194|1195|1196|1197|1198|1199|1200|1201|1202|1203|1204|1205|1206|1207|1208|1209|1210|1211|1212|1213|1214|1215|1216 +2.375000000000,512,1,38,1217|1218|1219|1220|1221|1222|1223|1224|1225|1226|1227|1228|1229|1230|1231|1232|1233|1234|1235|1236|1237|1238|1239|1240|1241|1242|1243|1244|1245|1246|1247|1248 +2.437500000000,512,1,39,1249|1250|1251|1252|1253|1254|1255|1256|1257|1258|1259|1260|1261|1262|1263|1264|1265|1266|1267|1268|1269|1270|1271|1272|1273|1274|1275|1276|1277|1278|1279|1280 +2.500000000000,512,1,40,1281|1282|1283|1284|1285|1286|1287|1288|1289|1290|1291|1292|1293|1294|1295|1296|1297|1298|1299|1300|1301|1302|1303|1304|1305|1306|1307|1308|1309|1310|1311|1312 +2.562500000000,512,1,41,1313|1314|1315|1316|1317|1318|1319|1320|1321|1322|1323|1324|1325|1326|1327|1328|1329|1330|1331|1332|1333|1334|1335|1336|1337|1338|1339|1340|1341|1342|1343|1344 +2.625000000000,512,1,42,1345|1346|1347|1348|1349|1350|1351|1352|1353|1354|1355|1356|1357|1358|1359|1360|1361|1362|1363|1364|1365|1366|1367|1368|1369|1370|1371|1372|1373|1374|1375|1376 +2.687500000000,512,1,43,1377|1378|1379|1380|1381|1382|1383|1384|1385|1386|1387|1388|1389|1390|1391|1392|1393|1394|1395|1396|1397|1398|1399|1400|1401|1402|1403|1404|1405|1406|1407|1408 +2.750000000000,512,1,44,1409|1410|1411|1412|1413|1414|1415|1416|1417|1418|1419|1420|1421|1422|1423|1424|1425|1426|1427|1428|1429|1430|1431|1432|1433|1434|1435|1436|1437|1438|1439|1440 +2.812500000000,512,1,45,1441|1442|1443|1444|1445|1446|1447|1448|1449|1450|1451|1452|1453|1454|1455|1456|1457|1458|1459|1460|1461|1462|1463|1464|1465|1466|1467|1468|1469|1470|1471|1472 +2.875000000000,512,1,46,1473|1474|1475|1476|1477|1478|1479|1480|1481|1482|1483|1484|1485|1486|1487|1488|1489|1490|1491|1492|1493|1494|1495|1496|1497|1498|1499|1500|1501|1502|1503|1504 +2.937500000000,512,1,47,1505|1506|1507|1508|1509|1510|1511|1512|1513|1514|1515|1516|1517|1518|1519|1520|1521|1522|1523|1524|1525|1526|1527|1528|1529|1530|1531|1532|1533|1534|1535|1536 +3.000000000000,512,1,48,1537|1538|1539|1540|1541|1542|1543|1544|1545|1546|1547|1548|1549|1550|1551|1552|1553|1554|1555|1556|1557|1558|1559|1560|1561|1562|1563|1564|1565|1566|1567|1568 +3.062500000000,512,1,49,1569|1570|1571|1572|1573|1574|1575|1576|1577|1578|1579|1580|1581|1582|1583|1584|1585|1586|1587|1588|1589|1590|1591|1592|1593|1594|1595|1596|1597|1598|1599|1600 +3.125000000000,512,1,50,1601|1602|1603|1604|1605|1606|1607|1608|1609|1610|1611|1612|1613|1614|1615|1616|1617|1618|1619|1620|1621|1622|1623|1624|1625|1626|1627|1628|1629|1630|1631|1632 +3.187500000000,512,1,51,1633|1634|1635|1636|1637|1638|1639|1640|1641|1642|1643|1644|1645|1646|1647|1648|1649|1650|1651|1652|1653|1654|1655|1656|1657|1658|1659|1660|1661|1662|1663|1664 +3.250000000000,512,1,52,1665|1666|1667|1668|1669|1670|1671|1672|1673|1674|1675|1676|1677|1678|1679|1680|1681|1682|1683|1684|1685|1686|1687|1688|1689|1690|1691|1692|1693|1694|1695|1696 +3.312500000000,512,1,53,1697|1698|1699|1700|1701|1702|1703|1704|1705|1706|1707|1708|1709|1710|1711|1712|1713|1714|1715|1716|1717|1718|1719|1720|1721|1722|1723|1724|1725|1726|1727|1728 +3.375000000000,512,1,54,1729|1730|1731|1732|1733|1734|1735|1736|1737|1738|1739|1740|1741|1742|1743|1744|1745|1746|1747|1748|1749|1750|1751|1752|1753|1754|1755|1756|1757|1758|1759|1760 +3.437500000000,512,1,55,1761|1762|1763|1764|1765|1766|1767|1768|1769|1770|1771|1772|1773|1774|1775|1776|1777|1778|1779|1780|1781|1782|1783|1784|1785|1786|1787|1788|1789|1790|1791|1792 +3.500000000000,512,1,56,1793|1794|1795|1796|1797|1798|1799|1800|1801|1802|1803|1804|1805|1806|1807|1808|1809|1810|1811|1812|1813|1814|1815|1816|1817|1818|1819|1820|1821|1822|1823|1824 +3.562500000000,512,1,57,1825|1826|1827|1828|1829|1830|1831|1832|1833|1834|1835|1836|1837|1838|1839|1840|1841|1842|1843|1844|1845|1846|1847|1848|1849|1850|1851|1852|1853|1854|1855|1856 +3.625000000000,512,1,58,1857|1858|1859|1860|1861|1862|1863|1864|1865|1866|1867|1868|1869|1870|1871|1872|1873|1874|1875|1876|1877|1878|1879|1880|1881|1882|1883|1884|1885|1886|1887|1888 +3.687500000000,512,1,59,1889|1890|1891|1892|1893|1894|1895|1896|1897|1898|1899|1900|1901|1902|1903|1904|1905|1906|1907|1908|1909|1910|1911|1912|1913|1914|1915|1916|1917|1918|1919|1920 +3.750000000000,512,1,60,1921|1922|1923|1924|1925|1926|1927|1928|1929|1930|1931|1932|1933|1934|1935|1936|1937|1938|1939|1940|1941|1942|1943|1944|1945|1946|1947|1948|1949|1950|1951|1952 +3.812500000000,512,1,61,1953|1954|1955|1956|1957|1958|1959|1960|1961|1962|1963|1964|1965|1966|1967|1968|1969|1970|1971|1972|1973|1974|1975|1976|1977|1978|1979|1980|1981|1982|1983|1984 +3.875000000000,512,1,62,1985|1986|1987|1988|1989|1990|1991|1992|1993|1994|1995|1996|1997|1998|1999|2000|2001|2002|2003|2004|2005|2006|2007|2008|2009|2010|2011|2012|2013|2014|2015|2016 +3.937500000000,512,1,63,2017|2018|2019|2020|2021|2022|2023|2024|2025|2026|2027|2028|2029|2030|2031|2032|2033|2034|2035|2036|2037|2038|2039|2040|2041|2042|2043|2044|2045|2046|2047|2048 diff --git a/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r256.csv b/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r256.csv new file mode 100644 index 0000000..c0b0a56 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r256.csv @@ -0,0 +1,65 @@ +arrived_at,num_prefill_tokens,num_decode_tokens,session_id,block_hash_ids +0.000000000000,512,1,0,1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32 +0.003906250000,512,1,1,33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58|59|60|61|62|63|64 +0.007812500000,512,1,2,65|66|67|68|69|70|71|72|73|74|75|76|77|78|79|80|81|82|83|84|85|86|87|88|89|90|91|92|93|94|95|96 +0.011718750000,512,1,3,97|98|99|100|101|102|103|104|105|106|107|108|109|110|111|112|113|114|115|116|117|118|119|120|121|122|123|124|125|126|127|128 +0.015625000000,512,1,4,129|130|131|132|133|134|135|136|137|138|139|140|141|142|143|144|145|146|147|148|149|150|151|152|153|154|155|156|157|158|159|160 +0.019531250000,512,1,5,161|162|163|164|165|166|167|168|169|170|171|172|173|174|175|176|177|178|179|180|181|182|183|184|185|186|187|188|189|190|191|192 +0.023437500000,512,1,6,193|194|195|196|197|198|199|200|201|202|203|204|205|206|207|208|209|210|211|212|213|214|215|216|217|218|219|220|221|222|223|224 +0.027343750000,512,1,7,225|226|227|228|229|230|231|232|233|234|235|236|237|238|239|240|241|242|243|244|245|246|247|248|249|250|251|252|253|254|255|256 +0.031250000000,512,1,8,257|258|259|260|261|262|263|264|265|266|267|268|269|270|271|272|273|274|275|276|277|278|279|280|281|282|283|284|285|286|287|288 +0.035156250000,512,1,9,289|290|291|292|293|294|295|296|297|298|299|300|301|302|303|304|305|306|307|308|309|310|311|312|313|314|315|316|317|318|319|320 +0.039062500000,512,1,10,321|322|323|324|325|326|327|328|329|330|331|332|333|334|335|336|337|338|339|340|341|342|343|344|345|346|347|348|349|350|351|352 +0.042968750000,512,1,11,353|354|355|356|357|358|359|360|361|362|363|364|365|366|367|368|369|370|371|372|373|374|375|376|377|378|379|380|381|382|383|384 +0.046875000000,512,1,12,385|386|387|388|389|390|391|392|393|394|395|396|397|398|399|400|401|402|403|404|405|406|407|408|409|410|411|412|413|414|415|416 +0.050781250000,512,1,13,417|418|419|420|421|422|423|424|425|426|427|428|429|430|431|432|433|434|435|436|437|438|439|440|441|442|443|444|445|446|447|448 +0.054687500000,512,1,14,449|450|451|452|453|454|455|456|457|458|459|460|461|462|463|464|465|466|467|468|469|470|471|472|473|474|475|476|477|478|479|480 +0.058593750000,512,1,15,481|482|483|484|485|486|487|488|489|490|491|492|493|494|495|496|497|498|499|500|501|502|503|504|505|506|507|508|509|510|511|512 +0.062500000000,512,1,16,513|514|515|516|517|518|519|520|521|522|523|524|525|526|527|528|529|530|531|532|533|534|535|536|537|538|539|540|541|542|543|544 +0.066406250000,512,1,17,545|546|547|548|549|550|551|552|553|554|555|556|557|558|559|560|561|562|563|564|565|566|567|568|569|570|571|572|573|574|575|576 +0.070312500000,512,1,18,577|578|579|580|581|582|583|584|585|586|587|588|589|590|591|592|593|594|595|596|597|598|599|600|601|602|603|604|605|606|607|608 +0.074218750000,512,1,19,609|610|611|612|613|614|615|616|617|618|619|620|621|622|623|624|625|626|627|628|629|630|631|632|633|634|635|636|637|638|639|640 +0.078125000000,512,1,20,641|642|643|644|645|646|647|648|649|650|651|652|653|654|655|656|657|658|659|660|661|662|663|664|665|666|667|668|669|670|671|672 +0.082031250000,512,1,21,673|674|675|676|677|678|679|680|681|682|683|684|685|686|687|688|689|690|691|692|693|694|695|696|697|698|699|700|701|702|703|704 +0.085937500000,512,1,22,705|706|707|708|709|710|711|712|713|714|715|716|717|718|719|720|721|722|723|724|725|726|727|728|729|730|731|732|733|734|735|736 +0.089843750000,512,1,23,737|738|739|740|741|742|743|744|745|746|747|748|749|750|751|752|753|754|755|756|757|758|759|760|761|762|763|764|765|766|767|768 +0.093750000000,512,1,24,769|770|771|772|773|774|775|776|777|778|779|780|781|782|783|784|785|786|787|788|789|790|791|792|793|794|795|796|797|798|799|800 +0.097656250000,512,1,25,801|802|803|804|805|806|807|808|809|810|811|812|813|814|815|816|817|818|819|820|821|822|823|824|825|826|827|828|829|830|831|832 +0.101562500000,512,1,26,833|834|835|836|837|838|839|840|841|842|843|844|845|846|847|848|849|850|851|852|853|854|855|856|857|858|859|860|861|862|863|864 +0.105468750000,512,1,27,865|866|867|868|869|870|871|872|873|874|875|876|877|878|879|880|881|882|883|884|885|886|887|888|889|890|891|892|893|894|895|896 +0.109375000000,512,1,28,897|898|899|900|901|902|903|904|905|906|907|908|909|910|911|912|913|914|915|916|917|918|919|920|921|922|923|924|925|926|927|928 +0.113281250000,512,1,29,929|930|931|932|933|934|935|936|937|938|939|940|941|942|943|944|945|946|947|948|949|950|951|952|953|954|955|956|957|958|959|960 +0.117187500000,512,1,30,961|962|963|964|965|966|967|968|969|970|971|972|973|974|975|976|977|978|979|980|981|982|983|984|985|986|987|988|989|990|991|992 +0.121093750000,512,1,31,993|994|995|996|997|998|999|1000|1001|1002|1003|1004|1005|1006|1007|1008|1009|1010|1011|1012|1013|1014|1015|1016|1017|1018|1019|1020|1021|1022|1023|1024 +0.125000000000,512,1,32,1025|1026|1027|1028|1029|1030|1031|1032|1033|1034|1035|1036|1037|1038|1039|1040|1041|1042|1043|1044|1045|1046|1047|1048|1049|1050|1051|1052|1053|1054|1055|1056 +0.128906250000,512,1,33,1057|1058|1059|1060|1061|1062|1063|1064|1065|1066|1067|1068|1069|1070|1071|1072|1073|1074|1075|1076|1077|1078|1079|1080|1081|1082|1083|1084|1085|1086|1087|1088 +0.132812500000,512,1,34,1089|1090|1091|1092|1093|1094|1095|1096|1097|1098|1099|1100|1101|1102|1103|1104|1105|1106|1107|1108|1109|1110|1111|1112|1113|1114|1115|1116|1117|1118|1119|1120 +0.136718750000,512,1,35,1121|1122|1123|1124|1125|1126|1127|1128|1129|1130|1131|1132|1133|1134|1135|1136|1137|1138|1139|1140|1141|1142|1143|1144|1145|1146|1147|1148|1149|1150|1151|1152 +0.140625000000,512,1,36,1153|1154|1155|1156|1157|1158|1159|1160|1161|1162|1163|1164|1165|1166|1167|1168|1169|1170|1171|1172|1173|1174|1175|1176|1177|1178|1179|1180|1181|1182|1183|1184 +0.144531250000,512,1,37,1185|1186|1187|1188|1189|1190|1191|1192|1193|1194|1195|1196|1197|1198|1199|1200|1201|1202|1203|1204|1205|1206|1207|1208|1209|1210|1211|1212|1213|1214|1215|1216 +0.148437500000,512,1,38,1217|1218|1219|1220|1221|1222|1223|1224|1225|1226|1227|1228|1229|1230|1231|1232|1233|1234|1235|1236|1237|1238|1239|1240|1241|1242|1243|1244|1245|1246|1247|1248 +0.152343750000,512,1,39,1249|1250|1251|1252|1253|1254|1255|1256|1257|1258|1259|1260|1261|1262|1263|1264|1265|1266|1267|1268|1269|1270|1271|1272|1273|1274|1275|1276|1277|1278|1279|1280 +0.156250000000,512,1,40,1281|1282|1283|1284|1285|1286|1287|1288|1289|1290|1291|1292|1293|1294|1295|1296|1297|1298|1299|1300|1301|1302|1303|1304|1305|1306|1307|1308|1309|1310|1311|1312 +0.160156250000,512,1,41,1313|1314|1315|1316|1317|1318|1319|1320|1321|1322|1323|1324|1325|1326|1327|1328|1329|1330|1331|1332|1333|1334|1335|1336|1337|1338|1339|1340|1341|1342|1343|1344 +0.164062500000,512,1,42,1345|1346|1347|1348|1349|1350|1351|1352|1353|1354|1355|1356|1357|1358|1359|1360|1361|1362|1363|1364|1365|1366|1367|1368|1369|1370|1371|1372|1373|1374|1375|1376 +0.167968750000,512,1,43,1377|1378|1379|1380|1381|1382|1383|1384|1385|1386|1387|1388|1389|1390|1391|1392|1393|1394|1395|1396|1397|1398|1399|1400|1401|1402|1403|1404|1405|1406|1407|1408 +0.171875000000,512,1,44,1409|1410|1411|1412|1413|1414|1415|1416|1417|1418|1419|1420|1421|1422|1423|1424|1425|1426|1427|1428|1429|1430|1431|1432|1433|1434|1435|1436|1437|1438|1439|1440 +0.175781250000,512,1,45,1441|1442|1443|1444|1445|1446|1447|1448|1449|1450|1451|1452|1453|1454|1455|1456|1457|1458|1459|1460|1461|1462|1463|1464|1465|1466|1467|1468|1469|1470|1471|1472 +0.179687500000,512,1,46,1473|1474|1475|1476|1477|1478|1479|1480|1481|1482|1483|1484|1485|1486|1487|1488|1489|1490|1491|1492|1493|1494|1495|1496|1497|1498|1499|1500|1501|1502|1503|1504 +0.183593750000,512,1,47,1505|1506|1507|1508|1509|1510|1511|1512|1513|1514|1515|1516|1517|1518|1519|1520|1521|1522|1523|1524|1525|1526|1527|1528|1529|1530|1531|1532|1533|1534|1535|1536 +0.187500000000,512,1,48,1537|1538|1539|1540|1541|1542|1543|1544|1545|1546|1547|1548|1549|1550|1551|1552|1553|1554|1555|1556|1557|1558|1559|1560|1561|1562|1563|1564|1565|1566|1567|1568 +0.191406250000,512,1,49,1569|1570|1571|1572|1573|1574|1575|1576|1577|1578|1579|1580|1581|1582|1583|1584|1585|1586|1587|1588|1589|1590|1591|1592|1593|1594|1595|1596|1597|1598|1599|1600 +0.195312500000,512,1,50,1601|1602|1603|1604|1605|1606|1607|1608|1609|1610|1611|1612|1613|1614|1615|1616|1617|1618|1619|1620|1621|1622|1623|1624|1625|1626|1627|1628|1629|1630|1631|1632 +0.199218750000,512,1,51,1633|1634|1635|1636|1637|1638|1639|1640|1641|1642|1643|1644|1645|1646|1647|1648|1649|1650|1651|1652|1653|1654|1655|1656|1657|1658|1659|1660|1661|1662|1663|1664 +0.203125000000,512,1,52,1665|1666|1667|1668|1669|1670|1671|1672|1673|1674|1675|1676|1677|1678|1679|1680|1681|1682|1683|1684|1685|1686|1687|1688|1689|1690|1691|1692|1693|1694|1695|1696 +0.207031250000,512,1,53,1697|1698|1699|1700|1701|1702|1703|1704|1705|1706|1707|1708|1709|1710|1711|1712|1713|1714|1715|1716|1717|1718|1719|1720|1721|1722|1723|1724|1725|1726|1727|1728 +0.210937500000,512,1,54,1729|1730|1731|1732|1733|1734|1735|1736|1737|1738|1739|1740|1741|1742|1743|1744|1745|1746|1747|1748|1749|1750|1751|1752|1753|1754|1755|1756|1757|1758|1759|1760 +0.214843750000,512,1,55,1761|1762|1763|1764|1765|1766|1767|1768|1769|1770|1771|1772|1773|1774|1775|1776|1777|1778|1779|1780|1781|1782|1783|1784|1785|1786|1787|1788|1789|1790|1791|1792 +0.218750000000,512,1,56,1793|1794|1795|1796|1797|1798|1799|1800|1801|1802|1803|1804|1805|1806|1807|1808|1809|1810|1811|1812|1813|1814|1815|1816|1817|1818|1819|1820|1821|1822|1823|1824 +0.222656250000,512,1,57,1825|1826|1827|1828|1829|1830|1831|1832|1833|1834|1835|1836|1837|1838|1839|1840|1841|1842|1843|1844|1845|1846|1847|1848|1849|1850|1851|1852|1853|1854|1855|1856 +0.226562500000,512,1,58,1857|1858|1859|1860|1861|1862|1863|1864|1865|1866|1867|1868|1869|1870|1871|1872|1873|1874|1875|1876|1877|1878|1879|1880|1881|1882|1883|1884|1885|1886|1887|1888 +0.230468750000,512,1,59,1889|1890|1891|1892|1893|1894|1895|1896|1897|1898|1899|1900|1901|1902|1903|1904|1905|1906|1907|1908|1909|1910|1911|1912|1913|1914|1915|1916|1917|1918|1919|1920 +0.234375000000,512,1,60,1921|1922|1923|1924|1925|1926|1927|1928|1929|1930|1931|1932|1933|1934|1935|1936|1937|1938|1939|1940|1941|1942|1943|1944|1945|1946|1947|1948|1949|1950|1951|1952 +0.238281250000,512,1,61,1953|1954|1955|1956|1957|1958|1959|1960|1961|1962|1963|1964|1965|1966|1967|1968|1969|1970|1971|1972|1973|1974|1975|1976|1977|1978|1979|1980|1981|1982|1983|1984 +0.242187500000,512,1,62,1985|1986|1987|1988|1989|1990|1991|1992|1993|1994|1995|1996|1997|1998|1999|2000|2001|2002|2003|2004|2005|2006|2007|2008|2009|2010|2011|2012|2013|2014|2015|2016 +0.246093750000,512,1,63,2017|2018|2019|2020|2021|2022|2023|2024|2025|2026|2027|2028|2029|2030|2031|2032|2033|2034|2035|2036|2037|2038|2039|2040|2041|2042|2043|2044|2045|2046|2047|2048 diff --git a/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r32.csv b/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r32.csv new file mode 100644 index 0000000..22a1ea6 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r32.csv @@ -0,0 +1,65 @@ +arrived_at,num_prefill_tokens,num_decode_tokens,session_id,block_hash_ids +0.000000000000,512,1,0,1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32 +0.031250000000,512,1,1,33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58|59|60|61|62|63|64 +0.062500000000,512,1,2,65|66|67|68|69|70|71|72|73|74|75|76|77|78|79|80|81|82|83|84|85|86|87|88|89|90|91|92|93|94|95|96 +0.093750000000,512,1,3,97|98|99|100|101|102|103|104|105|106|107|108|109|110|111|112|113|114|115|116|117|118|119|120|121|122|123|124|125|126|127|128 +0.125000000000,512,1,4,129|130|131|132|133|134|135|136|137|138|139|140|141|142|143|144|145|146|147|148|149|150|151|152|153|154|155|156|157|158|159|160 +0.156250000000,512,1,5,161|162|163|164|165|166|167|168|169|170|171|172|173|174|175|176|177|178|179|180|181|182|183|184|185|186|187|188|189|190|191|192 +0.187500000000,512,1,6,193|194|195|196|197|198|199|200|201|202|203|204|205|206|207|208|209|210|211|212|213|214|215|216|217|218|219|220|221|222|223|224 +0.218750000000,512,1,7,225|226|227|228|229|230|231|232|233|234|235|236|237|238|239|240|241|242|243|244|245|246|247|248|249|250|251|252|253|254|255|256 +0.250000000000,512,1,8,257|258|259|260|261|262|263|264|265|266|267|268|269|270|271|272|273|274|275|276|277|278|279|280|281|282|283|284|285|286|287|288 +0.281250000000,512,1,9,289|290|291|292|293|294|295|296|297|298|299|300|301|302|303|304|305|306|307|308|309|310|311|312|313|314|315|316|317|318|319|320 +0.312500000000,512,1,10,321|322|323|324|325|326|327|328|329|330|331|332|333|334|335|336|337|338|339|340|341|342|343|344|345|346|347|348|349|350|351|352 +0.343750000000,512,1,11,353|354|355|356|357|358|359|360|361|362|363|364|365|366|367|368|369|370|371|372|373|374|375|376|377|378|379|380|381|382|383|384 +0.375000000000,512,1,12,385|386|387|388|389|390|391|392|393|394|395|396|397|398|399|400|401|402|403|404|405|406|407|408|409|410|411|412|413|414|415|416 +0.406250000000,512,1,13,417|418|419|420|421|422|423|424|425|426|427|428|429|430|431|432|433|434|435|436|437|438|439|440|441|442|443|444|445|446|447|448 +0.437500000000,512,1,14,449|450|451|452|453|454|455|456|457|458|459|460|461|462|463|464|465|466|467|468|469|470|471|472|473|474|475|476|477|478|479|480 +0.468750000000,512,1,15,481|482|483|484|485|486|487|488|489|490|491|492|493|494|495|496|497|498|499|500|501|502|503|504|505|506|507|508|509|510|511|512 +0.500000000000,512,1,16,513|514|515|516|517|518|519|520|521|522|523|524|525|526|527|528|529|530|531|532|533|534|535|536|537|538|539|540|541|542|543|544 +0.531250000000,512,1,17,545|546|547|548|549|550|551|552|553|554|555|556|557|558|559|560|561|562|563|564|565|566|567|568|569|570|571|572|573|574|575|576 +0.562500000000,512,1,18,577|578|579|580|581|582|583|584|585|586|587|588|589|590|591|592|593|594|595|596|597|598|599|600|601|602|603|604|605|606|607|608 +0.593750000000,512,1,19,609|610|611|612|613|614|615|616|617|618|619|620|621|622|623|624|625|626|627|628|629|630|631|632|633|634|635|636|637|638|639|640 +0.625000000000,512,1,20,641|642|643|644|645|646|647|648|649|650|651|652|653|654|655|656|657|658|659|660|661|662|663|664|665|666|667|668|669|670|671|672 +0.656250000000,512,1,21,673|674|675|676|677|678|679|680|681|682|683|684|685|686|687|688|689|690|691|692|693|694|695|696|697|698|699|700|701|702|703|704 +0.687500000000,512,1,22,705|706|707|708|709|710|711|712|713|714|715|716|717|718|719|720|721|722|723|724|725|726|727|728|729|730|731|732|733|734|735|736 +0.718750000000,512,1,23,737|738|739|740|741|742|743|744|745|746|747|748|749|750|751|752|753|754|755|756|757|758|759|760|761|762|763|764|765|766|767|768 +0.750000000000,512,1,24,769|770|771|772|773|774|775|776|777|778|779|780|781|782|783|784|785|786|787|788|789|790|791|792|793|794|795|796|797|798|799|800 +0.781250000000,512,1,25,801|802|803|804|805|806|807|808|809|810|811|812|813|814|815|816|817|818|819|820|821|822|823|824|825|826|827|828|829|830|831|832 +0.812500000000,512,1,26,833|834|835|836|837|838|839|840|841|842|843|844|845|846|847|848|849|850|851|852|853|854|855|856|857|858|859|860|861|862|863|864 +0.843750000000,512,1,27,865|866|867|868|869|870|871|872|873|874|875|876|877|878|879|880|881|882|883|884|885|886|887|888|889|890|891|892|893|894|895|896 +0.875000000000,512,1,28,897|898|899|900|901|902|903|904|905|906|907|908|909|910|911|912|913|914|915|916|917|918|919|920|921|922|923|924|925|926|927|928 +0.906250000000,512,1,29,929|930|931|932|933|934|935|936|937|938|939|940|941|942|943|944|945|946|947|948|949|950|951|952|953|954|955|956|957|958|959|960 +0.937500000000,512,1,30,961|962|963|964|965|966|967|968|969|970|971|972|973|974|975|976|977|978|979|980|981|982|983|984|985|986|987|988|989|990|991|992 +0.968750000000,512,1,31,993|994|995|996|997|998|999|1000|1001|1002|1003|1004|1005|1006|1007|1008|1009|1010|1011|1012|1013|1014|1015|1016|1017|1018|1019|1020|1021|1022|1023|1024 +1.000000000000,512,1,32,1025|1026|1027|1028|1029|1030|1031|1032|1033|1034|1035|1036|1037|1038|1039|1040|1041|1042|1043|1044|1045|1046|1047|1048|1049|1050|1051|1052|1053|1054|1055|1056 +1.031250000000,512,1,33,1057|1058|1059|1060|1061|1062|1063|1064|1065|1066|1067|1068|1069|1070|1071|1072|1073|1074|1075|1076|1077|1078|1079|1080|1081|1082|1083|1084|1085|1086|1087|1088 +1.062500000000,512,1,34,1089|1090|1091|1092|1093|1094|1095|1096|1097|1098|1099|1100|1101|1102|1103|1104|1105|1106|1107|1108|1109|1110|1111|1112|1113|1114|1115|1116|1117|1118|1119|1120 +1.093750000000,512,1,35,1121|1122|1123|1124|1125|1126|1127|1128|1129|1130|1131|1132|1133|1134|1135|1136|1137|1138|1139|1140|1141|1142|1143|1144|1145|1146|1147|1148|1149|1150|1151|1152 +1.125000000000,512,1,36,1153|1154|1155|1156|1157|1158|1159|1160|1161|1162|1163|1164|1165|1166|1167|1168|1169|1170|1171|1172|1173|1174|1175|1176|1177|1178|1179|1180|1181|1182|1183|1184 +1.156250000000,512,1,37,1185|1186|1187|1188|1189|1190|1191|1192|1193|1194|1195|1196|1197|1198|1199|1200|1201|1202|1203|1204|1205|1206|1207|1208|1209|1210|1211|1212|1213|1214|1215|1216 +1.187500000000,512,1,38,1217|1218|1219|1220|1221|1222|1223|1224|1225|1226|1227|1228|1229|1230|1231|1232|1233|1234|1235|1236|1237|1238|1239|1240|1241|1242|1243|1244|1245|1246|1247|1248 +1.218750000000,512,1,39,1249|1250|1251|1252|1253|1254|1255|1256|1257|1258|1259|1260|1261|1262|1263|1264|1265|1266|1267|1268|1269|1270|1271|1272|1273|1274|1275|1276|1277|1278|1279|1280 +1.250000000000,512,1,40,1281|1282|1283|1284|1285|1286|1287|1288|1289|1290|1291|1292|1293|1294|1295|1296|1297|1298|1299|1300|1301|1302|1303|1304|1305|1306|1307|1308|1309|1310|1311|1312 +1.281250000000,512,1,41,1313|1314|1315|1316|1317|1318|1319|1320|1321|1322|1323|1324|1325|1326|1327|1328|1329|1330|1331|1332|1333|1334|1335|1336|1337|1338|1339|1340|1341|1342|1343|1344 +1.312500000000,512,1,42,1345|1346|1347|1348|1349|1350|1351|1352|1353|1354|1355|1356|1357|1358|1359|1360|1361|1362|1363|1364|1365|1366|1367|1368|1369|1370|1371|1372|1373|1374|1375|1376 +1.343750000000,512,1,43,1377|1378|1379|1380|1381|1382|1383|1384|1385|1386|1387|1388|1389|1390|1391|1392|1393|1394|1395|1396|1397|1398|1399|1400|1401|1402|1403|1404|1405|1406|1407|1408 +1.375000000000,512,1,44,1409|1410|1411|1412|1413|1414|1415|1416|1417|1418|1419|1420|1421|1422|1423|1424|1425|1426|1427|1428|1429|1430|1431|1432|1433|1434|1435|1436|1437|1438|1439|1440 +1.406250000000,512,1,45,1441|1442|1443|1444|1445|1446|1447|1448|1449|1450|1451|1452|1453|1454|1455|1456|1457|1458|1459|1460|1461|1462|1463|1464|1465|1466|1467|1468|1469|1470|1471|1472 +1.437500000000,512,1,46,1473|1474|1475|1476|1477|1478|1479|1480|1481|1482|1483|1484|1485|1486|1487|1488|1489|1490|1491|1492|1493|1494|1495|1496|1497|1498|1499|1500|1501|1502|1503|1504 +1.468750000000,512,1,47,1505|1506|1507|1508|1509|1510|1511|1512|1513|1514|1515|1516|1517|1518|1519|1520|1521|1522|1523|1524|1525|1526|1527|1528|1529|1530|1531|1532|1533|1534|1535|1536 +1.500000000000,512,1,48,1537|1538|1539|1540|1541|1542|1543|1544|1545|1546|1547|1548|1549|1550|1551|1552|1553|1554|1555|1556|1557|1558|1559|1560|1561|1562|1563|1564|1565|1566|1567|1568 +1.531250000000,512,1,49,1569|1570|1571|1572|1573|1574|1575|1576|1577|1578|1579|1580|1581|1582|1583|1584|1585|1586|1587|1588|1589|1590|1591|1592|1593|1594|1595|1596|1597|1598|1599|1600 +1.562500000000,512,1,50,1601|1602|1603|1604|1605|1606|1607|1608|1609|1610|1611|1612|1613|1614|1615|1616|1617|1618|1619|1620|1621|1622|1623|1624|1625|1626|1627|1628|1629|1630|1631|1632 +1.593750000000,512,1,51,1633|1634|1635|1636|1637|1638|1639|1640|1641|1642|1643|1644|1645|1646|1647|1648|1649|1650|1651|1652|1653|1654|1655|1656|1657|1658|1659|1660|1661|1662|1663|1664 +1.625000000000,512,1,52,1665|1666|1667|1668|1669|1670|1671|1672|1673|1674|1675|1676|1677|1678|1679|1680|1681|1682|1683|1684|1685|1686|1687|1688|1689|1690|1691|1692|1693|1694|1695|1696 +1.656250000000,512,1,53,1697|1698|1699|1700|1701|1702|1703|1704|1705|1706|1707|1708|1709|1710|1711|1712|1713|1714|1715|1716|1717|1718|1719|1720|1721|1722|1723|1724|1725|1726|1727|1728 +1.687500000000,512,1,54,1729|1730|1731|1732|1733|1734|1735|1736|1737|1738|1739|1740|1741|1742|1743|1744|1745|1746|1747|1748|1749|1750|1751|1752|1753|1754|1755|1756|1757|1758|1759|1760 +1.718750000000,512,1,55,1761|1762|1763|1764|1765|1766|1767|1768|1769|1770|1771|1772|1773|1774|1775|1776|1777|1778|1779|1780|1781|1782|1783|1784|1785|1786|1787|1788|1789|1790|1791|1792 +1.750000000000,512,1,56,1793|1794|1795|1796|1797|1798|1799|1800|1801|1802|1803|1804|1805|1806|1807|1808|1809|1810|1811|1812|1813|1814|1815|1816|1817|1818|1819|1820|1821|1822|1823|1824 +1.781250000000,512,1,57,1825|1826|1827|1828|1829|1830|1831|1832|1833|1834|1835|1836|1837|1838|1839|1840|1841|1842|1843|1844|1845|1846|1847|1848|1849|1850|1851|1852|1853|1854|1855|1856 +1.812500000000,512,1,58,1857|1858|1859|1860|1861|1862|1863|1864|1865|1866|1867|1868|1869|1870|1871|1872|1873|1874|1875|1876|1877|1878|1879|1880|1881|1882|1883|1884|1885|1886|1887|1888 +1.843750000000,512,1,59,1889|1890|1891|1892|1893|1894|1895|1896|1897|1898|1899|1900|1901|1902|1903|1904|1905|1906|1907|1908|1909|1910|1911|1912|1913|1914|1915|1916|1917|1918|1919|1920 +1.875000000000,512,1,60,1921|1922|1923|1924|1925|1926|1927|1928|1929|1930|1931|1932|1933|1934|1935|1936|1937|1938|1939|1940|1941|1942|1943|1944|1945|1946|1947|1948|1949|1950|1951|1952 +1.906250000000,512,1,61,1953|1954|1955|1956|1957|1958|1959|1960|1961|1962|1963|1964|1965|1966|1967|1968|1969|1970|1971|1972|1973|1974|1975|1976|1977|1978|1979|1980|1981|1982|1983|1984 +1.937500000000,512,1,62,1985|1986|1987|1988|1989|1990|1991|1992|1993|1994|1995|1996|1997|1998|1999|2000|2001|2002|2003|2004|2005|2006|2007|2008|2009|2010|2011|2012|2013|2014|2015|2016 +1.968750000000,512,1,63,2017|2018|2019|2020|2021|2022|2023|2024|2025|2026|2027|2028|2029|2030|2031|2032|2033|2034|2035|2036|2037|2038|2039|2040|2041|2042|2043|2044|2045|2046|2047|2048 diff --git a/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r512.csv b/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r512.csv new file mode 100644 index 0000000..bdbfbee --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r512.csv @@ -0,0 +1,65 @@ +arrived_at,num_prefill_tokens,num_decode_tokens,session_id,block_hash_ids +0.000000000000,512,1,0,1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32 +0.001953125000,512,1,1,33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58|59|60|61|62|63|64 +0.003906250000,512,1,2,65|66|67|68|69|70|71|72|73|74|75|76|77|78|79|80|81|82|83|84|85|86|87|88|89|90|91|92|93|94|95|96 +0.005859375000,512,1,3,97|98|99|100|101|102|103|104|105|106|107|108|109|110|111|112|113|114|115|116|117|118|119|120|121|122|123|124|125|126|127|128 +0.007812500000,512,1,4,129|130|131|132|133|134|135|136|137|138|139|140|141|142|143|144|145|146|147|148|149|150|151|152|153|154|155|156|157|158|159|160 +0.009765625000,512,1,5,161|162|163|164|165|166|167|168|169|170|171|172|173|174|175|176|177|178|179|180|181|182|183|184|185|186|187|188|189|190|191|192 +0.011718750000,512,1,6,193|194|195|196|197|198|199|200|201|202|203|204|205|206|207|208|209|210|211|212|213|214|215|216|217|218|219|220|221|222|223|224 +0.013671875000,512,1,7,225|226|227|228|229|230|231|232|233|234|235|236|237|238|239|240|241|242|243|244|245|246|247|248|249|250|251|252|253|254|255|256 +0.015625000000,512,1,8,257|258|259|260|261|262|263|264|265|266|267|268|269|270|271|272|273|274|275|276|277|278|279|280|281|282|283|284|285|286|287|288 +0.017578125000,512,1,9,289|290|291|292|293|294|295|296|297|298|299|300|301|302|303|304|305|306|307|308|309|310|311|312|313|314|315|316|317|318|319|320 +0.019531250000,512,1,10,321|322|323|324|325|326|327|328|329|330|331|332|333|334|335|336|337|338|339|340|341|342|343|344|345|346|347|348|349|350|351|352 +0.021484375000,512,1,11,353|354|355|356|357|358|359|360|361|362|363|364|365|366|367|368|369|370|371|372|373|374|375|376|377|378|379|380|381|382|383|384 +0.023437500000,512,1,12,385|386|387|388|389|390|391|392|393|394|395|396|397|398|399|400|401|402|403|404|405|406|407|408|409|410|411|412|413|414|415|416 +0.025390625000,512,1,13,417|418|419|420|421|422|423|424|425|426|427|428|429|430|431|432|433|434|435|436|437|438|439|440|441|442|443|444|445|446|447|448 +0.027343750000,512,1,14,449|450|451|452|453|454|455|456|457|458|459|460|461|462|463|464|465|466|467|468|469|470|471|472|473|474|475|476|477|478|479|480 +0.029296875000,512,1,15,481|482|483|484|485|486|487|488|489|490|491|492|493|494|495|496|497|498|499|500|501|502|503|504|505|506|507|508|509|510|511|512 +0.031250000000,512,1,16,513|514|515|516|517|518|519|520|521|522|523|524|525|526|527|528|529|530|531|532|533|534|535|536|537|538|539|540|541|542|543|544 +0.033203125000,512,1,17,545|546|547|548|549|550|551|552|553|554|555|556|557|558|559|560|561|562|563|564|565|566|567|568|569|570|571|572|573|574|575|576 +0.035156250000,512,1,18,577|578|579|580|581|582|583|584|585|586|587|588|589|590|591|592|593|594|595|596|597|598|599|600|601|602|603|604|605|606|607|608 +0.037109375000,512,1,19,609|610|611|612|613|614|615|616|617|618|619|620|621|622|623|624|625|626|627|628|629|630|631|632|633|634|635|636|637|638|639|640 +0.039062500000,512,1,20,641|642|643|644|645|646|647|648|649|650|651|652|653|654|655|656|657|658|659|660|661|662|663|664|665|666|667|668|669|670|671|672 +0.041015625000,512,1,21,673|674|675|676|677|678|679|680|681|682|683|684|685|686|687|688|689|690|691|692|693|694|695|696|697|698|699|700|701|702|703|704 +0.042968750000,512,1,22,705|706|707|708|709|710|711|712|713|714|715|716|717|718|719|720|721|722|723|724|725|726|727|728|729|730|731|732|733|734|735|736 +0.044921875000,512,1,23,737|738|739|740|741|742|743|744|745|746|747|748|749|750|751|752|753|754|755|756|757|758|759|760|761|762|763|764|765|766|767|768 +0.046875000000,512,1,24,769|770|771|772|773|774|775|776|777|778|779|780|781|782|783|784|785|786|787|788|789|790|791|792|793|794|795|796|797|798|799|800 +0.048828125000,512,1,25,801|802|803|804|805|806|807|808|809|810|811|812|813|814|815|816|817|818|819|820|821|822|823|824|825|826|827|828|829|830|831|832 +0.050781250000,512,1,26,833|834|835|836|837|838|839|840|841|842|843|844|845|846|847|848|849|850|851|852|853|854|855|856|857|858|859|860|861|862|863|864 +0.052734375000,512,1,27,865|866|867|868|869|870|871|872|873|874|875|876|877|878|879|880|881|882|883|884|885|886|887|888|889|890|891|892|893|894|895|896 +0.054687500000,512,1,28,897|898|899|900|901|902|903|904|905|906|907|908|909|910|911|912|913|914|915|916|917|918|919|920|921|922|923|924|925|926|927|928 +0.056640625000,512,1,29,929|930|931|932|933|934|935|936|937|938|939|940|941|942|943|944|945|946|947|948|949|950|951|952|953|954|955|956|957|958|959|960 +0.058593750000,512,1,30,961|962|963|964|965|966|967|968|969|970|971|972|973|974|975|976|977|978|979|980|981|982|983|984|985|986|987|988|989|990|991|992 +0.060546875000,512,1,31,993|994|995|996|997|998|999|1000|1001|1002|1003|1004|1005|1006|1007|1008|1009|1010|1011|1012|1013|1014|1015|1016|1017|1018|1019|1020|1021|1022|1023|1024 +0.062500000000,512,1,32,1025|1026|1027|1028|1029|1030|1031|1032|1033|1034|1035|1036|1037|1038|1039|1040|1041|1042|1043|1044|1045|1046|1047|1048|1049|1050|1051|1052|1053|1054|1055|1056 +0.064453125000,512,1,33,1057|1058|1059|1060|1061|1062|1063|1064|1065|1066|1067|1068|1069|1070|1071|1072|1073|1074|1075|1076|1077|1078|1079|1080|1081|1082|1083|1084|1085|1086|1087|1088 +0.066406250000,512,1,34,1089|1090|1091|1092|1093|1094|1095|1096|1097|1098|1099|1100|1101|1102|1103|1104|1105|1106|1107|1108|1109|1110|1111|1112|1113|1114|1115|1116|1117|1118|1119|1120 +0.068359375000,512,1,35,1121|1122|1123|1124|1125|1126|1127|1128|1129|1130|1131|1132|1133|1134|1135|1136|1137|1138|1139|1140|1141|1142|1143|1144|1145|1146|1147|1148|1149|1150|1151|1152 +0.070312500000,512,1,36,1153|1154|1155|1156|1157|1158|1159|1160|1161|1162|1163|1164|1165|1166|1167|1168|1169|1170|1171|1172|1173|1174|1175|1176|1177|1178|1179|1180|1181|1182|1183|1184 +0.072265625000,512,1,37,1185|1186|1187|1188|1189|1190|1191|1192|1193|1194|1195|1196|1197|1198|1199|1200|1201|1202|1203|1204|1205|1206|1207|1208|1209|1210|1211|1212|1213|1214|1215|1216 +0.074218750000,512,1,38,1217|1218|1219|1220|1221|1222|1223|1224|1225|1226|1227|1228|1229|1230|1231|1232|1233|1234|1235|1236|1237|1238|1239|1240|1241|1242|1243|1244|1245|1246|1247|1248 +0.076171875000,512,1,39,1249|1250|1251|1252|1253|1254|1255|1256|1257|1258|1259|1260|1261|1262|1263|1264|1265|1266|1267|1268|1269|1270|1271|1272|1273|1274|1275|1276|1277|1278|1279|1280 +0.078125000000,512,1,40,1281|1282|1283|1284|1285|1286|1287|1288|1289|1290|1291|1292|1293|1294|1295|1296|1297|1298|1299|1300|1301|1302|1303|1304|1305|1306|1307|1308|1309|1310|1311|1312 +0.080078125000,512,1,41,1313|1314|1315|1316|1317|1318|1319|1320|1321|1322|1323|1324|1325|1326|1327|1328|1329|1330|1331|1332|1333|1334|1335|1336|1337|1338|1339|1340|1341|1342|1343|1344 +0.082031250000,512,1,42,1345|1346|1347|1348|1349|1350|1351|1352|1353|1354|1355|1356|1357|1358|1359|1360|1361|1362|1363|1364|1365|1366|1367|1368|1369|1370|1371|1372|1373|1374|1375|1376 +0.083984375000,512,1,43,1377|1378|1379|1380|1381|1382|1383|1384|1385|1386|1387|1388|1389|1390|1391|1392|1393|1394|1395|1396|1397|1398|1399|1400|1401|1402|1403|1404|1405|1406|1407|1408 +0.085937500000,512,1,44,1409|1410|1411|1412|1413|1414|1415|1416|1417|1418|1419|1420|1421|1422|1423|1424|1425|1426|1427|1428|1429|1430|1431|1432|1433|1434|1435|1436|1437|1438|1439|1440 +0.087890625000,512,1,45,1441|1442|1443|1444|1445|1446|1447|1448|1449|1450|1451|1452|1453|1454|1455|1456|1457|1458|1459|1460|1461|1462|1463|1464|1465|1466|1467|1468|1469|1470|1471|1472 +0.089843750000,512,1,46,1473|1474|1475|1476|1477|1478|1479|1480|1481|1482|1483|1484|1485|1486|1487|1488|1489|1490|1491|1492|1493|1494|1495|1496|1497|1498|1499|1500|1501|1502|1503|1504 +0.091796875000,512,1,47,1505|1506|1507|1508|1509|1510|1511|1512|1513|1514|1515|1516|1517|1518|1519|1520|1521|1522|1523|1524|1525|1526|1527|1528|1529|1530|1531|1532|1533|1534|1535|1536 +0.093750000000,512,1,48,1537|1538|1539|1540|1541|1542|1543|1544|1545|1546|1547|1548|1549|1550|1551|1552|1553|1554|1555|1556|1557|1558|1559|1560|1561|1562|1563|1564|1565|1566|1567|1568 +0.095703125000,512,1,49,1569|1570|1571|1572|1573|1574|1575|1576|1577|1578|1579|1580|1581|1582|1583|1584|1585|1586|1587|1588|1589|1590|1591|1592|1593|1594|1595|1596|1597|1598|1599|1600 +0.097656250000,512,1,50,1601|1602|1603|1604|1605|1606|1607|1608|1609|1610|1611|1612|1613|1614|1615|1616|1617|1618|1619|1620|1621|1622|1623|1624|1625|1626|1627|1628|1629|1630|1631|1632 +0.099609375000,512,1,51,1633|1634|1635|1636|1637|1638|1639|1640|1641|1642|1643|1644|1645|1646|1647|1648|1649|1650|1651|1652|1653|1654|1655|1656|1657|1658|1659|1660|1661|1662|1663|1664 +0.101562500000,512,1,52,1665|1666|1667|1668|1669|1670|1671|1672|1673|1674|1675|1676|1677|1678|1679|1680|1681|1682|1683|1684|1685|1686|1687|1688|1689|1690|1691|1692|1693|1694|1695|1696 +0.103515625000,512,1,53,1697|1698|1699|1700|1701|1702|1703|1704|1705|1706|1707|1708|1709|1710|1711|1712|1713|1714|1715|1716|1717|1718|1719|1720|1721|1722|1723|1724|1725|1726|1727|1728 +0.105468750000,512,1,54,1729|1730|1731|1732|1733|1734|1735|1736|1737|1738|1739|1740|1741|1742|1743|1744|1745|1746|1747|1748|1749|1750|1751|1752|1753|1754|1755|1756|1757|1758|1759|1760 +0.107421875000,512,1,55,1761|1762|1763|1764|1765|1766|1767|1768|1769|1770|1771|1772|1773|1774|1775|1776|1777|1778|1779|1780|1781|1782|1783|1784|1785|1786|1787|1788|1789|1790|1791|1792 +0.109375000000,512,1,56,1793|1794|1795|1796|1797|1798|1799|1800|1801|1802|1803|1804|1805|1806|1807|1808|1809|1810|1811|1812|1813|1814|1815|1816|1817|1818|1819|1820|1821|1822|1823|1824 +0.111328125000,512,1,57,1825|1826|1827|1828|1829|1830|1831|1832|1833|1834|1835|1836|1837|1838|1839|1840|1841|1842|1843|1844|1845|1846|1847|1848|1849|1850|1851|1852|1853|1854|1855|1856 +0.113281250000,512,1,58,1857|1858|1859|1860|1861|1862|1863|1864|1865|1866|1867|1868|1869|1870|1871|1872|1873|1874|1875|1876|1877|1878|1879|1880|1881|1882|1883|1884|1885|1886|1887|1888 +0.115234375000,512,1,59,1889|1890|1891|1892|1893|1894|1895|1896|1897|1898|1899|1900|1901|1902|1903|1904|1905|1906|1907|1908|1909|1910|1911|1912|1913|1914|1915|1916|1917|1918|1919|1920 +0.117187500000,512,1,60,1921|1922|1923|1924|1925|1926|1927|1928|1929|1930|1931|1932|1933|1934|1935|1936|1937|1938|1939|1940|1941|1942|1943|1944|1945|1946|1947|1948|1949|1950|1951|1952 +0.119140625000,512,1,61,1953|1954|1955|1956|1957|1958|1959|1960|1961|1962|1963|1964|1965|1966|1967|1968|1969|1970|1971|1972|1973|1974|1975|1976|1977|1978|1979|1980|1981|1982|1983|1984 +0.121093750000,512,1,62,1985|1986|1987|1988|1989|1990|1991|1992|1993|1994|1995|1996|1997|1998|1999|2000|2001|2002|2003|2004|2005|2006|2007|2008|2009|2010|2011|2012|2013|2014|2015|2016 +0.123046875000,512,1,63,2017|2018|2019|2020|2021|2022|2023|2024|2025|2026|2027|2028|2029|2030|2031|2032|2033|2034|2035|2036|2037|2038|2039|2040|2041|2042|2043|2044|2045|2046|2047|2048 diff --git a/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r64.csv b/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r64.csv new file mode 100644 index 0000000..db33eff --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r64.csv @@ -0,0 +1,65 @@ +arrived_at,num_prefill_tokens,num_decode_tokens,session_id,block_hash_ids +0.000000000000,512,1,0,1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32 +0.015625000000,512,1,1,33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58|59|60|61|62|63|64 +0.031250000000,512,1,2,65|66|67|68|69|70|71|72|73|74|75|76|77|78|79|80|81|82|83|84|85|86|87|88|89|90|91|92|93|94|95|96 +0.046875000000,512,1,3,97|98|99|100|101|102|103|104|105|106|107|108|109|110|111|112|113|114|115|116|117|118|119|120|121|122|123|124|125|126|127|128 +0.062500000000,512,1,4,129|130|131|132|133|134|135|136|137|138|139|140|141|142|143|144|145|146|147|148|149|150|151|152|153|154|155|156|157|158|159|160 +0.078125000000,512,1,5,161|162|163|164|165|166|167|168|169|170|171|172|173|174|175|176|177|178|179|180|181|182|183|184|185|186|187|188|189|190|191|192 +0.093750000000,512,1,6,193|194|195|196|197|198|199|200|201|202|203|204|205|206|207|208|209|210|211|212|213|214|215|216|217|218|219|220|221|222|223|224 +0.109375000000,512,1,7,225|226|227|228|229|230|231|232|233|234|235|236|237|238|239|240|241|242|243|244|245|246|247|248|249|250|251|252|253|254|255|256 +0.125000000000,512,1,8,257|258|259|260|261|262|263|264|265|266|267|268|269|270|271|272|273|274|275|276|277|278|279|280|281|282|283|284|285|286|287|288 +0.140625000000,512,1,9,289|290|291|292|293|294|295|296|297|298|299|300|301|302|303|304|305|306|307|308|309|310|311|312|313|314|315|316|317|318|319|320 +0.156250000000,512,1,10,321|322|323|324|325|326|327|328|329|330|331|332|333|334|335|336|337|338|339|340|341|342|343|344|345|346|347|348|349|350|351|352 +0.171875000000,512,1,11,353|354|355|356|357|358|359|360|361|362|363|364|365|366|367|368|369|370|371|372|373|374|375|376|377|378|379|380|381|382|383|384 +0.187500000000,512,1,12,385|386|387|388|389|390|391|392|393|394|395|396|397|398|399|400|401|402|403|404|405|406|407|408|409|410|411|412|413|414|415|416 +0.203125000000,512,1,13,417|418|419|420|421|422|423|424|425|426|427|428|429|430|431|432|433|434|435|436|437|438|439|440|441|442|443|444|445|446|447|448 +0.218750000000,512,1,14,449|450|451|452|453|454|455|456|457|458|459|460|461|462|463|464|465|466|467|468|469|470|471|472|473|474|475|476|477|478|479|480 +0.234375000000,512,1,15,481|482|483|484|485|486|487|488|489|490|491|492|493|494|495|496|497|498|499|500|501|502|503|504|505|506|507|508|509|510|511|512 +0.250000000000,512,1,16,513|514|515|516|517|518|519|520|521|522|523|524|525|526|527|528|529|530|531|532|533|534|535|536|537|538|539|540|541|542|543|544 +0.265625000000,512,1,17,545|546|547|548|549|550|551|552|553|554|555|556|557|558|559|560|561|562|563|564|565|566|567|568|569|570|571|572|573|574|575|576 +0.281250000000,512,1,18,577|578|579|580|581|582|583|584|585|586|587|588|589|590|591|592|593|594|595|596|597|598|599|600|601|602|603|604|605|606|607|608 +0.296875000000,512,1,19,609|610|611|612|613|614|615|616|617|618|619|620|621|622|623|624|625|626|627|628|629|630|631|632|633|634|635|636|637|638|639|640 +0.312500000000,512,1,20,641|642|643|644|645|646|647|648|649|650|651|652|653|654|655|656|657|658|659|660|661|662|663|664|665|666|667|668|669|670|671|672 +0.328125000000,512,1,21,673|674|675|676|677|678|679|680|681|682|683|684|685|686|687|688|689|690|691|692|693|694|695|696|697|698|699|700|701|702|703|704 +0.343750000000,512,1,22,705|706|707|708|709|710|711|712|713|714|715|716|717|718|719|720|721|722|723|724|725|726|727|728|729|730|731|732|733|734|735|736 +0.359375000000,512,1,23,737|738|739|740|741|742|743|744|745|746|747|748|749|750|751|752|753|754|755|756|757|758|759|760|761|762|763|764|765|766|767|768 +0.375000000000,512,1,24,769|770|771|772|773|774|775|776|777|778|779|780|781|782|783|784|785|786|787|788|789|790|791|792|793|794|795|796|797|798|799|800 +0.390625000000,512,1,25,801|802|803|804|805|806|807|808|809|810|811|812|813|814|815|816|817|818|819|820|821|822|823|824|825|826|827|828|829|830|831|832 +0.406250000000,512,1,26,833|834|835|836|837|838|839|840|841|842|843|844|845|846|847|848|849|850|851|852|853|854|855|856|857|858|859|860|861|862|863|864 +0.421875000000,512,1,27,865|866|867|868|869|870|871|872|873|874|875|876|877|878|879|880|881|882|883|884|885|886|887|888|889|890|891|892|893|894|895|896 +0.437500000000,512,1,28,897|898|899|900|901|902|903|904|905|906|907|908|909|910|911|912|913|914|915|916|917|918|919|920|921|922|923|924|925|926|927|928 +0.453125000000,512,1,29,929|930|931|932|933|934|935|936|937|938|939|940|941|942|943|944|945|946|947|948|949|950|951|952|953|954|955|956|957|958|959|960 +0.468750000000,512,1,30,961|962|963|964|965|966|967|968|969|970|971|972|973|974|975|976|977|978|979|980|981|982|983|984|985|986|987|988|989|990|991|992 +0.484375000000,512,1,31,993|994|995|996|997|998|999|1000|1001|1002|1003|1004|1005|1006|1007|1008|1009|1010|1011|1012|1013|1014|1015|1016|1017|1018|1019|1020|1021|1022|1023|1024 +0.500000000000,512,1,32,1025|1026|1027|1028|1029|1030|1031|1032|1033|1034|1035|1036|1037|1038|1039|1040|1041|1042|1043|1044|1045|1046|1047|1048|1049|1050|1051|1052|1053|1054|1055|1056 +0.515625000000,512,1,33,1057|1058|1059|1060|1061|1062|1063|1064|1065|1066|1067|1068|1069|1070|1071|1072|1073|1074|1075|1076|1077|1078|1079|1080|1081|1082|1083|1084|1085|1086|1087|1088 +0.531250000000,512,1,34,1089|1090|1091|1092|1093|1094|1095|1096|1097|1098|1099|1100|1101|1102|1103|1104|1105|1106|1107|1108|1109|1110|1111|1112|1113|1114|1115|1116|1117|1118|1119|1120 +0.546875000000,512,1,35,1121|1122|1123|1124|1125|1126|1127|1128|1129|1130|1131|1132|1133|1134|1135|1136|1137|1138|1139|1140|1141|1142|1143|1144|1145|1146|1147|1148|1149|1150|1151|1152 +0.562500000000,512,1,36,1153|1154|1155|1156|1157|1158|1159|1160|1161|1162|1163|1164|1165|1166|1167|1168|1169|1170|1171|1172|1173|1174|1175|1176|1177|1178|1179|1180|1181|1182|1183|1184 +0.578125000000,512,1,37,1185|1186|1187|1188|1189|1190|1191|1192|1193|1194|1195|1196|1197|1198|1199|1200|1201|1202|1203|1204|1205|1206|1207|1208|1209|1210|1211|1212|1213|1214|1215|1216 +0.593750000000,512,1,38,1217|1218|1219|1220|1221|1222|1223|1224|1225|1226|1227|1228|1229|1230|1231|1232|1233|1234|1235|1236|1237|1238|1239|1240|1241|1242|1243|1244|1245|1246|1247|1248 +0.609375000000,512,1,39,1249|1250|1251|1252|1253|1254|1255|1256|1257|1258|1259|1260|1261|1262|1263|1264|1265|1266|1267|1268|1269|1270|1271|1272|1273|1274|1275|1276|1277|1278|1279|1280 +0.625000000000,512,1,40,1281|1282|1283|1284|1285|1286|1287|1288|1289|1290|1291|1292|1293|1294|1295|1296|1297|1298|1299|1300|1301|1302|1303|1304|1305|1306|1307|1308|1309|1310|1311|1312 +0.640625000000,512,1,41,1313|1314|1315|1316|1317|1318|1319|1320|1321|1322|1323|1324|1325|1326|1327|1328|1329|1330|1331|1332|1333|1334|1335|1336|1337|1338|1339|1340|1341|1342|1343|1344 +0.656250000000,512,1,42,1345|1346|1347|1348|1349|1350|1351|1352|1353|1354|1355|1356|1357|1358|1359|1360|1361|1362|1363|1364|1365|1366|1367|1368|1369|1370|1371|1372|1373|1374|1375|1376 +0.671875000000,512,1,43,1377|1378|1379|1380|1381|1382|1383|1384|1385|1386|1387|1388|1389|1390|1391|1392|1393|1394|1395|1396|1397|1398|1399|1400|1401|1402|1403|1404|1405|1406|1407|1408 +0.687500000000,512,1,44,1409|1410|1411|1412|1413|1414|1415|1416|1417|1418|1419|1420|1421|1422|1423|1424|1425|1426|1427|1428|1429|1430|1431|1432|1433|1434|1435|1436|1437|1438|1439|1440 +0.703125000000,512,1,45,1441|1442|1443|1444|1445|1446|1447|1448|1449|1450|1451|1452|1453|1454|1455|1456|1457|1458|1459|1460|1461|1462|1463|1464|1465|1466|1467|1468|1469|1470|1471|1472 +0.718750000000,512,1,46,1473|1474|1475|1476|1477|1478|1479|1480|1481|1482|1483|1484|1485|1486|1487|1488|1489|1490|1491|1492|1493|1494|1495|1496|1497|1498|1499|1500|1501|1502|1503|1504 +0.734375000000,512,1,47,1505|1506|1507|1508|1509|1510|1511|1512|1513|1514|1515|1516|1517|1518|1519|1520|1521|1522|1523|1524|1525|1526|1527|1528|1529|1530|1531|1532|1533|1534|1535|1536 +0.750000000000,512,1,48,1537|1538|1539|1540|1541|1542|1543|1544|1545|1546|1547|1548|1549|1550|1551|1552|1553|1554|1555|1556|1557|1558|1559|1560|1561|1562|1563|1564|1565|1566|1567|1568 +0.765625000000,512,1,49,1569|1570|1571|1572|1573|1574|1575|1576|1577|1578|1579|1580|1581|1582|1583|1584|1585|1586|1587|1588|1589|1590|1591|1592|1593|1594|1595|1596|1597|1598|1599|1600 +0.781250000000,512,1,50,1601|1602|1603|1604|1605|1606|1607|1608|1609|1610|1611|1612|1613|1614|1615|1616|1617|1618|1619|1620|1621|1622|1623|1624|1625|1626|1627|1628|1629|1630|1631|1632 +0.796875000000,512,1,51,1633|1634|1635|1636|1637|1638|1639|1640|1641|1642|1643|1644|1645|1646|1647|1648|1649|1650|1651|1652|1653|1654|1655|1656|1657|1658|1659|1660|1661|1662|1663|1664 +0.812500000000,512,1,52,1665|1666|1667|1668|1669|1670|1671|1672|1673|1674|1675|1676|1677|1678|1679|1680|1681|1682|1683|1684|1685|1686|1687|1688|1689|1690|1691|1692|1693|1694|1695|1696 +0.828125000000,512,1,53,1697|1698|1699|1700|1701|1702|1703|1704|1705|1706|1707|1708|1709|1710|1711|1712|1713|1714|1715|1716|1717|1718|1719|1720|1721|1722|1723|1724|1725|1726|1727|1728 +0.843750000000,512,1,54,1729|1730|1731|1732|1733|1734|1735|1736|1737|1738|1739|1740|1741|1742|1743|1744|1745|1746|1747|1748|1749|1750|1751|1752|1753|1754|1755|1756|1757|1758|1759|1760 +0.859375000000,512,1,55,1761|1762|1763|1764|1765|1766|1767|1768|1769|1770|1771|1772|1773|1774|1775|1776|1777|1778|1779|1780|1781|1782|1783|1784|1785|1786|1787|1788|1789|1790|1791|1792 +0.875000000000,512,1,56,1793|1794|1795|1796|1797|1798|1799|1800|1801|1802|1803|1804|1805|1806|1807|1808|1809|1810|1811|1812|1813|1814|1815|1816|1817|1818|1819|1820|1821|1822|1823|1824 +0.890625000000,512,1,57,1825|1826|1827|1828|1829|1830|1831|1832|1833|1834|1835|1836|1837|1838|1839|1840|1841|1842|1843|1844|1845|1846|1847|1848|1849|1850|1851|1852|1853|1854|1855|1856 +0.906250000000,512,1,58,1857|1858|1859|1860|1861|1862|1863|1864|1865|1866|1867|1868|1869|1870|1871|1872|1873|1874|1875|1876|1877|1878|1879|1880|1881|1882|1883|1884|1885|1886|1887|1888 +0.921875000000,512,1,59,1889|1890|1891|1892|1893|1894|1895|1896|1897|1898|1899|1900|1901|1902|1903|1904|1905|1906|1907|1908|1909|1910|1911|1912|1913|1914|1915|1916|1917|1918|1919|1920 +0.937500000000,512,1,60,1921|1922|1923|1924|1925|1926|1927|1928|1929|1930|1931|1932|1933|1934|1935|1936|1937|1938|1939|1940|1941|1942|1943|1944|1945|1946|1947|1948|1949|1950|1951|1952 +0.953125000000,512,1,61,1953|1954|1955|1956|1957|1958|1959|1960|1961|1962|1963|1964|1965|1966|1967|1968|1969|1970|1971|1972|1973|1974|1975|1976|1977|1978|1979|1980|1981|1982|1983|1984 +0.968750000000,512,1,62,1985|1986|1987|1988|1989|1990|1991|1992|1993|1994|1995|1996|1997|1998|1999|2000|2001|2002|2003|2004|2005|2006|2007|2008|2009|2010|2011|2012|2013|2014|2015|2016 +0.984375000000,512,1,63,2017|2018|2019|2020|2021|2022|2023|2024|2025|2026|2027|2028|2029|2030|2031|2032|2033|2034|2035|2036|2037|2038|2039|2040|2041|2042|2043|2044|2045|2046|2047|2048 diff --git a/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/manifest.json b/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/manifest.json new file mode 100644 index 0000000..6cac8e9 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/manifest.json @@ -0,0 +1,50 @@ +{ + "anchors": [ + { + "label": "r0p5", + "path": "r0p5.csv", + "rate": 0.5, + "sha256": "f8564275c4d7b4f2fa28c375b97ebf38634f623abd68ea88ba2a436061ecbb6d" + }, + { + "label": "r1", + "path": "r1.csv", + "rate": 1.0, + "sha256": "5e029026dc0e7dab6801dfa263975c2a9c2720d844ca41367b2aafd373bfea9e" + }, + { + "label": "r2", + "path": "r2.csv", + "rate": 2.0, + "sha256": "215997a1e318e4aea365e6c9cb87d36620342e128e2cae3e27ac3aef48908900" + }, + { + "label": "r4", + "path": "r4.csv", + "rate": 4.0, + "sha256": "d69a8a9d2b946e4c567ac77b75738bb07ec91b189651a23f3a323f90977cda8d" + }, + { + "label": "r8", + "path": "r8.csv", + "rate": 8.0, + "sha256": "51341a2520c0c2e14f5d7c110083d861b75d7b712ec628b44d07f785bc22a4d2" + }, + { + "label": "r16", + "path": "r16.csv", + "rate": 16.0, + "sha256": "f6d7522e766e0c049118245a9f7c0686fe4e6dc597a871ffc7cc447e0af90aaa" + } + ], + "contract": { + "arrival": "open_loop_uniform", + "block_size": 16, + "input_tokens": 2048, + "output_tokens": 128, + "prefix_caching": false, + "prefix_relation": "all request block identities are disjoint", + "requests": 64 + }, + "schema": "qwen30-fixed-frontier-traces-v1" +} diff --git a/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r0p5.csv b/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r0p5.csv new file mode 100644 index 0000000..1969175 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r0p5.csv @@ -0,0 +1,65 @@ +arrived_at,num_prefill_tokens,num_decode_tokens,session_id,block_hash_ids +0.000000000000,2048,128,0,1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58|59|60|61|62|63|64|65|66|67|68|69|70|71|72|73|74|75|76|77|78|79|80|81|82|83|84|85|86|87|88|89|90|91|92|93|94|95|96|97|98|99|100|101|102|103|104|105|106|107|108|109|110|111|112|113|114|115|116|117|118|119|120|121|122|123|124|125|126|127|128 +2.000000000000,2048,128,1,129|130|131|132|133|134|135|136|137|138|139|140|141|142|143|144|145|146|147|148|149|150|151|152|153|154|155|156|157|158|159|160|161|162|163|164|165|166|167|168|169|170|171|172|173|174|175|176|177|178|179|180|181|182|183|184|185|186|187|188|189|190|191|192|193|194|195|196|197|198|199|200|201|202|203|204|205|206|207|208|209|210|211|212|213|214|215|216|217|218|219|220|221|222|223|224|225|226|227|228|229|230|231|232|233|234|235|236|237|238|239|240|241|242|243|244|245|246|247|248|249|250|251|252|253|254|255|256 +4.000000000000,2048,128,2,257|258|259|260|261|262|263|264|265|266|267|268|269|270|271|272|273|274|275|276|277|278|279|280|281|282|283|284|285|286|287|288|289|290|291|292|293|294|295|296|297|298|299|300|301|302|303|304|305|306|307|308|309|310|311|312|313|314|315|316|317|318|319|320|321|322|323|324|325|326|327|328|329|330|331|332|333|334|335|336|337|338|339|340|341|342|343|344|345|346|347|348|349|350|351|352|353|354|355|356|357|358|359|360|361|362|363|364|365|366|367|368|369|370|371|372|373|374|375|376|377|378|379|380|381|382|383|384 +6.000000000000,2048,128,3,385|386|387|388|389|390|391|392|393|394|395|396|397|398|399|400|401|402|403|404|405|406|407|408|409|410|411|412|413|414|415|416|417|418|419|420|421|422|423|424|425|426|427|428|429|430|431|432|433|434|435|436|437|438|439|440|441|442|443|444|445|446|447|448|449|450|451|452|453|454|455|456|457|458|459|460|461|462|463|464|465|466|467|468|469|470|471|472|473|474|475|476|477|478|479|480|481|482|483|484|485|486|487|488|489|490|491|492|493|494|495|496|497|498|499|500|501|502|503|504|505|506|507|508|509|510|511|512 +8.000000000000,2048,128,4,513|514|515|516|517|518|519|520|521|522|523|524|525|526|527|528|529|530|531|532|533|534|535|536|537|538|539|540|541|542|543|544|545|546|547|548|549|550|551|552|553|554|555|556|557|558|559|560|561|562|563|564|565|566|567|568|569|570|571|572|573|574|575|576|577|578|579|580|581|582|583|584|585|586|587|588|589|590|591|592|593|594|595|596|597|598|599|600|601|602|603|604|605|606|607|608|609|610|611|612|613|614|615|616|617|618|619|620|621|622|623|624|625|626|627|628|629|630|631|632|633|634|635|636|637|638|639|640 +10.000000000000,2048,128,5,641|642|643|644|645|646|647|648|649|650|651|652|653|654|655|656|657|658|659|660|661|662|663|664|665|666|667|668|669|670|671|672|673|674|675|676|677|678|679|680|681|682|683|684|685|686|687|688|689|690|691|692|693|694|695|696|697|698|699|700|701|702|703|704|705|706|707|708|709|710|711|712|713|714|715|716|717|718|719|720|721|722|723|724|725|726|727|728|729|730|731|732|733|734|735|736|737|738|739|740|741|742|743|744|745|746|747|748|749|750|751|752|753|754|755|756|757|758|759|760|761|762|763|764|765|766|767|768 +12.000000000000,2048,128,6,769|770|771|772|773|774|775|776|777|778|779|780|781|782|783|784|785|786|787|788|789|790|791|792|793|794|795|796|797|798|799|800|801|802|803|804|805|806|807|808|809|810|811|812|813|814|815|816|817|818|819|820|821|822|823|824|825|826|827|828|829|830|831|832|833|834|835|836|837|838|839|840|841|842|843|844|845|846|847|848|849|850|851|852|853|854|855|856|857|858|859|860|861|862|863|864|865|866|867|868|869|870|871|872|873|874|875|876|877|878|879|880|881|882|883|884|885|886|887|888|889|890|891|892|893|894|895|896 +14.000000000000,2048,128,7,897|898|899|900|901|902|903|904|905|906|907|908|909|910|911|912|913|914|915|916|917|918|919|920|921|922|923|924|925|926|927|928|929|930|931|932|933|934|935|936|937|938|939|940|941|942|943|944|945|946|947|948|949|950|951|952|953|954|955|956|957|958|959|960|961|962|963|964|965|966|967|968|969|970|971|972|973|974|975|976|977|978|979|980|981|982|983|984|985|986|987|988|989|990|991|992|993|994|995|996|997|998|999|1000|1001|1002|1003|1004|1005|1006|1007|1008|1009|1010|1011|1012|1013|1014|1015|1016|1017|1018|1019|1020|1021|1022|1023|1024 +16.000000000000,2048,128,8,1025|1026|1027|1028|1029|1030|1031|1032|1033|1034|1035|1036|1037|1038|1039|1040|1041|1042|1043|1044|1045|1046|1047|1048|1049|1050|1051|1052|1053|1054|1055|1056|1057|1058|1059|1060|1061|1062|1063|1064|1065|1066|1067|1068|1069|1070|1071|1072|1073|1074|1075|1076|1077|1078|1079|1080|1081|1082|1083|1084|1085|1086|1087|1088|1089|1090|1091|1092|1093|1094|1095|1096|1097|1098|1099|1100|1101|1102|1103|1104|1105|1106|1107|1108|1109|1110|1111|1112|1113|1114|1115|1116|1117|1118|1119|1120|1121|1122|1123|1124|1125|1126|1127|1128|1129|1130|1131|1132|1133|1134|1135|1136|1137|1138|1139|1140|1141|1142|1143|1144|1145|1146|1147|1148|1149|1150|1151|1152 +18.000000000000,2048,128,9,1153|1154|1155|1156|1157|1158|1159|1160|1161|1162|1163|1164|1165|1166|1167|1168|1169|1170|1171|1172|1173|1174|1175|1176|1177|1178|1179|1180|1181|1182|1183|1184|1185|1186|1187|1188|1189|1190|1191|1192|1193|1194|1195|1196|1197|1198|1199|1200|1201|1202|1203|1204|1205|1206|1207|1208|1209|1210|1211|1212|1213|1214|1215|1216|1217|1218|1219|1220|1221|1222|1223|1224|1225|1226|1227|1228|1229|1230|1231|1232|1233|1234|1235|1236|1237|1238|1239|1240|1241|1242|1243|1244|1245|1246|1247|1248|1249|1250|1251|1252|1253|1254|1255|1256|1257|1258|1259|1260|1261|1262|1263|1264|1265|1266|1267|1268|1269|1270|1271|1272|1273|1274|1275|1276|1277|1278|1279|1280 +20.000000000000,2048,128,10,1281|1282|1283|1284|1285|1286|1287|1288|1289|1290|1291|1292|1293|1294|1295|1296|1297|1298|1299|1300|1301|1302|1303|1304|1305|1306|1307|1308|1309|1310|1311|1312|1313|1314|1315|1316|1317|1318|1319|1320|1321|1322|1323|1324|1325|1326|1327|1328|1329|1330|1331|1332|1333|1334|1335|1336|1337|1338|1339|1340|1341|1342|1343|1344|1345|1346|1347|1348|1349|1350|1351|1352|1353|1354|1355|1356|1357|1358|1359|1360|1361|1362|1363|1364|1365|1366|1367|1368|1369|1370|1371|1372|1373|1374|1375|1376|1377|1378|1379|1380|1381|1382|1383|1384|1385|1386|1387|1388|1389|1390|1391|1392|1393|1394|1395|1396|1397|1398|1399|1400|1401|1402|1403|1404|1405|1406|1407|1408 +22.000000000000,2048,128,11,1409|1410|1411|1412|1413|1414|1415|1416|1417|1418|1419|1420|1421|1422|1423|1424|1425|1426|1427|1428|1429|1430|1431|1432|1433|1434|1435|1436|1437|1438|1439|1440|1441|1442|1443|1444|1445|1446|1447|1448|1449|1450|1451|1452|1453|1454|1455|1456|1457|1458|1459|1460|1461|1462|1463|1464|1465|1466|1467|1468|1469|1470|1471|1472|1473|1474|1475|1476|1477|1478|1479|1480|1481|1482|1483|1484|1485|1486|1487|1488|1489|1490|1491|1492|1493|1494|1495|1496|1497|1498|1499|1500|1501|1502|1503|1504|1505|1506|1507|1508|1509|1510|1511|1512|1513|1514|1515|1516|1517|1518|1519|1520|1521|1522|1523|1524|1525|1526|1527|1528|1529|1530|1531|1532|1533|1534|1535|1536 +24.000000000000,2048,128,12,1537|1538|1539|1540|1541|1542|1543|1544|1545|1546|1547|1548|1549|1550|1551|1552|1553|1554|1555|1556|1557|1558|1559|1560|1561|1562|1563|1564|1565|1566|1567|1568|1569|1570|1571|1572|1573|1574|1575|1576|1577|1578|1579|1580|1581|1582|1583|1584|1585|1586|1587|1588|1589|1590|1591|1592|1593|1594|1595|1596|1597|1598|1599|1600|1601|1602|1603|1604|1605|1606|1607|1608|1609|1610|1611|1612|1613|1614|1615|1616|1617|1618|1619|1620|1621|1622|1623|1624|1625|1626|1627|1628|1629|1630|1631|1632|1633|1634|1635|1636|1637|1638|1639|1640|1641|1642|1643|1644|1645|1646|1647|1648|1649|1650|1651|1652|1653|1654|1655|1656|1657|1658|1659|1660|1661|1662|1663|1664 +26.000000000000,2048,128,13,1665|1666|1667|1668|1669|1670|1671|1672|1673|1674|1675|1676|1677|1678|1679|1680|1681|1682|1683|1684|1685|1686|1687|1688|1689|1690|1691|1692|1693|1694|1695|1696|1697|1698|1699|1700|1701|1702|1703|1704|1705|1706|1707|1708|1709|1710|1711|1712|1713|1714|1715|1716|1717|1718|1719|1720|1721|1722|1723|1724|1725|1726|1727|1728|1729|1730|1731|1732|1733|1734|1735|1736|1737|1738|1739|1740|1741|1742|1743|1744|1745|1746|1747|1748|1749|1750|1751|1752|1753|1754|1755|1756|1757|1758|1759|1760|1761|1762|1763|1764|1765|1766|1767|1768|1769|1770|1771|1772|1773|1774|1775|1776|1777|1778|1779|1780|1781|1782|1783|1784|1785|1786|1787|1788|1789|1790|1791|1792 +28.000000000000,2048,128,14,1793|1794|1795|1796|1797|1798|1799|1800|1801|1802|1803|1804|1805|1806|1807|1808|1809|1810|1811|1812|1813|1814|1815|1816|1817|1818|1819|1820|1821|1822|1823|1824|1825|1826|1827|1828|1829|1830|1831|1832|1833|1834|1835|1836|1837|1838|1839|1840|1841|1842|1843|1844|1845|1846|1847|1848|1849|1850|1851|1852|1853|1854|1855|1856|1857|1858|1859|1860|1861|1862|1863|1864|1865|1866|1867|1868|1869|1870|1871|1872|1873|1874|1875|1876|1877|1878|1879|1880|1881|1882|1883|1884|1885|1886|1887|1888|1889|1890|1891|1892|1893|1894|1895|1896|1897|1898|1899|1900|1901|1902|1903|1904|1905|1906|1907|1908|1909|1910|1911|1912|1913|1914|1915|1916|1917|1918|1919|1920 +30.000000000000,2048,128,15,1921|1922|1923|1924|1925|1926|1927|1928|1929|1930|1931|1932|1933|1934|1935|1936|1937|1938|1939|1940|1941|1942|1943|1944|1945|1946|1947|1948|1949|1950|1951|1952|1953|1954|1955|1956|1957|1958|1959|1960|1961|1962|1963|1964|1965|1966|1967|1968|1969|1970|1971|1972|1973|1974|1975|1976|1977|1978|1979|1980|1981|1982|1983|1984|1985|1986|1987|1988|1989|1990|1991|1992|1993|1994|1995|1996|1997|1998|1999|2000|2001|2002|2003|2004|2005|2006|2007|2008|2009|2010|2011|2012|2013|2014|2015|2016|2017|2018|2019|2020|2021|2022|2023|2024|2025|2026|2027|2028|2029|2030|2031|2032|2033|2034|2035|2036|2037|2038|2039|2040|2041|2042|2043|2044|2045|2046|2047|2048 +32.000000000000,2048,128,16,2049|2050|2051|2052|2053|2054|2055|2056|2057|2058|2059|2060|2061|2062|2063|2064|2065|2066|2067|2068|2069|2070|2071|2072|2073|2074|2075|2076|2077|2078|2079|2080|2081|2082|2083|2084|2085|2086|2087|2088|2089|2090|2091|2092|2093|2094|2095|2096|2097|2098|2099|2100|2101|2102|2103|2104|2105|2106|2107|2108|2109|2110|2111|2112|2113|2114|2115|2116|2117|2118|2119|2120|2121|2122|2123|2124|2125|2126|2127|2128|2129|2130|2131|2132|2133|2134|2135|2136|2137|2138|2139|2140|2141|2142|2143|2144|2145|2146|2147|2148|2149|2150|2151|2152|2153|2154|2155|2156|2157|2158|2159|2160|2161|2162|2163|2164|2165|2166|2167|2168|2169|2170|2171|2172|2173|2174|2175|2176 +34.000000000000,2048,128,17,2177|2178|2179|2180|2181|2182|2183|2184|2185|2186|2187|2188|2189|2190|2191|2192|2193|2194|2195|2196|2197|2198|2199|2200|2201|2202|2203|2204|2205|2206|2207|2208|2209|2210|2211|2212|2213|2214|2215|2216|2217|2218|2219|2220|2221|2222|2223|2224|2225|2226|2227|2228|2229|2230|2231|2232|2233|2234|2235|2236|2237|2238|2239|2240|2241|2242|2243|2244|2245|2246|2247|2248|2249|2250|2251|2252|2253|2254|2255|2256|2257|2258|2259|2260|2261|2262|2263|2264|2265|2266|2267|2268|2269|2270|2271|2272|2273|2274|2275|2276|2277|2278|2279|2280|2281|2282|2283|2284|2285|2286|2287|2288|2289|2290|2291|2292|2293|2294|2295|2296|2297|2298|2299|2300|2301|2302|2303|2304 +36.000000000000,2048,128,18,2305|2306|2307|2308|2309|2310|2311|2312|2313|2314|2315|2316|2317|2318|2319|2320|2321|2322|2323|2324|2325|2326|2327|2328|2329|2330|2331|2332|2333|2334|2335|2336|2337|2338|2339|2340|2341|2342|2343|2344|2345|2346|2347|2348|2349|2350|2351|2352|2353|2354|2355|2356|2357|2358|2359|2360|2361|2362|2363|2364|2365|2366|2367|2368|2369|2370|2371|2372|2373|2374|2375|2376|2377|2378|2379|2380|2381|2382|2383|2384|2385|2386|2387|2388|2389|2390|2391|2392|2393|2394|2395|2396|2397|2398|2399|2400|2401|2402|2403|2404|2405|2406|2407|2408|2409|2410|2411|2412|2413|2414|2415|2416|2417|2418|2419|2420|2421|2422|2423|2424|2425|2426|2427|2428|2429|2430|2431|2432 +38.000000000000,2048,128,19,2433|2434|2435|2436|2437|2438|2439|2440|2441|2442|2443|2444|2445|2446|2447|2448|2449|2450|2451|2452|2453|2454|2455|2456|2457|2458|2459|2460|2461|2462|2463|2464|2465|2466|2467|2468|2469|2470|2471|2472|2473|2474|2475|2476|2477|2478|2479|2480|2481|2482|2483|2484|2485|2486|2487|2488|2489|2490|2491|2492|2493|2494|2495|2496|2497|2498|2499|2500|2501|2502|2503|2504|2505|2506|2507|2508|2509|2510|2511|2512|2513|2514|2515|2516|2517|2518|2519|2520|2521|2522|2523|2524|2525|2526|2527|2528|2529|2530|2531|2532|2533|2534|2535|2536|2537|2538|2539|2540|2541|2542|2543|2544|2545|2546|2547|2548|2549|2550|2551|2552|2553|2554|2555|2556|2557|2558|2559|2560 +40.000000000000,2048,128,20,2561|2562|2563|2564|2565|2566|2567|2568|2569|2570|2571|2572|2573|2574|2575|2576|2577|2578|2579|2580|2581|2582|2583|2584|2585|2586|2587|2588|2589|2590|2591|2592|2593|2594|2595|2596|2597|2598|2599|2600|2601|2602|2603|2604|2605|2606|2607|2608|2609|2610|2611|2612|2613|2614|2615|2616|2617|2618|2619|2620|2621|2622|2623|2624|2625|2626|2627|2628|2629|2630|2631|2632|2633|2634|2635|2636|2637|2638|2639|2640|2641|2642|2643|2644|2645|2646|2647|2648|2649|2650|2651|2652|2653|2654|2655|2656|2657|2658|2659|2660|2661|2662|2663|2664|2665|2666|2667|2668|2669|2670|2671|2672|2673|2674|2675|2676|2677|2678|2679|2680|2681|2682|2683|2684|2685|2686|2687|2688 +42.000000000000,2048,128,21,2689|2690|2691|2692|2693|2694|2695|2696|2697|2698|2699|2700|2701|2702|2703|2704|2705|2706|2707|2708|2709|2710|2711|2712|2713|2714|2715|2716|2717|2718|2719|2720|2721|2722|2723|2724|2725|2726|2727|2728|2729|2730|2731|2732|2733|2734|2735|2736|2737|2738|2739|2740|2741|2742|2743|2744|2745|2746|2747|2748|2749|2750|2751|2752|2753|2754|2755|2756|2757|2758|2759|2760|2761|2762|2763|2764|2765|2766|2767|2768|2769|2770|2771|2772|2773|2774|2775|2776|2777|2778|2779|2780|2781|2782|2783|2784|2785|2786|2787|2788|2789|2790|2791|2792|2793|2794|2795|2796|2797|2798|2799|2800|2801|2802|2803|2804|2805|2806|2807|2808|2809|2810|2811|2812|2813|2814|2815|2816 +44.000000000000,2048,128,22,2817|2818|2819|2820|2821|2822|2823|2824|2825|2826|2827|2828|2829|2830|2831|2832|2833|2834|2835|2836|2837|2838|2839|2840|2841|2842|2843|2844|2845|2846|2847|2848|2849|2850|2851|2852|2853|2854|2855|2856|2857|2858|2859|2860|2861|2862|2863|2864|2865|2866|2867|2868|2869|2870|2871|2872|2873|2874|2875|2876|2877|2878|2879|2880|2881|2882|2883|2884|2885|2886|2887|2888|2889|2890|2891|2892|2893|2894|2895|2896|2897|2898|2899|2900|2901|2902|2903|2904|2905|2906|2907|2908|2909|2910|2911|2912|2913|2914|2915|2916|2917|2918|2919|2920|2921|2922|2923|2924|2925|2926|2927|2928|2929|2930|2931|2932|2933|2934|2935|2936|2937|2938|2939|2940|2941|2942|2943|2944 +46.000000000000,2048,128,23,2945|2946|2947|2948|2949|2950|2951|2952|2953|2954|2955|2956|2957|2958|2959|2960|2961|2962|2963|2964|2965|2966|2967|2968|2969|2970|2971|2972|2973|2974|2975|2976|2977|2978|2979|2980|2981|2982|2983|2984|2985|2986|2987|2988|2989|2990|2991|2992|2993|2994|2995|2996|2997|2998|2999|3000|3001|3002|3003|3004|3005|3006|3007|3008|3009|3010|3011|3012|3013|3014|3015|3016|3017|3018|3019|3020|3021|3022|3023|3024|3025|3026|3027|3028|3029|3030|3031|3032|3033|3034|3035|3036|3037|3038|3039|3040|3041|3042|3043|3044|3045|3046|3047|3048|3049|3050|3051|3052|3053|3054|3055|3056|3057|3058|3059|3060|3061|3062|3063|3064|3065|3066|3067|3068|3069|3070|3071|3072 +48.000000000000,2048,128,24,3073|3074|3075|3076|3077|3078|3079|3080|3081|3082|3083|3084|3085|3086|3087|3088|3089|3090|3091|3092|3093|3094|3095|3096|3097|3098|3099|3100|3101|3102|3103|3104|3105|3106|3107|3108|3109|3110|3111|3112|3113|3114|3115|3116|3117|3118|3119|3120|3121|3122|3123|3124|3125|3126|3127|3128|3129|3130|3131|3132|3133|3134|3135|3136|3137|3138|3139|3140|3141|3142|3143|3144|3145|3146|3147|3148|3149|3150|3151|3152|3153|3154|3155|3156|3157|3158|3159|3160|3161|3162|3163|3164|3165|3166|3167|3168|3169|3170|3171|3172|3173|3174|3175|3176|3177|3178|3179|3180|3181|3182|3183|3184|3185|3186|3187|3188|3189|3190|3191|3192|3193|3194|3195|3196|3197|3198|3199|3200 +50.000000000000,2048,128,25,3201|3202|3203|3204|3205|3206|3207|3208|3209|3210|3211|3212|3213|3214|3215|3216|3217|3218|3219|3220|3221|3222|3223|3224|3225|3226|3227|3228|3229|3230|3231|3232|3233|3234|3235|3236|3237|3238|3239|3240|3241|3242|3243|3244|3245|3246|3247|3248|3249|3250|3251|3252|3253|3254|3255|3256|3257|3258|3259|3260|3261|3262|3263|3264|3265|3266|3267|3268|3269|3270|3271|3272|3273|3274|3275|3276|3277|3278|3279|3280|3281|3282|3283|3284|3285|3286|3287|3288|3289|3290|3291|3292|3293|3294|3295|3296|3297|3298|3299|3300|3301|3302|3303|3304|3305|3306|3307|3308|3309|3310|3311|3312|3313|3314|3315|3316|3317|3318|3319|3320|3321|3322|3323|3324|3325|3326|3327|3328 +52.000000000000,2048,128,26,3329|3330|3331|3332|3333|3334|3335|3336|3337|3338|3339|3340|3341|3342|3343|3344|3345|3346|3347|3348|3349|3350|3351|3352|3353|3354|3355|3356|3357|3358|3359|3360|3361|3362|3363|3364|3365|3366|3367|3368|3369|3370|3371|3372|3373|3374|3375|3376|3377|3378|3379|3380|3381|3382|3383|3384|3385|3386|3387|3388|3389|3390|3391|3392|3393|3394|3395|3396|3397|3398|3399|3400|3401|3402|3403|3404|3405|3406|3407|3408|3409|3410|3411|3412|3413|3414|3415|3416|3417|3418|3419|3420|3421|3422|3423|3424|3425|3426|3427|3428|3429|3430|3431|3432|3433|3434|3435|3436|3437|3438|3439|3440|3441|3442|3443|3444|3445|3446|3447|3448|3449|3450|3451|3452|3453|3454|3455|3456 +54.000000000000,2048,128,27,3457|3458|3459|3460|3461|3462|3463|3464|3465|3466|3467|3468|3469|3470|3471|3472|3473|3474|3475|3476|3477|3478|3479|3480|3481|3482|3483|3484|3485|3486|3487|3488|3489|3490|3491|3492|3493|3494|3495|3496|3497|3498|3499|3500|3501|3502|3503|3504|3505|3506|3507|3508|3509|3510|3511|3512|3513|3514|3515|3516|3517|3518|3519|3520|3521|3522|3523|3524|3525|3526|3527|3528|3529|3530|3531|3532|3533|3534|3535|3536|3537|3538|3539|3540|3541|3542|3543|3544|3545|3546|3547|3548|3549|3550|3551|3552|3553|3554|3555|3556|3557|3558|3559|3560|3561|3562|3563|3564|3565|3566|3567|3568|3569|3570|3571|3572|3573|3574|3575|3576|3577|3578|3579|3580|3581|3582|3583|3584 +56.000000000000,2048,128,28,3585|3586|3587|3588|3589|3590|3591|3592|3593|3594|3595|3596|3597|3598|3599|3600|3601|3602|3603|3604|3605|3606|3607|3608|3609|3610|3611|3612|3613|3614|3615|3616|3617|3618|3619|3620|3621|3622|3623|3624|3625|3626|3627|3628|3629|3630|3631|3632|3633|3634|3635|3636|3637|3638|3639|3640|3641|3642|3643|3644|3645|3646|3647|3648|3649|3650|3651|3652|3653|3654|3655|3656|3657|3658|3659|3660|3661|3662|3663|3664|3665|3666|3667|3668|3669|3670|3671|3672|3673|3674|3675|3676|3677|3678|3679|3680|3681|3682|3683|3684|3685|3686|3687|3688|3689|3690|3691|3692|3693|3694|3695|3696|3697|3698|3699|3700|3701|3702|3703|3704|3705|3706|3707|3708|3709|3710|3711|3712 +58.000000000000,2048,128,29,3713|3714|3715|3716|3717|3718|3719|3720|3721|3722|3723|3724|3725|3726|3727|3728|3729|3730|3731|3732|3733|3734|3735|3736|3737|3738|3739|3740|3741|3742|3743|3744|3745|3746|3747|3748|3749|3750|3751|3752|3753|3754|3755|3756|3757|3758|3759|3760|3761|3762|3763|3764|3765|3766|3767|3768|3769|3770|3771|3772|3773|3774|3775|3776|3777|3778|3779|3780|3781|3782|3783|3784|3785|3786|3787|3788|3789|3790|3791|3792|3793|3794|3795|3796|3797|3798|3799|3800|3801|3802|3803|3804|3805|3806|3807|3808|3809|3810|3811|3812|3813|3814|3815|3816|3817|3818|3819|3820|3821|3822|3823|3824|3825|3826|3827|3828|3829|3830|3831|3832|3833|3834|3835|3836|3837|3838|3839|3840 +60.000000000000,2048,128,30,3841|3842|3843|3844|3845|3846|3847|3848|3849|3850|3851|3852|3853|3854|3855|3856|3857|3858|3859|3860|3861|3862|3863|3864|3865|3866|3867|3868|3869|3870|3871|3872|3873|3874|3875|3876|3877|3878|3879|3880|3881|3882|3883|3884|3885|3886|3887|3888|3889|3890|3891|3892|3893|3894|3895|3896|3897|3898|3899|3900|3901|3902|3903|3904|3905|3906|3907|3908|3909|3910|3911|3912|3913|3914|3915|3916|3917|3918|3919|3920|3921|3922|3923|3924|3925|3926|3927|3928|3929|3930|3931|3932|3933|3934|3935|3936|3937|3938|3939|3940|3941|3942|3943|3944|3945|3946|3947|3948|3949|3950|3951|3952|3953|3954|3955|3956|3957|3958|3959|3960|3961|3962|3963|3964|3965|3966|3967|3968 +62.000000000000,2048,128,31,3969|3970|3971|3972|3973|3974|3975|3976|3977|3978|3979|3980|3981|3982|3983|3984|3985|3986|3987|3988|3989|3990|3991|3992|3993|3994|3995|3996|3997|3998|3999|4000|4001|4002|4003|4004|4005|4006|4007|4008|4009|4010|4011|4012|4013|4014|4015|4016|4017|4018|4019|4020|4021|4022|4023|4024|4025|4026|4027|4028|4029|4030|4031|4032|4033|4034|4035|4036|4037|4038|4039|4040|4041|4042|4043|4044|4045|4046|4047|4048|4049|4050|4051|4052|4053|4054|4055|4056|4057|4058|4059|4060|4061|4062|4063|4064|4065|4066|4067|4068|4069|4070|4071|4072|4073|4074|4075|4076|4077|4078|4079|4080|4081|4082|4083|4084|4085|4086|4087|4088|4089|4090|4091|4092|4093|4094|4095|4096 +64.000000000000,2048,128,32,4097|4098|4099|4100|4101|4102|4103|4104|4105|4106|4107|4108|4109|4110|4111|4112|4113|4114|4115|4116|4117|4118|4119|4120|4121|4122|4123|4124|4125|4126|4127|4128|4129|4130|4131|4132|4133|4134|4135|4136|4137|4138|4139|4140|4141|4142|4143|4144|4145|4146|4147|4148|4149|4150|4151|4152|4153|4154|4155|4156|4157|4158|4159|4160|4161|4162|4163|4164|4165|4166|4167|4168|4169|4170|4171|4172|4173|4174|4175|4176|4177|4178|4179|4180|4181|4182|4183|4184|4185|4186|4187|4188|4189|4190|4191|4192|4193|4194|4195|4196|4197|4198|4199|4200|4201|4202|4203|4204|4205|4206|4207|4208|4209|4210|4211|4212|4213|4214|4215|4216|4217|4218|4219|4220|4221|4222|4223|4224 +66.000000000000,2048,128,33,4225|4226|4227|4228|4229|4230|4231|4232|4233|4234|4235|4236|4237|4238|4239|4240|4241|4242|4243|4244|4245|4246|4247|4248|4249|4250|4251|4252|4253|4254|4255|4256|4257|4258|4259|4260|4261|4262|4263|4264|4265|4266|4267|4268|4269|4270|4271|4272|4273|4274|4275|4276|4277|4278|4279|4280|4281|4282|4283|4284|4285|4286|4287|4288|4289|4290|4291|4292|4293|4294|4295|4296|4297|4298|4299|4300|4301|4302|4303|4304|4305|4306|4307|4308|4309|4310|4311|4312|4313|4314|4315|4316|4317|4318|4319|4320|4321|4322|4323|4324|4325|4326|4327|4328|4329|4330|4331|4332|4333|4334|4335|4336|4337|4338|4339|4340|4341|4342|4343|4344|4345|4346|4347|4348|4349|4350|4351|4352 +68.000000000000,2048,128,34,4353|4354|4355|4356|4357|4358|4359|4360|4361|4362|4363|4364|4365|4366|4367|4368|4369|4370|4371|4372|4373|4374|4375|4376|4377|4378|4379|4380|4381|4382|4383|4384|4385|4386|4387|4388|4389|4390|4391|4392|4393|4394|4395|4396|4397|4398|4399|4400|4401|4402|4403|4404|4405|4406|4407|4408|4409|4410|4411|4412|4413|4414|4415|4416|4417|4418|4419|4420|4421|4422|4423|4424|4425|4426|4427|4428|4429|4430|4431|4432|4433|4434|4435|4436|4437|4438|4439|4440|4441|4442|4443|4444|4445|4446|4447|4448|4449|4450|4451|4452|4453|4454|4455|4456|4457|4458|4459|4460|4461|4462|4463|4464|4465|4466|4467|4468|4469|4470|4471|4472|4473|4474|4475|4476|4477|4478|4479|4480 +70.000000000000,2048,128,35,4481|4482|4483|4484|4485|4486|4487|4488|4489|4490|4491|4492|4493|4494|4495|4496|4497|4498|4499|4500|4501|4502|4503|4504|4505|4506|4507|4508|4509|4510|4511|4512|4513|4514|4515|4516|4517|4518|4519|4520|4521|4522|4523|4524|4525|4526|4527|4528|4529|4530|4531|4532|4533|4534|4535|4536|4537|4538|4539|4540|4541|4542|4543|4544|4545|4546|4547|4548|4549|4550|4551|4552|4553|4554|4555|4556|4557|4558|4559|4560|4561|4562|4563|4564|4565|4566|4567|4568|4569|4570|4571|4572|4573|4574|4575|4576|4577|4578|4579|4580|4581|4582|4583|4584|4585|4586|4587|4588|4589|4590|4591|4592|4593|4594|4595|4596|4597|4598|4599|4600|4601|4602|4603|4604|4605|4606|4607|4608 +72.000000000000,2048,128,36,4609|4610|4611|4612|4613|4614|4615|4616|4617|4618|4619|4620|4621|4622|4623|4624|4625|4626|4627|4628|4629|4630|4631|4632|4633|4634|4635|4636|4637|4638|4639|4640|4641|4642|4643|4644|4645|4646|4647|4648|4649|4650|4651|4652|4653|4654|4655|4656|4657|4658|4659|4660|4661|4662|4663|4664|4665|4666|4667|4668|4669|4670|4671|4672|4673|4674|4675|4676|4677|4678|4679|4680|4681|4682|4683|4684|4685|4686|4687|4688|4689|4690|4691|4692|4693|4694|4695|4696|4697|4698|4699|4700|4701|4702|4703|4704|4705|4706|4707|4708|4709|4710|4711|4712|4713|4714|4715|4716|4717|4718|4719|4720|4721|4722|4723|4724|4725|4726|4727|4728|4729|4730|4731|4732|4733|4734|4735|4736 +74.000000000000,2048,128,37,4737|4738|4739|4740|4741|4742|4743|4744|4745|4746|4747|4748|4749|4750|4751|4752|4753|4754|4755|4756|4757|4758|4759|4760|4761|4762|4763|4764|4765|4766|4767|4768|4769|4770|4771|4772|4773|4774|4775|4776|4777|4778|4779|4780|4781|4782|4783|4784|4785|4786|4787|4788|4789|4790|4791|4792|4793|4794|4795|4796|4797|4798|4799|4800|4801|4802|4803|4804|4805|4806|4807|4808|4809|4810|4811|4812|4813|4814|4815|4816|4817|4818|4819|4820|4821|4822|4823|4824|4825|4826|4827|4828|4829|4830|4831|4832|4833|4834|4835|4836|4837|4838|4839|4840|4841|4842|4843|4844|4845|4846|4847|4848|4849|4850|4851|4852|4853|4854|4855|4856|4857|4858|4859|4860|4861|4862|4863|4864 +76.000000000000,2048,128,38,4865|4866|4867|4868|4869|4870|4871|4872|4873|4874|4875|4876|4877|4878|4879|4880|4881|4882|4883|4884|4885|4886|4887|4888|4889|4890|4891|4892|4893|4894|4895|4896|4897|4898|4899|4900|4901|4902|4903|4904|4905|4906|4907|4908|4909|4910|4911|4912|4913|4914|4915|4916|4917|4918|4919|4920|4921|4922|4923|4924|4925|4926|4927|4928|4929|4930|4931|4932|4933|4934|4935|4936|4937|4938|4939|4940|4941|4942|4943|4944|4945|4946|4947|4948|4949|4950|4951|4952|4953|4954|4955|4956|4957|4958|4959|4960|4961|4962|4963|4964|4965|4966|4967|4968|4969|4970|4971|4972|4973|4974|4975|4976|4977|4978|4979|4980|4981|4982|4983|4984|4985|4986|4987|4988|4989|4990|4991|4992 +78.000000000000,2048,128,39,4993|4994|4995|4996|4997|4998|4999|5000|5001|5002|5003|5004|5005|5006|5007|5008|5009|5010|5011|5012|5013|5014|5015|5016|5017|5018|5019|5020|5021|5022|5023|5024|5025|5026|5027|5028|5029|5030|5031|5032|5033|5034|5035|5036|5037|5038|5039|5040|5041|5042|5043|5044|5045|5046|5047|5048|5049|5050|5051|5052|5053|5054|5055|5056|5057|5058|5059|5060|5061|5062|5063|5064|5065|5066|5067|5068|5069|5070|5071|5072|5073|5074|5075|5076|5077|5078|5079|5080|5081|5082|5083|5084|5085|5086|5087|5088|5089|5090|5091|5092|5093|5094|5095|5096|5097|5098|5099|5100|5101|5102|5103|5104|5105|5106|5107|5108|5109|5110|5111|5112|5113|5114|5115|5116|5117|5118|5119|5120 +80.000000000000,2048,128,40,5121|5122|5123|5124|5125|5126|5127|5128|5129|5130|5131|5132|5133|5134|5135|5136|5137|5138|5139|5140|5141|5142|5143|5144|5145|5146|5147|5148|5149|5150|5151|5152|5153|5154|5155|5156|5157|5158|5159|5160|5161|5162|5163|5164|5165|5166|5167|5168|5169|5170|5171|5172|5173|5174|5175|5176|5177|5178|5179|5180|5181|5182|5183|5184|5185|5186|5187|5188|5189|5190|5191|5192|5193|5194|5195|5196|5197|5198|5199|5200|5201|5202|5203|5204|5205|5206|5207|5208|5209|5210|5211|5212|5213|5214|5215|5216|5217|5218|5219|5220|5221|5222|5223|5224|5225|5226|5227|5228|5229|5230|5231|5232|5233|5234|5235|5236|5237|5238|5239|5240|5241|5242|5243|5244|5245|5246|5247|5248 +82.000000000000,2048,128,41,5249|5250|5251|5252|5253|5254|5255|5256|5257|5258|5259|5260|5261|5262|5263|5264|5265|5266|5267|5268|5269|5270|5271|5272|5273|5274|5275|5276|5277|5278|5279|5280|5281|5282|5283|5284|5285|5286|5287|5288|5289|5290|5291|5292|5293|5294|5295|5296|5297|5298|5299|5300|5301|5302|5303|5304|5305|5306|5307|5308|5309|5310|5311|5312|5313|5314|5315|5316|5317|5318|5319|5320|5321|5322|5323|5324|5325|5326|5327|5328|5329|5330|5331|5332|5333|5334|5335|5336|5337|5338|5339|5340|5341|5342|5343|5344|5345|5346|5347|5348|5349|5350|5351|5352|5353|5354|5355|5356|5357|5358|5359|5360|5361|5362|5363|5364|5365|5366|5367|5368|5369|5370|5371|5372|5373|5374|5375|5376 +84.000000000000,2048,128,42,5377|5378|5379|5380|5381|5382|5383|5384|5385|5386|5387|5388|5389|5390|5391|5392|5393|5394|5395|5396|5397|5398|5399|5400|5401|5402|5403|5404|5405|5406|5407|5408|5409|5410|5411|5412|5413|5414|5415|5416|5417|5418|5419|5420|5421|5422|5423|5424|5425|5426|5427|5428|5429|5430|5431|5432|5433|5434|5435|5436|5437|5438|5439|5440|5441|5442|5443|5444|5445|5446|5447|5448|5449|5450|5451|5452|5453|5454|5455|5456|5457|5458|5459|5460|5461|5462|5463|5464|5465|5466|5467|5468|5469|5470|5471|5472|5473|5474|5475|5476|5477|5478|5479|5480|5481|5482|5483|5484|5485|5486|5487|5488|5489|5490|5491|5492|5493|5494|5495|5496|5497|5498|5499|5500|5501|5502|5503|5504 +86.000000000000,2048,128,43,5505|5506|5507|5508|5509|5510|5511|5512|5513|5514|5515|5516|5517|5518|5519|5520|5521|5522|5523|5524|5525|5526|5527|5528|5529|5530|5531|5532|5533|5534|5535|5536|5537|5538|5539|5540|5541|5542|5543|5544|5545|5546|5547|5548|5549|5550|5551|5552|5553|5554|5555|5556|5557|5558|5559|5560|5561|5562|5563|5564|5565|5566|5567|5568|5569|5570|5571|5572|5573|5574|5575|5576|5577|5578|5579|5580|5581|5582|5583|5584|5585|5586|5587|5588|5589|5590|5591|5592|5593|5594|5595|5596|5597|5598|5599|5600|5601|5602|5603|5604|5605|5606|5607|5608|5609|5610|5611|5612|5613|5614|5615|5616|5617|5618|5619|5620|5621|5622|5623|5624|5625|5626|5627|5628|5629|5630|5631|5632 +88.000000000000,2048,128,44,5633|5634|5635|5636|5637|5638|5639|5640|5641|5642|5643|5644|5645|5646|5647|5648|5649|5650|5651|5652|5653|5654|5655|5656|5657|5658|5659|5660|5661|5662|5663|5664|5665|5666|5667|5668|5669|5670|5671|5672|5673|5674|5675|5676|5677|5678|5679|5680|5681|5682|5683|5684|5685|5686|5687|5688|5689|5690|5691|5692|5693|5694|5695|5696|5697|5698|5699|5700|5701|5702|5703|5704|5705|5706|5707|5708|5709|5710|5711|5712|5713|5714|5715|5716|5717|5718|5719|5720|5721|5722|5723|5724|5725|5726|5727|5728|5729|5730|5731|5732|5733|5734|5735|5736|5737|5738|5739|5740|5741|5742|5743|5744|5745|5746|5747|5748|5749|5750|5751|5752|5753|5754|5755|5756|5757|5758|5759|5760 +90.000000000000,2048,128,45,5761|5762|5763|5764|5765|5766|5767|5768|5769|5770|5771|5772|5773|5774|5775|5776|5777|5778|5779|5780|5781|5782|5783|5784|5785|5786|5787|5788|5789|5790|5791|5792|5793|5794|5795|5796|5797|5798|5799|5800|5801|5802|5803|5804|5805|5806|5807|5808|5809|5810|5811|5812|5813|5814|5815|5816|5817|5818|5819|5820|5821|5822|5823|5824|5825|5826|5827|5828|5829|5830|5831|5832|5833|5834|5835|5836|5837|5838|5839|5840|5841|5842|5843|5844|5845|5846|5847|5848|5849|5850|5851|5852|5853|5854|5855|5856|5857|5858|5859|5860|5861|5862|5863|5864|5865|5866|5867|5868|5869|5870|5871|5872|5873|5874|5875|5876|5877|5878|5879|5880|5881|5882|5883|5884|5885|5886|5887|5888 +92.000000000000,2048,128,46,5889|5890|5891|5892|5893|5894|5895|5896|5897|5898|5899|5900|5901|5902|5903|5904|5905|5906|5907|5908|5909|5910|5911|5912|5913|5914|5915|5916|5917|5918|5919|5920|5921|5922|5923|5924|5925|5926|5927|5928|5929|5930|5931|5932|5933|5934|5935|5936|5937|5938|5939|5940|5941|5942|5943|5944|5945|5946|5947|5948|5949|5950|5951|5952|5953|5954|5955|5956|5957|5958|5959|5960|5961|5962|5963|5964|5965|5966|5967|5968|5969|5970|5971|5972|5973|5974|5975|5976|5977|5978|5979|5980|5981|5982|5983|5984|5985|5986|5987|5988|5989|5990|5991|5992|5993|5994|5995|5996|5997|5998|5999|6000|6001|6002|6003|6004|6005|6006|6007|6008|6009|6010|6011|6012|6013|6014|6015|6016 +94.000000000000,2048,128,47,6017|6018|6019|6020|6021|6022|6023|6024|6025|6026|6027|6028|6029|6030|6031|6032|6033|6034|6035|6036|6037|6038|6039|6040|6041|6042|6043|6044|6045|6046|6047|6048|6049|6050|6051|6052|6053|6054|6055|6056|6057|6058|6059|6060|6061|6062|6063|6064|6065|6066|6067|6068|6069|6070|6071|6072|6073|6074|6075|6076|6077|6078|6079|6080|6081|6082|6083|6084|6085|6086|6087|6088|6089|6090|6091|6092|6093|6094|6095|6096|6097|6098|6099|6100|6101|6102|6103|6104|6105|6106|6107|6108|6109|6110|6111|6112|6113|6114|6115|6116|6117|6118|6119|6120|6121|6122|6123|6124|6125|6126|6127|6128|6129|6130|6131|6132|6133|6134|6135|6136|6137|6138|6139|6140|6141|6142|6143|6144 +96.000000000000,2048,128,48,6145|6146|6147|6148|6149|6150|6151|6152|6153|6154|6155|6156|6157|6158|6159|6160|6161|6162|6163|6164|6165|6166|6167|6168|6169|6170|6171|6172|6173|6174|6175|6176|6177|6178|6179|6180|6181|6182|6183|6184|6185|6186|6187|6188|6189|6190|6191|6192|6193|6194|6195|6196|6197|6198|6199|6200|6201|6202|6203|6204|6205|6206|6207|6208|6209|6210|6211|6212|6213|6214|6215|6216|6217|6218|6219|6220|6221|6222|6223|6224|6225|6226|6227|6228|6229|6230|6231|6232|6233|6234|6235|6236|6237|6238|6239|6240|6241|6242|6243|6244|6245|6246|6247|6248|6249|6250|6251|6252|6253|6254|6255|6256|6257|6258|6259|6260|6261|6262|6263|6264|6265|6266|6267|6268|6269|6270|6271|6272 +98.000000000000,2048,128,49,6273|6274|6275|6276|6277|6278|6279|6280|6281|6282|6283|6284|6285|6286|6287|6288|6289|6290|6291|6292|6293|6294|6295|6296|6297|6298|6299|6300|6301|6302|6303|6304|6305|6306|6307|6308|6309|6310|6311|6312|6313|6314|6315|6316|6317|6318|6319|6320|6321|6322|6323|6324|6325|6326|6327|6328|6329|6330|6331|6332|6333|6334|6335|6336|6337|6338|6339|6340|6341|6342|6343|6344|6345|6346|6347|6348|6349|6350|6351|6352|6353|6354|6355|6356|6357|6358|6359|6360|6361|6362|6363|6364|6365|6366|6367|6368|6369|6370|6371|6372|6373|6374|6375|6376|6377|6378|6379|6380|6381|6382|6383|6384|6385|6386|6387|6388|6389|6390|6391|6392|6393|6394|6395|6396|6397|6398|6399|6400 +100.000000000000,2048,128,50,6401|6402|6403|6404|6405|6406|6407|6408|6409|6410|6411|6412|6413|6414|6415|6416|6417|6418|6419|6420|6421|6422|6423|6424|6425|6426|6427|6428|6429|6430|6431|6432|6433|6434|6435|6436|6437|6438|6439|6440|6441|6442|6443|6444|6445|6446|6447|6448|6449|6450|6451|6452|6453|6454|6455|6456|6457|6458|6459|6460|6461|6462|6463|6464|6465|6466|6467|6468|6469|6470|6471|6472|6473|6474|6475|6476|6477|6478|6479|6480|6481|6482|6483|6484|6485|6486|6487|6488|6489|6490|6491|6492|6493|6494|6495|6496|6497|6498|6499|6500|6501|6502|6503|6504|6505|6506|6507|6508|6509|6510|6511|6512|6513|6514|6515|6516|6517|6518|6519|6520|6521|6522|6523|6524|6525|6526|6527|6528 +102.000000000000,2048,128,51,6529|6530|6531|6532|6533|6534|6535|6536|6537|6538|6539|6540|6541|6542|6543|6544|6545|6546|6547|6548|6549|6550|6551|6552|6553|6554|6555|6556|6557|6558|6559|6560|6561|6562|6563|6564|6565|6566|6567|6568|6569|6570|6571|6572|6573|6574|6575|6576|6577|6578|6579|6580|6581|6582|6583|6584|6585|6586|6587|6588|6589|6590|6591|6592|6593|6594|6595|6596|6597|6598|6599|6600|6601|6602|6603|6604|6605|6606|6607|6608|6609|6610|6611|6612|6613|6614|6615|6616|6617|6618|6619|6620|6621|6622|6623|6624|6625|6626|6627|6628|6629|6630|6631|6632|6633|6634|6635|6636|6637|6638|6639|6640|6641|6642|6643|6644|6645|6646|6647|6648|6649|6650|6651|6652|6653|6654|6655|6656 +104.000000000000,2048,128,52,6657|6658|6659|6660|6661|6662|6663|6664|6665|6666|6667|6668|6669|6670|6671|6672|6673|6674|6675|6676|6677|6678|6679|6680|6681|6682|6683|6684|6685|6686|6687|6688|6689|6690|6691|6692|6693|6694|6695|6696|6697|6698|6699|6700|6701|6702|6703|6704|6705|6706|6707|6708|6709|6710|6711|6712|6713|6714|6715|6716|6717|6718|6719|6720|6721|6722|6723|6724|6725|6726|6727|6728|6729|6730|6731|6732|6733|6734|6735|6736|6737|6738|6739|6740|6741|6742|6743|6744|6745|6746|6747|6748|6749|6750|6751|6752|6753|6754|6755|6756|6757|6758|6759|6760|6761|6762|6763|6764|6765|6766|6767|6768|6769|6770|6771|6772|6773|6774|6775|6776|6777|6778|6779|6780|6781|6782|6783|6784 +106.000000000000,2048,128,53,6785|6786|6787|6788|6789|6790|6791|6792|6793|6794|6795|6796|6797|6798|6799|6800|6801|6802|6803|6804|6805|6806|6807|6808|6809|6810|6811|6812|6813|6814|6815|6816|6817|6818|6819|6820|6821|6822|6823|6824|6825|6826|6827|6828|6829|6830|6831|6832|6833|6834|6835|6836|6837|6838|6839|6840|6841|6842|6843|6844|6845|6846|6847|6848|6849|6850|6851|6852|6853|6854|6855|6856|6857|6858|6859|6860|6861|6862|6863|6864|6865|6866|6867|6868|6869|6870|6871|6872|6873|6874|6875|6876|6877|6878|6879|6880|6881|6882|6883|6884|6885|6886|6887|6888|6889|6890|6891|6892|6893|6894|6895|6896|6897|6898|6899|6900|6901|6902|6903|6904|6905|6906|6907|6908|6909|6910|6911|6912 +108.000000000000,2048,128,54,6913|6914|6915|6916|6917|6918|6919|6920|6921|6922|6923|6924|6925|6926|6927|6928|6929|6930|6931|6932|6933|6934|6935|6936|6937|6938|6939|6940|6941|6942|6943|6944|6945|6946|6947|6948|6949|6950|6951|6952|6953|6954|6955|6956|6957|6958|6959|6960|6961|6962|6963|6964|6965|6966|6967|6968|6969|6970|6971|6972|6973|6974|6975|6976|6977|6978|6979|6980|6981|6982|6983|6984|6985|6986|6987|6988|6989|6990|6991|6992|6993|6994|6995|6996|6997|6998|6999|7000|7001|7002|7003|7004|7005|7006|7007|7008|7009|7010|7011|7012|7013|7014|7015|7016|7017|7018|7019|7020|7021|7022|7023|7024|7025|7026|7027|7028|7029|7030|7031|7032|7033|7034|7035|7036|7037|7038|7039|7040 +110.000000000000,2048,128,55,7041|7042|7043|7044|7045|7046|7047|7048|7049|7050|7051|7052|7053|7054|7055|7056|7057|7058|7059|7060|7061|7062|7063|7064|7065|7066|7067|7068|7069|7070|7071|7072|7073|7074|7075|7076|7077|7078|7079|7080|7081|7082|7083|7084|7085|7086|7087|7088|7089|7090|7091|7092|7093|7094|7095|7096|7097|7098|7099|7100|7101|7102|7103|7104|7105|7106|7107|7108|7109|7110|7111|7112|7113|7114|7115|7116|7117|7118|7119|7120|7121|7122|7123|7124|7125|7126|7127|7128|7129|7130|7131|7132|7133|7134|7135|7136|7137|7138|7139|7140|7141|7142|7143|7144|7145|7146|7147|7148|7149|7150|7151|7152|7153|7154|7155|7156|7157|7158|7159|7160|7161|7162|7163|7164|7165|7166|7167|7168 +112.000000000000,2048,128,56,7169|7170|7171|7172|7173|7174|7175|7176|7177|7178|7179|7180|7181|7182|7183|7184|7185|7186|7187|7188|7189|7190|7191|7192|7193|7194|7195|7196|7197|7198|7199|7200|7201|7202|7203|7204|7205|7206|7207|7208|7209|7210|7211|7212|7213|7214|7215|7216|7217|7218|7219|7220|7221|7222|7223|7224|7225|7226|7227|7228|7229|7230|7231|7232|7233|7234|7235|7236|7237|7238|7239|7240|7241|7242|7243|7244|7245|7246|7247|7248|7249|7250|7251|7252|7253|7254|7255|7256|7257|7258|7259|7260|7261|7262|7263|7264|7265|7266|7267|7268|7269|7270|7271|7272|7273|7274|7275|7276|7277|7278|7279|7280|7281|7282|7283|7284|7285|7286|7287|7288|7289|7290|7291|7292|7293|7294|7295|7296 +114.000000000000,2048,128,57,7297|7298|7299|7300|7301|7302|7303|7304|7305|7306|7307|7308|7309|7310|7311|7312|7313|7314|7315|7316|7317|7318|7319|7320|7321|7322|7323|7324|7325|7326|7327|7328|7329|7330|7331|7332|7333|7334|7335|7336|7337|7338|7339|7340|7341|7342|7343|7344|7345|7346|7347|7348|7349|7350|7351|7352|7353|7354|7355|7356|7357|7358|7359|7360|7361|7362|7363|7364|7365|7366|7367|7368|7369|7370|7371|7372|7373|7374|7375|7376|7377|7378|7379|7380|7381|7382|7383|7384|7385|7386|7387|7388|7389|7390|7391|7392|7393|7394|7395|7396|7397|7398|7399|7400|7401|7402|7403|7404|7405|7406|7407|7408|7409|7410|7411|7412|7413|7414|7415|7416|7417|7418|7419|7420|7421|7422|7423|7424 +116.000000000000,2048,128,58,7425|7426|7427|7428|7429|7430|7431|7432|7433|7434|7435|7436|7437|7438|7439|7440|7441|7442|7443|7444|7445|7446|7447|7448|7449|7450|7451|7452|7453|7454|7455|7456|7457|7458|7459|7460|7461|7462|7463|7464|7465|7466|7467|7468|7469|7470|7471|7472|7473|7474|7475|7476|7477|7478|7479|7480|7481|7482|7483|7484|7485|7486|7487|7488|7489|7490|7491|7492|7493|7494|7495|7496|7497|7498|7499|7500|7501|7502|7503|7504|7505|7506|7507|7508|7509|7510|7511|7512|7513|7514|7515|7516|7517|7518|7519|7520|7521|7522|7523|7524|7525|7526|7527|7528|7529|7530|7531|7532|7533|7534|7535|7536|7537|7538|7539|7540|7541|7542|7543|7544|7545|7546|7547|7548|7549|7550|7551|7552 +118.000000000000,2048,128,59,7553|7554|7555|7556|7557|7558|7559|7560|7561|7562|7563|7564|7565|7566|7567|7568|7569|7570|7571|7572|7573|7574|7575|7576|7577|7578|7579|7580|7581|7582|7583|7584|7585|7586|7587|7588|7589|7590|7591|7592|7593|7594|7595|7596|7597|7598|7599|7600|7601|7602|7603|7604|7605|7606|7607|7608|7609|7610|7611|7612|7613|7614|7615|7616|7617|7618|7619|7620|7621|7622|7623|7624|7625|7626|7627|7628|7629|7630|7631|7632|7633|7634|7635|7636|7637|7638|7639|7640|7641|7642|7643|7644|7645|7646|7647|7648|7649|7650|7651|7652|7653|7654|7655|7656|7657|7658|7659|7660|7661|7662|7663|7664|7665|7666|7667|7668|7669|7670|7671|7672|7673|7674|7675|7676|7677|7678|7679|7680 +120.000000000000,2048,128,60,7681|7682|7683|7684|7685|7686|7687|7688|7689|7690|7691|7692|7693|7694|7695|7696|7697|7698|7699|7700|7701|7702|7703|7704|7705|7706|7707|7708|7709|7710|7711|7712|7713|7714|7715|7716|7717|7718|7719|7720|7721|7722|7723|7724|7725|7726|7727|7728|7729|7730|7731|7732|7733|7734|7735|7736|7737|7738|7739|7740|7741|7742|7743|7744|7745|7746|7747|7748|7749|7750|7751|7752|7753|7754|7755|7756|7757|7758|7759|7760|7761|7762|7763|7764|7765|7766|7767|7768|7769|7770|7771|7772|7773|7774|7775|7776|7777|7778|7779|7780|7781|7782|7783|7784|7785|7786|7787|7788|7789|7790|7791|7792|7793|7794|7795|7796|7797|7798|7799|7800|7801|7802|7803|7804|7805|7806|7807|7808 +122.000000000000,2048,128,61,7809|7810|7811|7812|7813|7814|7815|7816|7817|7818|7819|7820|7821|7822|7823|7824|7825|7826|7827|7828|7829|7830|7831|7832|7833|7834|7835|7836|7837|7838|7839|7840|7841|7842|7843|7844|7845|7846|7847|7848|7849|7850|7851|7852|7853|7854|7855|7856|7857|7858|7859|7860|7861|7862|7863|7864|7865|7866|7867|7868|7869|7870|7871|7872|7873|7874|7875|7876|7877|7878|7879|7880|7881|7882|7883|7884|7885|7886|7887|7888|7889|7890|7891|7892|7893|7894|7895|7896|7897|7898|7899|7900|7901|7902|7903|7904|7905|7906|7907|7908|7909|7910|7911|7912|7913|7914|7915|7916|7917|7918|7919|7920|7921|7922|7923|7924|7925|7926|7927|7928|7929|7930|7931|7932|7933|7934|7935|7936 +124.000000000000,2048,128,62,7937|7938|7939|7940|7941|7942|7943|7944|7945|7946|7947|7948|7949|7950|7951|7952|7953|7954|7955|7956|7957|7958|7959|7960|7961|7962|7963|7964|7965|7966|7967|7968|7969|7970|7971|7972|7973|7974|7975|7976|7977|7978|7979|7980|7981|7982|7983|7984|7985|7986|7987|7988|7989|7990|7991|7992|7993|7994|7995|7996|7997|7998|7999|8000|8001|8002|8003|8004|8005|8006|8007|8008|8009|8010|8011|8012|8013|8014|8015|8016|8017|8018|8019|8020|8021|8022|8023|8024|8025|8026|8027|8028|8029|8030|8031|8032|8033|8034|8035|8036|8037|8038|8039|8040|8041|8042|8043|8044|8045|8046|8047|8048|8049|8050|8051|8052|8053|8054|8055|8056|8057|8058|8059|8060|8061|8062|8063|8064 +126.000000000000,2048,128,63,8065|8066|8067|8068|8069|8070|8071|8072|8073|8074|8075|8076|8077|8078|8079|8080|8081|8082|8083|8084|8085|8086|8087|8088|8089|8090|8091|8092|8093|8094|8095|8096|8097|8098|8099|8100|8101|8102|8103|8104|8105|8106|8107|8108|8109|8110|8111|8112|8113|8114|8115|8116|8117|8118|8119|8120|8121|8122|8123|8124|8125|8126|8127|8128|8129|8130|8131|8132|8133|8134|8135|8136|8137|8138|8139|8140|8141|8142|8143|8144|8145|8146|8147|8148|8149|8150|8151|8152|8153|8154|8155|8156|8157|8158|8159|8160|8161|8162|8163|8164|8165|8166|8167|8168|8169|8170|8171|8172|8173|8174|8175|8176|8177|8178|8179|8180|8181|8182|8183|8184|8185|8186|8187|8188|8189|8190|8191|8192 diff --git a/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r1.csv b/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r1.csv new file mode 100644 index 0000000..2fa8c5e --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r1.csv @@ -0,0 +1,65 @@ +arrived_at,num_prefill_tokens,num_decode_tokens,session_id,block_hash_ids +0.000000000000,2048,128,0,1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58|59|60|61|62|63|64|65|66|67|68|69|70|71|72|73|74|75|76|77|78|79|80|81|82|83|84|85|86|87|88|89|90|91|92|93|94|95|96|97|98|99|100|101|102|103|104|105|106|107|108|109|110|111|112|113|114|115|116|117|118|119|120|121|122|123|124|125|126|127|128 +1.000000000000,2048,128,1,129|130|131|132|133|134|135|136|137|138|139|140|141|142|143|144|145|146|147|148|149|150|151|152|153|154|155|156|157|158|159|160|161|162|163|164|165|166|167|168|169|170|171|172|173|174|175|176|177|178|179|180|181|182|183|184|185|186|187|188|189|190|191|192|193|194|195|196|197|198|199|200|201|202|203|204|205|206|207|208|209|210|211|212|213|214|215|216|217|218|219|220|221|222|223|224|225|226|227|228|229|230|231|232|233|234|235|236|237|238|239|240|241|242|243|244|245|246|247|248|249|250|251|252|253|254|255|256 +2.000000000000,2048,128,2,257|258|259|260|261|262|263|264|265|266|267|268|269|270|271|272|273|274|275|276|277|278|279|280|281|282|283|284|285|286|287|288|289|290|291|292|293|294|295|296|297|298|299|300|301|302|303|304|305|306|307|308|309|310|311|312|313|314|315|316|317|318|319|320|321|322|323|324|325|326|327|328|329|330|331|332|333|334|335|336|337|338|339|340|341|342|343|344|345|346|347|348|349|350|351|352|353|354|355|356|357|358|359|360|361|362|363|364|365|366|367|368|369|370|371|372|373|374|375|376|377|378|379|380|381|382|383|384 +3.000000000000,2048,128,3,385|386|387|388|389|390|391|392|393|394|395|396|397|398|399|400|401|402|403|404|405|406|407|408|409|410|411|412|413|414|415|416|417|418|419|420|421|422|423|424|425|426|427|428|429|430|431|432|433|434|435|436|437|438|439|440|441|442|443|444|445|446|447|448|449|450|451|452|453|454|455|456|457|458|459|460|461|462|463|464|465|466|467|468|469|470|471|472|473|474|475|476|477|478|479|480|481|482|483|484|485|486|487|488|489|490|491|492|493|494|495|496|497|498|499|500|501|502|503|504|505|506|507|508|509|510|511|512 +4.000000000000,2048,128,4,513|514|515|516|517|518|519|520|521|522|523|524|525|526|527|528|529|530|531|532|533|534|535|536|537|538|539|540|541|542|543|544|545|546|547|548|549|550|551|552|553|554|555|556|557|558|559|560|561|562|563|564|565|566|567|568|569|570|571|572|573|574|575|576|577|578|579|580|581|582|583|584|585|586|587|588|589|590|591|592|593|594|595|596|597|598|599|600|601|602|603|604|605|606|607|608|609|610|611|612|613|614|615|616|617|618|619|620|621|622|623|624|625|626|627|628|629|630|631|632|633|634|635|636|637|638|639|640 +5.000000000000,2048,128,5,641|642|643|644|645|646|647|648|649|650|651|652|653|654|655|656|657|658|659|660|661|662|663|664|665|666|667|668|669|670|671|672|673|674|675|676|677|678|679|680|681|682|683|684|685|686|687|688|689|690|691|692|693|694|695|696|697|698|699|700|701|702|703|704|705|706|707|708|709|710|711|712|713|714|715|716|717|718|719|720|721|722|723|724|725|726|727|728|729|730|731|732|733|734|735|736|737|738|739|740|741|742|743|744|745|746|747|748|749|750|751|752|753|754|755|756|757|758|759|760|761|762|763|764|765|766|767|768 +6.000000000000,2048,128,6,769|770|771|772|773|774|775|776|777|778|779|780|781|782|783|784|785|786|787|788|789|790|791|792|793|794|795|796|797|798|799|800|801|802|803|804|805|806|807|808|809|810|811|812|813|814|815|816|817|818|819|820|821|822|823|824|825|826|827|828|829|830|831|832|833|834|835|836|837|838|839|840|841|842|843|844|845|846|847|848|849|850|851|852|853|854|855|856|857|858|859|860|861|862|863|864|865|866|867|868|869|870|871|872|873|874|875|876|877|878|879|880|881|882|883|884|885|886|887|888|889|890|891|892|893|894|895|896 +7.000000000000,2048,128,7,897|898|899|900|901|902|903|904|905|906|907|908|909|910|911|912|913|914|915|916|917|918|919|920|921|922|923|924|925|926|927|928|929|930|931|932|933|934|935|936|937|938|939|940|941|942|943|944|945|946|947|948|949|950|951|952|953|954|955|956|957|958|959|960|961|962|963|964|965|966|967|968|969|970|971|972|973|974|975|976|977|978|979|980|981|982|983|984|985|986|987|988|989|990|991|992|993|994|995|996|997|998|999|1000|1001|1002|1003|1004|1005|1006|1007|1008|1009|1010|1011|1012|1013|1014|1015|1016|1017|1018|1019|1020|1021|1022|1023|1024 +8.000000000000,2048,128,8,1025|1026|1027|1028|1029|1030|1031|1032|1033|1034|1035|1036|1037|1038|1039|1040|1041|1042|1043|1044|1045|1046|1047|1048|1049|1050|1051|1052|1053|1054|1055|1056|1057|1058|1059|1060|1061|1062|1063|1064|1065|1066|1067|1068|1069|1070|1071|1072|1073|1074|1075|1076|1077|1078|1079|1080|1081|1082|1083|1084|1085|1086|1087|1088|1089|1090|1091|1092|1093|1094|1095|1096|1097|1098|1099|1100|1101|1102|1103|1104|1105|1106|1107|1108|1109|1110|1111|1112|1113|1114|1115|1116|1117|1118|1119|1120|1121|1122|1123|1124|1125|1126|1127|1128|1129|1130|1131|1132|1133|1134|1135|1136|1137|1138|1139|1140|1141|1142|1143|1144|1145|1146|1147|1148|1149|1150|1151|1152 +9.000000000000,2048,128,9,1153|1154|1155|1156|1157|1158|1159|1160|1161|1162|1163|1164|1165|1166|1167|1168|1169|1170|1171|1172|1173|1174|1175|1176|1177|1178|1179|1180|1181|1182|1183|1184|1185|1186|1187|1188|1189|1190|1191|1192|1193|1194|1195|1196|1197|1198|1199|1200|1201|1202|1203|1204|1205|1206|1207|1208|1209|1210|1211|1212|1213|1214|1215|1216|1217|1218|1219|1220|1221|1222|1223|1224|1225|1226|1227|1228|1229|1230|1231|1232|1233|1234|1235|1236|1237|1238|1239|1240|1241|1242|1243|1244|1245|1246|1247|1248|1249|1250|1251|1252|1253|1254|1255|1256|1257|1258|1259|1260|1261|1262|1263|1264|1265|1266|1267|1268|1269|1270|1271|1272|1273|1274|1275|1276|1277|1278|1279|1280 +10.000000000000,2048,128,10,1281|1282|1283|1284|1285|1286|1287|1288|1289|1290|1291|1292|1293|1294|1295|1296|1297|1298|1299|1300|1301|1302|1303|1304|1305|1306|1307|1308|1309|1310|1311|1312|1313|1314|1315|1316|1317|1318|1319|1320|1321|1322|1323|1324|1325|1326|1327|1328|1329|1330|1331|1332|1333|1334|1335|1336|1337|1338|1339|1340|1341|1342|1343|1344|1345|1346|1347|1348|1349|1350|1351|1352|1353|1354|1355|1356|1357|1358|1359|1360|1361|1362|1363|1364|1365|1366|1367|1368|1369|1370|1371|1372|1373|1374|1375|1376|1377|1378|1379|1380|1381|1382|1383|1384|1385|1386|1387|1388|1389|1390|1391|1392|1393|1394|1395|1396|1397|1398|1399|1400|1401|1402|1403|1404|1405|1406|1407|1408 +11.000000000000,2048,128,11,1409|1410|1411|1412|1413|1414|1415|1416|1417|1418|1419|1420|1421|1422|1423|1424|1425|1426|1427|1428|1429|1430|1431|1432|1433|1434|1435|1436|1437|1438|1439|1440|1441|1442|1443|1444|1445|1446|1447|1448|1449|1450|1451|1452|1453|1454|1455|1456|1457|1458|1459|1460|1461|1462|1463|1464|1465|1466|1467|1468|1469|1470|1471|1472|1473|1474|1475|1476|1477|1478|1479|1480|1481|1482|1483|1484|1485|1486|1487|1488|1489|1490|1491|1492|1493|1494|1495|1496|1497|1498|1499|1500|1501|1502|1503|1504|1505|1506|1507|1508|1509|1510|1511|1512|1513|1514|1515|1516|1517|1518|1519|1520|1521|1522|1523|1524|1525|1526|1527|1528|1529|1530|1531|1532|1533|1534|1535|1536 +12.000000000000,2048,128,12,1537|1538|1539|1540|1541|1542|1543|1544|1545|1546|1547|1548|1549|1550|1551|1552|1553|1554|1555|1556|1557|1558|1559|1560|1561|1562|1563|1564|1565|1566|1567|1568|1569|1570|1571|1572|1573|1574|1575|1576|1577|1578|1579|1580|1581|1582|1583|1584|1585|1586|1587|1588|1589|1590|1591|1592|1593|1594|1595|1596|1597|1598|1599|1600|1601|1602|1603|1604|1605|1606|1607|1608|1609|1610|1611|1612|1613|1614|1615|1616|1617|1618|1619|1620|1621|1622|1623|1624|1625|1626|1627|1628|1629|1630|1631|1632|1633|1634|1635|1636|1637|1638|1639|1640|1641|1642|1643|1644|1645|1646|1647|1648|1649|1650|1651|1652|1653|1654|1655|1656|1657|1658|1659|1660|1661|1662|1663|1664 +13.000000000000,2048,128,13,1665|1666|1667|1668|1669|1670|1671|1672|1673|1674|1675|1676|1677|1678|1679|1680|1681|1682|1683|1684|1685|1686|1687|1688|1689|1690|1691|1692|1693|1694|1695|1696|1697|1698|1699|1700|1701|1702|1703|1704|1705|1706|1707|1708|1709|1710|1711|1712|1713|1714|1715|1716|1717|1718|1719|1720|1721|1722|1723|1724|1725|1726|1727|1728|1729|1730|1731|1732|1733|1734|1735|1736|1737|1738|1739|1740|1741|1742|1743|1744|1745|1746|1747|1748|1749|1750|1751|1752|1753|1754|1755|1756|1757|1758|1759|1760|1761|1762|1763|1764|1765|1766|1767|1768|1769|1770|1771|1772|1773|1774|1775|1776|1777|1778|1779|1780|1781|1782|1783|1784|1785|1786|1787|1788|1789|1790|1791|1792 +14.000000000000,2048,128,14,1793|1794|1795|1796|1797|1798|1799|1800|1801|1802|1803|1804|1805|1806|1807|1808|1809|1810|1811|1812|1813|1814|1815|1816|1817|1818|1819|1820|1821|1822|1823|1824|1825|1826|1827|1828|1829|1830|1831|1832|1833|1834|1835|1836|1837|1838|1839|1840|1841|1842|1843|1844|1845|1846|1847|1848|1849|1850|1851|1852|1853|1854|1855|1856|1857|1858|1859|1860|1861|1862|1863|1864|1865|1866|1867|1868|1869|1870|1871|1872|1873|1874|1875|1876|1877|1878|1879|1880|1881|1882|1883|1884|1885|1886|1887|1888|1889|1890|1891|1892|1893|1894|1895|1896|1897|1898|1899|1900|1901|1902|1903|1904|1905|1906|1907|1908|1909|1910|1911|1912|1913|1914|1915|1916|1917|1918|1919|1920 +15.000000000000,2048,128,15,1921|1922|1923|1924|1925|1926|1927|1928|1929|1930|1931|1932|1933|1934|1935|1936|1937|1938|1939|1940|1941|1942|1943|1944|1945|1946|1947|1948|1949|1950|1951|1952|1953|1954|1955|1956|1957|1958|1959|1960|1961|1962|1963|1964|1965|1966|1967|1968|1969|1970|1971|1972|1973|1974|1975|1976|1977|1978|1979|1980|1981|1982|1983|1984|1985|1986|1987|1988|1989|1990|1991|1992|1993|1994|1995|1996|1997|1998|1999|2000|2001|2002|2003|2004|2005|2006|2007|2008|2009|2010|2011|2012|2013|2014|2015|2016|2017|2018|2019|2020|2021|2022|2023|2024|2025|2026|2027|2028|2029|2030|2031|2032|2033|2034|2035|2036|2037|2038|2039|2040|2041|2042|2043|2044|2045|2046|2047|2048 +16.000000000000,2048,128,16,2049|2050|2051|2052|2053|2054|2055|2056|2057|2058|2059|2060|2061|2062|2063|2064|2065|2066|2067|2068|2069|2070|2071|2072|2073|2074|2075|2076|2077|2078|2079|2080|2081|2082|2083|2084|2085|2086|2087|2088|2089|2090|2091|2092|2093|2094|2095|2096|2097|2098|2099|2100|2101|2102|2103|2104|2105|2106|2107|2108|2109|2110|2111|2112|2113|2114|2115|2116|2117|2118|2119|2120|2121|2122|2123|2124|2125|2126|2127|2128|2129|2130|2131|2132|2133|2134|2135|2136|2137|2138|2139|2140|2141|2142|2143|2144|2145|2146|2147|2148|2149|2150|2151|2152|2153|2154|2155|2156|2157|2158|2159|2160|2161|2162|2163|2164|2165|2166|2167|2168|2169|2170|2171|2172|2173|2174|2175|2176 +17.000000000000,2048,128,17,2177|2178|2179|2180|2181|2182|2183|2184|2185|2186|2187|2188|2189|2190|2191|2192|2193|2194|2195|2196|2197|2198|2199|2200|2201|2202|2203|2204|2205|2206|2207|2208|2209|2210|2211|2212|2213|2214|2215|2216|2217|2218|2219|2220|2221|2222|2223|2224|2225|2226|2227|2228|2229|2230|2231|2232|2233|2234|2235|2236|2237|2238|2239|2240|2241|2242|2243|2244|2245|2246|2247|2248|2249|2250|2251|2252|2253|2254|2255|2256|2257|2258|2259|2260|2261|2262|2263|2264|2265|2266|2267|2268|2269|2270|2271|2272|2273|2274|2275|2276|2277|2278|2279|2280|2281|2282|2283|2284|2285|2286|2287|2288|2289|2290|2291|2292|2293|2294|2295|2296|2297|2298|2299|2300|2301|2302|2303|2304 +18.000000000000,2048,128,18,2305|2306|2307|2308|2309|2310|2311|2312|2313|2314|2315|2316|2317|2318|2319|2320|2321|2322|2323|2324|2325|2326|2327|2328|2329|2330|2331|2332|2333|2334|2335|2336|2337|2338|2339|2340|2341|2342|2343|2344|2345|2346|2347|2348|2349|2350|2351|2352|2353|2354|2355|2356|2357|2358|2359|2360|2361|2362|2363|2364|2365|2366|2367|2368|2369|2370|2371|2372|2373|2374|2375|2376|2377|2378|2379|2380|2381|2382|2383|2384|2385|2386|2387|2388|2389|2390|2391|2392|2393|2394|2395|2396|2397|2398|2399|2400|2401|2402|2403|2404|2405|2406|2407|2408|2409|2410|2411|2412|2413|2414|2415|2416|2417|2418|2419|2420|2421|2422|2423|2424|2425|2426|2427|2428|2429|2430|2431|2432 +19.000000000000,2048,128,19,2433|2434|2435|2436|2437|2438|2439|2440|2441|2442|2443|2444|2445|2446|2447|2448|2449|2450|2451|2452|2453|2454|2455|2456|2457|2458|2459|2460|2461|2462|2463|2464|2465|2466|2467|2468|2469|2470|2471|2472|2473|2474|2475|2476|2477|2478|2479|2480|2481|2482|2483|2484|2485|2486|2487|2488|2489|2490|2491|2492|2493|2494|2495|2496|2497|2498|2499|2500|2501|2502|2503|2504|2505|2506|2507|2508|2509|2510|2511|2512|2513|2514|2515|2516|2517|2518|2519|2520|2521|2522|2523|2524|2525|2526|2527|2528|2529|2530|2531|2532|2533|2534|2535|2536|2537|2538|2539|2540|2541|2542|2543|2544|2545|2546|2547|2548|2549|2550|2551|2552|2553|2554|2555|2556|2557|2558|2559|2560 +20.000000000000,2048,128,20,2561|2562|2563|2564|2565|2566|2567|2568|2569|2570|2571|2572|2573|2574|2575|2576|2577|2578|2579|2580|2581|2582|2583|2584|2585|2586|2587|2588|2589|2590|2591|2592|2593|2594|2595|2596|2597|2598|2599|2600|2601|2602|2603|2604|2605|2606|2607|2608|2609|2610|2611|2612|2613|2614|2615|2616|2617|2618|2619|2620|2621|2622|2623|2624|2625|2626|2627|2628|2629|2630|2631|2632|2633|2634|2635|2636|2637|2638|2639|2640|2641|2642|2643|2644|2645|2646|2647|2648|2649|2650|2651|2652|2653|2654|2655|2656|2657|2658|2659|2660|2661|2662|2663|2664|2665|2666|2667|2668|2669|2670|2671|2672|2673|2674|2675|2676|2677|2678|2679|2680|2681|2682|2683|2684|2685|2686|2687|2688 +21.000000000000,2048,128,21,2689|2690|2691|2692|2693|2694|2695|2696|2697|2698|2699|2700|2701|2702|2703|2704|2705|2706|2707|2708|2709|2710|2711|2712|2713|2714|2715|2716|2717|2718|2719|2720|2721|2722|2723|2724|2725|2726|2727|2728|2729|2730|2731|2732|2733|2734|2735|2736|2737|2738|2739|2740|2741|2742|2743|2744|2745|2746|2747|2748|2749|2750|2751|2752|2753|2754|2755|2756|2757|2758|2759|2760|2761|2762|2763|2764|2765|2766|2767|2768|2769|2770|2771|2772|2773|2774|2775|2776|2777|2778|2779|2780|2781|2782|2783|2784|2785|2786|2787|2788|2789|2790|2791|2792|2793|2794|2795|2796|2797|2798|2799|2800|2801|2802|2803|2804|2805|2806|2807|2808|2809|2810|2811|2812|2813|2814|2815|2816 +22.000000000000,2048,128,22,2817|2818|2819|2820|2821|2822|2823|2824|2825|2826|2827|2828|2829|2830|2831|2832|2833|2834|2835|2836|2837|2838|2839|2840|2841|2842|2843|2844|2845|2846|2847|2848|2849|2850|2851|2852|2853|2854|2855|2856|2857|2858|2859|2860|2861|2862|2863|2864|2865|2866|2867|2868|2869|2870|2871|2872|2873|2874|2875|2876|2877|2878|2879|2880|2881|2882|2883|2884|2885|2886|2887|2888|2889|2890|2891|2892|2893|2894|2895|2896|2897|2898|2899|2900|2901|2902|2903|2904|2905|2906|2907|2908|2909|2910|2911|2912|2913|2914|2915|2916|2917|2918|2919|2920|2921|2922|2923|2924|2925|2926|2927|2928|2929|2930|2931|2932|2933|2934|2935|2936|2937|2938|2939|2940|2941|2942|2943|2944 +23.000000000000,2048,128,23,2945|2946|2947|2948|2949|2950|2951|2952|2953|2954|2955|2956|2957|2958|2959|2960|2961|2962|2963|2964|2965|2966|2967|2968|2969|2970|2971|2972|2973|2974|2975|2976|2977|2978|2979|2980|2981|2982|2983|2984|2985|2986|2987|2988|2989|2990|2991|2992|2993|2994|2995|2996|2997|2998|2999|3000|3001|3002|3003|3004|3005|3006|3007|3008|3009|3010|3011|3012|3013|3014|3015|3016|3017|3018|3019|3020|3021|3022|3023|3024|3025|3026|3027|3028|3029|3030|3031|3032|3033|3034|3035|3036|3037|3038|3039|3040|3041|3042|3043|3044|3045|3046|3047|3048|3049|3050|3051|3052|3053|3054|3055|3056|3057|3058|3059|3060|3061|3062|3063|3064|3065|3066|3067|3068|3069|3070|3071|3072 +24.000000000000,2048,128,24,3073|3074|3075|3076|3077|3078|3079|3080|3081|3082|3083|3084|3085|3086|3087|3088|3089|3090|3091|3092|3093|3094|3095|3096|3097|3098|3099|3100|3101|3102|3103|3104|3105|3106|3107|3108|3109|3110|3111|3112|3113|3114|3115|3116|3117|3118|3119|3120|3121|3122|3123|3124|3125|3126|3127|3128|3129|3130|3131|3132|3133|3134|3135|3136|3137|3138|3139|3140|3141|3142|3143|3144|3145|3146|3147|3148|3149|3150|3151|3152|3153|3154|3155|3156|3157|3158|3159|3160|3161|3162|3163|3164|3165|3166|3167|3168|3169|3170|3171|3172|3173|3174|3175|3176|3177|3178|3179|3180|3181|3182|3183|3184|3185|3186|3187|3188|3189|3190|3191|3192|3193|3194|3195|3196|3197|3198|3199|3200 +25.000000000000,2048,128,25,3201|3202|3203|3204|3205|3206|3207|3208|3209|3210|3211|3212|3213|3214|3215|3216|3217|3218|3219|3220|3221|3222|3223|3224|3225|3226|3227|3228|3229|3230|3231|3232|3233|3234|3235|3236|3237|3238|3239|3240|3241|3242|3243|3244|3245|3246|3247|3248|3249|3250|3251|3252|3253|3254|3255|3256|3257|3258|3259|3260|3261|3262|3263|3264|3265|3266|3267|3268|3269|3270|3271|3272|3273|3274|3275|3276|3277|3278|3279|3280|3281|3282|3283|3284|3285|3286|3287|3288|3289|3290|3291|3292|3293|3294|3295|3296|3297|3298|3299|3300|3301|3302|3303|3304|3305|3306|3307|3308|3309|3310|3311|3312|3313|3314|3315|3316|3317|3318|3319|3320|3321|3322|3323|3324|3325|3326|3327|3328 +26.000000000000,2048,128,26,3329|3330|3331|3332|3333|3334|3335|3336|3337|3338|3339|3340|3341|3342|3343|3344|3345|3346|3347|3348|3349|3350|3351|3352|3353|3354|3355|3356|3357|3358|3359|3360|3361|3362|3363|3364|3365|3366|3367|3368|3369|3370|3371|3372|3373|3374|3375|3376|3377|3378|3379|3380|3381|3382|3383|3384|3385|3386|3387|3388|3389|3390|3391|3392|3393|3394|3395|3396|3397|3398|3399|3400|3401|3402|3403|3404|3405|3406|3407|3408|3409|3410|3411|3412|3413|3414|3415|3416|3417|3418|3419|3420|3421|3422|3423|3424|3425|3426|3427|3428|3429|3430|3431|3432|3433|3434|3435|3436|3437|3438|3439|3440|3441|3442|3443|3444|3445|3446|3447|3448|3449|3450|3451|3452|3453|3454|3455|3456 +27.000000000000,2048,128,27,3457|3458|3459|3460|3461|3462|3463|3464|3465|3466|3467|3468|3469|3470|3471|3472|3473|3474|3475|3476|3477|3478|3479|3480|3481|3482|3483|3484|3485|3486|3487|3488|3489|3490|3491|3492|3493|3494|3495|3496|3497|3498|3499|3500|3501|3502|3503|3504|3505|3506|3507|3508|3509|3510|3511|3512|3513|3514|3515|3516|3517|3518|3519|3520|3521|3522|3523|3524|3525|3526|3527|3528|3529|3530|3531|3532|3533|3534|3535|3536|3537|3538|3539|3540|3541|3542|3543|3544|3545|3546|3547|3548|3549|3550|3551|3552|3553|3554|3555|3556|3557|3558|3559|3560|3561|3562|3563|3564|3565|3566|3567|3568|3569|3570|3571|3572|3573|3574|3575|3576|3577|3578|3579|3580|3581|3582|3583|3584 +28.000000000000,2048,128,28,3585|3586|3587|3588|3589|3590|3591|3592|3593|3594|3595|3596|3597|3598|3599|3600|3601|3602|3603|3604|3605|3606|3607|3608|3609|3610|3611|3612|3613|3614|3615|3616|3617|3618|3619|3620|3621|3622|3623|3624|3625|3626|3627|3628|3629|3630|3631|3632|3633|3634|3635|3636|3637|3638|3639|3640|3641|3642|3643|3644|3645|3646|3647|3648|3649|3650|3651|3652|3653|3654|3655|3656|3657|3658|3659|3660|3661|3662|3663|3664|3665|3666|3667|3668|3669|3670|3671|3672|3673|3674|3675|3676|3677|3678|3679|3680|3681|3682|3683|3684|3685|3686|3687|3688|3689|3690|3691|3692|3693|3694|3695|3696|3697|3698|3699|3700|3701|3702|3703|3704|3705|3706|3707|3708|3709|3710|3711|3712 +29.000000000000,2048,128,29,3713|3714|3715|3716|3717|3718|3719|3720|3721|3722|3723|3724|3725|3726|3727|3728|3729|3730|3731|3732|3733|3734|3735|3736|3737|3738|3739|3740|3741|3742|3743|3744|3745|3746|3747|3748|3749|3750|3751|3752|3753|3754|3755|3756|3757|3758|3759|3760|3761|3762|3763|3764|3765|3766|3767|3768|3769|3770|3771|3772|3773|3774|3775|3776|3777|3778|3779|3780|3781|3782|3783|3784|3785|3786|3787|3788|3789|3790|3791|3792|3793|3794|3795|3796|3797|3798|3799|3800|3801|3802|3803|3804|3805|3806|3807|3808|3809|3810|3811|3812|3813|3814|3815|3816|3817|3818|3819|3820|3821|3822|3823|3824|3825|3826|3827|3828|3829|3830|3831|3832|3833|3834|3835|3836|3837|3838|3839|3840 +30.000000000000,2048,128,30,3841|3842|3843|3844|3845|3846|3847|3848|3849|3850|3851|3852|3853|3854|3855|3856|3857|3858|3859|3860|3861|3862|3863|3864|3865|3866|3867|3868|3869|3870|3871|3872|3873|3874|3875|3876|3877|3878|3879|3880|3881|3882|3883|3884|3885|3886|3887|3888|3889|3890|3891|3892|3893|3894|3895|3896|3897|3898|3899|3900|3901|3902|3903|3904|3905|3906|3907|3908|3909|3910|3911|3912|3913|3914|3915|3916|3917|3918|3919|3920|3921|3922|3923|3924|3925|3926|3927|3928|3929|3930|3931|3932|3933|3934|3935|3936|3937|3938|3939|3940|3941|3942|3943|3944|3945|3946|3947|3948|3949|3950|3951|3952|3953|3954|3955|3956|3957|3958|3959|3960|3961|3962|3963|3964|3965|3966|3967|3968 +31.000000000000,2048,128,31,3969|3970|3971|3972|3973|3974|3975|3976|3977|3978|3979|3980|3981|3982|3983|3984|3985|3986|3987|3988|3989|3990|3991|3992|3993|3994|3995|3996|3997|3998|3999|4000|4001|4002|4003|4004|4005|4006|4007|4008|4009|4010|4011|4012|4013|4014|4015|4016|4017|4018|4019|4020|4021|4022|4023|4024|4025|4026|4027|4028|4029|4030|4031|4032|4033|4034|4035|4036|4037|4038|4039|4040|4041|4042|4043|4044|4045|4046|4047|4048|4049|4050|4051|4052|4053|4054|4055|4056|4057|4058|4059|4060|4061|4062|4063|4064|4065|4066|4067|4068|4069|4070|4071|4072|4073|4074|4075|4076|4077|4078|4079|4080|4081|4082|4083|4084|4085|4086|4087|4088|4089|4090|4091|4092|4093|4094|4095|4096 +32.000000000000,2048,128,32,4097|4098|4099|4100|4101|4102|4103|4104|4105|4106|4107|4108|4109|4110|4111|4112|4113|4114|4115|4116|4117|4118|4119|4120|4121|4122|4123|4124|4125|4126|4127|4128|4129|4130|4131|4132|4133|4134|4135|4136|4137|4138|4139|4140|4141|4142|4143|4144|4145|4146|4147|4148|4149|4150|4151|4152|4153|4154|4155|4156|4157|4158|4159|4160|4161|4162|4163|4164|4165|4166|4167|4168|4169|4170|4171|4172|4173|4174|4175|4176|4177|4178|4179|4180|4181|4182|4183|4184|4185|4186|4187|4188|4189|4190|4191|4192|4193|4194|4195|4196|4197|4198|4199|4200|4201|4202|4203|4204|4205|4206|4207|4208|4209|4210|4211|4212|4213|4214|4215|4216|4217|4218|4219|4220|4221|4222|4223|4224 +33.000000000000,2048,128,33,4225|4226|4227|4228|4229|4230|4231|4232|4233|4234|4235|4236|4237|4238|4239|4240|4241|4242|4243|4244|4245|4246|4247|4248|4249|4250|4251|4252|4253|4254|4255|4256|4257|4258|4259|4260|4261|4262|4263|4264|4265|4266|4267|4268|4269|4270|4271|4272|4273|4274|4275|4276|4277|4278|4279|4280|4281|4282|4283|4284|4285|4286|4287|4288|4289|4290|4291|4292|4293|4294|4295|4296|4297|4298|4299|4300|4301|4302|4303|4304|4305|4306|4307|4308|4309|4310|4311|4312|4313|4314|4315|4316|4317|4318|4319|4320|4321|4322|4323|4324|4325|4326|4327|4328|4329|4330|4331|4332|4333|4334|4335|4336|4337|4338|4339|4340|4341|4342|4343|4344|4345|4346|4347|4348|4349|4350|4351|4352 +34.000000000000,2048,128,34,4353|4354|4355|4356|4357|4358|4359|4360|4361|4362|4363|4364|4365|4366|4367|4368|4369|4370|4371|4372|4373|4374|4375|4376|4377|4378|4379|4380|4381|4382|4383|4384|4385|4386|4387|4388|4389|4390|4391|4392|4393|4394|4395|4396|4397|4398|4399|4400|4401|4402|4403|4404|4405|4406|4407|4408|4409|4410|4411|4412|4413|4414|4415|4416|4417|4418|4419|4420|4421|4422|4423|4424|4425|4426|4427|4428|4429|4430|4431|4432|4433|4434|4435|4436|4437|4438|4439|4440|4441|4442|4443|4444|4445|4446|4447|4448|4449|4450|4451|4452|4453|4454|4455|4456|4457|4458|4459|4460|4461|4462|4463|4464|4465|4466|4467|4468|4469|4470|4471|4472|4473|4474|4475|4476|4477|4478|4479|4480 +35.000000000000,2048,128,35,4481|4482|4483|4484|4485|4486|4487|4488|4489|4490|4491|4492|4493|4494|4495|4496|4497|4498|4499|4500|4501|4502|4503|4504|4505|4506|4507|4508|4509|4510|4511|4512|4513|4514|4515|4516|4517|4518|4519|4520|4521|4522|4523|4524|4525|4526|4527|4528|4529|4530|4531|4532|4533|4534|4535|4536|4537|4538|4539|4540|4541|4542|4543|4544|4545|4546|4547|4548|4549|4550|4551|4552|4553|4554|4555|4556|4557|4558|4559|4560|4561|4562|4563|4564|4565|4566|4567|4568|4569|4570|4571|4572|4573|4574|4575|4576|4577|4578|4579|4580|4581|4582|4583|4584|4585|4586|4587|4588|4589|4590|4591|4592|4593|4594|4595|4596|4597|4598|4599|4600|4601|4602|4603|4604|4605|4606|4607|4608 +36.000000000000,2048,128,36,4609|4610|4611|4612|4613|4614|4615|4616|4617|4618|4619|4620|4621|4622|4623|4624|4625|4626|4627|4628|4629|4630|4631|4632|4633|4634|4635|4636|4637|4638|4639|4640|4641|4642|4643|4644|4645|4646|4647|4648|4649|4650|4651|4652|4653|4654|4655|4656|4657|4658|4659|4660|4661|4662|4663|4664|4665|4666|4667|4668|4669|4670|4671|4672|4673|4674|4675|4676|4677|4678|4679|4680|4681|4682|4683|4684|4685|4686|4687|4688|4689|4690|4691|4692|4693|4694|4695|4696|4697|4698|4699|4700|4701|4702|4703|4704|4705|4706|4707|4708|4709|4710|4711|4712|4713|4714|4715|4716|4717|4718|4719|4720|4721|4722|4723|4724|4725|4726|4727|4728|4729|4730|4731|4732|4733|4734|4735|4736 +37.000000000000,2048,128,37,4737|4738|4739|4740|4741|4742|4743|4744|4745|4746|4747|4748|4749|4750|4751|4752|4753|4754|4755|4756|4757|4758|4759|4760|4761|4762|4763|4764|4765|4766|4767|4768|4769|4770|4771|4772|4773|4774|4775|4776|4777|4778|4779|4780|4781|4782|4783|4784|4785|4786|4787|4788|4789|4790|4791|4792|4793|4794|4795|4796|4797|4798|4799|4800|4801|4802|4803|4804|4805|4806|4807|4808|4809|4810|4811|4812|4813|4814|4815|4816|4817|4818|4819|4820|4821|4822|4823|4824|4825|4826|4827|4828|4829|4830|4831|4832|4833|4834|4835|4836|4837|4838|4839|4840|4841|4842|4843|4844|4845|4846|4847|4848|4849|4850|4851|4852|4853|4854|4855|4856|4857|4858|4859|4860|4861|4862|4863|4864 +38.000000000000,2048,128,38,4865|4866|4867|4868|4869|4870|4871|4872|4873|4874|4875|4876|4877|4878|4879|4880|4881|4882|4883|4884|4885|4886|4887|4888|4889|4890|4891|4892|4893|4894|4895|4896|4897|4898|4899|4900|4901|4902|4903|4904|4905|4906|4907|4908|4909|4910|4911|4912|4913|4914|4915|4916|4917|4918|4919|4920|4921|4922|4923|4924|4925|4926|4927|4928|4929|4930|4931|4932|4933|4934|4935|4936|4937|4938|4939|4940|4941|4942|4943|4944|4945|4946|4947|4948|4949|4950|4951|4952|4953|4954|4955|4956|4957|4958|4959|4960|4961|4962|4963|4964|4965|4966|4967|4968|4969|4970|4971|4972|4973|4974|4975|4976|4977|4978|4979|4980|4981|4982|4983|4984|4985|4986|4987|4988|4989|4990|4991|4992 +39.000000000000,2048,128,39,4993|4994|4995|4996|4997|4998|4999|5000|5001|5002|5003|5004|5005|5006|5007|5008|5009|5010|5011|5012|5013|5014|5015|5016|5017|5018|5019|5020|5021|5022|5023|5024|5025|5026|5027|5028|5029|5030|5031|5032|5033|5034|5035|5036|5037|5038|5039|5040|5041|5042|5043|5044|5045|5046|5047|5048|5049|5050|5051|5052|5053|5054|5055|5056|5057|5058|5059|5060|5061|5062|5063|5064|5065|5066|5067|5068|5069|5070|5071|5072|5073|5074|5075|5076|5077|5078|5079|5080|5081|5082|5083|5084|5085|5086|5087|5088|5089|5090|5091|5092|5093|5094|5095|5096|5097|5098|5099|5100|5101|5102|5103|5104|5105|5106|5107|5108|5109|5110|5111|5112|5113|5114|5115|5116|5117|5118|5119|5120 +40.000000000000,2048,128,40,5121|5122|5123|5124|5125|5126|5127|5128|5129|5130|5131|5132|5133|5134|5135|5136|5137|5138|5139|5140|5141|5142|5143|5144|5145|5146|5147|5148|5149|5150|5151|5152|5153|5154|5155|5156|5157|5158|5159|5160|5161|5162|5163|5164|5165|5166|5167|5168|5169|5170|5171|5172|5173|5174|5175|5176|5177|5178|5179|5180|5181|5182|5183|5184|5185|5186|5187|5188|5189|5190|5191|5192|5193|5194|5195|5196|5197|5198|5199|5200|5201|5202|5203|5204|5205|5206|5207|5208|5209|5210|5211|5212|5213|5214|5215|5216|5217|5218|5219|5220|5221|5222|5223|5224|5225|5226|5227|5228|5229|5230|5231|5232|5233|5234|5235|5236|5237|5238|5239|5240|5241|5242|5243|5244|5245|5246|5247|5248 +41.000000000000,2048,128,41,5249|5250|5251|5252|5253|5254|5255|5256|5257|5258|5259|5260|5261|5262|5263|5264|5265|5266|5267|5268|5269|5270|5271|5272|5273|5274|5275|5276|5277|5278|5279|5280|5281|5282|5283|5284|5285|5286|5287|5288|5289|5290|5291|5292|5293|5294|5295|5296|5297|5298|5299|5300|5301|5302|5303|5304|5305|5306|5307|5308|5309|5310|5311|5312|5313|5314|5315|5316|5317|5318|5319|5320|5321|5322|5323|5324|5325|5326|5327|5328|5329|5330|5331|5332|5333|5334|5335|5336|5337|5338|5339|5340|5341|5342|5343|5344|5345|5346|5347|5348|5349|5350|5351|5352|5353|5354|5355|5356|5357|5358|5359|5360|5361|5362|5363|5364|5365|5366|5367|5368|5369|5370|5371|5372|5373|5374|5375|5376 +42.000000000000,2048,128,42,5377|5378|5379|5380|5381|5382|5383|5384|5385|5386|5387|5388|5389|5390|5391|5392|5393|5394|5395|5396|5397|5398|5399|5400|5401|5402|5403|5404|5405|5406|5407|5408|5409|5410|5411|5412|5413|5414|5415|5416|5417|5418|5419|5420|5421|5422|5423|5424|5425|5426|5427|5428|5429|5430|5431|5432|5433|5434|5435|5436|5437|5438|5439|5440|5441|5442|5443|5444|5445|5446|5447|5448|5449|5450|5451|5452|5453|5454|5455|5456|5457|5458|5459|5460|5461|5462|5463|5464|5465|5466|5467|5468|5469|5470|5471|5472|5473|5474|5475|5476|5477|5478|5479|5480|5481|5482|5483|5484|5485|5486|5487|5488|5489|5490|5491|5492|5493|5494|5495|5496|5497|5498|5499|5500|5501|5502|5503|5504 +43.000000000000,2048,128,43,5505|5506|5507|5508|5509|5510|5511|5512|5513|5514|5515|5516|5517|5518|5519|5520|5521|5522|5523|5524|5525|5526|5527|5528|5529|5530|5531|5532|5533|5534|5535|5536|5537|5538|5539|5540|5541|5542|5543|5544|5545|5546|5547|5548|5549|5550|5551|5552|5553|5554|5555|5556|5557|5558|5559|5560|5561|5562|5563|5564|5565|5566|5567|5568|5569|5570|5571|5572|5573|5574|5575|5576|5577|5578|5579|5580|5581|5582|5583|5584|5585|5586|5587|5588|5589|5590|5591|5592|5593|5594|5595|5596|5597|5598|5599|5600|5601|5602|5603|5604|5605|5606|5607|5608|5609|5610|5611|5612|5613|5614|5615|5616|5617|5618|5619|5620|5621|5622|5623|5624|5625|5626|5627|5628|5629|5630|5631|5632 +44.000000000000,2048,128,44,5633|5634|5635|5636|5637|5638|5639|5640|5641|5642|5643|5644|5645|5646|5647|5648|5649|5650|5651|5652|5653|5654|5655|5656|5657|5658|5659|5660|5661|5662|5663|5664|5665|5666|5667|5668|5669|5670|5671|5672|5673|5674|5675|5676|5677|5678|5679|5680|5681|5682|5683|5684|5685|5686|5687|5688|5689|5690|5691|5692|5693|5694|5695|5696|5697|5698|5699|5700|5701|5702|5703|5704|5705|5706|5707|5708|5709|5710|5711|5712|5713|5714|5715|5716|5717|5718|5719|5720|5721|5722|5723|5724|5725|5726|5727|5728|5729|5730|5731|5732|5733|5734|5735|5736|5737|5738|5739|5740|5741|5742|5743|5744|5745|5746|5747|5748|5749|5750|5751|5752|5753|5754|5755|5756|5757|5758|5759|5760 +45.000000000000,2048,128,45,5761|5762|5763|5764|5765|5766|5767|5768|5769|5770|5771|5772|5773|5774|5775|5776|5777|5778|5779|5780|5781|5782|5783|5784|5785|5786|5787|5788|5789|5790|5791|5792|5793|5794|5795|5796|5797|5798|5799|5800|5801|5802|5803|5804|5805|5806|5807|5808|5809|5810|5811|5812|5813|5814|5815|5816|5817|5818|5819|5820|5821|5822|5823|5824|5825|5826|5827|5828|5829|5830|5831|5832|5833|5834|5835|5836|5837|5838|5839|5840|5841|5842|5843|5844|5845|5846|5847|5848|5849|5850|5851|5852|5853|5854|5855|5856|5857|5858|5859|5860|5861|5862|5863|5864|5865|5866|5867|5868|5869|5870|5871|5872|5873|5874|5875|5876|5877|5878|5879|5880|5881|5882|5883|5884|5885|5886|5887|5888 +46.000000000000,2048,128,46,5889|5890|5891|5892|5893|5894|5895|5896|5897|5898|5899|5900|5901|5902|5903|5904|5905|5906|5907|5908|5909|5910|5911|5912|5913|5914|5915|5916|5917|5918|5919|5920|5921|5922|5923|5924|5925|5926|5927|5928|5929|5930|5931|5932|5933|5934|5935|5936|5937|5938|5939|5940|5941|5942|5943|5944|5945|5946|5947|5948|5949|5950|5951|5952|5953|5954|5955|5956|5957|5958|5959|5960|5961|5962|5963|5964|5965|5966|5967|5968|5969|5970|5971|5972|5973|5974|5975|5976|5977|5978|5979|5980|5981|5982|5983|5984|5985|5986|5987|5988|5989|5990|5991|5992|5993|5994|5995|5996|5997|5998|5999|6000|6001|6002|6003|6004|6005|6006|6007|6008|6009|6010|6011|6012|6013|6014|6015|6016 +47.000000000000,2048,128,47,6017|6018|6019|6020|6021|6022|6023|6024|6025|6026|6027|6028|6029|6030|6031|6032|6033|6034|6035|6036|6037|6038|6039|6040|6041|6042|6043|6044|6045|6046|6047|6048|6049|6050|6051|6052|6053|6054|6055|6056|6057|6058|6059|6060|6061|6062|6063|6064|6065|6066|6067|6068|6069|6070|6071|6072|6073|6074|6075|6076|6077|6078|6079|6080|6081|6082|6083|6084|6085|6086|6087|6088|6089|6090|6091|6092|6093|6094|6095|6096|6097|6098|6099|6100|6101|6102|6103|6104|6105|6106|6107|6108|6109|6110|6111|6112|6113|6114|6115|6116|6117|6118|6119|6120|6121|6122|6123|6124|6125|6126|6127|6128|6129|6130|6131|6132|6133|6134|6135|6136|6137|6138|6139|6140|6141|6142|6143|6144 +48.000000000000,2048,128,48,6145|6146|6147|6148|6149|6150|6151|6152|6153|6154|6155|6156|6157|6158|6159|6160|6161|6162|6163|6164|6165|6166|6167|6168|6169|6170|6171|6172|6173|6174|6175|6176|6177|6178|6179|6180|6181|6182|6183|6184|6185|6186|6187|6188|6189|6190|6191|6192|6193|6194|6195|6196|6197|6198|6199|6200|6201|6202|6203|6204|6205|6206|6207|6208|6209|6210|6211|6212|6213|6214|6215|6216|6217|6218|6219|6220|6221|6222|6223|6224|6225|6226|6227|6228|6229|6230|6231|6232|6233|6234|6235|6236|6237|6238|6239|6240|6241|6242|6243|6244|6245|6246|6247|6248|6249|6250|6251|6252|6253|6254|6255|6256|6257|6258|6259|6260|6261|6262|6263|6264|6265|6266|6267|6268|6269|6270|6271|6272 +49.000000000000,2048,128,49,6273|6274|6275|6276|6277|6278|6279|6280|6281|6282|6283|6284|6285|6286|6287|6288|6289|6290|6291|6292|6293|6294|6295|6296|6297|6298|6299|6300|6301|6302|6303|6304|6305|6306|6307|6308|6309|6310|6311|6312|6313|6314|6315|6316|6317|6318|6319|6320|6321|6322|6323|6324|6325|6326|6327|6328|6329|6330|6331|6332|6333|6334|6335|6336|6337|6338|6339|6340|6341|6342|6343|6344|6345|6346|6347|6348|6349|6350|6351|6352|6353|6354|6355|6356|6357|6358|6359|6360|6361|6362|6363|6364|6365|6366|6367|6368|6369|6370|6371|6372|6373|6374|6375|6376|6377|6378|6379|6380|6381|6382|6383|6384|6385|6386|6387|6388|6389|6390|6391|6392|6393|6394|6395|6396|6397|6398|6399|6400 +50.000000000000,2048,128,50,6401|6402|6403|6404|6405|6406|6407|6408|6409|6410|6411|6412|6413|6414|6415|6416|6417|6418|6419|6420|6421|6422|6423|6424|6425|6426|6427|6428|6429|6430|6431|6432|6433|6434|6435|6436|6437|6438|6439|6440|6441|6442|6443|6444|6445|6446|6447|6448|6449|6450|6451|6452|6453|6454|6455|6456|6457|6458|6459|6460|6461|6462|6463|6464|6465|6466|6467|6468|6469|6470|6471|6472|6473|6474|6475|6476|6477|6478|6479|6480|6481|6482|6483|6484|6485|6486|6487|6488|6489|6490|6491|6492|6493|6494|6495|6496|6497|6498|6499|6500|6501|6502|6503|6504|6505|6506|6507|6508|6509|6510|6511|6512|6513|6514|6515|6516|6517|6518|6519|6520|6521|6522|6523|6524|6525|6526|6527|6528 +51.000000000000,2048,128,51,6529|6530|6531|6532|6533|6534|6535|6536|6537|6538|6539|6540|6541|6542|6543|6544|6545|6546|6547|6548|6549|6550|6551|6552|6553|6554|6555|6556|6557|6558|6559|6560|6561|6562|6563|6564|6565|6566|6567|6568|6569|6570|6571|6572|6573|6574|6575|6576|6577|6578|6579|6580|6581|6582|6583|6584|6585|6586|6587|6588|6589|6590|6591|6592|6593|6594|6595|6596|6597|6598|6599|6600|6601|6602|6603|6604|6605|6606|6607|6608|6609|6610|6611|6612|6613|6614|6615|6616|6617|6618|6619|6620|6621|6622|6623|6624|6625|6626|6627|6628|6629|6630|6631|6632|6633|6634|6635|6636|6637|6638|6639|6640|6641|6642|6643|6644|6645|6646|6647|6648|6649|6650|6651|6652|6653|6654|6655|6656 +52.000000000000,2048,128,52,6657|6658|6659|6660|6661|6662|6663|6664|6665|6666|6667|6668|6669|6670|6671|6672|6673|6674|6675|6676|6677|6678|6679|6680|6681|6682|6683|6684|6685|6686|6687|6688|6689|6690|6691|6692|6693|6694|6695|6696|6697|6698|6699|6700|6701|6702|6703|6704|6705|6706|6707|6708|6709|6710|6711|6712|6713|6714|6715|6716|6717|6718|6719|6720|6721|6722|6723|6724|6725|6726|6727|6728|6729|6730|6731|6732|6733|6734|6735|6736|6737|6738|6739|6740|6741|6742|6743|6744|6745|6746|6747|6748|6749|6750|6751|6752|6753|6754|6755|6756|6757|6758|6759|6760|6761|6762|6763|6764|6765|6766|6767|6768|6769|6770|6771|6772|6773|6774|6775|6776|6777|6778|6779|6780|6781|6782|6783|6784 +53.000000000000,2048,128,53,6785|6786|6787|6788|6789|6790|6791|6792|6793|6794|6795|6796|6797|6798|6799|6800|6801|6802|6803|6804|6805|6806|6807|6808|6809|6810|6811|6812|6813|6814|6815|6816|6817|6818|6819|6820|6821|6822|6823|6824|6825|6826|6827|6828|6829|6830|6831|6832|6833|6834|6835|6836|6837|6838|6839|6840|6841|6842|6843|6844|6845|6846|6847|6848|6849|6850|6851|6852|6853|6854|6855|6856|6857|6858|6859|6860|6861|6862|6863|6864|6865|6866|6867|6868|6869|6870|6871|6872|6873|6874|6875|6876|6877|6878|6879|6880|6881|6882|6883|6884|6885|6886|6887|6888|6889|6890|6891|6892|6893|6894|6895|6896|6897|6898|6899|6900|6901|6902|6903|6904|6905|6906|6907|6908|6909|6910|6911|6912 +54.000000000000,2048,128,54,6913|6914|6915|6916|6917|6918|6919|6920|6921|6922|6923|6924|6925|6926|6927|6928|6929|6930|6931|6932|6933|6934|6935|6936|6937|6938|6939|6940|6941|6942|6943|6944|6945|6946|6947|6948|6949|6950|6951|6952|6953|6954|6955|6956|6957|6958|6959|6960|6961|6962|6963|6964|6965|6966|6967|6968|6969|6970|6971|6972|6973|6974|6975|6976|6977|6978|6979|6980|6981|6982|6983|6984|6985|6986|6987|6988|6989|6990|6991|6992|6993|6994|6995|6996|6997|6998|6999|7000|7001|7002|7003|7004|7005|7006|7007|7008|7009|7010|7011|7012|7013|7014|7015|7016|7017|7018|7019|7020|7021|7022|7023|7024|7025|7026|7027|7028|7029|7030|7031|7032|7033|7034|7035|7036|7037|7038|7039|7040 +55.000000000000,2048,128,55,7041|7042|7043|7044|7045|7046|7047|7048|7049|7050|7051|7052|7053|7054|7055|7056|7057|7058|7059|7060|7061|7062|7063|7064|7065|7066|7067|7068|7069|7070|7071|7072|7073|7074|7075|7076|7077|7078|7079|7080|7081|7082|7083|7084|7085|7086|7087|7088|7089|7090|7091|7092|7093|7094|7095|7096|7097|7098|7099|7100|7101|7102|7103|7104|7105|7106|7107|7108|7109|7110|7111|7112|7113|7114|7115|7116|7117|7118|7119|7120|7121|7122|7123|7124|7125|7126|7127|7128|7129|7130|7131|7132|7133|7134|7135|7136|7137|7138|7139|7140|7141|7142|7143|7144|7145|7146|7147|7148|7149|7150|7151|7152|7153|7154|7155|7156|7157|7158|7159|7160|7161|7162|7163|7164|7165|7166|7167|7168 +56.000000000000,2048,128,56,7169|7170|7171|7172|7173|7174|7175|7176|7177|7178|7179|7180|7181|7182|7183|7184|7185|7186|7187|7188|7189|7190|7191|7192|7193|7194|7195|7196|7197|7198|7199|7200|7201|7202|7203|7204|7205|7206|7207|7208|7209|7210|7211|7212|7213|7214|7215|7216|7217|7218|7219|7220|7221|7222|7223|7224|7225|7226|7227|7228|7229|7230|7231|7232|7233|7234|7235|7236|7237|7238|7239|7240|7241|7242|7243|7244|7245|7246|7247|7248|7249|7250|7251|7252|7253|7254|7255|7256|7257|7258|7259|7260|7261|7262|7263|7264|7265|7266|7267|7268|7269|7270|7271|7272|7273|7274|7275|7276|7277|7278|7279|7280|7281|7282|7283|7284|7285|7286|7287|7288|7289|7290|7291|7292|7293|7294|7295|7296 +57.000000000000,2048,128,57,7297|7298|7299|7300|7301|7302|7303|7304|7305|7306|7307|7308|7309|7310|7311|7312|7313|7314|7315|7316|7317|7318|7319|7320|7321|7322|7323|7324|7325|7326|7327|7328|7329|7330|7331|7332|7333|7334|7335|7336|7337|7338|7339|7340|7341|7342|7343|7344|7345|7346|7347|7348|7349|7350|7351|7352|7353|7354|7355|7356|7357|7358|7359|7360|7361|7362|7363|7364|7365|7366|7367|7368|7369|7370|7371|7372|7373|7374|7375|7376|7377|7378|7379|7380|7381|7382|7383|7384|7385|7386|7387|7388|7389|7390|7391|7392|7393|7394|7395|7396|7397|7398|7399|7400|7401|7402|7403|7404|7405|7406|7407|7408|7409|7410|7411|7412|7413|7414|7415|7416|7417|7418|7419|7420|7421|7422|7423|7424 +58.000000000000,2048,128,58,7425|7426|7427|7428|7429|7430|7431|7432|7433|7434|7435|7436|7437|7438|7439|7440|7441|7442|7443|7444|7445|7446|7447|7448|7449|7450|7451|7452|7453|7454|7455|7456|7457|7458|7459|7460|7461|7462|7463|7464|7465|7466|7467|7468|7469|7470|7471|7472|7473|7474|7475|7476|7477|7478|7479|7480|7481|7482|7483|7484|7485|7486|7487|7488|7489|7490|7491|7492|7493|7494|7495|7496|7497|7498|7499|7500|7501|7502|7503|7504|7505|7506|7507|7508|7509|7510|7511|7512|7513|7514|7515|7516|7517|7518|7519|7520|7521|7522|7523|7524|7525|7526|7527|7528|7529|7530|7531|7532|7533|7534|7535|7536|7537|7538|7539|7540|7541|7542|7543|7544|7545|7546|7547|7548|7549|7550|7551|7552 +59.000000000000,2048,128,59,7553|7554|7555|7556|7557|7558|7559|7560|7561|7562|7563|7564|7565|7566|7567|7568|7569|7570|7571|7572|7573|7574|7575|7576|7577|7578|7579|7580|7581|7582|7583|7584|7585|7586|7587|7588|7589|7590|7591|7592|7593|7594|7595|7596|7597|7598|7599|7600|7601|7602|7603|7604|7605|7606|7607|7608|7609|7610|7611|7612|7613|7614|7615|7616|7617|7618|7619|7620|7621|7622|7623|7624|7625|7626|7627|7628|7629|7630|7631|7632|7633|7634|7635|7636|7637|7638|7639|7640|7641|7642|7643|7644|7645|7646|7647|7648|7649|7650|7651|7652|7653|7654|7655|7656|7657|7658|7659|7660|7661|7662|7663|7664|7665|7666|7667|7668|7669|7670|7671|7672|7673|7674|7675|7676|7677|7678|7679|7680 +60.000000000000,2048,128,60,7681|7682|7683|7684|7685|7686|7687|7688|7689|7690|7691|7692|7693|7694|7695|7696|7697|7698|7699|7700|7701|7702|7703|7704|7705|7706|7707|7708|7709|7710|7711|7712|7713|7714|7715|7716|7717|7718|7719|7720|7721|7722|7723|7724|7725|7726|7727|7728|7729|7730|7731|7732|7733|7734|7735|7736|7737|7738|7739|7740|7741|7742|7743|7744|7745|7746|7747|7748|7749|7750|7751|7752|7753|7754|7755|7756|7757|7758|7759|7760|7761|7762|7763|7764|7765|7766|7767|7768|7769|7770|7771|7772|7773|7774|7775|7776|7777|7778|7779|7780|7781|7782|7783|7784|7785|7786|7787|7788|7789|7790|7791|7792|7793|7794|7795|7796|7797|7798|7799|7800|7801|7802|7803|7804|7805|7806|7807|7808 +61.000000000000,2048,128,61,7809|7810|7811|7812|7813|7814|7815|7816|7817|7818|7819|7820|7821|7822|7823|7824|7825|7826|7827|7828|7829|7830|7831|7832|7833|7834|7835|7836|7837|7838|7839|7840|7841|7842|7843|7844|7845|7846|7847|7848|7849|7850|7851|7852|7853|7854|7855|7856|7857|7858|7859|7860|7861|7862|7863|7864|7865|7866|7867|7868|7869|7870|7871|7872|7873|7874|7875|7876|7877|7878|7879|7880|7881|7882|7883|7884|7885|7886|7887|7888|7889|7890|7891|7892|7893|7894|7895|7896|7897|7898|7899|7900|7901|7902|7903|7904|7905|7906|7907|7908|7909|7910|7911|7912|7913|7914|7915|7916|7917|7918|7919|7920|7921|7922|7923|7924|7925|7926|7927|7928|7929|7930|7931|7932|7933|7934|7935|7936 +62.000000000000,2048,128,62,7937|7938|7939|7940|7941|7942|7943|7944|7945|7946|7947|7948|7949|7950|7951|7952|7953|7954|7955|7956|7957|7958|7959|7960|7961|7962|7963|7964|7965|7966|7967|7968|7969|7970|7971|7972|7973|7974|7975|7976|7977|7978|7979|7980|7981|7982|7983|7984|7985|7986|7987|7988|7989|7990|7991|7992|7993|7994|7995|7996|7997|7998|7999|8000|8001|8002|8003|8004|8005|8006|8007|8008|8009|8010|8011|8012|8013|8014|8015|8016|8017|8018|8019|8020|8021|8022|8023|8024|8025|8026|8027|8028|8029|8030|8031|8032|8033|8034|8035|8036|8037|8038|8039|8040|8041|8042|8043|8044|8045|8046|8047|8048|8049|8050|8051|8052|8053|8054|8055|8056|8057|8058|8059|8060|8061|8062|8063|8064 +63.000000000000,2048,128,63,8065|8066|8067|8068|8069|8070|8071|8072|8073|8074|8075|8076|8077|8078|8079|8080|8081|8082|8083|8084|8085|8086|8087|8088|8089|8090|8091|8092|8093|8094|8095|8096|8097|8098|8099|8100|8101|8102|8103|8104|8105|8106|8107|8108|8109|8110|8111|8112|8113|8114|8115|8116|8117|8118|8119|8120|8121|8122|8123|8124|8125|8126|8127|8128|8129|8130|8131|8132|8133|8134|8135|8136|8137|8138|8139|8140|8141|8142|8143|8144|8145|8146|8147|8148|8149|8150|8151|8152|8153|8154|8155|8156|8157|8158|8159|8160|8161|8162|8163|8164|8165|8166|8167|8168|8169|8170|8171|8172|8173|8174|8175|8176|8177|8178|8179|8180|8181|8182|8183|8184|8185|8186|8187|8188|8189|8190|8191|8192 diff --git a/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r16.csv b/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r16.csv new file mode 100644 index 0000000..5d32a58 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r16.csv @@ -0,0 +1,65 @@ +arrived_at,num_prefill_tokens,num_decode_tokens,session_id,block_hash_ids +0.000000000000,2048,128,0,1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58|59|60|61|62|63|64|65|66|67|68|69|70|71|72|73|74|75|76|77|78|79|80|81|82|83|84|85|86|87|88|89|90|91|92|93|94|95|96|97|98|99|100|101|102|103|104|105|106|107|108|109|110|111|112|113|114|115|116|117|118|119|120|121|122|123|124|125|126|127|128 +0.062500000000,2048,128,1,129|130|131|132|133|134|135|136|137|138|139|140|141|142|143|144|145|146|147|148|149|150|151|152|153|154|155|156|157|158|159|160|161|162|163|164|165|166|167|168|169|170|171|172|173|174|175|176|177|178|179|180|181|182|183|184|185|186|187|188|189|190|191|192|193|194|195|196|197|198|199|200|201|202|203|204|205|206|207|208|209|210|211|212|213|214|215|216|217|218|219|220|221|222|223|224|225|226|227|228|229|230|231|232|233|234|235|236|237|238|239|240|241|242|243|244|245|246|247|248|249|250|251|252|253|254|255|256 +0.125000000000,2048,128,2,257|258|259|260|261|262|263|264|265|266|267|268|269|270|271|272|273|274|275|276|277|278|279|280|281|282|283|284|285|286|287|288|289|290|291|292|293|294|295|296|297|298|299|300|301|302|303|304|305|306|307|308|309|310|311|312|313|314|315|316|317|318|319|320|321|322|323|324|325|326|327|328|329|330|331|332|333|334|335|336|337|338|339|340|341|342|343|344|345|346|347|348|349|350|351|352|353|354|355|356|357|358|359|360|361|362|363|364|365|366|367|368|369|370|371|372|373|374|375|376|377|378|379|380|381|382|383|384 +0.187500000000,2048,128,3,385|386|387|388|389|390|391|392|393|394|395|396|397|398|399|400|401|402|403|404|405|406|407|408|409|410|411|412|413|414|415|416|417|418|419|420|421|422|423|424|425|426|427|428|429|430|431|432|433|434|435|436|437|438|439|440|441|442|443|444|445|446|447|448|449|450|451|452|453|454|455|456|457|458|459|460|461|462|463|464|465|466|467|468|469|470|471|472|473|474|475|476|477|478|479|480|481|482|483|484|485|486|487|488|489|490|491|492|493|494|495|496|497|498|499|500|501|502|503|504|505|506|507|508|509|510|511|512 +0.250000000000,2048,128,4,513|514|515|516|517|518|519|520|521|522|523|524|525|526|527|528|529|530|531|532|533|534|535|536|537|538|539|540|541|542|543|544|545|546|547|548|549|550|551|552|553|554|555|556|557|558|559|560|561|562|563|564|565|566|567|568|569|570|571|572|573|574|575|576|577|578|579|580|581|582|583|584|585|586|587|588|589|590|591|592|593|594|595|596|597|598|599|600|601|602|603|604|605|606|607|608|609|610|611|612|613|614|615|616|617|618|619|620|621|622|623|624|625|626|627|628|629|630|631|632|633|634|635|636|637|638|639|640 +0.312500000000,2048,128,5,641|642|643|644|645|646|647|648|649|650|651|652|653|654|655|656|657|658|659|660|661|662|663|664|665|666|667|668|669|670|671|672|673|674|675|676|677|678|679|680|681|682|683|684|685|686|687|688|689|690|691|692|693|694|695|696|697|698|699|700|701|702|703|704|705|706|707|708|709|710|711|712|713|714|715|716|717|718|719|720|721|722|723|724|725|726|727|728|729|730|731|732|733|734|735|736|737|738|739|740|741|742|743|744|745|746|747|748|749|750|751|752|753|754|755|756|757|758|759|760|761|762|763|764|765|766|767|768 +0.375000000000,2048,128,6,769|770|771|772|773|774|775|776|777|778|779|780|781|782|783|784|785|786|787|788|789|790|791|792|793|794|795|796|797|798|799|800|801|802|803|804|805|806|807|808|809|810|811|812|813|814|815|816|817|818|819|820|821|822|823|824|825|826|827|828|829|830|831|832|833|834|835|836|837|838|839|840|841|842|843|844|845|846|847|848|849|850|851|852|853|854|855|856|857|858|859|860|861|862|863|864|865|866|867|868|869|870|871|872|873|874|875|876|877|878|879|880|881|882|883|884|885|886|887|888|889|890|891|892|893|894|895|896 +0.437500000000,2048,128,7,897|898|899|900|901|902|903|904|905|906|907|908|909|910|911|912|913|914|915|916|917|918|919|920|921|922|923|924|925|926|927|928|929|930|931|932|933|934|935|936|937|938|939|940|941|942|943|944|945|946|947|948|949|950|951|952|953|954|955|956|957|958|959|960|961|962|963|964|965|966|967|968|969|970|971|972|973|974|975|976|977|978|979|980|981|982|983|984|985|986|987|988|989|990|991|992|993|994|995|996|997|998|999|1000|1001|1002|1003|1004|1005|1006|1007|1008|1009|1010|1011|1012|1013|1014|1015|1016|1017|1018|1019|1020|1021|1022|1023|1024 +0.500000000000,2048,128,8,1025|1026|1027|1028|1029|1030|1031|1032|1033|1034|1035|1036|1037|1038|1039|1040|1041|1042|1043|1044|1045|1046|1047|1048|1049|1050|1051|1052|1053|1054|1055|1056|1057|1058|1059|1060|1061|1062|1063|1064|1065|1066|1067|1068|1069|1070|1071|1072|1073|1074|1075|1076|1077|1078|1079|1080|1081|1082|1083|1084|1085|1086|1087|1088|1089|1090|1091|1092|1093|1094|1095|1096|1097|1098|1099|1100|1101|1102|1103|1104|1105|1106|1107|1108|1109|1110|1111|1112|1113|1114|1115|1116|1117|1118|1119|1120|1121|1122|1123|1124|1125|1126|1127|1128|1129|1130|1131|1132|1133|1134|1135|1136|1137|1138|1139|1140|1141|1142|1143|1144|1145|1146|1147|1148|1149|1150|1151|1152 +0.562500000000,2048,128,9,1153|1154|1155|1156|1157|1158|1159|1160|1161|1162|1163|1164|1165|1166|1167|1168|1169|1170|1171|1172|1173|1174|1175|1176|1177|1178|1179|1180|1181|1182|1183|1184|1185|1186|1187|1188|1189|1190|1191|1192|1193|1194|1195|1196|1197|1198|1199|1200|1201|1202|1203|1204|1205|1206|1207|1208|1209|1210|1211|1212|1213|1214|1215|1216|1217|1218|1219|1220|1221|1222|1223|1224|1225|1226|1227|1228|1229|1230|1231|1232|1233|1234|1235|1236|1237|1238|1239|1240|1241|1242|1243|1244|1245|1246|1247|1248|1249|1250|1251|1252|1253|1254|1255|1256|1257|1258|1259|1260|1261|1262|1263|1264|1265|1266|1267|1268|1269|1270|1271|1272|1273|1274|1275|1276|1277|1278|1279|1280 +0.625000000000,2048,128,10,1281|1282|1283|1284|1285|1286|1287|1288|1289|1290|1291|1292|1293|1294|1295|1296|1297|1298|1299|1300|1301|1302|1303|1304|1305|1306|1307|1308|1309|1310|1311|1312|1313|1314|1315|1316|1317|1318|1319|1320|1321|1322|1323|1324|1325|1326|1327|1328|1329|1330|1331|1332|1333|1334|1335|1336|1337|1338|1339|1340|1341|1342|1343|1344|1345|1346|1347|1348|1349|1350|1351|1352|1353|1354|1355|1356|1357|1358|1359|1360|1361|1362|1363|1364|1365|1366|1367|1368|1369|1370|1371|1372|1373|1374|1375|1376|1377|1378|1379|1380|1381|1382|1383|1384|1385|1386|1387|1388|1389|1390|1391|1392|1393|1394|1395|1396|1397|1398|1399|1400|1401|1402|1403|1404|1405|1406|1407|1408 +0.687500000000,2048,128,11,1409|1410|1411|1412|1413|1414|1415|1416|1417|1418|1419|1420|1421|1422|1423|1424|1425|1426|1427|1428|1429|1430|1431|1432|1433|1434|1435|1436|1437|1438|1439|1440|1441|1442|1443|1444|1445|1446|1447|1448|1449|1450|1451|1452|1453|1454|1455|1456|1457|1458|1459|1460|1461|1462|1463|1464|1465|1466|1467|1468|1469|1470|1471|1472|1473|1474|1475|1476|1477|1478|1479|1480|1481|1482|1483|1484|1485|1486|1487|1488|1489|1490|1491|1492|1493|1494|1495|1496|1497|1498|1499|1500|1501|1502|1503|1504|1505|1506|1507|1508|1509|1510|1511|1512|1513|1514|1515|1516|1517|1518|1519|1520|1521|1522|1523|1524|1525|1526|1527|1528|1529|1530|1531|1532|1533|1534|1535|1536 +0.750000000000,2048,128,12,1537|1538|1539|1540|1541|1542|1543|1544|1545|1546|1547|1548|1549|1550|1551|1552|1553|1554|1555|1556|1557|1558|1559|1560|1561|1562|1563|1564|1565|1566|1567|1568|1569|1570|1571|1572|1573|1574|1575|1576|1577|1578|1579|1580|1581|1582|1583|1584|1585|1586|1587|1588|1589|1590|1591|1592|1593|1594|1595|1596|1597|1598|1599|1600|1601|1602|1603|1604|1605|1606|1607|1608|1609|1610|1611|1612|1613|1614|1615|1616|1617|1618|1619|1620|1621|1622|1623|1624|1625|1626|1627|1628|1629|1630|1631|1632|1633|1634|1635|1636|1637|1638|1639|1640|1641|1642|1643|1644|1645|1646|1647|1648|1649|1650|1651|1652|1653|1654|1655|1656|1657|1658|1659|1660|1661|1662|1663|1664 +0.812500000000,2048,128,13,1665|1666|1667|1668|1669|1670|1671|1672|1673|1674|1675|1676|1677|1678|1679|1680|1681|1682|1683|1684|1685|1686|1687|1688|1689|1690|1691|1692|1693|1694|1695|1696|1697|1698|1699|1700|1701|1702|1703|1704|1705|1706|1707|1708|1709|1710|1711|1712|1713|1714|1715|1716|1717|1718|1719|1720|1721|1722|1723|1724|1725|1726|1727|1728|1729|1730|1731|1732|1733|1734|1735|1736|1737|1738|1739|1740|1741|1742|1743|1744|1745|1746|1747|1748|1749|1750|1751|1752|1753|1754|1755|1756|1757|1758|1759|1760|1761|1762|1763|1764|1765|1766|1767|1768|1769|1770|1771|1772|1773|1774|1775|1776|1777|1778|1779|1780|1781|1782|1783|1784|1785|1786|1787|1788|1789|1790|1791|1792 +0.875000000000,2048,128,14,1793|1794|1795|1796|1797|1798|1799|1800|1801|1802|1803|1804|1805|1806|1807|1808|1809|1810|1811|1812|1813|1814|1815|1816|1817|1818|1819|1820|1821|1822|1823|1824|1825|1826|1827|1828|1829|1830|1831|1832|1833|1834|1835|1836|1837|1838|1839|1840|1841|1842|1843|1844|1845|1846|1847|1848|1849|1850|1851|1852|1853|1854|1855|1856|1857|1858|1859|1860|1861|1862|1863|1864|1865|1866|1867|1868|1869|1870|1871|1872|1873|1874|1875|1876|1877|1878|1879|1880|1881|1882|1883|1884|1885|1886|1887|1888|1889|1890|1891|1892|1893|1894|1895|1896|1897|1898|1899|1900|1901|1902|1903|1904|1905|1906|1907|1908|1909|1910|1911|1912|1913|1914|1915|1916|1917|1918|1919|1920 +0.937500000000,2048,128,15,1921|1922|1923|1924|1925|1926|1927|1928|1929|1930|1931|1932|1933|1934|1935|1936|1937|1938|1939|1940|1941|1942|1943|1944|1945|1946|1947|1948|1949|1950|1951|1952|1953|1954|1955|1956|1957|1958|1959|1960|1961|1962|1963|1964|1965|1966|1967|1968|1969|1970|1971|1972|1973|1974|1975|1976|1977|1978|1979|1980|1981|1982|1983|1984|1985|1986|1987|1988|1989|1990|1991|1992|1993|1994|1995|1996|1997|1998|1999|2000|2001|2002|2003|2004|2005|2006|2007|2008|2009|2010|2011|2012|2013|2014|2015|2016|2017|2018|2019|2020|2021|2022|2023|2024|2025|2026|2027|2028|2029|2030|2031|2032|2033|2034|2035|2036|2037|2038|2039|2040|2041|2042|2043|2044|2045|2046|2047|2048 +1.000000000000,2048,128,16,2049|2050|2051|2052|2053|2054|2055|2056|2057|2058|2059|2060|2061|2062|2063|2064|2065|2066|2067|2068|2069|2070|2071|2072|2073|2074|2075|2076|2077|2078|2079|2080|2081|2082|2083|2084|2085|2086|2087|2088|2089|2090|2091|2092|2093|2094|2095|2096|2097|2098|2099|2100|2101|2102|2103|2104|2105|2106|2107|2108|2109|2110|2111|2112|2113|2114|2115|2116|2117|2118|2119|2120|2121|2122|2123|2124|2125|2126|2127|2128|2129|2130|2131|2132|2133|2134|2135|2136|2137|2138|2139|2140|2141|2142|2143|2144|2145|2146|2147|2148|2149|2150|2151|2152|2153|2154|2155|2156|2157|2158|2159|2160|2161|2162|2163|2164|2165|2166|2167|2168|2169|2170|2171|2172|2173|2174|2175|2176 +1.062500000000,2048,128,17,2177|2178|2179|2180|2181|2182|2183|2184|2185|2186|2187|2188|2189|2190|2191|2192|2193|2194|2195|2196|2197|2198|2199|2200|2201|2202|2203|2204|2205|2206|2207|2208|2209|2210|2211|2212|2213|2214|2215|2216|2217|2218|2219|2220|2221|2222|2223|2224|2225|2226|2227|2228|2229|2230|2231|2232|2233|2234|2235|2236|2237|2238|2239|2240|2241|2242|2243|2244|2245|2246|2247|2248|2249|2250|2251|2252|2253|2254|2255|2256|2257|2258|2259|2260|2261|2262|2263|2264|2265|2266|2267|2268|2269|2270|2271|2272|2273|2274|2275|2276|2277|2278|2279|2280|2281|2282|2283|2284|2285|2286|2287|2288|2289|2290|2291|2292|2293|2294|2295|2296|2297|2298|2299|2300|2301|2302|2303|2304 +1.125000000000,2048,128,18,2305|2306|2307|2308|2309|2310|2311|2312|2313|2314|2315|2316|2317|2318|2319|2320|2321|2322|2323|2324|2325|2326|2327|2328|2329|2330|2331|2332|2333|2334|2335|2336|2337|2338|2339|2340|2341|2342|2343|2344|2345|2346|2347|2348|2349|2350|2351|2352|2353|2354|2355|2356|2357|2358|2359|2360|2361|2362|2363|2364|2365|2366|2367|2368|2369|2370|2371|2372|2373|2374|2375|2376|2377|2378|2379|2380|2381|2382|2383|2384|2385|2386|2387|2388|2389|2390|2391|2392|2393|2394|2395|2396|2397|2398|2399|2400|2401|2402|2403|2404|2405|2406|2407|2408|2409|2410|2411|2412|2413|2414|2415|2416|2417|2418|2419|2420|2421|2422|2423|2424|2425|2426|2427|2428|2429|2430|2431|2432 +1.187500000000,2048,128,19,2433|2434|2435|2436|2437|2438|2439|2440|2441|2442|2443|2444|2445|2446|2447|2448|2449|2450|2451|2452|2453|2454|2455|2456|2457|2458|2459|2460|2461|2462|2463|2464|2465|2466|2467|2468|2469|2470|2471|2472|2473|2474|2475|2476|2477|2478|2479|2480|2481|2482|2483|2484|2485|2486|2487|2488|2489|2490|2491|2492|2493|2494|2495|2496|2497|2498|2499|2500|2501|2502|2503|2504|2505|2506|2507|2508|2509|2510|2511|2512|2513|2514|2515|2516|2517|2518|2519|2520|2521|2522|2523|2524|2525|2526|2527|2528|2529|2530|2531|2532|2533|2534|2535|2536|2537|2538|2539|2540|2541|2542|2543|2544|2545|2546|2547|2548|2549|2550|2551|2552|2553|2554|2555|2556|2557|2558|2559|2560 +1.250000000000,2048,128,20,2561|2562|2563|2564|2565|2566|2567|2568|2569|2570|2571|2572|2573|2574|2575|2576|2577|2578|2579|2580|2581|2582|2583|2584|2585|2586|2587|2588|2589|2590|2591|2592|2593|2594|2595|2596|2597|2598|2599|2600|2601|2602|2603|2604|2605|2606|2607|2608|2609|2610|2611|2612|2613|2614|2615|2616|2617|2618|2619|2620|2621|2622|2623|2624|2625|2626|2627|2628|2629|2630|2631|2632|2633|2634|2635|2636|2637|2638|2639|2640|2641|2642|2643|2644|2645|2646|2647|2648|2649|2650|2651|2652|2653|2654|2655|2656|2657|2658|2659|2660|2661|2662|2663|2664|2665|2666|2667|2668|2669|2670|2671|2672|2673|2674|2675|2676|2677|2678|2679|2680|2681|2682|2683|2684|2685|2686|2687|2688 +1.312500000000,2048,128,21,2689|2690|2691|2692|2693|2694|2695|2696|2697|2698|2699|2700|2701|2702|2703|2704|2705|2706|2707|2708|2709|2710|2711|2712|2713|2714|2715|2716|2717|2718|2719|2720|2721|2722|2723|2724|2725|2726|2727|2728|2729|2730|2731|2732|2733|2734|2735|2736|2737|2738|2739|2740|2741|2742|2743|2744|2745|2746|2747|2748|2749|2750|2751|2752|2753|2754|2755|2756|2757|2758|2759|2760|2761|2762|2763|2764|2765|2766|2767|2768|2769|2770|2771|2772|2773|2774|2775|2776|2777|2778|2779|2780|2781|2782|2783|2784|2785|2786|2787|2788|2789|2790|2791|2792|2793|2794|2795|2796|2797|2798|2799|2800|2801|2802|2803|2804|2805|2806|2807|2808|2809|2810|2811|2812|2813|2814|2815|2816 +1.375000000000,2048,128,22,2817|2818|2819|2820|2821|2822|2823|2824|2825|2826|2827|2828|2829|2830|2831|2832|2833|2834|2835|2836|2837|2838|2839|2840|2841|2842|2843|2844|2845|2846|2847|2848|2849|2850|2851|2852|2853|2854|2855|2856|2857|2858|2859|2860|2861|2862|2863|2864|2865|2866|2867|2868|2869|2870|2871|2872|2873|2874|2875|2876|2877|2878|2879|2880|2881|2882|2883|2884|2885|2886|2887|2888|2889|2890|2891|2892|2893|2894|2895|2896|2897|2898|2899|2900|2901|2902|2903|2904|2905|2906|2907|2908|2909|2910|2911|2912|2913|2914|2915|2916|2917|2918|2919|2920|2921|2922|2923|2924|2925|2926|2927|2928|2929|2930|2931|2932|2933|2934|2935|2936|2937|2938|2939|2940|2941|2942|2943|2944 +1.437500000000,2048,128,23,2945|2946|2947|2948|2949|2950|2951|2952|2953|2954|2955|2956|2957|2958|2959|2960|2961|2962|2963|2964|2965|2966|2967|2968|2969|2970|2971|2972|2973|2974|2975|2976|2977|2978|2979|2980|2981|2982|2983|2984|2985|2986|2987|2988|2989|2990|2991|2992|2993|2994|2995|2996|2997|2998|2999|3000|3001|3002|3003|3004|3005|3006|3007|3008|3009|3010|3011|3012|3013|3014|3015|3016|3017|3018|3019|3020|3021|3022|3023|3024|3025|3026|3027|3028|3029|3030|3031|3032|3033|3034|3035|3036|3037|3038|3039|3040|3041|3042|3043|3044|3045|3046|3047|3048|3049|3050|3051|3052|3053|3054|3055|3056|3057|3058|3059|3060|3061|3062|3063|3064|3065|3066|3067|3068|3069|3070|3071|3072 +1.500000000000,2048,128,24,3073|3074|3075|3076|3077|3078|3079|3080|3081|3082|3083|3084|3085|3086|3087|3088|3089|3090|3091|3092|3093|3094|3095|3096|3097|3098|3099|3100|3101|3102|3103|3104|3105|3106|3107|3108|3109|3110|3111|3112|3113|3114|3115|3116|3117|3118|3119|3120|3121|3122|3123|3124|3125|3126|3127|3128|3129|3130|3131|3132|3133|3134|3135|3136|3137|3138|3139|3140|3141|3142|3143|3144|3145|3146|3147|3148|3149|3150|3151|3152|3153|3154|3155|3156|3157|3158|3159|3160|3161|3162|3163|3164|3165|3166|3167|3168|3169|3170|3171|3172|3173|3174|3175|3176|3177|3178|3179|3180|3181|3182|3183|3184|3185|3186|3187|3188|3189|3190|3191|3192|3193|3194|3195|3196|3197|3198|3199|3200 +1.562500000000,2048,128,25,3201|3202|3203|3204|3205|3206|3207|3208|3209|3210|3211|3212|3213|3214|3215|3216|3217|3218|3219|3220|3221|3222|3223|3224|3225|3226|3227|3228|3229|3230|3231|3232|3233|3234|3235|3236|3237|3238|3239|3240|3241|3242|3243|3244|3245|3246|3247|3248|3249|3250|3251|3252|3253|3254|3255|3256|3257|3258|3259|3260|3261|3262|3263|3264|3265|3266|3267|3268|3269|3270|3271|3272|3273|3274|3275|3276|3277|3278|3279|3280|3281|3282|3283|3284|3285|3286|3287|3288|3289|3290|3291|3292|3293|3294|3295|3296|3297|3298|3299|3300|3301|3302|3303|3304|3305|3306|3307|3308|3309|3310|3311|3312|3313|3314|3315|3316|3317|3318|3319|3320|3321|3322|3323|3324|3325|3326|3327|3328 +1.625000000000,2048,128,26,3329|3330|3331|3332|3333|3334|3335|3336|3337|3338|3339|3340|3341|3342|3343|3344|3345|3346|3347|3348|3349|3350|3351|3352|3353|3354|3355|3356|3357|3358|3359|3360|3361|3362|3363|3364|3365|3366|3367|3368|3369|3370|3371|3372|3373|3374|3375|3376|3377|3378|3379|3380|3381|3382|3383|3384|3385|3386|3387|3388|3389|3390|3391|3392|3393|3394|3395|3396|3397|3398|3399|3400|3401|3402|3403|3404|3405|3406|3407|3408|3409|3410|3411|3412|3413|3414|3415|3416|3417|3418|3419|3420|3421|3422|3423|3424|3425|3426|3427|3428|3429|3430|3431|3432|3433|3434|3435|3436|3437|3438|3439|3440|3441|3442|3443|3444|3445|3446|3447|3448|3449|3450|3451|3452|3453|3454|3455|3456 +1.687500000000,2048,128,27,3457|3458|3459|3460|3461|3462|3463|3464|3465|3466|3467|3468|3469|3470|3471|3472|3473|3474|3475|3476|3477|3478|3479|3480|3481|3482|3483|3484|3485|3486|3487|3488|3489|3490|3491|3492|3493|3494|3495|3496|3497|3498|3499|3500|3501|3502|3503|3504|3505|3506|3507|3508|3509|3510|3511|3512|3513|3514|3515|3516|3517|3518|3519|3520|3521|3522|3523|3524|3525|3526|3527|3528|3529|3530|3531|3532|3533|3534|3535|3536|3537|3538|3539|3540|3541|3542|3543|3544|3545|3546|3547|3548|3549|3550|3551|3552|3553|3554|3555|3556|3557|3558|3559|3560|3561|3562|3563|3564|3565|3566|3567|3568|3569|3570|3571|3572|3573|3574|3575|3576|3577|3578|3579|3580|3581|3582|3583|3584 +1.750000000000,2048,128,28,3585|3586|3587|3588|3589|3590|3591|3592|3593|3594|3595|3596|3597|3598|3599|3600|3601|3602|3603|3604|3605|3606|3607|3608|3609|3610|3611|3612|3613|3614|3615|3616|3617|3618|3619|3620|3621|3622|3623|3624|3625|3626|3627|3628|3629|3630|3631|3632|3633|3634|3635|3636|3637|3638|3639|3640|3641|3642|3643|3644|3645|3646|3647|3648|3649|3650|3651|3652|3653|3654|3655|3656|3657|3658|3659|3660|3661|3662|3663|3664|3665|3666|3667|3668|3669|3670|3671|3672|3673|3674|3675|3676|3677|3678|3679|3680|3681|3682|3683|3684|3685|3686|3687|3688|3689|3690|3691|3692|3693|3694|3695|3696|3697|3698|3699|3700|3701|3702|3703|3704|3705|3706|3707|3708|3709|3710|3711|3712 +1.812500000000,2048,128,29,3713|3714|3715|3716|3717|3718|3719|3720|3721|3722|3723|3724|3725|3726|3727|3728|3729|3730|3731|3732|3733|3734|3735|3736|3737|3738|3739|3740|3741|3742|3743|3744|3745|3746|3747|3748|3749|3750|3751|3752|3753|3754|3755|3756|3757|3758|3759|3760|3761|3762|3763|3764|3765|3766|3767|3768|3769|3770|3771|3772|3773|3774|3775|3776|3777|3778|3779|3780|3781|3782|3783|3784|3785|3786|3787|3788|3789|3790|3791|3792|3793|3794|3795|3796|3797|3798|3799|3800|3801|3802|3803|3804|3805|3806|3807|3808|3809|3810|3811|3812|3813|3814|3815|3816|3817|3818|3819|3820|3821|3822|3823|3824|3825|3826|3827|3828|3829|3830|3831|3832|3833|3834|3835|3836|3837|3838|3839|3840 +1.875000000000,2048,128,30,3841|3842|3843|3844|3845|3846|3847|3848|3849|3850|3851|3852|3853|3854|3855|3856|3857|3858|3859|3860|3861|3862|3863|3864|3865|3866|3867|3868|3869|3870|3871|3872|3873|3874|3875|3876|3877|3878|3879|3880|3881|3882|3883|3884|3885|3886|3887|3888|3889|3890|3891|3892|3893|3894|3895|3896|3897|3898|3899|3900|3901|3902|3903|3904|3905|3906|3907|3908|3909|3910|3911|3912|3913|3914|3915|3916|3917|3918|3919|3920|3921|3922|3923|3924|3925|3926|3927|3928|3929|3930|3931|3932|3933|3934|3935|3936|3937|3938|3939|3940|3941|3942|3943|3944|3945|3946|3947|3948|3949|3950|3951|3952|3953|3954|3955|3956|3957|3958|3959|3960|3961|3962|3963|3964|3965|3966|3967|3968 +1.937500000000,2048,128,31,3969|3970|3971|3972|3973|3974|3975|3976|3977|3978|3979|3980|3981|3982|3983|3984|3985|3986|3987|3988|3989|3990|3991|3992|3993|3994|3995|3996|3997|3998|3999|4000|4001|4002|4003|4004|4005|4006|4007|4008|4009|4010|4011|4012|4013|4014|4015|4016|4017|4018|4019|4020|4021|4022|4023|4024|4025|4026|4027|4028|4029|4030|4031|4032|4033|4034|4035|4036|4037|4038|4039|4040|4041|4042|4043|4044|4045|4046|4047|4048|4049|4050|4051|4052|4053|4054|4055|4056|4057|4058|4059|4060|4061|4062|4063|4064|4065|4066|4067|4068|4069|4070|4071|4072|4073|4074|4075|4076|4077|4078|4079|4080|4081|4082|4083|4084|4085|4086|4087|4088|4089|4090|4091|4092|4093|4094|4095|4096 +2.000000000000,2048,128,32,4097|4098|4099|4100|4101|4102|4103|4104|4105|4106|4107|4108|4109|4110|4111|4112|4113|4114|4115|4116|4117|4118|4119|4120|4121|4122|4123|4124|4125|4126|4127|4128|4129|4130|4131|4132|4133|4134|4135|4136|4137|4138|4139|4140|4141|4142|4143|4144|4145|4146|4147|4148|4149|4150|4151|4152|4153|4154|4155|4156|4157|4158|4159|4160|4161|4162|4163|4164|4165|4166|4167|4168|4169|4170|4171|4172|4173|4174|4175|4176|4177|4178|4179|4180|4181|4182|4183|4184|4185|4186|4187|4188|4189|4190|4191|4192|4193|4194|4195|4196|4197|4198|4199|4200|4201|4202|4203|4204|4205|4206|4207|4208|4209|4210|4211|4212|4213|4214|4215|4216|4217|4218|4219|4220|4221|4222|4223|4224 +2.062500000000,2048,128,33,4225|4226|4227|4228|4229|4230|4231|4232|4233|4234|4235|4236|4237|4238|4239|4240|4241|4242|4243|4244|4245|4246|4247|4248|4249|4250|4251|4252|4253|4254|4255|4256|4257|4258|4259|4260|4261|4262|4263|4264|4265|4266|4267|4268|4269|4270|4271|4272|4273|4274|4275|4276|4277|4278|4279|4280|4281|4282|4283|4284|4285|4286|4287|4288|4289|4290|4291|4292|4293|4294|4295|4296|4297|4298|4299|4300|4301|4302|4303|4304|4305|4306|4307|4308|4309|4310|4311|4312|4313|4314|4315|4316|4317|4318|4319|4320|4321|4322|4323|4324|4325|4326|4327|4328|4329|4330|4331|4332|4333|4334|4335|4336|4337|4338|4339|4340|4341|4342|4343|4344|4345|4346|4347|4348|4349|4350|4351|4352 +2.125000000000,2048,128,34,4353|4354|4355|4356|4357|4358|4359|4360|4361|4362|4363|4364|4365|4366|4367|4368|4369|4370|4371|4372|4373|4374|4375|4376|4377|4378|4379|4380|4381|4382|4383|4384|4385|4386|4387|4388|4389|4390|4391|4392|4393|4394|4395|4396|4397|4398|4399|4400|4401|4402|4403|4404|4405|4406|4407|4408|4409|4410|4411|4412|4413|4414|4415|4416|4417|4418|4419|4420|4421|4422|4423|4424|4425|4426|4427|4428|4429|4430|4431|4432|4433|4434|4435|4436|4437|4438|4439|4440|4441|4442|4443|4444|4445|4446|4447|4448|4449|4450|4451|4452|4453|4454|4455|4456|4457|4458|4459|4460|4461|4462|4463|4464|4465|4466|4467|4468|4469|4470|4471|4472|4473|4474|4475|4476|4477|4478|4479|4480 +2.187500000000,2048,128,35,4481|4482|4483|4484|4485|4486|4487|4488|4489|4490|4491|4492|4493|4494|4495|4496|4497|4498|4499|4500|4501|4502|4503|4504|4505|4506|4507|4508|4509|4510|4511|4512|4513|4514|4515|4516|4517|4518|4519|4520|4521|4522|4523|4524|4525|4526|4527|4528|4529|4530|4531|4532|4533|4534|4535|4536|4537|4538|4539|4540|4541|4542|4543|4544|4545|4546|4547|4548|4549|4550|4551|4552|4553|4554|4555|4556|4557|4558|4559|4560|4561|4562|4563|4564|4565|4566|4567|4568|4569|4570|4571|4572|4573|4574|4575|4576|4577|4578|4579|4580|4581|4582|4583|4584|4585|4586|4587|4588|4589|4590|4591|4592|4593|4594|4595|4596|4597|4598|4599|4600|4601|4602|4603|4604|4605|4606|4607|4608 +2.250000000000,2048,128,36,4609|4610|4611|4612|4613|4614|4615|4616|4617|4618|4619|4620|4621|4622|4623|4624|4625|4626|4627|4628|4629|4630|4631|4632|4633|4634|4635|4636|4637|4638|4639|4640|4641|4642|4643|4644|4645|4646|4647|4648|4649|4650|4651|4652|4653|4654|4655|4656|4657|4658|4659|4660|4661|4662|4663|4664|4665|4666|4667|4668|4669|4670|4671|4672|4673|4674|4675|4676|4677|4678|4679|4680|4681|4682|4683|4684|4685|4686|4687|4688|4689|4690|4691|4692|4693|4694|4695|4696|4697|4698|4699|4700|4701|4702|4703|4704|4705|4706|4707|4708|4709|4710|4711|4712|4713|4714|4715|4716|4717|4718|4719|4720|4721|4722|4723|4724|4725|4726|4727|4728|4729|4730|4731|4732|4733|4734|4735|4736 +2.312500000000,2048,128,37,4737|4738|4739|4740|4741|4742|4743|4744|4745|4746|4747|4748|4749|4750|4751|4752|4753|4754|4755|4756|4757|4758|4759|4760|4761|4762|4763|4764|4765|4766|4767|4768|4769|4770|4771|4772|4773|4774|4775|4776|4777|4778|4779|4780|4781|4782|4783|4784|4785|4786|4787|4788|4789|4790|4791|4792|4793|4794|4795|4796|4797|4798|4799|4800|4801|4802|4803|4804|4805|4806|4807|4808|4809|4810|4811|4812|4813|4814|4815|4816|4817|4818|4819|4820|4821|4822|4823|4824|4825|4826|4827|4828|4829|4830|4831|4832|4833|4834|4835|4836|4837|4838|4839|4840|4841|4842|4843|4844|4845|4846|4847|4848|4849|4850|4851|4852|4853|4854|4855|4856|4857|4858|4859|4860|4861|4862|4863|4864 +2.375000000000,2048,128,38,4865|4866|4867|4868|4869|4870|4871|4872|4873|4874|4875|4876|4877|4878|4879|4880|4881|4882|4883|4884|4885|4886|4887|4888|4889|4890|4891|4892|4893|4894|4895|4896|4897|4898|4899|4900|4901|4902|4903|4904|4905|4906|4907|4908|4909|4910|4911|4912|4913|4914|4915|4916|4917|4918|4919|4920|4921|4922|4923|4924|4925|4926|4927|4928|4929|4930|4931|4932|4933|4934|4935|4936|4937|4938|4939|4940|4941|4942|4943|4944|4945|4946|4947|4948|4949|4950|4951|4952|4953|4954|4955|4956|4957|4958|4959|4960|4961|4962|4963|4964|4965|4966|4967|4968|4969|4970|4971|4972|4973|4974|4975|4976|4977|4978|4979|4980|4981|4982|4983|4984|4985|4986|4987|4988|4989|4990|4991|4992 +2.437500000000,2048,128,39,4993|4994|4995|4996|4997|4998|4999|5000|5001|5002|5003|5004|5005|5006|5007|5008|5009|5010|5011|5012|5013|5014|5015|5016|5017|5018|5019|5020|5021|5022|5023|5024|5025|5026|5027|5028|5029|5030|5031|5032|5033|5034|5035|5036|5037|5038|5039|5040|5041|5042|5043|5044|5045|5046|5047|5048|5049|5050|5051|5052|5053|5054|5055|5056|5057|5058|5059|5060|5061|5062|5063|5064|5065|5066|5067|5068|5069|5070|5071|5072|5073|5074|5075|5076|5077|5078|5079|5080|5081|5082|5083|5084|5085|5086|5087|5088|5089|5090|5091|5092|5093|5094|5095|5096|5097|5098|5099|5100|5101|5102|5103|5104|5105|5106|5107|5108|5109|5110|5111|5112|5113|5114|5115|5116|5117|5118|5119|5120 +2.500000000000,2048,128,40,5121|5122|5123|5124|5125|5126|5127|5128|5129|5130|5131|5132|5133|5134|5135|5136|5137|5138|5139|5140|5141|5142|5143|5144|5145|5146|5147|5148|5149|5150|5151|5152|5153|5154|5155|5156|5157|5158|5159|5160|5161|5162|5163|5164|5165|5166|5167|5168|5169|5170|5171|5172|5173|5174|5175|5176|5177|5178|5179|5180|5181|5182|5183|5184|5185|5186|5187|5188|5189|5190|5191|5192|5193|5194|5195|5196|5197|5198|5199|5200|5201|5202|5203|5204|5205|5206|5207|5208|5209|5210|5211|5212|5213|5214|5215|5216|5217|5218|5219|5220|5221|5222|5223|5224|5225|5226|5227|5228|5229|5230|5231|5232|5233|5234|5235|5236|5237|5238|5239|5240|5241|5242|5243|5244|5245|5246|5247|5248 +2.562500000000,2048,128,41,5249|5250|5251|5252|5253|5254|5255|5256|5257|5258|5259|5260|5261|5262|5263|5264|5265|5266|5267|5268|5269|5270|5271|5272|5273|5274|5275|5276|5277|5278|5279|5280|5281|5282|5283|5284|5285|5286|5287|5288|5289|5290|5291|5292|5293|5294|5295|5296|5297|5298|5299|5300|5301|5302|5303|5304|5305|5306|5307|5308|5309|5310|5311|5312|5313|5314|5315|5316|5317|5318|5319|5320|5321|5322|5323|5324|5325|5326|5327|5328|5329|5330|5331|5332|5333|5334|5335|5336|5337|5338|5339|5340|5341|5342|5343|5344|5345|5346|5347|5348|5349|5350|5351|5352|5353|5354|5355|5356|5357|5358|5359|5360|5361|5362|5363|5364|5365|5366|5367|5368|5369|5370|5371|5372|5373|5374|5375|5376 +2.625000000000,2048,128,42,5377|5378|5379|5380|5381|5382|5383|5384|5385|5386|5387|5388|5389|5390|5391|5392|5393|5394|5395|5396|5397|5398|5399|5400|5401|5402|5403|5404|5405|5406|5407|5408|5409|5410|5411|5412|5413|5414|5415|5416|5417|5418|5419|5420|5421|5422|5423|5424|5425|5426|5427|5428|5429|5430|5431|5432|5433|5434|5435|5436|5437|5438|5439|5440|5441|5442|5443|5444|5445|5446|5447|5448|5449|5450|5451|5452|5453|5454|5455|5456|5457|5458|5459|5460|5461|5462|5463|5464|5465|5466|5467|5468|5469|5470|5471|5472|5473|5474|5475|5476|5477|5478|5479|5480|5481|5482|5483|5484|5485|5486|5487|5488|5489|5490|5491|5492|5493|5494|5495|5496|5497|5498|5499|5500|5501|5502|5503|5504 +2.687500000000,2048,128,43,5505|5506|5507|5508|5509|5510|5511|5512|5513|5514|5515|5516|5517|5518|5519|5520|5521|5522|5523|5524|5525|5526|5527|5528|5529|5530|5531|5532|5533|5534|5535|5536|5537|5538|5539|5540|5541|5542|5543|5544|5545|5546|5547|5548|5549|5550|5551|5552|5553|5554|5555|5556|5557|5558|5559|5560|5561|5562|5563|5564|5565|5566|5567|5568|5569|5570|5571|5572|5573|5574|5575|5576|5577|5578|5579|5580|5581|5582|5583|5584|5585|5586|5587|5588|5589|5590|5591|5592|5593|5594|5595|5596|5597|5598|5599|5600|5601|5602|5603|5604|5605|5606|5607|5608|5609|5610|5611|5612|5613|5614|5615|5616|5617|5618|5619|5620|5621|5622|5623|5624|5625|5626|5627|5628|5629|5630|5631|5632 +2.750000000000,2048,128,44,5633|5634|5635|5636|5637|5638|5639|5640|5641|5642|5643|5644|5645|5646|5647|5648|5649|5650|5651|5652|5653|5654|5655|5656|5657|5658|5659|5660|5661|5662|5663|5664|5665|5666|5667|5668|5669|5670|5671|5672|5673|5674|5675|5676|5677|5678|5679|5680|5681|5682|5683|5684|5685|5686|5687|5688|5689|5690|5691|5692|5693|5694|5695|5696|5697|5698|5699|5700|5701|5702|5703|5704|5705|5706|5707|5708|5709|5710|5711|5712|5713|5714|5715|5716|5717|5718|5719|5720|5721|5722|5723|5724|5725|5726|5727|5728|5729|5730|5731|5732|5733|5734|5735|5736|5737|5738|5739|5740|5741|5742|5743|5744|5745|5746|5747|5748|5749|5750|5751|5752|5753|5754|5755|5756|5757|5758|5759|5760 +2.812500000000,2048,128,45,5761|5762|5763|5764|5765|5766|5767|5768|5769|5770|5771|5772|5773|5774|5775|5776|5777|5778|5779|5780|5781|5782|5783|5784|5785|5786|5787|5788|5789|5790|5791|5792|5793|5794|5795|5796|5797|5798|5799|5800|5801|5802|5803|5804|5805|5806|5807|5808|5809|5810|5811|5812|5813|5814|5815|5816|5817|5818|5819|5820|5821|5822|5823|5824|5825|5826|5827|5828|5829|5830|5831|5832|5833|5834|5835|5836|5837|5838|5839|5840|5841|5842|5843|5844|5845|5846|5847|5848|5849|5850|5851|5852|5853|5854|5855|5856|5857|5858|5859|5860|5861|5862|5863|5864|5865|5866|5867|5868|5869|5870|5871|5872|5873|5874|5875|5876|5877|5878|5879|5880|5881|5882|5883|5884|5885|5886|5887|5888 +2.875000000000,2048,128,46,5889|5890|5891|5892|5893|5894|5895|5896|5897|5898|5899|5900|5901|5902|5903|5904|5905|5906|5907|5908|5909|5910|5911|5912|5913|5914|5915|5916|5917|5918|5919|5920|5921|5922|5923|5924|5925|5926|5927|5928|5929|5930|5931|5932|5933|5934|5935|5936|5937|5938|5939|5940|5941|5942|5943|5944|5945|5946|5947|5948|5949|5950|5951|5952|5953|5954|5955|5956|5957|5958|5959|5960|5961|5962|5963|5964|5965|5966|5967|5968|5969|5970|5971|5972|5973|5974|5975|5976|5977|5978|5979|5980|5981|5982|5983|5984|5985|5986|5987|5988|5989|5990|5991|5992|5993|5994|5995|5996|5997|5998|5999|6000|6001|6002|6003|6004|6005|6006|6007|6008|6009|6010|6011|6012|6013|6014|6015|6016 +2.937500000000,2048,128,47,6017|6018|6019|6020|6021|6022|6023|6024|6025|6026|6027|6028|6029|6030|6031|6032|6033|6034|6035|6036|6037|6038|6039|6040|6041|6042|6043|6044|6045|6046|6047|6048|6049|6050|6051|6052|6053|6054|6055|6056|6057|6058|6059|6060|6061|6062|6063|6064|6065|6066|6067|6068|6069|6070|6071|6072|6073|6074|6075|6076|6077|6078|6079|6080|6081|6082|6083|6084|6085|6086|6087|6088|6089|6090|6091|6092|6093|6094|6095|6096|6097|6098|6099|6100|6101|6102|6103|6104|6105|6106|6107|6108|6109|6110|6111|6112|6113|6114|6115|6116|6117|6118|6119|6120|6121|6122|6123|6124|6125|6126|6127|6128|6129|6130|6131|6132|6133|6134|6135|6136|6137|6138|6139|6140|6141|6142|6143|6144 +3.000000000000,2048,128,48,6145|6146|6147|6148|6149|6150|6151|6152|6153|6154|6155|6156|6157|6158|6159|6160|6161|6162|6163|6164|6165|6166|6167|6168|6169|6170|6171|6172|6173|6174|6175|6176|6177|6178|6179|6180|6181|6182|6183|6184|6185|6186|6187|6188|6189|6190|6191|6192|6193|6194|6195|6196|6197|6198|6199|6200|6201|6202|6203|6204|6205|6206|6207|6208|6209|6210|6211|6212|6213|6214|6215|6216|6217|6218|6219|6220|6221|6222|6223|6224|6225|6226|6227|6228|6229|6230|6231|6232|6233|6234|6235|6236|6237|6238|6239|6240|6241|6242|6243|6244|6245|6246|6247|6248|6249|6250|6251|6252|6253|6254|6255|6256|6257|6258|6259|6260|6261|6262|6263|6264|6265|6266|6267|6268|6269|6270|6271|6272 +3.062500000000,2048,128,49,6273|6274|6275|6276|6277|6278|6279|6280|6281|6282|6283|6284|6285|6286|6287|6288|6289|6290|6291|6292|6293|6294|6295|6296|6297|6298|6299|6300|6301|6302|6303|6304|6305|6306|6307|6308|6309|6310|6311|6312|6313|6314|6315|6316|6317|6318|6319|6320|6321|6322|6323|6324|6325|6326|6327|6328|6329|6330|6331|6332|6333|6334|6335|6336|6337|6338|6339|6340|6341|6342|6343|6344|6345|6346|6347|6348|6349|6350|6351|6352|6353|6354|6355|6356|6357|6358|6359|6360|6361|6362|6363|6364|6365|6366|6367|6368|6369|6370|6371|6372|6373|6374|6375|6376|6377|6378|6379|6380|6381|6382|6383|6384|6385|6386|6387|6388|6389|6390|6391|6392|6393|6394|6395|6396|6397|6398|6399|6400 +3.125000000000,2048,128,50,6401|6402|6403|6404|6405|6406|6407|6408|6409|6410|6411|6412|6413|6414|6415|6416|6417|6418|6419|6420|6421|6422|6423|6424|6425|6426|6427|6428|6429|6430|6431|6432|6433|6434|6435|6436|6437|6438|6439|6440|6441|6442|6443|6444|6445|6446|6447|6448|6449|6450|6451|6452|6453|6454|6455|6456|6457|6458|6459|6460|6461|6462|6463|6464|6465|6466|6467|6468|6469|6470|6471|6472|6473|6474|6475|6476|6477|6478|6479|6480|6481|6482|6483|6484|6485|6486|6487|6488|6489|6490|6491|6492|6493|6494|6495|6496|6497|6498|6499|6500|6501|6502|6503|6504|6505|6506|6507|6508|6509|6510|6511|6512|6513|6514|6515|6516|6517|6518|6519|6520|6521|6522|6523|6524|6525|6526|6527|6528 +3.187500000000,2048,128,51,6529|6530|6531|6532|6533|6534|6535|6536|6537|6538|6539|6540|6541|6542|6543|6544|6545|6546|6547|6548|6549|6550|6551|6552|6553|6554|6555|6556|6557|6558|6559|6560|6561|6562|6563|6564|6565|6566|6567|6568|6569|6570|6571|6572|6573|6574|6575|6576|6577|6578|6579|6580|6581|6582|6583|6584|6585|6586|6587|6588|6589|6590|6591|6592|6593|6594|6595|6596|6597|6598|6599|6600|6601|6602|6603|6604|6605|6606|6607|6608|6609|6610|6611|6612|6613|6614|6615|6616|6617|6618|6619|6620|6621|6622|6623|6624|6625|6626|6627|6628|6629|6630|6631|6632|6633|6634|6635|6636|6637|6638|6639|6640|6641|6642|6643|6644|6645|6646|6647|6648|6649|6650|6651|6652|6653|6654|6655|6656 +3.250000000000,2048,128,52,6657|6658|6659|6660|6661|6662|6663|6664|6665|6666|6667|6668|6669|6670|6671|6672|6673|6674|6675|6676|6677|6678|6679|6680|6681|6682|6683|6684|6685|6686|6687|6688|6689|6690|6691|6692|6693|6694|6695|6696|6697|6698|6699|6700|6701|6702|6703|6704|6705|6706|6707|6708|6709|6710|6711|6712|6713|6714|6715|6716|6717|6718|6719|6720|6721|6722|6723|6724|6725|6726|6727|6728|6729|6730|6731|6732|6733|6734|6735|6736|6737|6738|6739|6740|6741|6742|6743|6744|6745|6746|6747|6748|6749|6750|6751|6752|6753|6754|6755|6756|6757|6758|6759|6760|6761|6762|6763|6764|6765|6766|6767|6768|6769|6770|6771|6772|6773|6774|6775|6776|6777|6778|6779|6780|6781|6782|6783|6784 +3.312500000000,2048,128,53,6785|6786|6787|6788|6789|6790|6791|6792|6793|6794|6795|6796|6797|6798|6799|6800|6801|6802|6803|6804|6805|6806|6807|6808|6809|6810|6811|6812|6813|6814|6815|6816|6817|6818|6819|6820|6821|6822|6823|6824|6825|6826|6827|6828|6829|6830|6831|6832|6833|6834|6835|6836|6837|6838|6839|6840|6841|6842|6843|6844|6845|6846|6847|6848|6849|6850|6851|6852|6853|6854|6855|6856|6857|6858|6859|6860|6861|6862|6863|6864|6865|6866|6867|6868|6869|6870|6871|6872|6873|6874|6875|6876|6877|6878|6879|6880|6881|6882|6883|6884|6885|6886|6887|6888|6889|6890|6891|6892|6893|6894|6895|6896|6897|6898|6899|6900|6901|6902|6903|6904|6905|6906|6907|6908|6909|6910|6911|6912 +3.375000000000,2048,128,54,6913|6914|6915|6916|6917|6918|6919|6920|6921|6922|6923|6924|6925|6926|6927|6928|6929|6930|6931|6932|6933|6934|6935|6936|6937|6938|6939|6940|6941|6942|6943|6944|6945|6946|6947|6948|6949|6950|6951|6952|6953|6954|6955|6956|6957|6958|6959|6960|6961|6962|6963|6964|6965|6966|6967|6968|6969|6970|6971|6972|6973|6974|6975|6976|6977|6978|6979|6980|6981|6982|6983|6984|6985|6986|6987|6988|6989|6990|6991|6992|6993|6994|6995|6996|6997|6998|6999|7000|7001|7002|7003|7004|7005|7006|7007|7008|7009|7010|7011|7012|7013|7014|7015|7016|7017|7018|7019|7020|7021|7022|7023|7024|7025|7026|7027|7028|7029|7030|7031|7032|7033|7034|7035|7036|7037|7038|7039|7040 +3.437500000000,2048,128,55,7041|7042|7043|7044|7045|7046|7047|7048|7049|7050|7051|7052|7053|7054|7055|7056|7057|7058|7059|7060|7061|7062|7063|7064|7065|7066|7067|7068|7069|7070|7071|7072|7073|7074|7075|7076|7077|7078|7079|7080|7081|7082|7083|7084|7085|7086|7087|7088|7089|7090|7091|7092|7093|7094|7095|7096|7097|7098|7099|7100|7101|7102|7103|7104|7105|7106|7107|7108|7109|7110|7111|7112|7113|7114|7115|7116|7117|7118|7119|7120|7121|7122|7123|7124|7125|7126|7127|7128|7129|7130|7131|7132|7133|7134|7135|7136|7137|7138|7139|7140|7141|7142|7143|7144|7145|7146|7147|7148|7149|7150|7151|7152|7153|7154|7155|7156|7157|7158|7159|7160|7161|7162|7163|7164|7165|7166|7167|7168 +3.500000000000,2048,128,56,7169|7170|7171|7172|7173|7174|7175|7176|7177|7178|7179|7180|7181|7182|7183|7184|7185|7186|7187|7188|7189|7190|7191|7192|7193|7194|7195|7196|7197|7198|7199|7200|7201|7202|7203|7204|7205|7206|7207|7208|7209|7210|7211|7212|7213|7214|7215|7216|7217|7218|7219|7220|7221|7222|7223|7224|7225|7226|7227|7228|7229|7230|7231|7232|7233|7234|7235|7236|7237|7238|7239|7240|7241|7242|7243|7244|7245|7246|7247|7248|7249|7250|7251|7252|7253|7254|7255|7256|7257|7258|7259|7260|7261|7262|7263|7264|7265|7266|7267|7268|7269|7270|7271|7272|7273|7274|7275|7276|7277|7278|7279|7280|7281|7282|7283|7284|7285|7286|7287|7288|7289|7290|7291|7292|7293|7294|7295|7296 +3.562500000000,2048,128,57,7297|7298|7299|7300|7301|7302|7303|7304|7305|7306|7307|7308|7309|7310|7311|7312|7313|7314|7315|7316|7317|7318|7319|7320|7321|7322|7323|7324|7325|7326|7327|7328|7329|7330|7331|7332|7333|7334|7335|7336|7337|7338|7339|7340|7341|7342|7343|7344|7345|7346|7347|7348|7349|7350|7351|7352|7353|7354|7355|7356|7357|7358|7359|7360|7361|7362|7363|7364|7365|7366|7367|7368|7369|7370|7371|7372|7373|7374|7375|7376|7377|7378|7379|7380|7381|7382|7383|7384|7385|7386|7387|7388|7389|7390|7391|7392|7393|7394|7395|7396|7397|7398|7399|7400|7401|7402|7403|7404|7405|7406|7407|7408|7409|7410|7411|7412|7413|7414|7415|7416|7417|7418|7419|7420|7421|7422|7423|7424 +3.625000000000,2048,128,58,7425|7426|7427|7428|7429|7430|7431|7432|7433|7434|7435|7436|7437|7438|7439|7440|7441|7442|7443|7444|7445|7446|7447|7448|7449|7450|7451|7452|7453|7454|7455|7456|7457|7458|7459|7460|7461|7462|7463|7464|7465|7466|7467|7468|7469|7470|7471|7472|7473|7474|7475|7476|7477|7478|7479|7480|7481|7482|7483|7484|7485|7486|7487|7488|7489|7490|7491|7492|7493|7494|7495|7496|7497|7498|7499|7500|7501|7502|7503|7504|7505|7506|7507|7508|7509|7510|7511|7512|7513|7514|7515|7516|7517|7518|7519|7520|7521|7522|7523|7524|7525|7526|7527|7528|7529|7530|7531|7532|7533|7534|7535|7536|7537|7538|7539|7540|7541|7542|7543|7544|7545|7546|7547|7548|7549|7550|7551|7552 +3.687500000000,2048,128,59,7553|7554|7555|7556|7557|7558|7559|7560|7561|7562|7563|7564|7565|7566|7567|7568|7569|7570|7571|7572|7573|7574|7575|7576|7577|7578|7579|7580|7581|7582|7583|7584|7585|7586|7587|7588|7589|7590|7591|7592|7593|7594|7595|7596|7597|7598|7599|7600|7601|7602|7603|7604|7605|7606|7607|7608|7609|7610|7611|7612|7613|7614|7615|7616|7617|7618|7619|7620|7621|7622|7623|7624|7625|7626|7627|7628|7629|7630|7631|7632|7633|7634|7635|7636|7637|7638|7639|7640|7641|7642|7643|7644|7645|7646|7647|7648|7649|7650|7651|7652|7653|7654|7655|7656|7657|7658|7659|7660|7661|7662|7663|7664|7665|7666|7667|7668|7669|7670|7671|7672|7673|7674|7675|7676|7677|7678|7679|7680 +3.750000000000,2048,128,60,7681|7682|7683|7684|7685|7686|7687|7688|7689|7690|7691|7692|7693|7694|7695|7696|7697|7698|7699|7700|7701|7702|7703|7704|7705|7706|7707|7708|7709|7710|7711|7712|7713|7714|7715|7716|7717|7718|7719|7720|7721|7722|7723|7724|7725|7726|7727|7728|7729|7730|7731|7732|7733|7734|7735|7736|7737|7738|7739|7740|7741|7742|7743|7744|7745|7746|7747|7748|7749|7750|7751|7752|7753|7754|7755|7756|7757|7758|7759|7760|7761|7762|7763|7764|7765|7766|7767|7768|7769|7770|7771|7772|7773|7774|7775|7776|7777|7778|7779|7780|7781|7782|7783|7784|7785|7786|7787|7788|7789|7790|7791|7792|7793|7794|7795|7796|7797|7798|7799|7800|7801|7802|7803|7804|7805|7806|7807|7808 +3.812500000000,2048,128,61,7809|7810|7811|7812|7813|7814|7815|7816|7817|7818|7819|7820|7821|7822|7823|7824|7825|7826|7827|7828|7829|7830|7831|7832|7833|7834|7835|7836|7837|7838|7839|7840|7841|7842|7843|7844|7845|7846|7847|7848|7849|7850|7851|7852|7853|7854|7855|7856|7857|7858|7859|7860|7861|7862|7863|7864|7865|7866|7867|7868|7869|7870|7871|7872|7873|7874|7875|7876|7877|7878|7879|7880|7881|7882|7883|7884|7885|7886|7887|7888|7889|7890|7891|7892|7893|7894|7895|7896|7897|7898|7899|7900|7901|7902|7903|7904|7905|7906|7907|7908|7909|7910|7911|7912|7913|7914|7915|7916|7917|7918|7919|7920|7921|7922|7923|7924|7925|7926|7927|7928|7929|7930|7931|7932|7933|7934|7935|7936 +3.875000000000,2048,128,62,7937|7938|7939|7940|7941|7942|7943|7944|7945|7946|7947|7948|7949|7950|7951|7952|7953|7954|7955|7956|7957|7958|7959|7960|7961|7962|7963|7964|7965|7966|7967|7968|7969|7970|7971|7972|7973|7974|7975|7976|7977|7978|7979|7980|7981|7982|7983|7984|7985|7986|7987|7988|7989|7990|7991|7992|7993|7994|7995|7996|7997|7998|7999|8000|8001|8002|8003|8004|8005|8006|8007|8008|8009|8010|8011|8012|8013|8014|8015|8016|8017|8018|8019|8020|8021|8022|8023|8024|8025|8026|8027|8028|8029|8030|8031|8032|8033|8034|8035|8036|8037|8038|8039|8040|8041|8042|8043|8044|8045|8046|8047|8048|8049|8050|8051|8052|8053|8054|8055|8056|8057|8058|8059|8060|8061|8062|8063|8064 +3.937500000000,2048,128,63,8065|8066|8067|8068|8069|8070|8071|8072|8073|8074|8075|8076|8077|8078|8079|8080|8081|8082|8083|8084|8085|8086|8087|8088|8089|8090|8091|8092|8093|8094|8095|8096|8097|8098|8099|8100|8101|8102|8103|8104|8105|8106|8107|8108|8109|8110|8111|8112|8113|8114|8115|8116|8117|8118|8119|8120|8121|8122|8123|8124|8125|8126|8127|8128|8129|8130|8131|8132|8133|8134|8135|8136|8137|8138|8139|8140|8141|8142|8143|8144|8145|8146|8147|8148|8149|8150|8151|8152|8153|8154|8155|8156|8157|8158|8159|8160|8161|8162|8163|8164|8165|8166|8167|8168|8169|8170|8171|8172|8173|8174|8175|8176|8177|8178|8179|8180|8181|8182|8183|8184|8185|8186|8187|8188|8189|8190|8191|8192 diff --git a/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r2.csv b/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r2.csv new file mode 100644 index 0000000..5a55e4e --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r2.csv @@ -0,0 +1,65 @@ +arrived_at,num_prefill_tokens,num_decode_tokens,session_id,block_hash_ids +0.000000000000,2048,128,0,1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58|59|60|61|62|63|64|65|66|67|68|69|70|71|72|73|74|75|76|77|78|79|80|81|82|83|84|85|86|87|88|89|90|91|92|93|94|95|96|97|98|99|100|101|102|103|104|105|106|107|108|109|110|111|112|113|114|115|116|117|118|119|120|121|122|123|124|125|126|127|128 +0.500000000000,2048,128,1,129|130|131|132|133|134|135|136|137|138|139|140|141|142|143|144|145|146|147|148|149|150|151|152|153|154|155|156|157|158|159|160|161|162|163|164|165|166|167|168|169|170|171|172|173|174|175|176|177|178|179|180|181|182|183|184|185|186|187|188|189|190|191|192|193|194|195|196|197|198|199|200|201|202|203|204|205|206|207|208|209|210|211|212|213|214|215|216|217|218|219|220|221|222|223|224|225|226|227|228|229|230|231|232|233|234|235|236|237|238|239|240|241|242|243|244|245|246|247|248|249|250|251|252|253|254|255|256 +1.000000000000,2048,128,2,257|258|259|260|261|262|263|264|265|266|267|268|269|270|271|272|273|274|275|276|277|278|279|280|281|282|283|284|285|286|287|288|289|290|291|292|293|294|295|296|297|298|299|300|301|302|303|304|305|306|307|308|309|310|311|312|313|314|315|316|317|318|319|320|321|322|323|324|325|326|327|328|329|330|331|332|333|334|335|336|337|338|339|340|341|342|343|344|345|346|347|348|349|350|351|352|353|354|355|356|357|358|359|360|361|362|363|364|365|366|367|368|369|370|371|372|373|374|375|376|377|378|379|380|381|382|383|384 +1.500000000000,2048,128,3,385|386|387|388|389|390|391|392|393|394|395|396|397|398|399|400|401|402|403|404|405|406|407|408|409|410|411|412|413|414|415|416|417|418|419|420|421|422|423|424|425|426|427|428|429|430|431|432|433|434|435|436|437|438|439|440|441|442|443|444|445|446|447|448|449|450|451|452|453|454|455|456|457|458|459|460|461|462|463|464|465|466|467|468|469|470|471|472|473|474|475|476|477|478|479|480|481|482|483|484|485|486|487|488|489|490|491|492|493|494|495|496|497|498|499|500|501|502|503|504|505|506|507|508|509|510|511|512 +2.000000000000,2048,128,4,513|514|515|516|517|518|519|520|521|522|523|524|525|526|527|528|529|530|531|532|533|534|535|536|537|538|539|540|541|542|543|544|545|546|547|548|549|550|551|552|553|554|555|556|557|558|559|560|561|562|563|564|565|566|567|568|569|570|571|572|573|574|575|576|577|578|579|580|581|582|583|584|585|586|587|588|589|590|591|592|593|594|595|596|597|598|599|600|601|602|603|604|605|606|607|608|609|610|611|612|613|614|615|616|617|618|619|620|621|622|623|624|625|626|627|628|629|630|631|632|633|634|635|636|637|638|639|640 +2.500000000000,2048,128,5,641|642|643|644|645|646|647|648|649|650|651|652|653|654|655|656|657|658|659|660|661|662|663|664|665|666|667|668|669|670|671|672|673|674|675|676|677|678|679|680|681|682|683|684|685|686|687|688|689|690|691|692|693|694|695|696|697|698|699|700|701|702|703|704|705|706|707|708|709|710|711|712|713|714|715|716|717|718|719|720|721|722|723|724|725|726|727|728|729|730|731|732|733|734|735|736|737|738|739|740|741|742|743|744|745|746|747|748|749|750|751|752|753|754|755|756|757|758|759|760|761|762|763|764|765|766|767|768 +3.000000000000,2048,128,6,769|770|771|772|773|774|775|776|777|778|779|780|781|782|783|784|785|786|787|788|789|790|791|792|793|794|795|796|797|798|799|800|801|802|803|804|805|806|807|808|809|810|811|812|813|814|815|816|817|818|819|820|821|822|823|824|825|826|827|828|829|830|831|832|833|834|835|836|837|838|839|840|841|842|843|844|845|846|847|848|849|850|851|852|853|854|855|856|857|858|859|860|861|862|863|864|865|866|867|868|869|870|871|872|873|874|875|876|877|878|879|880|881|882|883|884|885|886|887|888|889|890|891|892|893|894|895|896 +3.500000000000,2048,128,7,897|898|899|900|901|902|903|904|905|906|907|908|909|910|911|912|913|914|915|916|917|918|919|920|921|922|923|924|925|926|927|928|929|930|931|932|933|934|935|936|937|938|939|940|941|942|943|944|945|946|947|948|949|950|951|952|953|954|955|956|957|958|959|960|961|962|963|964|965|966|967|968|969|970|971|972|973|974|975|976|977|978|979|980|981|982|983|984|985|986|987|988|989|990|991|992|993|994|995|996|997|998|999|1000|1001|1002|1003|1004|1005|1006|1007|1008|1009|1010|1011|1012|1013|1014|1015|1016|1017|1018|1019|1020|1021|1022|1023|1024 +4.000000000000,2048,128,8,1025|1026|1027|1028|1029|1030|1031|1032|1033|1034|1035|1036|1037|1038|1039|1040|1041|1042|1043|1044|1045|1046|1047|1048|1049|1050|1051|1052|1053|1054|1055|1056|1057|1058|1059|1060|1061|1062|1063|1064|1065|1066|1067|1068|1069|1070|1071|1072|1073|1074|1075|1076|1077|1078|1079|1080|1081|1082|1083|1084|1085|1086|1087|1088|1089|1090|1091|1092|1093|1094|1095|1096|1097|1098|1099|1100|1101|1102|1103|1104|1105|1106|1107|1108|1109|1110|1111|1112|1113|1114|1115|1116|1117|1118|1119|1120|1121|1122|1123|1124|1125|1126|1127|1128|1129|1130|1131|1132|1133|1134|1135|1136|1137|1138|1139|1140|1141|1142|1143|1144|1145|1146|1147|1148|1149|1150|1151|1152 +4.500000000000,2048,128,9,1153|1154|1155|1156|1157|1158|1159|1160|1161|1162|1163|1164|1165|1166|1167|1168|1169|1170|1171|1172|1173|1174|1175|1176|1177|1178|1179|1180|1181|1182|1183|1184|1185|1186|1187|1188|1189|1190|1191|1192|1193|1194|1195|1196|1197|1198|1199|1200|1201|1202|1203|1204|1205|1206|1207|1208|1209|1210|1211|1212|1213|1214|1215|1216|1217|1218|1219|1220|1221|1222|1223|1224|1225|1226|1227|1228|1229|1230|1231|1232|1233|1234|1235|1236|1237|1238|1239|1240|1241|1242|1243|1244|1245|1246|1247|1248|1249|1250|1251|1252|1253|1254|1255|1256|1257|1258|1259|1260|1261|1262|1263|1264|1265|1266|1267|1268|1269|1270|1271|1272|1273|1274|1275|1276|1277|1278|1279|1280 +5.000000000000,2048,128,10,1281|1282|1283|1284|1285|1286|1287|1288|1289|1290|1291|1292|1293|1294|1295|1296|1297|1298|1299|1300|1301|1302|1303|1304|1305|1306|1307|1308|1309|1310|1311|1312|1313|1314|1315|1316|1317|1318|1319|1320|1321|1322|1323|1324|1325|1326|1327|1328|1329|1330|1331|1332|1333|1334|1335|1336|1337|1338|1339|1340|1341|1342|1343|1344|1345|1346|1347|1348|1349|1350|1351|1352|1353|1354|1355|1356|1357|1358|1359|1360|1361|1362|1363|1364|1365|1366|1367|1368|1369|1370|1371|1372|1373|1374|1375|1376|1377|1378|1379|1380|1381|1382|1383|1384|1385|1386|1387|1388|1389|1390|1391|1392|1393|1394|1395|1396|1397|1398|1399|1400|1401|1402|1403|1404|1405|1406|1407|1408 +5.500000000000,2048,128,11,1409|1410|1411|1412|1413|1414|1415|1416|1417|1418|1419|1420|1421|1422|1423|1424|1425|1426|1427|1428|1429|1430|1431|1432|1433|1434|1435|1436|1437|1438|1439|1440|1441|1442|1443|1444|1445|1446|1447|1448|1449|1450|1451|1452|1453|1454|1455|1456|1457|1458|1459|1460|1461|1462|1463|1464|1465|1466|1467|1468|1469|1470|1471|1472|1473|1474|1475|1476|1477|1478|1479|1480|1481|1482|1483|1484|1485|1486|1487|1488|1489|1490|1491|1492|1493|1494|1495|1496|1497|1498|1499|1500|1501|1502|1503|1504|1505|1506|1507|1508|1509|1510|1511|1512|1513|1514|1515|1516|1517|1518|1519|1520|1521|1522|1523|1524|1525|1526|1527|1528|1529|1530|1531|1532|1533|1534|1535|1536 +6.000000000000,2048,128,12,1537|1538|1539|1540|1541|1542|1543|1544|1545|1546|1547|1548|1549|1550|1551|1552|1553|1554|1555|1556|1557|1558|1559|1560|1561|1562|1563|1564|1565|1566|1567|1568|1569|1570|1571|1572|1573|1574|1575|1576|1577|1578|1579|1580|1581|1582|1583|1584|1585|1586|1587|1588|1589|1590|1591|1592|1593|1594|1595|1596|1597|1598|1599|1600|1601|1602|1603|1604|1605|1606|1607|1608|1609|1610|1611|1612|1613|1614|1615|1616|1617|1618|1619|1620|1621|1622|1623|1624|1625|1626|1627|1628|1629|1630|1631|1632|1633|1634|1635|1636|1637|1638|1639|1640|1641|1642|1643|1644|1645|1646|1647|1648|1649|1650|1651|1652|1653|1654|1655|1656|1657|1658|1659|1660|1661|1662|1663|1664 +6.500000000000,2048,128,13,1665|1666|1667|1668|1669|1670|1671|1672|1673|1674|1675|1676|1677|1678|1679|1680|1681|1682|1683|1684|1685|1686|1687|1688|1689|1690|1691|1692|1693|1694|1695|1696|1697|1698|1699|1700|1701|1702|1703|1704|1705|1706|1707|1708|1709|1710|1711|1712|1713|1714|1715|1716|1717|1718|1719|1720|1721|1722|1723|1724|1725|1726|1727|1728|1729|1730|1731|1732|1733|1734|1735|1736|1737|1738|1739|1740|1741|1742|1743|1744|1745|1746|1747|1748|1749|1750|1751|1752|1753|1754|1755|1756|1757|1758|1759|1760|1761|1762|1763|1764|1765|1766|1767|1768|1769|1770|1771|1772|1773|1774|1775|1776|1777|1778|1779|1780|1781|1782|1783|1784|1785|1786|1787|1788|1789|1790|1791|1792 +7.000000000000,2048,128,14,1793|1794|1795|1796|1797|1798|1799|1800|1801|1802|1803|1804|1805|1806|1807|1808|1809|1810|1811|1812|1813|1814|1815|1816|1817|1818|1819|1820|1821|1822|1823|1824|1825|1826|1827|1828|1829|1830|1831|1832|1833|1834|1835|1836|1837|1838|1839|1840|1841|1842|1843|1844|1845|1846|1847|1848|1849|1850|1851|1852|1853|1854|1855|1856|1857|1858|1859|1860|1861|1862|1863|1864|1865|1866|1867|1868|1869|1870|1871|1872|1873|1874|1875|1876|1877|1878|1879|1880|1881|1882|1883|1884|1885|1886|1887|1888|1889|1890|1891|1892|1893|1894|1895|1896|1897|1898|1899|1900|1901|1902|1903|1904|1905|1906|1907|1908|1909|1910|1911|1912|1913|1914|1915|1916|1917|1918|1919|1920 +7.500000000000,2048,128,15,1921|1922|1923|1924|1925|1926|1927|1928|1929|1930|1931|1932|1933|1934|1935|1936|1937|1938|1939|1940|1941|1942|1943|1944|1945|1946|1947|1948|1949|1950|1951|1952|1953|1954|1955|1956|1957|1958|1959|1960|1961|1962|1963|1964|1965|1966|1967|1968|1969|1970|1971|1972|1973|1974|1975|1976|1977|1978|1979|1980|1981|1982|1983|1984|1985|1986|1987|1988|1989|1990|1991|1992|1993|1994|1995|1996|1997|1998|1999|2000|2001|2002|2003|2004|2005|2006|2007|2008|2009|2010|2011|2012|2013|2014|2015|2016|2017|2018|2019|2020|2021|2022|2023|2024|2025|2026|2027|2028|2029|2030|2031|2032|2033|2034|2035|2036|2037|2038|2039|2040|2041|2042|2043|2044|2045|2046|2047|2048 +8.000000000000,2048,128,16,2049|2050|2051|2052|2053|2054|2055|2056|2057|2058|2059|2060|2061|2062|2063|2064|2065|2066|2067|2068|2069|2070|2071|2072|2073|2074|2075|2076|2077|2078|2079|2080|2081|2082|2083|2084|2085|2086|2087|2088|2089|2090|2091|2092|2093|2094|2095|2096|2097|2098|2099|2100|2101|2102|2103|2104|2105|2106|2107|2108|2109|2110|2111|2112|2113|2114|2115|2116|2117|2118|2119|2120|2121|2122|2123|2124|2125|2126|2127|2128|2129|2130|2131|2132|2133|2134|2135|2136|2137|2138|2139|2140|2141|2142|2143|2144|2145|2146|2147|2148|2149|2150|2151|2152|2153|2154|2155|2156|2157|2158|2159|2160|2161|2162|2163|2164|2165|2166|2167|2168|2169|2170|2171|2172|2173|2174|2175|2176 +8.500000000000,2048,128,17,2177|2178|2179|2180|2181|2182|2183|2184|2185|2186|2187|2188|2189|2190|2191|2192|2193|2194|2195|2196|2197|2198|2199|2200|2201|2202|2203|2204|2205|2206|2207|2208|2209|2210|2211|2212|2213|2214|2215|2216|2217|2218|2219|2220|2221|2222|2223|2224|2225|2226|2227|2228|2229|2230|2231|2232|2233|2234|2235|2236|2237|2238|2239|2240|2241|2242|2243|2244|2245|2246|2247|2248|2249|2250|2251|2252|2253|2254|2255|2256|2257|2258|2259|2260|2261|2262|2263|2264|2265|2266|2267|2268|2269|2270|2271|2272|2273|2274|2275|2276|2277|2278|2279|2280|2281|2282|2283|2284|2285|2286|2287|2288|2289|2290|2291|2292|2293|2294|2295|2296|2297|2298|2299|2300|2301|2302|2303|2304 +9.000000000000,2048,128,18,2305|2306|2307|2308|2309|2310|2311|2312|2313|2314|2315|2316|2317|2318|2319|2320|2321|2322|2323|2324|2325|2326|2327|2328|2329|2330|2331|2332|2333|2334|2335|2336|2337|2338|2339|2340|2341|2342|2343|2344|2345|2346|2347|2348|2349|2350|2351|2352|2353|2354|2355|2356|2357|2358|2359|2360|2361|2362|2363|2364|2365|2366|2367|2368|2369|2370|2371|2372|2373|2374|2375|2376|2377|2378|2379|2380|2381|2382|2383|2384|2385|2386|2387|2388|2389|2390|2391|2392|2393|2394|2395|2396|2397|2398|2399|2400|2401|2402|2403|2404|2405|2406|2407|2408|2409|2410|2411|2412|2413|2414|2415|2416|2417|2418|2419|2420|2421|2422|2423|2424|2425|2426|2427|2428|2429|2430|2431|2432 +9.500000000000,2048,128,19,2433|2434|2435|2436|2437|2438|2439|2440|2441|2442|2443|2444|2445|2446|2447|2448|2449|2450|2451|2452|2453|2454|2455|2456|2457|2458|2459|2460|2461|2462|2463|2464|2465|2466|2467|2468|2469|2470|2471|2472|2473|2474|2475|2476|2477|2478|2479|2480|2481|2482|2483|2484|2485|2486|2487|2488|2489|2490|2491|2492|2493|2494|2495|2496|2497|2498|2499|2500|2501|2502|2503|2504|2505|2506|2507|2508|2509|2510|2511|2512|2513|2514|2515|2516|2517|2518|2519|2520|2521|2522|2523|2524|2525|2526|2527|2528|2529|2530|2531|2532|2533|2534|2535|2536|2537|2538|2539|2540|2541|2542|2543|2544|2545|2546|2547|2548|2549|2550|2551|2552|2553|2554|2555|2556|2557|2558|2559|2560 +10.000000000000,2048,128,20,2561|2562|2563|2564|2565|2566|2567|2568|2569|2570|2571|2572|2573|2574|2575|2576|2577|2578|2579|2580|2581|2582|2583|2584|2585|2586|2587|2588|2589|2590|2591|2592|2593|2594|2595|2596|2597|2598|2599|2600|2601|2602|2603|2604|2605|2606|2607|2608|2609|2610|2611|2612|2613|2614|2615|2616|2617|2618|2619|2620|2621|2622|2623|2624|2625|2626|2627|2628|2629|2630|2631|2632|2633|2634|2635|2636|2637|2638|2639|2640|2641|2642|2643|2644|2645|2646|2647|2648|2649|2650|2651|2652|2653|2654|2655|2656|2657|2658|2659|2660|2661|2662|2663|2664|2665|2666|2667|2668|2669|2670|2671|2672|2673|2674|2675|2676|2677|2678|2679|2680|2681|2682|2683|2684|2685|2686|2687|2688 +10.500000000000,2048,128,21,2689|2690|2691|2692|2693|2694|2695|2696|2697|2698|2699|2700|2701|2702|2703|2704|2705|2706|2707|2708|2709|2710|2711|2712|2713|2714|2715|2716|2717|2718|2719|2720|2721|2722|2723|2724|2725|2726|2727|2728|2729|2730|2731|2732|2733|2734|2735|2736|2737|2738|2739|2740|2741|2742|2743|2744|2745|2746|2747|2748|2749|2750|2751|2752|2753|2754|2755|2756|2757|2758|2759|2760|2761|2762|2763|2764|2765|2766|2767|2768|2769|2770|2771|2772|2773|2774|2775|2776|2777|2778|2779|2780|2781|2782|2783|2784|2785|2786|2787|2788|2789|2790|2791|2792|2793|2794|2795|2796|2797|2798|2799|2800|2801|2802|2803|2804|2805|2806|2807|2808|2809|2810|2811|2812|2813|2814|2815|2816 +11.000000000000,2048,128,22,2817|2818|2819|2820|2821|2822|2823|2824|2825|2826|2827|2828|2829|2830|2831|2832|2833|2834|2835|2836|2837|2838|2839|2840|2841|2842|2843|2844|2845|2846|2847|2848|2849|2850|2851|2852|2853|2854|2855|2856|2857|2858|2859|2860|2861|2862|2863|2864|2865|2866|2867|2868|2869|2870|2871|2872|2873|2874|2875|2876|2877|2878|2879|2880|2881|2882|2883|2884|2885|2886|2887|2888|2889|2890|2891|2892|2893|2894|2895|2896|2897|2898|2899|2900|2901|2902|2903|2904|2905|2906|2907|2908|2909|2910|2911|2912|2913|2914|2915|2916|2917|2918|2919|2920|2921|2922|2923|2924|2925|2926|2927|2928|2929|2930|2931|2932|2933|2934|2935|2936|2937|2938|2939|2940|2941|2942|2943|2944 +11.500000000000,2048,128,23,2945|2946|2947|2948|2949|2950|2951|2952|2953|2954|2955|2956|2957|2958|2959|2960|2961|2962|2963|2964|2965|2966|2967|2968|2969|2970|2971|2972|2973|2974|2975|2976|2977|2978|2979|2980|2981|2982|2983|2984|2985|2986|2987|2988|2989|2990|2991|2992|2993|2994|2995|2996|2997|2998|2999|3000|3001|3002|3003|3004|3005|3006|3007|3008|3009|3010|3011|3012|3013|3014|3015|3016|3017|3018|3019|3020|3021|3022|3023|3024|3025|3026|3027|3028|3029|3030|3031|3032|3033|3034|3035|3036|3037|3038|3039|3040|3041|3042|3043|3044|3045|3046|3047|3048|3049|3050|3051|3052|3053|3054|3055|3056|3057|3058|3059|3060|3061|3062|3063|3064|3065|3066|3067|3068|3069|3070|3071|3072 +12.000000000000,2048,128,24,3073|3074|3075|3076|3077|3078|3079|3080|3081|3082|3083|3084|3085|3086|3087|3088|3089|3090|3091|3092|3093|3094|3095|3096|3097|3098|3099|3100|3101|3102|3103|3104|3105|3106|3107|3108|3109|3110|3111|3112|3113|3114|3115|3116|3117|3118|3119|3120|3121|3122|3123|3124|3125|3126|3127|3128|3129|3130|3131|3132|3133|3134|3135|3136|3137|3138|3139|3140|3141|3142|3143|3144|3145|3146|3147|3148|3149|3150|3151|3152|3153|3154|3155|3156|3157|3158|3159|3160|3161|3162|3163|3164|3165|3166|3167|3168|3169|3170|3171|3172|3173|3174|3175|3176|3177|3178|3179|3180|3181|3182|3183|3184|3185|3186|3187|3188|3189|3190|3191|3192|3193|3194|3195|3196|3197|3198|3199|3200 +12.500000000000,2048,128,25,3201|3202|3203|3204|3205|3206|3207|3208|3209|3210|3211|3212|3213|3214|3215|3216|3217|3218|3219|3220|3221|3222|3223|3224|3225|3226|3227|3228|3229|3230|3231|3232|3233|3234|3235|3236|3237|3238|3239|3240|3241|3242|3243|3244|3245|3246|3247|3248|3249|3250|3251|3252|3253|3254|3255|3256|3257|3258|3259|3260|3261|3262|3263|3264|3265|3266|3267|3268|3269|3270|3271|3272|3273|3274|3275|3276|3277|3278|3279|3280|3281|3282|3283|3284|3285|3286|3287|3288|3289|3290|3291|3292|3293|3294|3295|3296|3297|3298|3299|3300|3301|3302|3303|3304|3305|3306|3307|3308|3309|3310|3311|3312|3313|3314|3315|3316|3317|3318|3319|3320|3321|3322|3323|3324|3325|3326|3327|3328 +13.000000000000,2048,128,26,3329|3330|3331|3332|3333|3334|3335|3336|3337|3338|3339|3340|3341|3342|3343|3344|3345|3346|3347|3348|3349|3350|3351|3352|3353|3354|3355|3356|3357|3358|3359|3360|3361|3362|3363|3364|3365|3366|3367|3368|3369|3370|3371|3372|3373|3374|3375|3376|3377|3378|3379|3380|3381|3382|3383|3384|3385|3386|3387|3388|3389|3390|3391|3392|3393|3394|3395|3396|3397|3398|3399|3400|3401|3402|3403|3404|3405|3406|3407|3408|3409|3410|3411|3412|3413|3414|3415|3416|3417|3418|3419|3420|3421|3422|3423|3424|3425|3426|3427|3428|3429|3430|3431|3432|3433|3434|3435|3436|3437|3438|3439|3440|3441|3442|3443|3444|3445|3446|3447|3448|3449|3450|3451|3452|3453|3454|3455|3456 +13.500000000000,2048,128,27,3457|3458|3459|3460|3461|3462|3463|3464|3465|3466|3467|3468|3469|3470|3471|3472|3473|3474|3475|3476|3477|3478|3479|3480|3481|3482|3483|3484|3485|3486|3487|3488|3489|3490|3491|3492|3493|3494|3495|3496|3497|3498|3499|3500|3501|3502|3503|3504|3505|3506|3507|3508|3509|3510|3511|3512|3513|3514|3515|3516|3517|3518|3519|3520|3521|3522|3523|3524|3525|3526|3527|3528|3529|3530|3531|3532|3533|3534|3535|3536|3537|3538|3539|3540|3541|3542|3543|3544|3545|3546|3547|3548|3549|3550|3551|3552|3553|3554|3555|3556|3557|3558|3559|3560|3561|3562|3563|3564|3565|3566|3567|3568|3569|3570|3571|3572|3573|3574|3575|3576|3577|3578|3579|3580|3581|3582|3583|3584 +14.000000000000,2048,128,28,3585|3586|3587|3588|3589|3590|3591|3592|3593|3594|3595|3596|3597|3598|3599|3600|3601|3602|3603|3604|3605|3606|3607|3608|3609|3610|3611|3612|3613|3614|3615|3616|3617|3618|3619|3620|3621|3622|3623|3624|3625|3626|3627|3628|3629|3630|3631|3632|3633|3634|3635|3636|3637|3638|3639|3640|3641|3642|3643|3644|3645|3646|3647|3648|3649|3650|3651|3652|3653|3654|3655|3656|3657|3658|3659|3660|3661|3662|3663|3664|3665|3666|3667|3668|3669|3670|3671|3672|3673|3674|3675|3676|3677|3678|3679|3680|3681|3682|3683|3684|3685|3686|3687|3688|3689|3690|3691|3692|3693|3694|3695|3696|3697|3698|3699|3700|3701|3702|3703|3704|3705|3706|3707|3708|3709|3710|3711|3712 +14.500000000000,2048,128,29,3713|3714|3715|3716|3717|3718|3719|3720|3721|3722|3723|3724|3725|3726|3727|3728|3729|3730|3731|3732|3733|3734|3735|3736|3737|3738|3739|3740|3741|3742|3743|3744|3745|3746|3747|3748|3749|3750|3751|3752|3753|3754|3755|3756|3757|3758|3759|3760|3761|3762|3763|3764|3765|3766|3767|3768|3769|3770|3771|3772|3773|3774|3775|3776|3777|3778|3779|3780|3781|3782|3783|3784|3785|3786|3787|3788|3789|3790|3791|3792|3793|3794|3795|3796|3797|3798|3799|3800|3801|3802|3803|3804|3805|3806|3807|3808|3809|3810|3811|3812|3813|3814|3815|3816|3817|3818|3819|3820|3821|3822|3823|3824|3825|3826|3827|3828|3829|3830|3831|3832|3833|3834|3835|3836|3837|3838|3839|3840 +15.000000000000,2048,128,30,3841|3842|3843|3844|3845|3846|3847|3848|3849|3850|3851|3852|3853|3854|3855|3856|3857|3858|3859|3860|3861|3862|3863|3864|3865|3866|3867|3868|3869|3870|3871|3872|3873|3874|3875|3876|3877|3878|3879|3880|3881|3882|3883|3884|3885|3886|3887|3888|3889|3890|3891|3892|3893|3894|3895|3896|3897|3898|3899|3900|3901|3902|3903|3904|3905|3906|3907|3908|3909|3910|3911|3912|3913|3914|3915|3916|3917|3918|3919|3920|3921|3922|3923|3924|3925|3926|3927|3928|3929|3930|3931|3932|3933|3934|3935|3936|3937|3938|3939|3940|3941|3942|3943|3944|3945|3946|3947|3948|3949|3950|3951|3952|3953|3954|3955|3956|3957|3958|3959|3960|3961|3962|3963|3964|3965|3966|3967|3968 +15.500000000000,2048,128,31,3969|3970|3971|3972|3973|3974|3975|3976|3977|3978|3979|3980|3981|3982|3983|3984|3985|3986|3987|3988|3989|3990|3991|3992|3993|3994|3995|3996|3997|3998|3999|4000|4001|4002|4003|4004|4005|4006|4007|4008|4009|4010|4011|4012|4013|4014|4015|4016|4017|4018|4019|4020|4021|4022|4023|4024|4025|4026|4027|4028|4029|4030|4031|4032|4033|4034|4035|4036|4037|4038|4039|4040|4041|4042|4043|4044|4045|4046|4047|4048|4049|4050|4051|4052|4053|4054|4055|4056|4057|4058|4059|4060|4061|4062|4063|4064|4065|4066|4067|4068|4069|4070|4071|4072|4073|4074|4075|4076|4077|4078|4079|4080|4081|4082|4083|4084|4085|4086|4087|4088|4089|4090|4091|4092|4093|4094|4095|4096 +16.000000000000,2048,128,32,4097|4098|4099|4100|4101|4102|4103|4104|4105|4106|4107|4108|4109|4110|4111|4112|4113|4114|4115|4116|4117|4118|4119|4120|4121|4122|4123|4124|4125|4126|4127|4128|4129|4130|4131|4132|4133|4134|4135|4136|4137|4138|4139|4140|4141|4142|4143|4144|4145|4146|4147|4148|4149|4150|4151|4152|4153|4154|4155|4156|4157|4158|4159|4160|4161|4162|4163|4164|4165|4166|4167|4168|4169|4170|4171|4172|4173|4174|4175|4176|4177|4178|4179|4180|4181|4182|4183|4184|4185|4186|4187|4188|4189|4190|4191|4192|4193|4194|4195|4196|4197|4198|4199|4200|4201|4202|4203|4204|4205|4206|4207|4208|4209|4210|4211|4212|4213|4214|4215|4216|4217|4218|4219|4220|4221|4222|4223|4224 +16.500000000000,2048,128,33,4225|4226|4227|4228|4229|4230|4231|4232|4233|4234|4235|4236|4237|4238|4239|4240|4241|4242|4243|4244|4245|4246|4247|4248|4249|4250|4251|4252|4253|4254|4255|4256|4257|4258|4259|4260|4261|4262|4263|4264|4265|4266|4267|4268|4269|4270|4271|4272|4273|4274|4275|4276|4277|4278|4279|4280|4281|4282|4283|4284|4285|4286|4287|4288|4289|4290|4291|4292|4293|4294|4295|4296|4297|4298|4299|4300|4301|4302|4303|4304|4305|4306|4307|4308|4309|4310|4311|4312|4313|4314|4315|4316|4317|4318|4319|4320|4321|4322|4323|4324|4325|4326|4327|4328|4329|4330|4331|4332|4333|4334|4335|4336|4337|4338|4339|4340|4341|4342|4343|4344|4345|4346|4347|4348|4349|4350|4351|4352 +17.000000000000,2048,128,34,4353|4354|4355|4356|4357|4358|4359|4360|4361|4362|4363|4364|4365|4366|4367|4368|4369|4370|4371|4372|4373|4374|4375|4376|4377|4378|4379|4380|4381|4382|4383|4384|4385|4386|4387|4388|4389|4390|4391|4392|4393|4394|4395|4396|4397|4398|4399|4400|4401|4402|4403|4404|4405|4406|4407|4408|4409|4410|4411|4412|4413|4414|4415|4416|4417|4418|4419|4420|4421|4422|4423|4424|4425|4426|4427|4428|4429|4430|4431|4432|4433|4434|4435|4436|4437|4438|4439|4440|4441|4442|4443|4444|4445|4446|4447|4448|4449|4450|4451|4452|4453|4454|4455|4456|4457|4458|4459|4460|4461|4462|4463|4464|4465|4466|4467|4468|4469|4470|4471|4472|4473|4474|4475|4476|4477|4478|4479|4480 +17.500000000000,2048,128,35,4481|4482|4483|4484|4485|4486|4487|4488|4489|4490|4491|4492|4493|4494|4495|4496|4497|4498|4499|4500|4501|4502|4503|4504|4505|4506|4507|4508|4509|4510|4511|4512|4513|4514|4515|4516|4517|4518|4519|4520|4521|4522|4523|4524|4525|4526|4527|4528|4529|4530|4531|4532|4533|4534|4535|4536|4537|4538|4539|4540|4541|4542|4543|4544|4545|4546|4547|4548|4549|4550|4551|4552|4553|4554|4555|4556|4557|4558|4559|4560|4561|4562|4563|4564|4565|4566|4567|4568|4569|4570|4571|4572|4573|4574|4575|4576|4577|4578|4579|4580|4581|4582|4583|4584|4585|4586|4587|4588|4589|4590|4591|4592|4593|4594|4595|4596|4597|4598|4599|4600|4601|4602|4603|4604|4605|4606|4607|4608 +18.000000000000,2048,128,36,4609|4610|4611|4612|4613|4614|4615|4616|4617|4618|4619|4620|4621|4622|4623|4624|4625|4626|4627|4628|4629|4630|4631|4632|4633|4634|4635|4636|4637|4638|4639|4640|4641|4642|4643|4644|4645|4646|4647|4648|4649|4650|4651|4652|4653|4654|4655|4656|4657|4658|4659|4660|4661|4662|4663|4664|4665|4666|4667|4668|4669|4670|4671|4672|4673|4674|4675|4676|4677|4678|4679|4680|4681|4682|4683|4684|4685|4686|4687|4688|4689|4690|4691|4692|4693|4694|4695|4696|4697|4698|4699|4700|4701|4702|4703|4704|4705|4706|4707|4708|4709|4710|4711|4712|4713|4714|4715|4716|4717|4718|4719|4720|4721|4722|4723|4724|4725|4726|4727|4728|4729|4730|4731|4732|4733|4734|4735|4736 +18.500000000000,2048,128,37,4737|4738|4739|4740|4741|4742|4743|4744|4745|4746|4747|4748|4749|4750|4751|4752|4753|4754|4755|4756|4757|4758|4759|4760|4761|4762|4763|4764|4765|4766|4767|4768|4769|4770|4771|4772|4773|4774|4775|4776|4777|4778|4779|4780|4781|4782|4783|4784|4785|4786|4787|4788|4789|4790|4791|4792|4793|4794|4795|4796|4797|4798|4799|4800|4801|4802|4803|4804|4805|4806|4807|4808|4809|4810|4811|4812|4813|4814|4815|4816|4817|4818|4819|4820|4821|4822|4823|4824|4825|4826|4827|4828|4829|4830|4831|4832|4833|4834|4835|4836|4837|4838|4839|4840|4841|4842|4843|4844|4845|4846|4847|4848|4849|4850|4851|4852|4853|4854|4855|4856|4857|4858|4859|4860|4861|4862|4863|4864 +19.000000000000,2048,128,38,4865|4866|4867|4868|4869|4870|4871|4872|4873|4874|4875|4876|4877|4878|4879|4880|4881|4882|4883|4884|4885|4886|4887|4888|4889|4890|4891|4892|4893|4894|4895|4896|4897|4898|4899|4900|4901|4902|4903|4904|4905|4906|4907|4908|4909|4910|4911|4912|4913|4914|4915|4916|4917|4918|4919|4920|4921|4922|4923|4924|4925|4926|4927|4928|4929|4930|4931|4932|4933|4934|4935|4936|4937|4938|4939|4940|4941|4942|4943|4944|4945|4946|4947|4948|4949|4950|4951|4952|4953|4954|4955|4956|4957|4958|4959|4960|4961|4962|4963|4964|4965|4966|4967|4968|4969|4970|4971|4972|4973|4974|4975|4976|4977|4978|4979|4980|4981|4982|4983|4984|4985|4986|4987|4988|4989|4990|4991|4992 +19.500000000000,2048,128,39,4993|4994|4995|4996|4997|4998|4999|5000|5001|5002|5003|5004|5005|5006|5007|5008|5009|5010|5011|5012|5013|5014|5015|5016|5017|5018|5019|5020|5021|5022|5023|5024|5025|5026|5027|5028|5029|5030|5031|5032|5033|5034|5035|5036|5037|5038|5039|5040|5041|5042|5043|5044|5045|5046|5047|5048|5049|5050|5051|5052|5053|5054|5055|5056|5057|5058|5059|5060|5061|5062|5063|5064|5065|5066|5067|5068|5069|5070|5071|5072|5073|5074|5075|5076|5077|5078|5079|5080|5081|5082|5083|5084|5085|5086|5087|5088|5089|5090|5091|5092|5093|5094|5095|5096|5097|5098|5099|5100|5101|5102|5103|5104|5105|5106|5107|5108|5109|5110|5111|5112|5113|5114|5115|5116|5117|5118|5119|5120 +20.000000000000,2048,128,40,5121|5122|5123|5124|5125|5126|5127|5128|5129|5130|5131|5132|5133|5134|5135|5136|5137|5138|5139|5140|5141|5142|5143|5144|5145|5146|5147|5148|5149|5150|5151|5152|5153|5154|5155|5156|5157|5158|5159|5160|5161|5162|5163|5164|5165|5166|5167|5168|5169|5170|5171|5172|5173|5174|5175|5176|5177|5178|5179|5180|5181|5182|5183|5184|5185|5186|5187|5188|5189|5190|5191|5192|5193|5194|5195|5196|5197|5198|5199|5200|5201|5202|5203|5204|5205|5206|5207|5208|5209|5210|5211|5212|5213|5214|5215|5216|5217|5218|5219|5220|5221|5222|5223|5224|5225|5226|5227|5228|5229|5230|5231|5232|5233|5234|5235|5236|5237|5238|5239|5240|5241|5242|5243|5244|5245|5246|5247|5248 +20.500000000000,2048,128,41,5249|5250|5251|5252|5253|5254|5255|5256|5257|5258|5259|5260|5261|5262|5263|5264|5265|5266|5267|5268|5269|5270|5271|5272|5273|5274|5275|5276|5277|5278|5279|5280|5281|5282|5283|5284|5285|5286|5287|5288|5289|5290|5291|5292|5293|5294|5295|5296|5297|5298|5299|5300|5301|5302|5303|5304|5305|5306|5307|5308|5309|5310|5311|5312|5313|5314|5315|5316|5317|5318|5319|5320|5321|5322|5323|5324|5325|5326|5327|5328|5329|5330|5331|5332|5333|5334|5335|5336|5337|5338|5339|5340|5341|5342|5343|5344|5345|5346|5347|5348|5349|5350|5351|5352|5353|5354|5355|5356|5357|5358|5359|5360|5361|5362|5363|5364|5365|5366|5367|5368|5369|5370|5371|5372|5373|5374|5375|5376 +21.000000000000,2048,128,42,5377|5378|5379|5380|5381|5382|5383|5384|5385|5386|5387|5388|5389|5390|5391|5392|5393|5394|5395|5396|5397|5398|5399|5400|5401|5402|5403|5404|5405|5406|5407|5408|5409|5410|5411|5412|5413|5414|5415|5416|5417|5418|5419|5420|5421|5422|5423|5424|5425|5426|5427|5428|5429|5430|5431|5432|5433|5434|5435|5436|5437|5438|5439|5440|5441|5442|5443|5444|5445|5446|5447|5448|5449|5450|5451|5452|5453|5454|5455|5456|5457|5458|5459|5460|5461|5462|5463|5464|5465|5466|5467|5468|5469|5470|5471|5472|5473|5474|5475|5476|5477|5478|5479|5480|5481|5482|5483|5484|5485|5486|5487|5488|5489|5490|5491|5492|5493|5494|5495|5496|5497|5498|5499|5500|5501|5502|5503|5504 +21.500000000000,2048,128,43,5505|5506|5507|5508|5509|5510|5511|5512|5513|5514|5515|5516|5517|5518|5519|5520|5521|5522|5523|5524|5525|5526|5527|5528|5529|5530|5531|5532|5533|5534|5535|5536|5537|5538|5539|5540|5541|5542|5543|5544|5545|5546|5547|5548|5549|5550|5551|5552|5553|5554|5555|5556|5557|5558|5559|5560|5561|5562|5563|5564|5565|5566|5567|5568|5569|5570|5571|5572|5573|5574|5575|5576|5577|5578|5579|5580|5581|5582|5583|5584|5585|5586|5587|5588|5589|5590|5591|5592|5593|5594|5595|5596|5597|5598|5599|5600|5601|5602|5603|5604|5605|5606|5607|5608|5609|5610|5611|5612|5613|5614|5615|5616|5617|5618|5619|5620|5621|5622|5623|5624|5625|5626|5627|5628|5629|5630|5631|5632 +22.000000000000,2048,128,44,5633|5634|5635|5636|5637|5638|5639|5640|5641|5642|5643|5644|5645|5646|5647|5648|5649|5650|5651|5652|5653|5654|5655|5656|5657|5658|5659|5660|5661|5662|5663|5664|5665|5666|5667|5668|5669|5670|5671|5672|5673|5674|5675|5676|5677|5678|5679|5680|5681|5682|5683|5684|5685|5686|5687|5688|5689|5690|5691|5692|5693|5694|5695|5696|5697|5698|5699|5700|5701|5702|5703|5704|5705|5706|5707|5708|5709|5710|5711|5712|5713|5714|5715|5716|5717|5718|5719|5720|5721|5722|5723|5724|5725|5726|5727|5728|5729|5730|5731|5732|5733|5734|5735|5736|5737|5738|5739|5740|5741|5742|5743|5744|5745|5746|5747|5748|5749|5750|5751|5752|5753|5754|5755|5756|5757|5758|5759|5760 +22.500000000000,2048,128,45,5761|5762|5763|5764|5765|5766|5767|5768|5769|5770|5771|5772|5773|5774|5775|5776|5777|5778|5779|5780|5781|5782|5783|5784|5785|5786|5787|5788|5789|5790|5791|5792|5793|5794|5795|5796|5797|5798|5799|5800|5801|5802|5803|5804|5805|5806|5807|5808|5809|5810|5811|5812|5813|5814|5815|5816|5817|5818|5819|5820|5821|5822|5823|5824|5825|5826|5827|5828|5829|5830|5831|5832|5833|5834|5835|5836|5837|5838|5839|5840|5841|5842|5843|5844|5845|5846|5847|5848|5849|5850|5851|5852|5853|5854|5855|5856|5857|5858|5859|5860|5861|5862|5863|5864|5865|5866|5867|5868|5869|5870|5871|5872|5873|5874|5875|5876|5877|5878|5879|5880|5881|5882|5883|5884|5885|5886|5887|5888 +23.000000000000,2048,128,46,5889|5890|5891|5892|5893|5894|5895|5896|5897|5898|5899|5900|5901|5902|5903|5904|5905|5906|5907|5908|5909|5910|5911|5912|5913|5914|5915|5916|5917|5918|5919|5920|5921|5922|5923|5924|5925|5926|5927|5928|5929|5930|5931|5932|5933|5934|5935|5936|5937|5938|5939|5940|5941|5942|5943|5944|5945|5946|5947|5948|5949|5950|5951|5952|5953|5954|5955|5956|5957|5958|5959|5960|5961|5962|5963|5964|5965|5966|5967|5968|5969|5970|5971|5972|5973|5974|5975|5976|5977|5978|5979|5980|5981|5982|5983|5984|5985|5986|5987|5988|5989|5990|5991|5992|5993|5994|5995|5996|5997|5998|5999|6000|6001|6002|6003|6004|6005|6006|6007|6008|6009|6010|6011|6012|6013|6014|6015|6016 +23.500000000000,2048,128,47,6017|6018|6019|6020|6021|6022|6023|6024|6025|6026|6027|6028|6029|6030|6031|6032|6033|6034|6035|6036|6037|6038|6039|6040|6041|6042|6043|6044|6045|6046|6047|6048|6049|6050|6051|6052|6053|6054|6055|6056|6057|6058|6059|6060|6061|6062|6063|6064|6065|6066|6067|6068|6069|6070|6071|6072|6073|6074|6075|6076|6077|6078|6079|6080|6081|6082|6083|6084|6085|6086|6087|6088|6089|6090|6091|6092|6093|6094|6095|6096|6097|6098|6099|6100|6101|6102|6103|6104|6105|6106|6107|6108|6109|6110|6111|6112|6113|6114|6115|6116|6117|6118|6119|6120|6121|6122|6123|6124|6125|6126|6127|6128|6129|6130|6131|6132|6133|6134|6135|6136|6137|6138|6139|6140|6141|6142|6143|6144 +24.000000000000,2048,128,48,6145|6146|6147|6148|6149|6150|6151|6152|6153|6154|6155|6156|6157|6158|6159|6160|6161|6162|6163|6164|6165|6166|6167|6168|6169|6170|6171|6172|6173|6174|6175|6176|6177|6178|6179|6180|6181|6182|6183|6184|6185|6186|6187|6188|6189|6190|6191|6192|6193|6194|6195|6196|6197|6198|6199|6200|6201|6202|6203|6204|6205|6206|6207|6208|6209|6210|6211|6212|6213|6214|6215|6216|6217|6218|6219|6220|6221|6222|6223|6224|6225|6226|6227|6228|6229|6230|6231|6232|6233|6234|6235|6236|6237|6238|6239|6240|6241|6242|6243|6244|6245|6246|6247|6248|6249|6250|6251|6252|6253|6254|6255|6256|6257|6258|6259|6260|6261|6262|6263|6264|6265|6266|6267|6268|6269|6270|6271|6272 +24.500000000000,2048,128,49,6273|6274|6275|6276|6277|6278|6279|6280|6281|6282|6283|6284|6285|6286|6287|6288|6289|6290|6291|6292|6293|6294|6295|6296|6297|6298|6299|6300|6301|6302|6303|6304|6305|6306|6307|6308|6309|6310|6311|6312|6313|6314|6315|6316|6317|6318|6319|6320|6321|6322|6323|6324|6325|6326|6327|6328|6329|6330|6331|6332|6333|6334|6335|6336|6337|6338|6339|6340|6341|6342|6343|6344|6345|6346|6347|6348|6349|6350|6351|6352|6353|6354|6355|6356|6357|6358|6359|6360|6361|6362|6363|6364|6365|6366|6367|6368|6369|6370|6371|6372|6373|6374|6375|6376|6377|6378|6379|6380|6381|6382|6383|6384|6385|6386|6387|6388|6389|6390|6391|6392|6393|6394|6395|6396|6397|6398|6399|6400 +25.000000000000,2048,128,50,6401|6402|6403|6404|6405|6406|6407|6408|6409|6410|6411|6412|6413|6414|6415|6416|6417|6418|6419|6420|6421|6422|6423|6424|6425|6426|6427|6428|6429|6430|6431|6432|6433|6434|6435|6436|6437|6438|6439|6440|6441|6442|6443|6444|6445|6446|6447|6448|6449|6450|6451|6452|6453|6454|6455|6456|6457|6458|6459|6460|6461|6462|6463|6464|6465|6466|6467|6468|6469|6470|6471|6472|6473|6474|6475|6476|6477|6478|6479|6480|6481|6482|6483|6484|6485|6486|6487|6488|6489|6490|6491|6492|6493|6494|6495|6496|6497|6498|6499|6500|6501|6502|6503|6504|6505|6506|6507|6508|6509|6510|6511|6512|6513|6514|6515|6516|6517|6518|6519|6520|6521|6522|6523|6524|6525|6526|6527|6528 +25.500000000000,2048,128,51,6529|6530|6531|6532|6533|6534|6535|6536|6537|6538|6539|6540|6541|6542|6543|6544|6545|6546|6547|6548|6549|6550|6551|6552|6553|6554|6555|6556|6557|6558|6559|6560|6561|6562|6563|6564|6565|6566|6567|6568|6569|6570|6571|6572|6573|6574|6575|6576|6577|6578|6579|6580|6581|6582|6583|6584|6585|6586|6587|6588|6589|6590|6591|6592|6593|6594|6595|6596|6597|6598|6599|6600|6601|6602|6603|6604|6605|6606|6607|6608|6609|6610|6611|6612|6613|6614|6615|6616|6617|6618|6619|6620|6621|6622|6623|6624|6625|6626|6627|6628|6629|6630|6631|6632|6633|6634|6635|6636|6637|6638|6639|6640|6641|6642|6643|6644|6645|6646|6647|6648|6649|6650|6651|6652|6653|6654|6655|6656 +26.000000000000,2048,128,52,6657|6658|6659|6660|6661|6662|6663|6664|6665|6666|6667|6668|6669|6670|6671|6672|6673|6674|6675|6676|6677|6678|6679|6680|6681|6682|6683|6684|6685|6686|6687|6688|6689|6690|6691|6692|6693|6694|6695|6696|6697|6698|6699|6700|6701|6702|6703|6704|6705|6706|6707|6708|6709|6710|6711|6712|6713|6714|6715|6716|6717|6718|6719|6720|6721|6722|6723|6724|6725|6726|6727|6728|6729|6730|6731|6732|6733|6734|6735|6736|6737|6738|6739|6740|6741|6742|6743|6744|6745|6746|6747|6748|6749|6750|6751|6752|6753|6754|6755|6756|6757|6758|6759|6760|6761|6762|6763|6764|6765|6766|6767|6768|6769|6770|6771|6772|6773|6774|6775|6776|6777|6778|6779|6780|6781|6782|6783|6784 +26.500000000000,2048,128,53,6785|6786|6787|6788|6789|6790|6791|6792|6793|6794|6795|6796|6797|6798|6799|6800|6801|6802|6803|6804|6805|6806|6807|6808|6809|6810|6811|6812|6813|6814|6815|6816|6817|6818|6819|6820|6821|6822|6823|6824|6825|6826|6827|6828|6829|6830|6831|6832|6833|6834|6835|6836|6837|6838|6839|6840|6841|6842|6843|6844|6845|6846|6847|6848|6849|6850|6851|6852|6853|6854|6855|6856|6857|6858|6859|6860|6861|6862|6863|6864|6865|6866|6867|6868|6869|6870|6871|6872|6873|6874|6875|6876|6877|6878|6879|6880|6881|6882|6883|6884|6885|6886|6887|6888|6889|6890|6891|6892|6893|6894|6895|6896|6897|6898|6899|6900|6901|6902|6903|6904|6905|6906|6907|6908|6909|6910|6911|6912 +27.000000000000,2048,128,54,6913|6914|6915|6916|6917|6918|6919|6920|6921|6922|6923|6924|6925|6926|6927|6928|6929|6930|6931|6932|6933|6934|6935|6936|6937|6938|6939|6940|6941|6942|6943|6944|6945|6946|6947|6948|6949|6950|6951|6952|6953|6954|6955|6956|6957|6958|6959|6960|6961|6962|6963|6964|6965|6966|6967|6968|6969|6970|6971|6972|6973|6974|6975|6976|6977|6978|6979|6980|6981|6982|6983|6984|6985|6986|6987|6988|6989|6990|6991|6992|6993|6994|6995|6996|6997|6998|6999|7000|7001|7002|7003|7004|7005|7006|7007|7008|7009|7010|7011|7012|7013|7014|7015|7016|7017|7018|7019|7020|7021|7022|7023|7024|7025|7026|7027|7028|7029|7030|7031|7032|7033|7034|7035|7036|7037|7038|7039|7040 +27.500000000000,2048,128,55,7041|7042|7043|7044|7045|7046|7047|7048|7049|7050|7051|7052|7053|7054|7055|7056|7057|7058|7059|7060|7061|7062|7063|7064|7065|7066|7067|7068|7069|7070|7071|7072|7073|7074|7075|7076|7077|7078|7079|7080|7081|7082|7083|7084|7085|7086|7087|7088|7089|7090|7091|7092|7093|7094|7095|7096|7097|7098|7099|7100|7101|7102|7103|7104|7105|7106|7107|7108|7109|7110|7111|7112|7113|7114|7115|7116|7117|7118|7119|7120|7121|7122|7123|7124|7125|7126|7127|7128|7129|7130|7131|7132|7133|7134|7135|7136|7137|7138|7139|7140|7141|7142|7143|7144|7145|7146|7147|7148|7149|7150|7151|7152|7153|7154|7155|7156|7157|7158|7159|7160|7161|7162|7163|7164|7165|7166|7167|7168 +28.000000000000,2048,128,56,7169|7170|7171|7172|7173|7174|7175|7176|7177|7178|7179|7180|7181|7182|7183|7184|7185|7186|7187|7188|7189|7190|7191|7192|7193|7194|7195|7196|7197|7198|7199|7200|7201|7202|7203|7204|7205|7206|7207|7208|7209|7210|7211|7212|7213|7214|7215|7216|7217|7218|7219|7220|7221|7222|7223|7224|7225|7226|7227|7228|7229|7230|7231|7232|7233|7234|7235|7236|7237|7238|7239|7240|7241|7242|7243|7244|7245|7246|7247|7248|7249|7250|7251|7252|7253|7254|7255|7256|7257|7258|7259|7260|7261|7262|7263|7264|7265|7266|7267|7268|7269|7270|7271|7272|7273|7274|7275|7276|7277|7278|7279|7280|7281|7282|7283|7284|7285|7286|7287|7288|7289|7290|7291|7292|7293|7294|7295|7296 +28.500000000000,2048,128,57,7297|7298|7299|7300|7301|7302|7303|7304|7305|7306|7307|7308|7309|7310|7311|7312|7313|7314|7315|7316|7317|7318|7319|7320|7321|7322|7323|7324|7325|7326|7327|7328|7329|7330|7331|7332|7333|7334|7335|7336|7337|7338|7339|7340|7341|7342|7343|7344|7345|7346|7347|7348|7349|7350|7351|7352|7353|7354|7355|7356|7357|7358|7359|7360|7361|7362|7363|7364|7365|7366|7367|7368|7369|7370|7371|7372|7373|7374|7375|7376|7377|7378|7379|7380|7381|7382|7383|7384|7385|7386|7387|7388|7389|7390|7391|7392|7393|7394|7395|7396|7397|7398|7399|7400|7401|7402|7403|7404|7405|7406|7407|7408|7409|7410|7411|7412|7413|7414|7415|7416|7417|7418|7419|7420|7421|7422|7423|7424 +29.000000000000,2048,128,58,7425|7426|7427|7428|7429|7430|7431|7432|7433|7434|7435|7436|7437|7438|7439|7440|7441|7442|7443|7444|7445|7446|7447|7448|7449|7450|7451|7452|7453|7454|7455|7456|7457|7458|7459|7460|7461|7462|7463|7464|7465|7466|7467|7468|7469|7470|7471|7472|7473|7474|7475|7476|7477|7478|7479|7480|7481|7482|7483|7484|7485|7486|7487|7488|7489|7490|7491|7492|7493|7494|7495|7496|7497|7498|7499|7500|7501|7502|7503|7504|7505|7506|7507|7508|7509|7510|7511|7512|7513|7514|7515|7516|7517|7518|7519|7520|7521|7522|7523|7524|7525|7526|7527|7528|7529|7530|7531|7532|7533|7534|7535|7536|7537|7538|7539|7540|7541|7542|7543|7544|7545|7546|7547|7548|7549|7550|7551|7552 +29.500000000000,2048,128,59,7553|7554|7555|7556|7557|7558|7559|7560|7561|7562|7563|7564|7565|7566|7567|7568|7569|7570|7571|7572|7573|7574|7575|7576|7577|7578|7579|7580|7581|7582|7583|7584|7585|7586|7587|7588|7589|7590|7591|7592|7593|7594|7595|7596|7597|7598|7599|7600|7601|7602|7603|7604|7605|7606|7607|7608|7609|7610|7611|7612|7613|7614|7615|7616|7617|7618|7619|7620|7621|7622|7623|7624|7625|7626|7627|7628|7629|7630|7631|7632|7633|7634|7635|7636|7637|7638|7639|7640|7641|7642|7643|7644|7645|7646|7647|7648|7649|7650|7651|7652|7653|7654|7655|7656|7657|7658|7659|7660|7661|7662|7663|7664|7665|7666|7667|7668|7669|7670|7671|7672|7673|7674|7675|7676|7677|7678|7679|7680 +30.000000000000,2048,128,60,7681|7682|7683|7684|7685|7686|7687|7688|7689|7690|7691|7692|7693|7694|7695|7696|7697|7698|7699|7700|7701|7702|7703|7704|7705|7706|7707|7708|7709|7710|7711|7712|7713|7714|7715|7716|7717|7718|7719|7720|7721|7722|7723|7724|7725|7726|7727|7728|7729|7730|7731|7732|7733|7734|7735|7736|7737|7738|7739|7740|7741|7742|7743|7744|7745|7746|7747|7748|7749|7750|7751|7752|7753|7754|7755|7756|7757|7758|7759|7760|7761|7762|7763|7764|7765|7766|7767|7768|7769|7770|7771|7772|7773|7774|7775|7776|7777|7778|7779|7780|7781|7782|7783|7784|7785|7786|7787|7788|7789|7790|7791|7792|7793|7794|7795|7796|7797|7798|7799|7800|7801|7802|7803|7804|7805|7806|7807|7808 +30.500000000000,2048,128,61,7809|7810|7811|7812|7813|7814|7815|7816|7817|7818|7819|7820|7821|7822|7823|7824|7825|7826|7827|7828|7829|7830|7831|7832|7833|7834|7835|7836|7837|7838|7839|7840|7841|7842|7843|7844|7845|7846|7847|7848|7849|7850|7851|7852|7853|7854|7855|7856|7857|7858|7859|7860|7861|7862|7863|7864|7865|7866|7867|7868|7869|7870|7871|7872|7873|7874|7875|7876|7877|7878|7879|7880|7881|7882|7883|7884|7885|7886|7887|7888|7889|7890|7891|7892|7893|7894|7895|7896|7897|7898|7899|7900|7901|7902|7903|7904|7905|7906|7907|7908|7909|7910|7911|7912|7913|7914|7915|7916|7917|7918|7919|7920|7921|7922|7923|7924|7925|7926|7927|7928|7929|7930|7931|7932|7933|7934|7935|7936 +31.000000000000,2048,128,62,7937|7938|7939|7940|7941|7942|7943|7944|7945|7946|7947|7948|7949|7950|7951|7952|7953|7954|7955|7956|7957|7958|7959|7960|7961|7962|7963|7964|7965|7966|7967|7968|7969|7970|7971|7972|7973|7974|7975|7976|7977|7978|7979|7980|7981|7982|7983|7984|7985|7986|7987|7988|7989|7990|7991|7992|7993|7994|7995|7996|7997|7998|7999|8000|8001|8002|8003|8004|8005|8006|8007|8008|8009|8010|8011|8012|8013|8014|8015|8016|8017|8018|8019|8020|8021|8022|8023|8024|8025|8026|8027|8028|8029|8030|8031|8032|8033|8034|8035|8036|8037|8038|8039|8040|8041|8042|8043|8044|8045|8046|8047|8048|8049|8050|8051|8052|8053|8054|8055|8056|8057|8058|8059|8060|8061|8062|8063|8064 +31.500000000000,2048,128,63,8065|8066|8067|8068|8069|8070|8071|8072|8073|8074|8075|8076|8077|8078|8079|8080|8081|8082|8083|8084|8085|8086|8087|8088|8089|8090|8091|8092|8093|8094|8095|8096|8097|8098|8099|8100|8101|8102|8103|8104|8105|8106|8107|8108|8109|8110|8111|8112|8113|8114|8115|8116|8117|8118|8119|8120|8121|8122|8123|8124|8125|8126|8127|8128|8129|8130|8131|8132|8133|8134|8135|8136|8137|8138|8139|8140|8141|8142|8143|8144|8145|8146|8147|8148|8149|8150|8151|8152|8153|8154|8155|8156|8157|8158|8159|8160|8161|8162|8163|8164|8165|8166|8167|8168|8169|8170|8171|8172|8173|8174|8175|8176|8177|8178|8179|8180|8181|8182|8183|8184|8185|8186|8187|8188|8189|8190|8191|8192 diff --git a/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r4.csv b/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r4.csv new file mode 100644 index 0000000..859ffcd --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r4.csv @@ -0,0 +1,65 @@ +arrived_at,num_prefill_tokens,num_decode_tokens,session_id,block_hash_ids +0.000000000000,2048,128,0,1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58|59|60|61|62|63|64|65|66|67|68|69|70|71|72|73|74|75|76|77|78|79|80|81|82|83|84|85|86|87|88|89|90|91|92|93|94|95|96|97|98|99|100|101|102|103|104|105|106|107|108|109|110|111|112|113|114|115|116|117|118|119|120|121|122|123|124|125|126|127|128 +0.250000000000,2048,128,1,129|130|131|132|133|134|135|136|137|138|139|140|141|142|143|144|145|146|147|148|149|150|151|152|153|154|155|156|157|158|159|160|161|162|163|164|165|166|167|168|169|170|171|172|173|174|175|176|177|178|179|180|181|182|183|184|185|186|187|188|189|190|191|192|193|194|195|196|197|198|199|200|201|202|203|204|205|206|207|208|209|210|211|212|213|214|215|216|217|218|219|220|221|222|223|224|225|226|227|228|229|230|231|232|233|234|235|236|237|238|239|240|241|242|243|244|245|246|247|248|249|250|251|252|253|254|255|256 +0.500000000000,2048,128,2,257|258|259|260|261|262|263|264|265|266|267|268|269|270|271|272|273|274|275|276|277|278|279|280|281|282|283|284|285|286|287|288|289|290|291|292|293|294|295|296|297|298|299|300|301|302|303|304|305|306|307|308|309|310|311|312|313|314|315|316|317|318|319|320|321|322|323|324|325|326|327|328|329|330|331|332|333|334|335|336|337|338|339|340|341|342|343|344|345|346|347|348|349|350|351|352|353|354|355|356|357|358|359|360|361|362|363|364|365|366|367|368|369|370|371|372|373|374|375|376|377|378|379|380|381|382|383|384 +0.750000000000,2048,128,3,385|386|387|388|389|390|391|392|393|394|395|396|397|398|399|400|401|402|403|404|405|406|407|408|409|410|411|412|413|414|415|416|417|418|419|420|421|422|423|424|425|426|427|428|429|430|431|432|433|434|435|436|437|438|439|440|441|442|443|444|445|446|447|448|449|450|451|452|453|454|455|456|457|458|459|460|461|462|463|464|465|466|467|468|469|470|471|472|473|474|475|476|477|478|479|480|481|482|483|484|485|486|487|488|489|490|491|492|493|494|495|496|497|498|499|500|501|502|503|504|505|506|507|508|509|510|511|512 +1.000000000000,2048,128,4,513|514|515|516|517|518|519|520|521|522|523|524|525|526|527|528|529|530|531|532|533|534|535|536|537|538|539|540|541|542|543|544|545|546|547|548|549|550|551|552|553|554|555|556|557|558|559|560|561|562|563|564|565|566|567|568|569|570|571|572|573|574|575|576|577|578|579|580|581|582|583|584|585|586|587|588|589|590|591|592|593|594|595|596|597|598|599|600|601|602|603|604|605|606|607|608|609|610|611|612|613|614|615|616|617|618|619|620|621|622|623|624|625|626|627|628|629|630|631|632|633|634|635|636|637|638|639|640 +1.250000000000,2048,128,5,641|642|643|644|645|646|647|648|649|650|651|652|653|654|655|656|657|658|659|660|661|662|663|664|665|666|667|668|669|670|671|672|673|674|675|676|677|678|679|680|681|682|683|684|685|686|687|688|689|690|691|692|693|694|695|696|697|698|699|700|701|702|703|704|705|706|707|708|709|710|711|712|713|714|715|716|717|718|719|720|721|722|723|724|725|726|727|728|729|730|731|732|733|734|735|736|737|738|739|740|741|742|743|744|745|746|747|748|749|750|751|752|753|754|755|756|757|758|759|760|761|762|763|764|765|766|767|768 +1.500000000000,2048,128,6,769|770|771|772|773|774|775|776|777|778|779|780|781|782|783|784|785|786|787|788|789|790|791|792|793|794|795|796|797|798|799|800|801|802|803|804|805|806|807|808|809|810|811|812|813|814|815|816|817|818|819|820|821|822|823|824|825|826|827|828|829|830|831|832|833|834|835|836|837|838|839|840|841|842|843|844|845|846|847|848|849|850|851|852|853|854|855|856|857|858|859|860|861|862|863|864|865|866|867|868|869|870|871|872|873|874|875|876|877|878|879|880|881|882|883|884|885|886|887|888|889|890|891|892|893|894|895|896 +1.750000000000,2048,128,7,897|898|899|900|901|902|903|904|905|906|907|908|909|910|911|912|913|914|915|916|917|918|919|920|921|922|923|924|925|926|927|928|929|930|931|932|933|934|935|936|937|938|939|940|941|942|943|944|945|946|947|948|949|950|951|952|953|954|955|956|957|958|959|960|961|962|963|964|965|966|967|968|969|970|971|972|973|974|975|976|977|978|979|980|981|982|983|984|985|986|987|988|989|990|991|992|993|994|995|996|997|998|999|1000|1001|1002|1003|1004|1005|1006|1007|1008|1009|1010|1011|1012|1013|1014|1015|1016|1017|1018|1019|1020|1021|1022|1023|1024 +2.000000000000,2048,128,8,1025|1026|1027|1028|1029|1030|1031|1032|1033|1034|1035|1036|1037|1038|1039|1040|1041|1042|1043|1044|1045|1046|1047|1048|1049|1050|1051|1052|1053|1054|1055|1056|1057|1058|1059|1060|1061|1062|1063|1064|1065|1066|1067|1068|1069|1070|1071|1072|1073|1074|1075|1076|1077|1078|1079|1080|1081|1082|1083|1084|1085|1086|1087|1088|1089|1090|1091|1092|1093|1094|1095|1096|1097|1098|1099|1100|1101|1102|1103|1104|1105|1106|1107|1108|1109|1110|1111|1112|1113|1114|1115|1116|1117|1118|1119|1120|1121|1122|1123|1124|1125|1126|1127|1128|1129|1130|1131|1132|1133|1134|1135|1136|1137|1138|1139|1140|1141|1142|1143|1144|1145|1146|1147|1148|1149|1150|1151|1152 +2.250000000000,2048,128,9,1153|1154|1155|1156|1157|1158|1159|1160|1161|1162|1163|1164|1165|1166|1167|1168|1169|1170|1171|1172|1173|1174|1175|1176|1177|1178|1179|1180|1181|1182|1183|1184|1185|1186|1187|1188|1189|1190|1191|1192|1193|1194|1195|1196|1197|1198|1199|1200|1201|1202|1203|1204|1205|1206|1207|1208|1209|1210|1211|1212|1213|1214|1215|1216|1217|1218|1219|1220|1221|1222|1223|1224|1225|1226|1227|1228|1229|1230|1231|1232|1233|1234|1235|1236|1237|1238|1239|1240|1241|1242|1243|1244|1245|1246|1247|1248|1249|1250|1251|1252|1253|1254|1255|1256|1257|1258|1259|1260|1261|1262|1263|1264|1265|1266|1267|1268|1269|1270|1271|1272|1273|1274|1275|1276|1277|1278|1279|1280 +2.500000000000,2048,128,10,1281|1282|1283|1284|1285|1286|1287|1288|1289|1290|1291|1292|1293|1294|1295|1296|1297|1298|1299|1300|1301|1302|1303|1304|1305|1306|1307|1308|1309|1310|1311|1312|1313|1314|1315|1316|1317|1318|1319|1320|1321|1322|1323|1324|1325|1326|1327|1328|1329|1330|1331|1332|1333|1334|1335|1336|1337|1338|1339|1340|1341|1342|1343|1344|1345|1346|1347|1348|1349|1350|1351|1352|1353|1354|1355|1356|1357|1358|1359|1360|1361|1362|1363|1364|1365|1366|1367|1368|1369|1370|1371|1372|1373|1374|1375|1376|1377|1378|1379|1380|1381|1382|1383|1384|1385|1386|1387|1388|1389|1390|1391|1392|1393|1394|1395|1396|1397|1398|1399|1400|1401|1402|1403|1404|1405|1406|1407|1408 +2.750000000000,2048,128,11,1409|1410|1411|1412|1413|1414|1415|1416|1417|1418|1419|1420|1421|1422|1423|1424|1425|1426|1427|1428|1429|1430|1431|1432|1433|1434|1435|1436|1437|1438|1439|1440|1441|1442|1443|1444|1445|1446|1447|1448|1449|1450|1451|1452|1453|1454|1455|1456|1457|1458|1459|1460|1461|1462|1463|1464|1465|1466|1467|1468|1469|1470|1471|1472|1473|1474|1475|1476|1477|1478|1479|1480|1481|1482|1483|1484|1485|1486|1487|1488|1489|1490|1491|1492|1493|1494|1495|1496|1497|1498|1499|1500|1501|1502|1503|1504|1505|1506|1507|1508|1509|1510|1511|1512|1513|1514|1515|1516|1517|1518|1519|1520|1521|1522|1523|1524|1525|1526|1527|1528|1529|1530|1531|1532|1533|1534|1535|1536 +3.000000000000,2048,128,12,1537|1538|1539|1540|1541|1542|1543|1544|1545|1546|1547|1548|1549|1550|1551|1552|1553|1554|1555|1556|1557|1558|1559|1560|1561|1562|1563|1564|1565|1566|1567|1568|1569|1570|1571|1572|1573|1574|1575|1576|1577|1578|1579|1580|1581|1582|1583|1584|1585|1586|1587|1588|1589|1590|1591|1592|1593|1594|1595|1596|1597|1598|1599|1600|1601|1602|1603|1604|1605|1606|1607|1608|1609|1610|1611|1612|1613|1614|1615|1616|1617|1618|1619|1620|1621|1622|1623|1624|1625|1626|1627|1628|1629|1630|1631|1632|1633|1634|1635|1636|1637|1638|1639|1640|1641|1642|1643|1644|1645|1646|1647|1648|1649|1650|1651|1652|1653|1654|1655|1656|1657|1658|1659|1660|1661|1662|1663|1664 +3.250000000000,2048,128,13,1665|1666|1667|1668|1669|1670|1671|1672|1673|1674|1675|1676|1677|1678|1679|1680|1681|1682|1683|1684|1685|1686|1687|1688|1689|1690|1691|1692|1693|1694|1695|1696|1697|1698|1699|1700|1701|1702|1703|1704|1705|1706|1707|1708|1709|1710|1711|1712|1713|1714|1715|1716|1717|1718|1719|1720|1721|1722|1723|1724|1725|1726|1727|1728|1729|1730|1731|1732|1733|1734|1735|1736|1737|1738|1739|1740|1741|1742|1743|1744|1745|1746|1747|1748|1749|1750|1751|1752|1753|1754|1755|1756|1757|1758|1759|1760|1761|1762|1763|1764|1765|1766|1767|1768|1769|1770|1771|1772|1773|1774|1775|1776|1777|1778|1779|1780|1781|1782|1783|1784|1785|1786|1787|1788|1789|1790|1791|1792 +3.500000000000,2048,128,14,1793|1794|1795|1796|1797|1798|1799|1800|1801|1802|1803|1804|1805|1806|1807|1808|1809|1810|1811|1812|1813|1814|1815|1816|1817|1818|1819|1820|1821|1822|1823|1824|1825|1826|1827|1828|1829|1830|1831|1832|1833|1834|1835|1836|1837|1838|1839|1840|1841|1842|1843|1844|1845|1846|1847|1848|1849|1850|1851|1852|1853|1854|1855|1856|1857|1858|1859|1860|1861|1862|1863|1864|1865|1866|1867|1868|1869|1870|1871|1872|1873|1874|1875|1876|1877|1878|1879|1880|1881|1882|1883|1884|1885|1886|1887|1888|1889|1890|1891|1892|1893|1894|1895|1896|1897|1898|1899|1900|1901|1902|1903|1904|1905|1906|1907|1908|1909|1910|1911|1912|1913|1914|1915|1916|1917|1918|1919|1920 +3.750000000000,2048,128,15,1921|1922|1923|1924|1925|1926|1927|1928|1929|1930|1931|1932|1933|1934|1935|1936|1937|1938|1939|1940|1941|1942|1943|1944|1945|1946|1947|1948|1949|1950|1951|1952|1953|1954|1955|1956|1957|1958|1959|1960|1961|1962|1963|1964|1965|1966|1967|1968|1969|1970|1971|1972|1973|1974|1975|1976|1977|1978|1979|1980|1981|1982|1983|1984|1985|1986|1987|1988|1989|1990|1991|1992|1993|1994|1995|1996|1997|1998|1999|2000|2001|2002|2003|2004|2005|2006|2007|2008|2009|2010|2011|2012|2013|2014|2015|2016|2017|2018|2019|2020|2021|2022|2023|2024|2025|2026|2027|2028|2029|2030|2031|2032|2033|2034|2035|2036|2037|2038|2039|2040|2041|2042|2043|2044|2045|2046|2047|2048 +4.000000000000,2048,128,16,2049|2050|2051|2052|2053|2054|2055|2056|2057|2058|2059|2060|2061|2062|2063|2064|2065|2066|2067|2068|2069|2070|2071|2072|2073|2074|2075|2076|2077|2078|2079|2080|2081|2082|2083|2084|2085|2086|2087|2088|2089|2090|2091|2092|2093|2094|2095|2096|2097|2098|2099|2100|2101|2102|2103|2104|2105|2106|2107|2108|2109|2110|2111|2112|2113|2114|2115|2116|2117|2118|2119|2120|2121|2122|2123|2124|2125|2126|2127|2128|2129|2130|2131|2132|2133|2134|2135|2136|2137|2138|2139|2140|2141|2142|2143|2144|2145|2146|2147|2148|2149|2150|2151|2152|2153|2154|2155|2156|2157|2158|2159|2160|2161|2162|2163|2164|2165|2166|2167|2168|2169|2170|2171|2172|2173|2174|2175|2176 +4.250000000000,2048,128,17,2177|2178|2179|2180|2181|2182|2183|2184|2185|2186|2187|2188|2189|2190|2191|2192|2193|2194|2195|2196|2197|2198|2199|2200|2201|2202|2203|2204|2205|2206|2207|2208|2209|2210|2211|2212|2213|2214|2215|2216|2217|2218|2219|2220|2221|2222|2223|2224|2225|2226|2227|2228|2229|2230|2231|2232|2233|2234|2235|2236|2237|2238|2239|2240|2241|2242|2243|2244|2245|2246|2247|2248|2249|2250|2251|2252|2253|2254|2255|2256|2257|2258|2259|2260|2261|2262|2263|2264|2265|2266|2267|2268|2269|2270|2271|2272|2273|2274|2275|2276|2277|2278|2279|2280|2281|2282|2283|2284|2285|2286|2287|2288|2289|2290|2291|2292|2293|2294|2295|2296|2297|2298|2299|2300|2301|2302|2303|2304 +4.500000000000,2048,128,18,2305|2306|2307|2308|2309|2310|2311|2312|2313|2314|2315|2316|2317|2318|2319|2320|2321|2322|2323|2324|2325|2326|2327|2328|2329|2330|2331|2332|2333|2334|2335|2336|2337|2338|2339|2340|2341|2342|2343|2344|2345|2346|2347|2348|2349|2350|2351|2352|2353|2354|2355|2356|2357|2358|2359|2360|2361|2362|2363|2364|2365|2366|2367|2368|2369|2370|2371|2372|2373|2374|2375|2376|2377|2378|2379|2380|2381|2382|2383|2384|2385|2386|2387|2388|2389|2390|2391|2392|2393|2394|2395|2396|2397|2398|2399|2400|2401|2402|2403|2404|2405|2406|2407|2408|2409|2410|2411|2412|2413|2414|2415|2416|2417|2418|2419|2420|2421|2422|2423|2424|2425|2426|2427|2428|2429|2430|2431|2432 +4.750000000000,2048,128,19,2433|2434|2435|2436|2437|2438|2439|2440|2441|2442|2443|2444|2445|2446|2447|2448|2449|2450|2451|2452|2453|2454|2455|2456|2457|2458|2459|2460|2461|2462|2463|2464|2465|2466|2467|2468|2469|2470|2471|2472|2473|2474|2475|2476|2477|2478|2479|2480|2481|2482|2483|2484|2485|2486|2487|2488|2489|2490|2491|2492|2493|2494|2495|2496|2497|2498|2499|2500|2501|2502|2503|2504|2505|2506|2507|2508|2509|2510|2511|2512|2513|2514|2515|2516|2517|2518|2519|2520|2521|2522|2523|2524|2525|2526|2527|2528|2529|2530|2531|2532|2533|2534|2535|2536|2537|2538|2539|2540|2541|2542|2543|2544|2545|2546|2547|2548|2549|2550|2551|2552|2553|2554|2555|2556|2557|2558|2559|2560 +5.000000000000,2048,128,20,2561|2562|2563|2564|2565|2566|2567|2568|2569|2570|2571|2572|2573|2574|2575|2576|2577|2578|2579|2580|2581|2582|2583|2584|2585|2586|2587|2588|2589|2590|2591|2592|2593|2594|2595|2596|2597|2598|2599|2600|2601|2602|2603|2604|2605|2606|2607|2608|2609|2610|2611|2612|2613|2614|2615|2616|2617|2618|2619|2620|2621|2622|2623|2624|2625|2626|2627|2628|2629|2630|2631|2632|2633|2634|2635|2636|2637|2638|2639|2640|2641|2642|2643|2644|2645|2646|2647|2648|2649|2650|2651|2652|2653|2654|2655|2656|2657|2658|2659|2660|2661|2662|2663|2664|2665|2666|2667|2668|2669|2670|2671|2672|2673|2674|2675|2676|2677|2678|2679|2680|2681|2682|2683|2684|2685|2686|2687|2688 +5.250000000000,2048,128,21,2689|2690|2691|2692|2693|2694|2695|2696|2697|2698|2699|2700|2701|2702|2703|2704|2705|2706|2707|2708|2709|2710|2711|2712|2713|2714|2715|2716|2717|2718|2719|2720|2721|2722|2723|2724|2725|2726|2727|2728|2729|2730|2731|2732|2733|2734|2735|2736|2737|2738|2739|2740|2741|2742|2743|2744|2745|2746|2747|2748|2749|2750|2751|2752|2753|2754|2755|2756|2757|2758|2759|2760|2761|2762|2763|2764|2765|2766|2767|2768|2769|2770|2771|2772|2773|2774|2775|2776|2777|2778|2779|2780|2781|2782|2783|2784|2785|2786|2787|2788|2789|2790|2791|2792|2793|2794|2795|2796|2797|2798|2799|2800|2801|2802|2803|2804|2805|2806|2807|2808|2809|2810|2811|2812|2813|2814|2815|2816 +5.500000000000,2048,128,22,2817|2818|2819|2820|2821|2822|2823|2824|2825|2826|2827|2828|2829|2830|2831|2832|2833|2834|2835|2836|2837|2838|2839|2840|2841|2842|2843|2844|2845|2846|2847|2848|2849|2850|2851|2852|2853|2854|2855|2856|2857|2858|2859|2860|2861|2862|2863|2864|2865|2866|2867|2868|2869|2870|2871|2872|2873|2874|2875|2876|2877|2878|2879|2880|2881|2882|2883|2884|2885|2886|2887|2888|2889|2890|2891|2892|2893|2894|2895|2896|2897|2898|2899|2900|2901|2902|2903|2904|2905|2906|2907|2908|2909|2910|2911|2912|2913|2914|2915|2916|2917|2918|2919|2920|2921|2922|2923|2924|2925|2926|2927|2928|2929|2930|2931|2932|2933|2934|2935|2936|2937|2938|2939|2940|2941|2942|2943|2944 +5.750000000000,2048,128,23,2945|2946|2947|2948|2949|2950|2951|2952|2953|2954|2955|2956|2957|2958|2959|2960|2961|2962|2963|2964|2965|2966|2967|2968|2969|2970|2971|2972|2973|2974|2975|2976|2977|2978|2979|2980|2981|2982|2983|2984|2985|2986|2987|2988|2989|2990|2991|2992|2993|2994|2995|2996|2997|2998|2999|3000|3001|3002|3003|3004|3005|3006|3007|3008|3009|3010|3011|3012|3013|3014|3015|3016|3017|3018|3019|3020|3021|3022|3023|3024|3025|3026|3027|3028|3029|3030|3031|3032|3033|3034|3035|3036|3037|3038|3039|3040|3041|3042|3043|3044|3045|3046|3047|3048|3049|3050|3051|3052|3053|3054|3055|3056|3057|3058|3059|3060|3061|3062|3063|3064|3065|3066|3067|3068|3069|3070|3071|3072 +6.000000000000,2048,128,24,3073|3074|3075|3076|3077|3078|3079|3080|3081|3082|3083|3084|3085|3086|3087|3088|3089|3090|3091|3092|3093|3094|3095|3096|3097|3098|3099|3100|3101|3102|3103|3104|3105|3106|3107|3108|3109|3110|3111|3112|3113|3114|3115|3116|3117|3118|3119|3120|3121|3122|3123|3124|3125|3126|3127|3128|3129|3130|3131|3132|3133|3134|3135|3136|3137|3138|3139|3140|3141|3142|3143|3144|3145|3146|3147|3148|3149|3150|3151|3152|3153|3154|3155|3156|3157|3158|3159|3160|3161|3162|3163|3164|3165|3166|3167|3168|3169|3170|3171|3172|3173|3174|3175|3176|3177|3178|3179|3180|3181|3182|3183|3184|3185|3186|3187|3188|3189|3190|3191|3192|3193|3194|3195|3196|3197|3198|3199|3200 +6.250000000000,2048,128,25,3201|3202|3203|3204|3205|3206|3207|3208|3209|3210|3211|3212|3213|3214|3215|3216|3217|3218|3219|3220|3221|3222|3223|3224|3225|3226|3227|3228|3229|3230|3231|3232|3233|3234|3235|3236|3237|3238|3239|3240|3241|3242|3243|3244|3245|3246|3247|3248|3249|3250|3251|3252|3253|3254|3255|3256|3257|3258|3259|3260|3261|3262|3263|3264|3265|3266|3267|3268|3269|3270|3271|3272|3273|3274|3275|3276|3277|3278|3279|3280|3281|3282|3283|3284|3285|3286|3287|3288|3289|3290|3291|3292|3293|3294|3295|3296|3297|3298|3299|3300|3301|3302|3303|3304|3305|3306|3307|3308|3309|3310|3311|3312|3313|3314|3315|3316|3317|3318|3319|3320|3321|3322|3323|3324|3325|3326|3327|3328 +6.500000000000,2048,128,26,3329|3330|3331|3332|3333|3334|3335|3336|3337|3338|3339|3340|3341|3342|3343|3344|3345|3346|3347|3348|3349|3350|3351|3352|3353|3354|3355|3356|3357|3358|3359|3360|3361|3362|3363|3364|3365|3366|3367|3368|3369|3370|3371|3372|3373|3374|3375|3376|3377|3378|3379|3380|3381|3382|3383|3384|3385|3386|3387|3388|3389|3390|3391|3392|3393|3394|3395|3396|3397|3398|3399|3400|3401|3402|3403|3404|3405|3406|3407|3408|3409|3410|3411|3412|3413|3414|3415|3416|3417|3418|3419|3420|3421|3422|3423|3424|3425|3426|3427|3428|3429|3430|3431|3432|3433|3434|3435|3436|3437|3438|3439|3440|3441|3442|3443|3444|3445|3446|3447|3448|3449|3450|3451|3452|3453|3454|3455|3456 +6.750000000000,2048,128,27,3457|3458|3459|3460|3461|3462|3463|3464|3465|3466|3467|3468|3469|3470|3471|3472|3473|3474|3475|3476|3477|3478|3479|3480|3481|3482|3483|3484|3485|3486|3487|3488|3489|3490|3491|3492|3493|3494|3495|3496|3497|3498|3499|3500|3501|3502|3503|3504|3505|3506|3507|3508|3509|3510|3511|3512|3513|3514|3515|3516|3517|3518|3519|3520|3521|3522|3523|3524|3525|3526|3527|3528|3529|3530|3531|3532|3533|3534|3535|3536|3537|3538|3539|3540|3541|3542|3543|3544|3545|3546|3547|3548|3549|3550|3551|3552|3553|3554|3555|3556|3557|3558|3559|3560|3561|3562|3563|3564|3565|3566|3567|3568|3569|3570|3571|3572|3573|3574|3575|3576|3577|3578|3579|3580|3581|3582|3583|3584 +7.000000000000,2048,128,28,3585|3586|3587|3588|3589|3590|3591|3592|3593|3594|3595|3596|3597|3598|3599|3600|3601|3602|3603|3604|3605|3606|3607|3608|3609|3610|3611|3612|3613|3614|3615|3616|3617|3618|3619|3620|3621|3622|3623|3624|3625|3626|3627|3628|3629|3630|3631|3632|3633|3634|3635|3636|3637|3638|3639|3640|3641|3642|3643|3644|3645|3646|3647|3648|3649|3650|3651|3652|3653|3654|3655|3656|3657|3658|3659|3660|3661|3662|3663|3664|3665|3666|3667|3668|3669|3670|3671|3672|3673|3674|3675|3676|3677|3678|3679|3680|3681|3682|3683|3684|3685|3686|3687|3688|3689|3690|3691|3692|3693|3694|3695|3696|3697|3698|3699|3700|3701|3702|3703|3704|3705|3706|3707|3708|3709|3710|3711|3712 +7.250000000000,2048,128,29,3713|3714|3715|3716|3717|3718|3719|3720|3721|3722|3723|3724|3725|3726|3727|3728|3729|3730|3731|3732|3733|3734|3735|3736|3737|3738|3739|3740|3741|3742|3743|3744|3745|3746|3747|3748|3749|3750|3751|3752|3753|3754|3755|3756|3757|3758|3759|3760|3761|3762|3763|3764|3765|3766|3767|3768|3769|3770|3771|3772|3773|3774|3775|3776|3777|3778|3779|3780|3781|3782|3783|3784|3785|3786|3787|3788|3789|3790|3791|3792|3793|3794|3795|3796|3797|3798|3799|3800|3801|3802|3803|3804|3805|3806|3807|3808|3809|3810|3811|3812|3813|3814|3815|3816|3817|3818|3819|3820|3821|3822|3823|3824|3825|3826|3827|3828|3829|3830|3831|3832|3833|3834|3835|3836|3837|3838|3839|3840 +7.500000000000,2048,128,30,3841|3842|3843|3844|3845|3846|3847|3848|3849|3850|3851|3852|3853|3854|3855|3856|3857|3858|3859|3860|3861|3862|3863|3864|3865|3866|3867|3868|3869|3870|3871|3872|3873|3874|3875|3876|3877|3878|3879|3880|3881|3882|3883|3884|3885|3886|3887|3888|3889|3890|3891|3892|3893|3894|3895|3896|3897|3898|3899|3900|3901|3902|3903|3904|3905|3906|3907|3908|3909|3910|3911|3912|3913|3914|3915|3916|3917|3918|3919|3920|3921|3922|3923|3924|3925|3926|3927|3928|3929|3930|3931|3932|3933|3934|3935|3936|3937|3938|3939|3940|3941|3942|3943|3944|3945|3946|3947|3948|3949|3950|3951|3952|3953|3954|3955|3956|3957|3958|3959|3960|3961|3962|3963|3964|3965|3966|3967|3968 +7.750000000000,2048,128,31,3969|3970|3971|3972|3973|3974|3975|3976|3977|3978|3979|3980|3981|3982|3983|3984|3985|3986|3987|3988|3989|3990|3991|3992|3993|3994|3995|3996|3997|3998|3999|4000|4001|4002|4003|4004|4005|4006|4007|4008|4009|4010|4011|4012|4013|4014|4015|4016|4017|4018|4019|4020|4021|4022|4023|4024|4025|4026|4027|4028|4029|4030|4031|4032|4033|4034|4035|4036|4037|4038|4039|4040|4041|4042|4043|4044|4045|4046|4047|4048|4049|4050|4051|4052|4053|4054|4055|4056|4057|4058|4059|4060|4061|4062|4063|4064|4065|4066|4067|4068|4069|4070|4071|4072|4073|4074|4075|4076|4077|4078|4079|4080|4081|4082|4083|4084|4085|4086|4087|4088|4089|4090|4091|4092|4093|4094|4095|4096 +8.000000000000,2048,128,32,4097|4098|4099|4100|4101|4102|4103|4104|4105|4106|4107|4108|4109|4110|4111|4112|4113|4114|4115|4116|4117|4118|4119|4120|4121|4122|4123|4124|4125|4126|4127|4128|4129|4130|4131|4132|4133|4134|4135|4136|4137|4138|4139|4140|4141|4142|4143|4144|4145|4146|4147|4148|4149|4150|4151|4152|4153|4154|4155|4156|4157|4158|4159|4160|4161|4162|4163|4164|4165|4166|4167|4168|4169|4170|4171|4172|4173|4174|4175|4176|4177|4178|4179|4180|4181|4182|4183|4184|4185|4186|4187|4188|4189|4190|4191|4192|4193|4194|4195|4196|4197|4198|4199|4200|4201|4202|4203|4204|4205|4206|4207|4208|4209|4210|4211|4212|4213|4214|4215|4216|4217|4218|4219|4220|4221|4222|4223|4224 +8.250000000000,2048,128,33,4225|4226|4227|4228|4229|4230|4231|4232|4233|4234|4235|4236|4237|4238|4239|4240|4241|4242|4243|4244|4245|4246|4247|4248|4249|4250|4251|4252|4253|4254|4255|4256|4257|4258|4259|4260|4261|4262|4263|4264|4265|4266|4267|4268|4269|4270|4271|4272|4273|4274|4275|4276|4277|4278|4279|4280|4281|4282|4283|4284|4285|4286|4287|4288|4289|4290|4291|4292|4293|4294|4295|4296|4297|4298|4299|4300|4301|4302|4303|4304|4305|4306|4307|4308|4309|4310|4311|4312|4313|4314|4315|4316|4317|4318|4319|4320|4321|4322|4323|4324|4325|4326|4327|4328|4329|4330|4331|4332|4333|4334|4335|4336|4337|4338|4339|4340|4341|4342|4343|4344|4345|4346|4347|4348|4349|4350|4351|4352 +8.500000000000,2048,128,34,4353|4354|4355|4356|4357|4358|4359|4360|4361|4362|4363|4364|4365|4366|4367|4368|4369|4370|4371|4372|4373|4374|4375|4376|4377|4378|4379|4380|4381|4382|4383|4384|4385|4386|4387|4388|4389|4390|4391|4392|4393|4394|4395|4396|4397|4398|4399|4400|4401|4402|4403|4404|4405|4406|4407|4408|4409|4410|4411|4412|4413|4414|4415|4416|4417|4418|4419|4420|4421|4422|4423|4424|4425|4426|4427|4428|4429|4430|4431|4432|4433|4434|4435|4436|4437|4438|4439|4440|4441|4442|4443|4444|4445|4446|4447|4448|4449|4450|4451|4452|4453|4454|4455|4456|4457|4458|4459|4460|4461|4462|4463|4464|4465|4466|4467|4468|4469|4470|4471|4472|4473|4474|4475|4476|4477|4478|4479|4480 +8.750000000000,2048,128,35,4481|4482|4483|4484|4485|4486|4487|4488|4489|4490|4491|4492|4493|4494|4495|4496|4497|4498|4499|4500|4501|4502|4503|4504|4505|4506|4507|4508|4509|4510|4511|4512|4513|4514|4515|4516|4517|4518|4519|4520|4521|4522|4523|4524|4525|4526|4527|4528|4529|4530|4531|4532|4533|4534|4535|4536|4537|4538|4539|4540|4541|4542|4543|4544|4545|4546|4547|4548|4549|4550|4551|4552|4553|4554|4555|4556|4557|4558|4559|4560|4561|4562|4563|4564|4565|4566|4567|4568|4569|4570|4571|4572|4573|4574|4575|4576|4577|4578|4579|4580|4581|4582|4583|4584|4585|4586|4587|4588|4589|4590|4591|4592|4593|4594|4595|4596|4597|4598|4599|4600|4601|4602|4603|4604|4605|4606|4607|4608 +9.000000000000,2048,128,36,4609|4610|4611|4612|4613|4614|4615|4616|4617|4618|4619|4620|4621|4622|4623|4624|4625|4626|4627|4628|4629|4630|4631|4632|4633|4634|4635|4636|4637|4638|4639|4640|4641|4642|4643|4644|4645|4646|4647|4648|4649|4650|4651|4652|4653|4654|4655|4656|4657|4658|4659|4660|4661|4662|4663|4664|4665|4666|4667|4668|4669|4670|4671|4672|4673|4674|4675|4676|4677|4678|4679|4680|4681|4682|4683|4684|4685|4686|4687|4688|4689|4690|4691|4692|4693|4694|4695|4696|4697|4698|4699|4700|4701|4702|4703|4704|4705|4706|4707|4708|4709|4710|4711|4712|4713|4714|4715|4716|4717|4718|4719|4720|4721|4722|4723|4724|4725|4726|4727|4728|4729|4730|4731|4732|4733|4734|4735|4736 +9.250000000000,2048,128,37,4737|4738|4739|4740|4741|4742|4743|4744|4745|4746|4747|4748|4749|4750|4751|4752|4753|4754|4755|4756|4757|4758|4759|4760|4761|4762|4763|4764|4765|4766|4767|4768|4769|4770|4771|4772|4773|4774|4775|4776|4777|4778|4779|4780|4781|4782|4783|4784|4785|4786|4787|4788|4789|4790|4791|4792|4793|4794|4795|4796|4797|4798|4799|4800|4801|4802|4803|4804|4805|4806|4807|4808|4809|4810|4811|4812|4813|4814|4815|4816|4817|4818|4819|4820|4821|4822|4823|4824|4825|4826|4827|4828|4829|4830|4831|4832|4833|4834|4835|4836|4837|4838|4839|4840|4841|4842|4843|4844|4845|4846|4847|4848|4849|4850|4851|4852|4853|4854|4855|4856|4857|4858|4859|4860|4861|4862|4863|4864 +9.500000000000,2048,128,38,4865|4866|4867|4868|4869|4870|4871|4872|4873|4874|4875|4876|4877|4878|4879|4880|4881|4882|4883|4884|4885|4886|4887|4888|4889|4890|4891|4892|4893|4894|4895|4896|4897|4898|4899|4900|4901|4902|4903|4904|4905|4906|4907|4908|4909|4910|4911|4912|4913|4914|4915|4916|4917|4918|4919|4920|4921|4922|4923|4924|4925|4926|4927|4928|4929|4930|4931|4932|4933|4934|4935|4936|4937|4938|4939|4940|4941|4942|4943|4944|4945|4946|4947|4948|4949|4950|4951|4952|4953|4954|4955|4956|4957|4958|4959|4960|4961|4962|4963|4964|4965|4966|4967|4968|4969|4970|4971|4972|4973|4974|4975|4976|4977|4978|4979|4980|4981|4982|4983|4984|4985|4986|4987|4988|4989|4990|4991|4992 +9.750000000000,2048,128,39,4993|4994|4995|4996|4997|4998|4999|5000|5001|5002|5003|5004|5005|5006|5007|5008|5009|5010|5011|5012|5013|5014|5015|5016|5017|5018|5019|5020|5021|5022|5023|5024|5025|5026|5027|5028|5029|5030|5031|5032|5033|5034|5035|5036|5037|5038|5039|5040|5041|5042|5043|5044|5045|5046|5047|5048|5049|5050|5051|5052|5053|5054|5055|5056|5057|5058|5059|5060|5061|5062|5063|5064|5065|5066|5067|5068|5069|5070|5071|5072|5073|5074|5075|5076|5077|5078|5079|5080|5081|5082|5083|5084|5085|5086|5087|5088|5089|5090|5091|5092|5093|5094|5095|5096|5097|5098|5099|5100|5101|5102|5103|5104|5105|5106|5107|5108|5109|5110|5111|5112|5113|5114|5115|5116|5117|5118|5119|5120 +10.000000000000,2048,128,40,5121|5122|5123|5124|5125|5126|5127|5128|5129|5130|5131|5132|5133|5134|5135|5136|5137|5138|5139|5140|5141|5142|5143|5144|5145|5146|5147|5148|5149|5150|5151|5152|5153|5154|5155|5156|5157|5158|5159|5160|5161|5162|5163|5164|5165|5166|5167|5168|5169|5170|5171|5172|5173|5174|5175|5176|5177|5178|5179|5180|5181|5182|5183|5184|5185|5186|5187|5188|5189|5190|5191|5192|5193|5194|5195|5196|5197|5198|5199|5200|5201|5202|5203|5204|5205|5206|5207|5208|5209|5210|5211|5212|5213|5214|5215|5216|5217|5218|5219|5220|5221|5222|5223|5224|5225|5226|5227|5228|5229|5230|5231|5232|5233|5234|5235|5236|5237|5238|5239|5240|5241|5242|5243|5244|5245|5246|5247|5248 +10.250000000000,2048,128,41,5249|5250|5251|5252|5253|5254|5255|5256|5257|5258|5259|5260|5261|5262|5263|5264|5265|5266|5267|5268|5269|5270|5271|5272|5273|5274|5275|5276|5277|5278|5279|5280|5281|5282|5283|5284|5285|5286|5287|5288|5289|5290|5291|5292|5293|5294|5295|5296|5297|5298|5299|5300|5301|5302|5303|5304|5305|5306|5307|5308|5309|5310|5311|5312|5313|5314|5315|5316|5317|5318|5319|5320|5321|5322|5323|5324|5325|5326|5327|5328|5329|5330|5331|5332|5333|5334|5335|5336|5337|5338|5339|5340|5341|5342|5343|5344|5345|5346|5347|5348|5349|5350|5351|5352|5353|5354|5355|5356|5357|5358|5359|5360|5361|5362|5363|5364|5365|5366|5367|5368|5369|5370|5371|5372|5373|5374|5375|5376 +10.500000000000,2048,128,42,5377|5378|5379|5380|5381|5382|5383|5384|5385|5386|5387|5388|5389|5390|5391|5392|5393|5394|5395|5396|5397|5398|5399|5400|5401|5402|5403|5404|5405|5406|5407|5408|5409|5410|5411|5412|5413|5414|5415|5416|5417|5418|5419|5420|5421|5422|5423|5424|5425|5426|5427|5428|5429|5430|5431|5432|5433|5434|5435|5436|5437|5438|5439|5440|5441|5442|5443|5444|5445|5446|5447|5448|5449|5450|5451|5452|5453|5454|5455|5456|5457|5458|5459|5460|5461|5462|5463|5464|5465|5466|5467|5468|5469|5470|5471|5472|5473|5474|5475|5476|5477|5478|5479|5480|5481|5482|5483|5484|5485|5486|5487|5488|5489|5490|5491|5492|5493|5494|5495|5496|5497|5498|5499|5500|5501|5502|5503|5504 +10.750000000000,2048,128,43,5505|5506|5507|5508|5509|5510|5511|5512|5513|5514|5515|5516|5517|5518|5519|5520|5521|5522|5523|5524|5525|5526|5527|5528|5529|5530|5531|5532|5533|5534|5535|5536|5537|5538|5539|5540|5541|5542|5543|5544|5545|5546|5547|5548|5549|5550|5551|5552|5553|5554|5555|5556|5557|5558|5559|5560|5561|5562|5563|5564|5565|5566|5567|5568|5569|5570|5571|5572|5573|5574|5575|5576|5577|5578|5579|5580|5581|5582|5583|5584|5585|5586|5587|5588|5589|5590|5591|5592|5593|5594|5595|5596|5597|5598|5599|5600|5601|5602|5603|5604|5605|5606|5607|5608|5609|5610|5611|5612|5613|5614|5615|5616|5617|5618|5619|5620|5621|5622|5623|5624|5625|5626|5627|5628|5629|5630|5631|5632 +11.000000000000,2048,128,44,5633|5634|5635|5636|5637|5638|5639|5640|5641|5642|5643|5644|5645|5646|5647|5648|5649|5650|5651|5652|5653|5654|5655|5656|5657|5658|5659|5660|5661|5662|5663|5664|5665|5666|5667|5668|5669|5670|5671|5672|5673|5674|5675|5676|5677|5678|5679|5680|5681|5682|5683|5684|5685|5686|5687|5688|5689|5690|5691|5692|5693|5694|5695|5696|5697|5698|5699|5700|5701|5702|5703|5704|5705|5706|5707|5708|5709|5710|5711|5712|5713|5714|5715|5716|5717|5718|5719|5720|5721|5722|5723|5724|5725|5726|5727|5728|5729|5730|5731|5732|5733|5734|5735|5736|5737|5738|5739|5740|5741|5742|5743|5744|5745|5746|5747|5748|5749|5750|5751|5752|5753|5754|5755|5756|5757|5758|5759|5760 +11.250000000000,2048,128,45,5761|5762|5763|5764|5765|5766|5767|5768|5769|5770|5771|5772|5773|5774|5775|5776|5777|5778|5779|5780|5781|5782|5783|5784|5785|5786|5787|5788|5789|5790|5791|5792|5793|5794|5795|5796|5797|5798|5799|5800|5801|5802|5803|5804|5805|5806|5807|5808|5809|5810|5811|5812|5813|5814|5815|5816|5817|5818|5819|5820|5821|5822|5823|5824|5825|5826|5827|5828|5829|5830|5831|5832|5833|5834|5835|5836|5837|5838|5839|5840|5841|5842|5843|5844|5845|5846|5847|5848|5849|5850|5851|5852|5853|5854|5855|5856|5857|5858|5859|5860|5861|5862|5863|5864|5865|5866|5867|5868|5869|5870|5871|5872|5873|5874|5875|5876|5877|5878|5879|5880|5881|5882|5883|5884|5885|5886|5887|5888 +11.500000000000,2048,128,46,5889|5890|5891|5892|5893|5894|5895|5896|5897|5898|5899|5900|5901|5902|5903|5904|5905|5906|5907|5908|5909|5910|5911|5912|5913|5914|5915|5916|5917|5918|5919|5920|5921|5922|5923|5924|5925|5926|5927|5928|5929|5930|5931|5932|5933|5934|5935|5936|5937|5938|5939|5940|5941|5942|5943|5944|5945|5946|5947|5948|5949|5950|5951|5952|5953|5954|5955|5956|5957|5958|5959|5960|5961|5962|5963|5964|5965|5966|5967|5968|5969|5970|5971|5972|5973|5974|5975|5976|5977|5978|5979|5980|5981|5982|5983|5984|5985|5986|5987|5988|5989|5990|5991|5992|5993|5994|5995|5996|5997|5998|5999|6000|6001|6002|6003|6004|6005|6006|6007|6008|6009|6010|6011|6012|6013|6014|6015|6016 +11.750000000000,2048,128,47,6017|6018|6019|6020|6021|6022|6023|6024|6025|6026|6027|6028|6029|6030|6031|6032|6033|6034|6035|6036|6037|6038|6039|6040|6041|6042|6043|6044|6045|6046|6047|6048|6049|6050|6051|6052|6053|6054|6055|6056|6057|6058|6059|6060|6061|6062|6063|6064|6065|6066|6067|6068|6069|6070|6071|6072|6073|6074|6075|6076|6077|6078|6079|6080|6081|6082|6083|6084|6085|6086|6087|6088|6089|6090|6091|6092|6093|6094|6095|6096|6097|6098|6099|6100|6101|6102|6103|6104|6105|6106|6107|6108|6109|6110|6111|6112|6113|6114|6115|6116|6117|6118|6119|6120|6121|6122|6123|6124|6125|6126|6127|6128|6129|6130|6131|6132|6133|6134|6135|6136|6137|6138|6139|6140|6141|6142|6143|6144 +12.000000000000,2048,128,48,6145|6146|6147|6148|6149|6150|6151|6152|6153|6154|6155|6156|6157|6158|6159|6160|6161|6162|6163|6164|6165|6166|6167|6168|6169|6170|6171|6172|6173|6174|6175|6176|6177|6178|6179|6180|6181|6182|6183|6184|6185|6186|6187|6188|6189|6190|6191|6192|6193|6194|6195|6196|6197|6198|6199|6200|6201|6202|6203|6204|6205|6206|6207|6208|6209|6210|6211|6212|6213|6214|6215|6216|6217|6218|6219|6220|6221|6222|6223|6224|6225|6226|6227|6228|6229|6230|6231|6232|6233|6234|6235|6236|6237|6238|6239|6240|6241|6242|6243|6244|6245|6246|6247|6248|6249|6250|6251|6252|6253|6254|6255|6256|6257|6258|6259|6260|6261|6262|6263|6264|6265|6266|6267|6268|6269|6270|6271|6272 +12.250000000000,2048,128,49,6273|6274|6275|6276|6277|6278|6279|6280|6281|6282|6283|6284|6285|6286|6287|6288|6289|6290|6291|6292|6293|6294|6295|6296|6297|6298|6299|6300|6301|6302|6303|6304|6305|6306|6307|6308|6309|6310|6311|6312|6313|6314|6315|6316|6317|6318|6319|6320|6321|6322|6323|6324|6325|6326|6327|6328|6329|6330|6331|6332|6333|6334|6335|6336|6337|6338|6339|6340|6341|6342|6343|6344|6345|6346|6347|6348|6349|6350|6351|6352|6353|6354|6355|6356|6357|6358|6359|6360|6361|6362|6363|6364|6365|6366|6367|6368|6369|6370|6371|6372|6373|6374|6375|6376|6377|6378|6379|6380|6381|6382|6383|6384|6385|6386|6387|6388|6389|6390|6391|6392|6393|6394|6395|6396|6397|6398|6399|6400 +12.500000000000,2048,128,50,6401|6402|6403|6404|6405|6406|6407|6408|6409|6410|6411|6412|6413|6414|6415|6416|6417|6418|6419|6420|6421|6422|6423|6424|6425|6426|6427|6428|6429|6430|6431|6432|6433|6434|6435|6436|6437|6438|6439|6440|6441|6442|6443|6444|6445|6446|6447|6448|6449|6450|6451|6452|6453|6454|6455|6456|6457|6458|6459|6460|6461|6462|6463|6464|6465|6466|6467|6468|6469|6470|6471|6472|6473|6474|6475|6476|6477|6478|6479|6480|6481|6482|6483|6484|6485|6486|6487|6488|6489|6490|6491|6492|6493|6494|6495|6496|6497|6498|6499|6500|6501|6502|6503|6504|6505|6506|6507|6508|6509|6510|6511|6512|6513|6514|6515|6516|6517|6518|6519|6520|6521|6522|6523|6524|6525|6526|6527|6528 +12.750000000000,2048,128,51,6529|6530|6531|6532|6533|6534|6535|6536|6537|6538|6539|6540|6541|6542|6543|6544|6545|6546|6547|6548|6549|6550|6551|6552|6553|6554|6555|6556|6557|6558|6559|6560|6561|6562|6563|6564|6565|6566|6567|6568|6569|6570|6571|6572|6573|6574|6575|6576|6577|6578|6579|6580|6581|6582|6583|6584|6585|6586|6587|6588|6589|6590|6591|6592|6593|6594|6595|6596|6597|6598|6599|6600|6601|6602|6603|6604|6605|6606|6607|6608|6609|6610|6611|6612|6613|6614|6615|6616|6617|6618|6619|6620|6621|6622|6623|6624|6625|6626|6627|6628|6629|6630|6631|6632|6633|6634|6635|6636|6637|6638|6639|6640|6641|6642|6643|6644|6645|6646|6647|6648|6649|6650|6651|6652|6653|6654|6655|6656 +13.000000000000,2048,128,52,6657|6658|6659|6660|6661|6662|6663|6664|6665|6666|6667|6668|6669|6670|6671|6672|6673|6674|6675|6676|6677|6678|6679|6680|6681|6682|6683|6684|6685|6686|6687|6688|6689|6690|6691|6692|6693|6694|6695|6696|6697|6698|6699|6700|6701|6702|6703|6704|6705|6706|6707|6708|6709|6710|6711|6712|6713|6714|6715|6716|6717|6718|6719|6720|6721|6722|6723|6724|6725|6726|6727|6728|6729|6730|6731|6732|6733|6734|6735|6736|6737|6738|6739|6740|6741|6742|6743|6744|6745|6746|6747|6748|6749|6750|6751|6752|6753|6754|6755|6756|6757|6758|6759|6760|6761|6762|6763|6764|6765|6766|6767|6768|6769|6770|6771|6772|6773|6774|6775|6776|6777|6778|6779|6780|6781|6782|6783|6784 +13.250000000000,2048,128,53,6785|6786|6787|6788|6789|6790|6791|6792|6793|6794|6795|6796|6797|6798|6799|6800|6801|6802|6803|6804|6805|6806|6807|6808|6809|6810|6811|6812|6813|6814|6815|6816|6817|6818|6819|6820|6821|6822|6823|6824|6825|6826|6827|6828|6829|6830|6831|6832|6833|6834|6835|6836|6837|6838|6839|6840|6841|6842|6843|6844|6845|6846|6847|6848|6849|6850|6851|6852|6853|6854|6855|6856|6857|6858|6859|6860|6861|6862|6863|6864|6865|6866|6867|6868|6869|6870|6871|6872|6873|6874|6875|6876|6877|6878|6879|6880|6881|6882|6883|6884|6885|6886|6887|6888|6889|6890|6891|6892|6893|6894|6895|6896|6897|6898|6899|6900|6901|6902|6903|6904|6905|6906|6907|6908|6909|6910|6911|6912 +13.500000000000,2048,128,54,6913|6914|6915|6916|6917|6918|6919|6920|6921|6922|6923|6924|6925|6926|6927|6928|6929|6930|6931|6932|6933|6934|6935|6936|6937|6938|6939|6940|6941|6942|6943|6944|6945|6946|6947|6948|6949|6950|6951|6952|6953|6954|6955|6956|6957|6958|6959|6960|6961|6962|6963|6964|6965|6966|6967|6968|6969|6970|6971|6972|6973|6974|6975|6976|6977|6978|6979|6980|6981|6982|6983|6984|6985|6986|6987|6988|6989|6990|6991|6992|6993|6994|6995|6996|6997|6998|6999|7000|7001|7002|7003|7004|7005|7006|7007|7008|7009|7010|7011|7012|7013|7014|7015|7016|7017|7018|7019|7020|7021|7022|7023|7024|7025|7026|7027|7028|7029|7030|7031|7032|7033|7034|7035|7036|7037|7038|7039|7040 +13.750000000000,2048,128,55,7041|7042|7043|7044|7045|7046|7047|7048|7049|7050|7051|7052|7053|7054|7055|7056|7057|7058|7059|7060|7061|7062|7063|7064|7065|7066|7067|7068|7069|7070|7071|7072|7073|7074|7075|7076|7077|7078|7079|7080|7081|7082|7083|7084|7085|7086|7087|7088|7089|7090|7091|7092|7093|7094|7095|7096|7097|7098|7099|7100|7101|7102|7103|7104|7105|7106|7107|7108|7109|7110|7111|7112|7113|7114|7115|7116|7117|7118|7119|7120|7121|7122|7123|7124|7125|7126|7127|7128|7129|7130|7131|7132|7133|7134|7135|7136|7137|7138|7139|7140|7141|7142|7143|7144|7145|7146|7147|7148|7149|7150|7151|7152|7153|7154|7155|7156|7157|7158|7159|7160|7161|7162|7163|7164|7165|7166|7167|7168 +14.000000000000,2048,128,56,7169|7170|7171|7172|7173|7174|7175|7176|7177|7178|7179|7180|7181|7182|7183|7184|7185|7186|7187|7188|7189|7190|7191|7192|7193|7194|7195|7196|7197|7198|7199|7200|7201|7202|7203|7204|7205|7206|7207|7208|7209|7210|7211|7212|7213|7214|7215|7216|7217|7218|7219|7220|7221|7222|7223|7224|7225|7226|7227|7228|7229|7230|7231|7232|7233|7234|7235|7236|7237|7238|7239|7240|7241|7242|7243|7244|7245|7246|7247|7248|7249|7250|7251|7252|7253|7254|7255|7256|7257|7258|7259|7260|7261|7262|7263|7264|7265|7266|7267|7268|7269|7270|7271|7272|7273|7274|7275|7276|7277|7278|7279|7280|7281|7282|7283|7284|7285|7286|7287|7288|7289|7290|7291|7292|7293|7294|7295|7296 +14.250000000000,2048,128,57,7297|7298|7299|7300|7301|7302|7303|7304|7305|7306|7307|7308|7309|7310|7311|7312|7313|7314|7315|7316|7317|7318|7319|7320|7321|7322|7323|7324|7325|7326|7327|7328|7329|7330|7331|7332|7333|7334|7335|7336|7337|7338|7339|7340|7341|7342|7343|7344|7345|7346|7347|7348|7349|7350|7351|7352|7353|7354|7355|7356|7357|7358|7359|7360|7361|7362|7363|7364|7365|7366|7367|7368|7369|7370|7371|7372|7373|7374|7375|7376|7377|7378|7379|7380|7381|7382|7383|7384|7385|7386|7387|7388|7389|7390|7391|7392|7393|7394|7395|7396|7397|7398|7399|7400|7401|7402|7403|7404|7405|7406|7407|7408|7409|7410|7411|7412|7413|7414|7415|7416|7417|7418|7419|7420|7421|7422|7423|7424 +14.500000000000,2048,128,58,7425|7426|7427|7428|7429|7430|7431|7432|7433|7434|7435|7436|7437|7438|7439|7440|7441|7442|7443|7444|7445|7446|7447|7448|7449|7450|7451|7452|7453|7454|7455|7456|7457|7458|7459|7460|7461|7462|7463|7464|7465|7466|7467|7468|7469|7470|7471|7472|7473|7474|7475|7476|7477|7478|7479|7480|7481|7482|7483|7484|7485|7486|7487|7488|7489|7490|7491|7492|7493|7494|7495|7496|7497|7498|7499|7500|7501|7502|7503|7504|7505|7506|7507|7508|7509|7510|7511|7512|7513|7514|7515|7516|7517|7518|7519|7520|7521|7522|7523|7524|7525|7526|7527|7528|7529|7530|7531|7532|7533|7534|7535|7536|7537|7538|7539|7540|7541|7542|7543|7544|7545|7546|7547|7548|7549|7550|7551|7552 +14.750000000000,2048,128,59,7553|7554|7555|7556|7557|7558|7559|7560|7561|7562|7563|7564|7565|7566|7567|7568|7569|7570|7571|7572|7573|7574|7575|7576|7577|7578|7579|7580|7581|7582|7583|7584|7585|7586|7587|7588|7589|7590|7591|7592|7593|7594|7595|7596|7597|7598|7599|7600|7601|7602|7603|7604|7605|7606|7607|7608|7609|7610|7611|7612|7613|7614|7615|7616|7617|7618|7619|7620|7621|7622|7623|7624|7625|7626|7627|7628|7629|7630|7631|7632|7633|7634|7635|7636|7637|7638|7639|7640|7641|7642|7643|7644|7645|7646|7647|7648|7649|7650|7651|7652|7653|7654|7655|7656|7657|7658|7659|7660|7661|7662|7663|7664|7665|7666|7667|7668|7669|7670|7671|7672|7673|7674|7675|7676|7677|7678|7679|7680 +15.000000000000,2048,128,60,7681|7682|7683|7684|7685|7686|7687|7688|7689|7690|7691|7692|7693|7694|7695|7696|7697|7698|7699|7700|7701|7702|7703|7704|7705|7706|7707|7708|7709|7710|7711|7712|7713|7714|7715|7716|7717|7718|7719|7720|7721|7722|7723|7724|7725|7726|7727|7728|7729|7730|7731|7732|7733|7734|7735|7736|7737|7738|7739|7740|7741|7742|7743|7744|7745|7746|7747|7748|7749|7750|7751|7752|7753|7754|7755|7756|7757|7758|7759|7760|7761|7762|7763|7764|7765|7766|7767|7768|7769|7770|7771|7772|7773|7774|7775|7776|7777|7778|7779|7780|7781|7782|7783|7784|7785|7786|7787|7788|7789|7790|7791|7792|7793|7794|7795|7796|7797|7798|7799|7800|7801|7802|7803|7804|7805|7806|7807|7808 +15.250000000000,2048,128,61,7809|7810|7811|7812|7813|7814|7815|7816|7817|7818|7819|7820|7821|7822|7823|7824|7825|7826|7827|7828|7829|7830|7831|7832|7833|7834|7835|7836|7837|7838|7839|7840|7841|7842|7843|7844|7845|7846|7847|7848|7849|7850|7851|7852|7853|7854|7855|7856|7857|7858|7859|7860|7861|7862|7863|7864|7865|7866|7867|7868|7869|7870|7871|7872|7873|7874|7875|7876|7877|7878|7879|7880|7881|7882|7883|7884|7885|7886|7887|7888|7889|7890|7891|7892|7893|7894|7895|7896|7897|7898|7899|7900|7901|7902|7903|7904|7905|7906|7907|7908|7909|7910|7911|7912|7913|7914|7915|7916|7917|7918|7919|7920|7921|7922|7923|7924|7925|7926|7927|7928|7929|7930|7931|7932|7933|7934|7935|7936 +15.500000000000,2048,128,62,7937|7938|7939|7940|7941|7942|7943|7944|7945|7946|7947|7948|7949|7950|7951|7952|7953|7954|7955|7956|7957|7958|7959|7960|7961|7962|7963|7964|7965|7966|7967|7968|7969|7970|7971|7972|7973|7974|7975|7976|7977|7978|7979|7980|7981|7982|7983|7984|7985|7986|7987|7988|7989|7990|7991|7992|7993|7994|7995|7996|7997|7998|7999|8000|8001|8002|8003|8004|8005|8006|8007|8008|8009|8010|8011|8012|8013|8014|8015|8016|8017|8018|8019|8020|8021|8022|8023|8024|8025|8026|8027|8028|8029|8030|8031|8032|8033|8034|8035|8036|8037|8038|8039|8040|8041|8042|8043|8044|8045|8046|8047|8048|8049|8050|8051|8052|8053|8054|8055|8056|8057|8058|8059|8060|8061|8062|8063|8064 +15.750000000000,2048,128,63,8065|8066|8067|8068|8069|8070|8071|8072|8073|8074|8075|8076|8077|8078|8079|8080|8081|8082|8083|8084|8085|8086|8087|8088|8089|8090|8091|8092|8093|8094|8095|8096|8097|8098|8099|8100|8101|8102|8103|8104|8105|8106|8107|8108|8109|8110|8111|8112|8113|8114|8115|8116|8117|8118|8119|8120|8121|8122|8123|8124|8125|8126|8127|8128|8129|8130|8131|8132|8133|8134|8135|8136|8137|8138|8139|8140|8141|8142|8143|8144|8145|8146|8147|8148|8149|8150|8151|8152|8153|8154|8155|8156|8157|8158|8159|8160|8161|8162|8163|8164|8165|8166|8167|8168|8169|8170|8171|8172|8173|8174|8175|8176|8177|8178|8179|8180|8181|8182|8183|8184|8185|8186|8187|8188|8189|8190|8191|8192 diff --git a/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r8.csv b/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r8.csv new file mode 100644 index 0000000..5469301 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/workloads/f2-isl2048-osl128/r8.csv @@ -0,0 +1,65 @@ +arrived_at,num_prefill_tokens,num_decode_tokens,session_id,block_hash_ids +0.000000000000,2048,128,0,1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58|59|60|61|62|63|64|65|66|67|68|69|70|71|72|73|74|75|76|77|78|79|80|81|82|83|84|85|86|87|88|89|90|91|92|93|94|95|96|97|98|99|100|101|102|103|104|105|106|107|108|109|110|111|112|113|114|115|116|117|118|119|120|121|122|123|124|125|126|127|128 +0.125000000000,2048,128,1,129|130|131|132|133|134|135|136|137|138|139|140|141|142|143|144|145|146|147|148|149|150|151|152|153|154|155|156|157|158|159|160|161|162|163|164|165|166|167|168|169|170|171|172|173|174|175|176|177|178|179|180|181|182|183|184|185|186|187|188|189|190|191|192|193|194|195|196|197|198|199|200|201|202|203|204|205|206|207|208|209|210|211|212|213|214|215|216|217|218|219|220|221|222|223|224|225|226|227|228|229|230|231|232|233|234|235|236|237|238|239|240|241|242|243|244|245|246|247|248|249|250|251|252|253|254|255|256 +0.250000000000,2048,128,2,257|258|259|260|261|262|263|264|265|266|267|268|269|270|271|272|273|274|275|276|277|278|279|280|281|282|283|284|285|286|287|288|289|290|291|292|293|294|295|296|297|298|299|300|301|302|303|304|305|306|307|308|309|310|311|312|313|314|315|316|317|318|319|320|321|322|323|324|325|326|327|328|329|330|331|332|333|334|335|336|337|338|339|340|341|342|343|344|345|346|347|348|349|350|351|352|353|354|355|356|357|358|359|360|361|362|363|364|365|366|367|368|369|370|371|372|373|374|375|376|377|378|379|380|381|382|383|384 +0.375000000000,2048,128,3,385|386|387|388|389|390|391|392|393|394|395|396|397|398|399|400|401|402|403|404|405|406|407|408|409|410|411|412|413|414|415|416|417|418|419|420|421|422|423|424|425|426|427|428|429|430|431|432|433|434|435|436|437|438|439|440|441|442|443|444|445|446|447|448|449|450|451|452|453|454|455|456|457|458|459|460|461|462|463|464|465|466|467|468|469|470|471|472|473|474|475|476|477|478|479|480|481|482|483|484|485|486|487|488|489|490|491|492|493|494|495|496|497|498|499|500|501|502|503|504|505|506|507|508|509|510|511|512 +0.500000000000,2048,128,4,513|514|515|516|517|518|519|520|521|522|523|524|525|526|527|528|529|530|531|532|533|534|535|536|537|538|539|540|541|542|543|544|545|546|547|548|549|550|551|552|553|554|555|556|557|558|559|560|561|562|563|564|565|566|567|568|569|570|571|572|573|574|575|576|577|578|579|580|581|582|583|584|585|586|587|588|589|590|591|592|593|594|595|596|597|598|599|600|601|602|603|604|605|606|607|608|609|610|611|612|613|614|615|616|617|618|619|620|621|622|623|624|625|626|627|628|629|630|631|632|633|634|635|636|637|638|639|640 +0.625000000000,2048,128,5,641|642|643|644|645|646|647|648|649|650|651|652|653|654|655|656|657|658|659|660|661|662|663|664|665|666|667|668|669|670|671|672|673|674|675|676|677|678|679|680|681|682|683|684|685|686|687|688|689|690|691|692|693|694|695|696|697|698|699|700|701|702|703|704|705|706|707|708|709|710|711|712|713|714|715|716|717|718|719|720|721|722|723|724|725|726|727|728|729|730|731|732|733|734|735|736|737|738|739|740|741|742|743|744|745|746|747|748|749|750|751|752|753|754|755|756|757|758|759|760|761|762|763|764|765|766|767|768 +0.750000000000,2048,128,6,769|770|771|772|773|774|775|776|777|778|779|780|781|782|783|784|785|786|787|788|789|790|791|792|793|794|795|796|797|798|799|800|801|802|803|804|805|806|807|808|809|810|811|812|813|814|815|816|817|818|819|820|821|822|823|824|825|826|827|828|829|830|831|832|833|834|835|836|837|838|839|840|841|842|843|844|845|846|847|848|849|850|851|852|853|854|855|856|857|858|859|860|861|862|863|864|865|866|867|868|869|870|871|872|873|874|875|876|877|878|879|880|881|882|883|884|885|886|887|888|889|890|891|892|893|894|895|896 +0.875000000000,2048,128,7,897|898|899|900|901|902|903|904|905|906|907|908|909|910|911|912|913|914|915|916|917|918|919|920|921|922|923|924|925|926|927|928|929|930|931|932|933|934|935|936|937|938|939|940|941|942|943|944|945|946|947|948|949|950|951|952|953|954|955|956|957|958|959|960|961|962|963|964|965|966|967|968|969|970|971|972|973|974|975|976|977|978|979|980|981|982|983|984|985|986|987|988|989|990|991|992|993|994|995|996|997|998|999|1000|1001|1002|1003|1004|1005|1006|1007|1008|1009|1010|1011|1012|1013|1014|1015|1016|1017|1018|1019|1020|1021|1022|1023|1024 +1.000000000000,2048,128,8,1025|1026|1027|1028|1029|1030|1031|1032|1033|1034|1035|1036|1037|1038|1039|1040|1041|1042|1043|1044|1045|1046|1047|1048|1049|1050|1051|1052|1053|1054|1055|1056|1057|1058|1059|1060|1061|1062|1063|1064|1065|1066|1067|1068|1069|1070|1071|1072|1073|1074|1075|1076|1077|1078|1079|1080|1081|1082|1083|1084|1085|1086|1087|1088|1089|1090|1091|1092|1093|1094|1095|1096|1097|1098|1099|1100|1101|1102|1103|1104|1105|1106|1107|1108|1109|1110|1111|1112|1113|1114|1115|1116|1117|1118|1119|1120|1121|1122|1123|1124|1125|1126|1127|1128|1129|1130|1131|1132|1133|1134|1135|1136|1137|1138|1139|1140|1141|1142|1143|1144|1145|1146|1147|1148|1149|1150|1151|1152 +1.125000000000,2048,128,9,1153|1154|1155|1156|1157|1158|1159|1160|1161|1162|1163|1164|1165|1166|1167|1168|1169|1170|1171|1172|1173|1174|1175|1176|1177|1178|1179|1180|1181|1182|1183|1184|1185|1186|1187|1188|1189|1190|1191|1192|1193|1194|1195|1196|1197|1198|1199|1200|1201|1202|1203|1204|1205|1206|1207|1208|1209|1210|1211|1212|1213|1214|1215|1216|1217|1218|1219|1220|1221|1222|1223|1224|1225|1226|1227|1228|1229|1230|1231|1232|1233|1234|1235|1236|1237|1238|1239|1240|1241|1242|1243|1244|1245|1246|1247|1248|1249|1250|1251|1252|1253|1254|1255|1256|1257|1258|1259|1260|1261|1262|1263|1264|1265|1266|1267|1268|1269|1270|1271|1272|1273|1274|1275|1276|1277|1278|1279|1280 +1.250000000000,2048,128,10,1281|1282|1283|1284|1285|1286|1287|1288|1289|1290|1291|1292|1293|1294|1295|1296|1297|1298|1299|1300|1301|1302|1303|1304|1305|1306|1307|1308|1309|1310|1311|1312|1313|1314|1315|1316|1317|1318|1319|1320|1321|1322|1323|1324|1325|1326|1327|1328|1329|1330|1331|1332|1333|1334|1335|1336|1337|1338|1339|1340|1341|1342|1343|1344|1345|1346|1347|1348|1349|1350|1351|1352|1353|1354|1355|1356|1357|1358|1359|1360|1361|1362|1363|1364|1365|1366|1367|1368|1369|1370|1371|1372|1373|1374|1375|1376|1377|1378|1379|1380|1381|1382|1383|1384|1385|1386|1387|1388|1389|1390|1391|1392|1393|1394|1395|1396|1397|1398|1399|1400|1401|1402|1403|1404|1405|1406|1407|1408 +1.375000000000,2048,128,11,1409|1410|1411|1412|1413|1414|1415|1416|1417|1418|1419|1420|1421|1422|1423|1424|1425|1426|1427|1428|1429|1430|1431|1432|1433|1434|1435|1436|1437|1438|1439|1440|1441|1442|1443|1444|1445|1446|1447|1448|1449|1450|1451|1452|1453|1454|1455|1456|1457|1458|1459|1460|1461|1462|1463|1464|1465|1466|1467|1468|1469|1470|1471|1472|1473|1474|1475|1476|1477|1478|1479|1480|1481|1482|1483|1484|1485|1486|1487|1488|1489|1490|1491|1492|1493|1494|1495|1496|1497|1498|1499|1500|1501|1502|1503|1504|1505|1506|1507|1508|1509|1510|1511|1512|1513|1514|1515|1516|1517|1518|1519|1520|1521|1522|1523|1524|1525|1526|1527|1528|1529|1530|1531|1532|1533|1534|1535|1536 +1.500000000000,2048,128,12,1537|1538|1539|1540|1541|1542|1543|1544|1545|1546|1547|1548|1549|1550|1551|1552|1553|1554|1555|1556|1557|1558|1559|1560|1561|1562|1563|1564|1565|1566|1567|1568|1569|1570|1571|1572|1573|1574|1575|1576|1577|1578|1579|1580|1581|1582|1583|1584|1585|1586|1587|1588|1589|1590|1591|1592|1593|1594|1595|1596|1597|1598|1599|1600|1601|1602|1603|1604|1605|1606|1607|1608|1609|1610|1611|1612|1613|1614|1615|1616|1617|1618|1619|1620|1621|1622|1623|1624|1625|1626|1627|1628|1629|1630|1631|1632|1633|1634|1635|1636|1637|1638|1639|1640|1641|1642|1643|1644|1645|1646|1647|1648|1649|1650|1651|1652|1653|1654|1655|1656|1657|1658|1659|1660|1661|1662|1663|1664 +1.625000000000,2048,128,13,1665|1666|1667|1668|1669|1670|1671|1672|1673|1674|1675|1676|1677|1678|1679|1680|1681|1682|1683|1684|1685|1686|1687|1688|1689|1690|1691|1692|1693|1694|1695|1696|1697|1698|1699|1700|1701|1702|1703|1704|1705|1706|1707|1708|1709|1710|1711|1712|1713|1714|1715|1716|1717|1718|1719|1720|1721|1722|1723|1724|1725|1726|1727|1728|1729|1730|1731|1732|1733|1734|1735|1736|1737|1738|1739|1740|1741|1742|1743|1744|1745|1746|1747|1748|1749|1750|1751|1752|1753|1754|1755|1756|1757|1758|1759|1760|1761|1762|1763|1764|1765|1766|1767|1768|1769|1770|1771|1772|1773|1774|1775|1776|1777|1778|1779|1780|1781|1782|1783|1784|1785|1786|1787|1788|1789|1790|1791|1792 +1.750000000000,2048,128,14,1793|1794|1795|1796|1797|1798|1799|1800|1801|1802|1803|1804|1805|1806|1807|1808|1809|1810|1811|1812|1813|1814|1815|1816|1817|1818|1819|1820|1821|1822|1823|1824|1825|1826|1827|1828|1829|1830|1831|1832|1833|1834|1835|1836|1837|1838|1839|1840|1841|1842|1843|1844|1845|1846|1847|1848|1849|1850|1851|1852|1853|1854|1855|1856|1857|1858|1859|1860|1861|1862|1863|1864|1865|1866|1867|1868|1869|1870|1871|1872|1873|1874|1875|1876|1877|1878|1879|1880|1881|1882|1883|1884|1885|1886|1887|1888|1889|1890|1891|1892|1893|1894|1895|1896|1897|1898|1899|1900|1901|1902|1903|1904|1905|1906|1907|1908|1909|1910|1911|1912|1913|1914|1915|1916|1917|1918|1919|1920 +1.875000000000,2048,128,15,1921|1922|1923|1924|1925|1926|1927|1928|1929|1930|1931|1932|1933|1934|1935|1936|1937|1938|1939|1940|1941|1942|1943|1944|1945|1946|1947|1948|1949|1950|1951|1952|1953|1954|1955|1956|1957|1958|1959|1960|1961|1962|1963|1964|1965|1966|1967|1968|1969|1970|1971|1972|1973|1974|1975|1976|1977|1978|1979|1980|1981|1982|1983|1984|1985|1986|1987|1988|1989|1990|1991|1992|1993|1994|1995|1996|1997|1998|1999|2000|2001|2002|2003|2004|2005|2006|2007|2008|2009|2010|2011|2012|2013|2014|2015|2016|2017|2018|2019|2020|2021|2022|2023|2024|2025|2026|2027|2028|2029|2030|2031|2032|2033|2034|2035|2036|2037|2038|2039|2040|2041|2042|2043|2044|2045|2046|2047|2048 +2.000000000000,2048,128,16,2049|2050|2051|2052|2053|2054|2055|2056|2057|2058|2059|2060|2061|2062|2063|2064|2065|2066|2067|2068|2069|2070|2071|2072|2073|2074|2075|2076|2077|2078|2079|2080|2081|2082|2083|2084|2085|2086|2087|2088|2089|2090|2091|2092|2093|2094|2095|2096|2097|2098|2099|2100|2101|2102|2103|2104|2105|2106|2107|2108|2109|2110|2111|2112|2113|2114|2115|2116|2117|2118|2119|2120|2121|2122|2123|2124|2125|2126|2127|2128|2129|2130|2131|2132|2133|2134|2135|2136|2137|2138|2139|2140|2141|2142|2143|2144|2145|2146|2147|2148|2149|2150|2151|2152|2153|2154|2155|2156|2157|2158|2159|2160|2161|2162|2163|2164|2165|2166|2167|2168|2169|2170|2171|2172|2173|2174|2175|2176 +2.125000000000,2048,128,17,2177|2178|2179|2180|2181|2182|2183|2184|2185|2186|2187|2188|2189|2190|2191|2192|2193|2194|2195|2196|2197|2198|2199|2200|2201|2202|2203|2204|2205|2206|2207|2208|2209|2210|2211|2212|2213|2214|2215|2216|2217|2218|2219|2220|2221|2222|2223|2224|2225|2226|2227|2228|2229|2230|2231|2232|2233|2234|2235|2236|2237|2238|2239|2240|2241|2242|2243|2244|2245|2246|2247|2248|2249|2250|2251|2252|2253|2254|2255|2256|2257|2258|2259|2260|2261|2262|2263|2264|2265|2266|2267|2268|2269|2270|2271|2272|2273|2274|2275|2276|2277|2278|2279|2280|2281|2282|2283|2284|2285|2286|2287|2288|2289|2290|2291|2292|2293|2294|2295|2296|2297|2298|2299|2300|2301|2302|2303|2304 +2.250000000000,2048,128,18,2305|2306|2307|2308|2309|2310|2311|2312|2313|2314|2315|2316|2317|2318|2319|2320|2321|2322|2323|2324|2325|2326|2327|2328|2329|2330|2331|2332|2333|2334|2335|2336|2337|2338|2339|2340|2341|2342|2343|2344|2345|2346|2347|2348|2349|2350|2351|2352|2353|2354|2355|2356|2357|2358|2359|2360|2361|2362|2363|2364|2365|2366|2367|2368|2369|2370|2371|2372|2373|2374|2375|2376|2377|2378|2379|2380|2381|2382|2383|2384|2385|2386|2387|2388|2389|2390|2391|2392|2393|2394|2395|2396|2397|2398|2399|2400|2401|2402|2403|2404|2405|2406|2407|2408|2409|2410|2411|2412|2413|2414|2415|2416|2417|2418|2419|2420|2421|2422|2423|2424|2425|2426|2427|2428|2429|2430|2431|2432 +2.375000000000,2048,128,19,2433|2434|2435|2436|2437|2438|2439|2440|2441|2442|2443|2444|2445|2446|2447|2448|2449|2450|2451|2452|2453|2454|2455|2456|2457|2458|2459|2460|2461|2462|2463|2464|2465|2466|2467|2468|2469|2470|2471|2472|2473|2474|2475|2476|2477|2478|2479|2480|2481|2482|2483|2484|2485|2486|2487|2488|2489|2490|2491|2492|2493|2494|2495|2496|2497|2498|2499|2500|2501|2502|2503|2504|2505|2506|2507|2508|2509|2510|2511|2512|2513|2514|2515|2516|2517|2518|2519|2520|2521|2522|2523|2524|2525|2526|2527|2528|2529|2530|2531|2532|2533|2534|2535|2536|2537|2538|2539|2540|2541|2542|2543|2544|2545|2546|2547|2548|2549|2550|2551|2552|2553|2554|2555|2556|2557|2558|2559|2560 +2.500000000000,2048,128,20,2561|2562|2563|2564|2565|2566|2567|2568|2569|2570|2571|2572|2573|2574|2575|2576|2577|2578|2579|2580|2581|2582|2583|2584|2585|2586|2587|2588|2589|2590|2591|2592|2593|2594|2595|2596|2597|2598|2599|2600|2601|2602|2603|2604|2605|2606|2607|2608|2609|2610|2611|2612|2613|2614|2615|2616|2617|2618|2619|2620|2621|2622|2623|2624|2625|2626|2627|2628|2629|2630|2631|2632|2633|2634|2635|2636|2637|2638|2639|2640|2641|2642|2643|2644|2645|2646|2647|2648|2649|2650|2651|2652|2653|2654|2655|2656|2657|2658|2659|2660|2661|2662|2663|2664|2665|2666|2667|2668|2669|2670|2671|2672|2673|2674|2675|2676|2677|2678|2679|2680|2681|2682|2683|2684|2685|2686|2687|2688 +2.625000000000,2048,128,21,2689|2690|2691|2692|2693|2694|2695|2696|2697|2698|2699|2700|2701|2702|2703|2704|2705|2706|2707|2708|2709|2710|2711|2712|2713|2714|2715|2716|2717|2718|2719|2720|2721|2722|2723|2724|2725|2726|2727|2728|2729|2730|2731|2732|2733|2734|2735|2736|2737|2738|2739|2740|2741|2742|2743|2744|2745|2746|2747|2748|2749|2750|2751|2752|2753|2754|2755|2756|2757|2758|2759|2760|2761|2762|2763|2764|2765|2766|2767|2768|2769|2770|2771|2772|2773|2774|2775|2776|2777|2778|2779|2780|2781|2782|2783|2784|2785|2786|2787|2788|2789|2790|2791|2792|2793|2794|2795|2796|2797|2798|2799|2800|2801|2802|2803|2804|2805|2806|2807|2808|2809|2810|2811|2812|2813|2814|2815|2816 +2.750000000000,2048,128,22,2817|2818|2819|2820|2821|2822|2823|2824|2825|2826|2827|2828|2829|2830|2831|2832|2833|2834|2835|2836|2837|2838|2839|2840|2841|2842|2843|2844|2845|2846|2847|2848|2849|2850|2851|2852|2853|2854|2855|2856|2857|2858|2859|2860|2861|2862|2863|2864|2865|2866|2867|2868|2869|2870|2871|2872|2873|2874|2875|2876|2877|2878|2879|2880|2881|2882|2883|2884|2885|2886|2887|2888|2889|2890|2891|2892|2893|2894|2895|2896|2897|2898|2899|2900|2901|2902|2903|2904|2905|2906|2907|2908|2909|2910|2911|2912|2913|2914|2915|2916|2917|2918|2919|2920|2921|2922|2923|2924|2925|2926|2927|2928|2929|2930|2931|2932|2933|2934|2935|2936|2937|2938|2939|2940|2941|2942|2943|2944 +2.875000000000,2048,128,23,2945|2946|2947|2948|2949|2950|2951|2952|2953|2954|2955|2956|2957|2958|2959|2960|2961|2962|2963|2964|2965|2966|2967|2968|2969|2970|2971|2972|2973|2974|2975|2976|2977|2978|2979|2980|2981|2982|2983|2984|2985|2986|2987|2988|2989|2990|2991|2992|2993|2994|2995|2996|2997|2998|2999|3000|3001|3002|3003|3004|3005|3006|3007|3008|3009|3010|3011|3012|3013|3014|3015|3016|3017|3018|3019|3020|3021|3022|3023|3024|3025|3026|3027|3028|3029|3030|3031|3032|3033|3034|3035|3036|3037|3038|3039|3040|3041|3042|3043|3044|3045|3046|3047|3048|3049|3050|3051|3052|3053|3054|3055|3056|3057|3058|3059|3060|3061|3062|3063|3064|3065|3066|3067|3068|3069|3070|3071|3072 +3.000000000000,2048,128,24,3073|3074|3075|3076|3077|3078|3079|3080|3081|3082|3083|3084|3085|3086|3087|3088|3089|3090|3091|3092|3093|3094|3095|3096|3097|3098|3099|3100|3101|3102|3103|3104|3105|3106|3107|3108|3109|3110|3111|3112|3113|3114|3115|3116|3117|3118|3119|3120|3121|3122|3123|3124|3125|3126|3127|3128|3129|3130|3131|3132|3133|3134|3135|3136|3137|3138|3139|3140|3141|3142|3143|3144|3145|3146|3147|3148|3149|3150|3151|3152|3153|3154|3155|3156|3157|3158|3159|3160|3161|3162|3163|3164|3165|3166|3167|3168|3169|3170|3171|3172|3173|3174|3175|3176|3177|3178|3179|3180|3181|3182|3183|3184|3185|3186|3187|3188|3189|3190|3191|3192|3193|3194|3195|3196|3197|3198|3199|3200 +3.125000000000,2048,128,25,3201|3202|3203|3204|3205|3206|3207|3208|3209|3210|3211|3212|3213|3214|3215|3216|3217|3218|3219|3220|3221|3222|3223|3224|3225|3226|3227|3228|3229|3230|3231|3232|3233|3234|3235|3236|3237|3238|3239|3240|3241|3242|3243|3244|3245|3246|3247|3248|3249|3250|3251|3252|3253|3254|3255|3256|3257|3258|3259|3260|3261|3262|3263|3264|3265|3266|3267|3268|3269|3270|3271|3272|3273|3274|3275|3276|3277|3278|3279|3280|3281|3282|3283|3284|3285|3286|3287|3288|3289|3290|3291|3292|3293|3294|3295|3296|3297|3298|3299|3300|3301|3302|3303|3304|3305|3306|3307|3308|3309|3310|3311|3312|3313|3314|3315|3316|3317|3318|3319|3320|3321|3322|3323|3324|3325|3326|3327|3328 +3.250000000000,2048,128,26,3329|3330|3331|3332|3333|3334|3335|3336|3337|3338|3339|3340|3341|3342|3343|3344|3345|3346|3347|3348|3349|3350|3351|3352|3353|3354|3355|3356|3357|3358|3359|3360|3361|3362|3363|3364|3365|3366|3367|3368|3369|3370|3371|3372|3373|3374|3375|3376|3377|3378|3379|3380|3381|3382|3383|3384|3385|3386|3387|3388|3389|3390|3391|3392|3393|3394|3395|3396|3397|3398|3399|3400|3401|3402|3403|3404|3405|3406|3407|3408|3409|3410|3411|3412|3413|3414|3415|3416|3417|3418|3419|3420|3421|3422|3423|3424|3425|3426|3427|3428|3429|3430|3431|3432|3433|3434|3435|3436|3437|3438|3439|3440|3441|3442|3443|3444|3445|3446|3447|3448|3449|3450|3451|3452|3453|3454|3455|3456 +3.375000000000,2048,128,27,3457|3458|3459|3460|3461|3462|3463|3464|3465|3466|3467|3468|3469|3470|3471|3472|3473|3474|3475|3476|3477|3478|3479|3480|3481|3482|3483|3484|3485|3486|3487|3488|3489|3490|3491|3492|3493|3494|3495|3496|3497|3498|3499|3500|3501|3502|3503|3504|3505|3506|3507|3508|3509|3510|3511|3512|3513|3514|3515|3516|3517|3518|3519|3520|3521|3522|3523|3524|3525|3526|3527|3528|3529|3530|3531|3532|3533|3534|3535|3536|3537|3538|3539|3540|3541|3542|3543|3544|3545|3546|3547|3548|3549|3550|3551|3552|3553|3554|3555|3556|3557|3558|3559|3560|3561|3562|3563|3564|3565|3566|3567|3568|3569|3570|3571|3572|3573|3574|3575|3576|3577|3578|3579|3580|3581|3582|3583|3584 +3.500000000000,2048,128,28,3585|3586|3587|3588|3589|3590|3591|3592|3593|3594|3595|3596|3597|3598|3599|3600|3601|3602|3603|3604|3605|3606|3607|3608|3609|3610|3611|3612|3613|3614|3615|3616|3617|3618|3619|3620|3621|3622|3623|3624|3625|3626|3627|3628|3629|3630|3631|3632|3633|3634|3635|3636|3637|3638|3639|3640|3641|3642|3643|3644|3645|3646|3647|3648|3649|3650|3651|3652|3653|3654|3655|3656|3657|3658|3659|3660|3661|3662|3663|3664|3665|3666|3667|3668|3669|3670|3671|3672|3673|3674|3675|3676|3677|3678|3679|3680|3681|3682|3683|3684|3685|3686|3687|3688|3689|3690|3691|3692|3693|3694|3695|3696|3697|3698|3699|3700|3701|3702|3703|3704|3705|3706|3707|3708|3709|3710|3711|3712 +3.625000000000,2048,128,29,3713|3714|3715|3716|3717|3718|3719|3720|3721|3722|3723|3724|3725|3726|3727|3728|3729|3730|3731|3732|3733|3734|3735|3736|3737|3738|3739|3740|3741|3742|3743|3744|3745|3746|3747|3748|3749|3750|3751|3752|3753|3754|3755|3756|3757|3758|3759|3760|3761|3762|3763|3764|3765|3766|3767|3768|3769|3770|3771|3772|3773|3774|3775|3776|3777|3778|3779|3780|3781|3782|3783|3784|3785|3786|3787|3788|3789|3790|3791|3792|3793|3794|3795|3796|3797|3798|3799|3800|3801|3802|3803|3804|3805|3806|3807|3808|3809|3810|3811|3812|3813|3814|3815|3816|3817|3818|3819|3820|3821|3822|3823|3824|3825|3826|3827|3828|3829|3830|3831|3832|3833|3834|3835|3836|3837|3838|3839|3840 +3.750000000000,2048,128,30,3841|3842|3843|3844|3845|3846|3847|3848|3849|3850|3851|3852|3853|3854|3855|3856|3857|3858|3859|3860|3861|3862|3863|3864|3865|3866|3867|3868|3869|3870|3871|3872|3873|3874|3875|3876|3877|3878|3879|3880|3881|3882|3883|3884|3885|3886|3887|3888|3889|3890|3891|3892|3893|3894|3895|3896|3897|3898|3899|3900|3901|3902|3903|3904|3905|3906|3907|3908|3909|3910|3911|3912|3913|3914|3915|3916|3917|3918|3919|3920|3921|3922|3923|3924|3925|3926|3927|3928|3929|3930|3931|3932|3933|3934|3935|3936|3937|3938|3939|3940|3941|3942|3943|3944|3945|3946|3947|3948|3949|3950|3951|3952|3953|3954|3955|3956|3957|3958|3959|3960|3961|3962|3963|3964|3965|3966|3967|3968 +3.875000000000,2048,128,31,3969|3970|3971|3972|3973|3974|3975|3976|3977|3978|3979|3980|3981|3982|3983|3984|3985|3986|3987|3988|3989|3990|3991|3992|3993|3994|3995|3996|3997|3998|3999|4000|4001|4002|4003|4004|4005|4006|4007|4008|4009|4010|4011|4012|4013|4014|4015|4016|4017|4018|4019|4020|4021|4022|4023|4024|4025|4026|4027|4028|4029|4030|4031|4032|4033|4034|4035|4036|4037|4038|4039|4040|4041|4042|4043|4044|4045|4046|4047|4048|4049|4050|4051|4052|4053|4054|4055|4056|4057|4058|4059|4060|4061|4062|4063|4064|4065|4066|4067|4068|4069|4070|4071|4072|4073|4074|4075|4076|4077|4078|4079|4080|4081|4082|4083|4084|4085|4086|4087|4088|4089|4090|4091|4092|4093|4094|4095|4096 +4.000000000000,2048,128,32,4097|4098|4099|4100|4101|4102|4103|4104|4105|4106|4107|4108|4109|4110|4111|4112|4113|4114|4115|4116|4117|4118|4119|4120|4121|4122|4123|4124|4125|4126|4127|4128|4129|4130|4131|4132|4133|4134|4135|4136|4137|4138|4139|4140|4141|4142|4143|4144|4145|4146|4147|4148|4149|4150|4151|4152|4153|4154|4155|4156|4157|4158|4159|4160|4161|4162|4163|4164|4165|4166|4167|4168|4169|4170|4171|4172|4173|4174|4175|4176|4177|4178|4179|4180|4181|4182|4183|4184|4185|4186|4187|4188|4189|4190|4191|4192|4193|4194|4195|4196|4197|4198|4199|4200|4201|4202|4203|4204|4205|4206|4207|4208|4209|4210|4211|4212|4213|4214|4215|4216|4217|4218|4219|4220|4221|4222|4223|4224 +4.125000000000,2048,128,33,4225|4226|4227|4228|4229|4230|4231|4232|4233|4234|4235|4236|4237|4238|4239|4240|4241|4242|4243|4244|4245|4246|4247|4248|4249|4250|4251|4252|4253|4254|4255|4256|4257|4258|4259|4260|4261|4262|4263|4264|4265|4266|4267|4268|4269|4270|4271|4272|4273|4274|4275|4276|4277|4278|4279|4280|4281|4282|4283|4284|4285|4286|4287|4288|4289|4290|4291|4292|4293|4294|4295|4296|4297|4298|4299|4300|4301|4302|4303|4304|4305|4306|4307|4308|4309|4310|4311|4312|4313|4314|4315|4316|4317|4318|4319|4320|4321|4322|4323|4324|4325|4326|4327|4328|4329|4330|4331|4332|4333|4334|4335|4336|4337|4338|4339|4340|4341|4342|4343|4344|4345|4346|4347|4348|4349|4350|4351|4352 +4.250000000000,2048,128,34,4353|4354|4355|4356|4357|4358|4359|4360|4361|4362|4363|4364|4365|4366|4367|4368|4369|4370|4371|4372|4373|4374|4375|4376|4377|4378|4379|4380|4381|4382|4383|4384|4385|4386|4387|4388|4389|4390|4391|4392|4393|4394|4395|4396|4397|4398|4399|4400|4401|4402|4403|4404|4405|4406|4407|4408|4409|4410|4411|4412|4413|4414|4415|4416|4417|4418|4419|4420|4421|4422|4423|4424|4425|4426|4427|4428|4429|4430|4431|4432|4433|4434|4435|4436|4437|4438|4439|4440|4441|4442|4443|4444|4445|4446|4447|4448|4449|4450|4451|4452|4453|4454|4455|4456|4457|4458|4459|4460|4461|4462|4463|4464|4465|4466|4467|4468|4469|4470|4471|4472|4473|4474|4475|4476|4477|4478|4479|4480 +4.375000000000,2048,128,35,4481|4482|4483|4484|4485|4486|4487|4488|4489|4490|4491|4492|4493|4494|4495|4496|4497|4498|4499|4500|4501|4502|4503|4504|4505|4506|4507|4508|4509|4510|4511|4512|4513|4514|4515|4516|4517|4518|4519|4520|4521|4522|4523|4524|4525|4526|4527|4528|4529|4530|4531|4532|4533|4534|4535|4536|4537|4538|4539|4540|4541|4542|4543|4544|4545|4546|4547|4548|4549|4550|4551|4552|4553|4554|4555|4556|4557|4558|4559|4560|4561|4562|4563|4564|4565|4566|4567|4568|4569|4570|4571|4572|4573|4574|4575|4576|4577|4578|4579|4580|4581|4582|4583|4584|4585|4586|4587|4588|4589|4590|4591|4592|4593|4594|4595|4596|4597|4598|4599|4600|4601|4602|4603|4604|4605|4606|4607|4608 +4.500000000000,2048,128,36,4609|4610|4611|4612|4613|4614|4615|4616|4617|4618|4619|4620|4621|4622|4623|4624|4625|4626|4627|4628|4629|4630|4631|4632|4633|4634|4635|4636|4637|4638|4639|4640|4641|4642|4643|4644|4645|4646|4647|4648|4649|4650|4651|4652|4653|4654|4655|4656|4657|4658|4659|4660|4661|4662|4663|4664|4665|4666|4667|4668|4669|4670|4671|4672|4673|4674|4675|4676|4677|4678|4679|4680|4681|4682|4683|4684|4685|4686|4687|4688|4689|4690|4691|4692|4693|4694|4695|4696|4697|4698|4699|4700|4701|4702|4703|4704|4705|4706|4707|4708|4709|4710|4711|4712|4713|4714|4715|4716|4717|4718|4719|4720|4721|4722|4723|4724|4725|4726|4727|4728|4729|4730|4731|4732|4733|4734|4735|4736 +4.625000000000,2048,128,37,4737|4738|4739|4740|4741|4742|4743|4744|4745|4746|4747|4748|4749|4750|4751|4752|4753|4754|4755|4756|4757|4758|4759|4760|4761|4762|4763|4764|4765|4766|4767|4768|4769|4770|4771|4772|4773|4774|4775|4776|4777|4778|4779|4780|4781|4782|4783|4784|4785|4786|4787|4788|4789|4790|4791|4792|4793|4794|4795|4796|4797|4798|4799|4800|4801|4802|4803|4804|4805|4806|4807|4808|4809|4810|4811|4812|4813|4814|4815|4816|4817|4818|4819|4820|4821|4822|4823|4824|4825|4826|4827|4828|4829|4830|4831|4832|4833|4834|4835|4836|4837|4838|4839|4840|4841|4842|4843|4844|4845|4846|4847|4848|4849|4850|4851|4852|4853|4854|4855|4856|4857|4858|4859|4860|4861|4862|4863|4864 +4.750000000000,2048,128,38,4865|4866|4867|4868|4869|4870|4871|4872|4873|4874|4875|4876|4877|4878|4879|4880|4881|4882|4883|4884|4885|4886|4887|4888|4889|4890|4891|4892|4893|4894|4895|4896|4897|4898|4899|4900|4901|4902|4903|4904|4905|4906|4907|4908|4909|4910|4911|4912|4913|4914|4915|4916|4917|4918|4919|4920|4921|4922|4923|4924|4925|4926|4927|4928|4929|4930|4931|4932|4933|4934|4935|4936|4937|4938|4939|4940|4941|4942|4943|4944|4945|4946|4947|4948|4949|4950|4951|4952|4953|4954|4955|4956|4957|4958|4959|4960|4961|4962|4963|4964|4965|4966|4967|4968|4969|4970|4971|4972|4973|4974|4975|4976|4977|4978|4979|4980|4981|4982|4983|4984|4985|4986|4987|4988|4989|4990|4991|4992 +4.875000000000,2048,128,39,4993|4994|4995|4996|4997|4998|4999|5000|5001|5002|5003|5004|5005|5006|5007|5008|5009|5010|5011|5012|5013|5014|5015|5016|5017|5018|5019|5020|5021|5022|5023|5024|5025|5026|5027|5028|5029|5030|5031|5032|5033|5034|5035|5036|5037|5038|5039|5040|5041|5042|5043|5044|5045|5046|5047|5048|5049|5050|5051|5052|5053|5054|5055|5056|5057|5058|5059|5060|5061|5062|5063|5064|5065|5066|5067|5068|5069|5070|5071|5072|5073|5074|5075|5076|5077|5078|5079|5080|5081|5082|5083|5084|5085|5086|5087|5088|5089|5090|5091|5092|5093|5094|5095|5096|5097|5098|5099|5100|5101|5102|5103|5104|5105|5106|5107|5108|5109|5110|5111|5112|5113|5114|5115|5116|5117|5118|5119|5120 +5.000000000000,2048,128,40,5121|5122|5123|5124|5125|5126|5127|5128|5129|5130|5131|5132|5133|5134|5135|5136|5137|5138|5139|5140|5141|5142|5143|5144|5145|5146|5147|5148|5149|5150|5151|5152|5153|5154|5155|5156|5157|5158|5159|5160|5161|5162|5163|5164|5165|5166|5167|5168|5169|5170|5171|5172|5173|5174|5175|5176|5177|5178|5179|5180|5181|5182|5183|5184|5185|5186|5187|5188|5189|5190|5191|5192|5193|5194|5195|5196|5197|5198|5199|5200|5201|5202|5203|5204|5205|5206|5207|5208|5209|5210|5211|5212|5213|5214|5215|5216|5217|5218|5219|5220|5221|5222|5223|5224|5225|5226|5227|5228|5229|5230|5231|5232|5233|5234|5235|5236|5237|5238|5239|5240|5241|5242|5243|5244|5245|5246|5247|5248 +5.125000000000,2048,128,41,5249|5250|5251|5252|5253|5254|5255|5256|5257|5258|5259|5260|5261|5262|5263|5264|5265|5266|5267|5268|5269|5270|5271|5272|5273|5274|5275|5276|5277|5278|5279|5280|5281|5282|5283|5284|5285|5286|5287|5288|5289|5290|5291|5292|5293|5294|5295|5296|5297|5298|5299|5300|5301|5302|5303|5304|5305|5306|5307|5308|5309|5310|5311|5312|5313|5314|5315|5316|5317|5318|5319|5320|5321|5322|5323|5324|5325|5326|5327|5328|5329|5330|5331|5332|5333|5334|5335|5336|5337|5338|5339|5340|5341|5342|5343|5344|5345|5346|5347|5348|5349|5350|5351|5352|5353|5354|5355|5356|5357|5358|5359|5360|5361|5362|5363|5364|5365|5366|5367|5368|5369|5370|5371|5372|5373|5374|5375|5376 +5.250000000000,2048,128,42,5377|5378|5379|5380|5381|5382|5383|5384|5385|5386|5387|5388|5389|5390|5391|5392|5393|5394|5395|5396|5397|5398|5399|5400|5401|5402|5403|5404|5405|5406|5407|5408|5409|5410|5411|5412|5413|5414|5415|5416|5417|5418|5419|5420|5421|5422|5423|5424|5425|5426|5427|5428|5429|5430|5431|5432|5433|5434|5435|5436|5437|5438|5439|5440|5441|5442|5443|5444|5445|5446|5447|5448|5449|5450|5451|5452|5453|5454|5455|5456|5457|5458|5459|5460|5461|5462|5463|5464|5465|5466|5467|5468|5469|5470|5471|5472|5473|5474|5475|5476|5477|5478|5479|5480|5481|5482|5483|5484|5485|5486|5487|5488|5489|5490|5491|5492|5493|5494|5495|5496|5497|5498|5499|5500|5501|5502|5503|5504 +5.375000000000,2048,128,43,5505|5506|5507|5508|5509|5510|5511|5512|5513|5514|5515|5516|5517|5518|5519|5520|5521|5522|5523|5524|5525|5526|5527|5528|5529|5530|5531|5532|5533|5534|5535|5536|5537|5538|5539|5540|5541|5542|5543|5544|5545|5546|5547|5548|5549|5550|5551|5552|5553|5554|5555|5556|5557|5558|5559|5560|5561|5562|5563|5564|5565|5566|5567|5568|5569|5570|5571|5572|5573|5574|5575|5576|5577|5578|5579|5580|5581|5582|5583|5584|5585|5586|5587|5588|5589|5590|5591|5592|5593|5594|5595|5596|5597|5598|5599|5600|5601|5602|5603|5604|5605|5606|5607|5608|5609|5610|5611|5612|5613|5614|5615|5616|5617|5618|5619|5620|5621|5622|5623|5624|5625|5626|5627|5628|5629|5630|5631|5632 +5.500000000000,2048,128,44,5633|5634|5635|5636|5637|5638|5639|5640|5641|5642|5643|5644|5645|5646|5647|5648|5649|5650|5651|5652|5653|5654|5655|5656|5657|5658|5659|5660|5661|5662|5663|5664|5665|5666|5667|5668|5669|5670|5671|5672|5673|5674|5675|5676|5677|5678|5679|5680|5681|5682|5683|5684|5685|5686|5687|5688|5689|5690|5691|5692|5693|5694|5695|5696|5697|5698|5699|5700|5701|5702|5703|5704|5705|5706|5707|5708|5709|5710|5711|5712|5713|5714|5715|5716|5717|5718|5719|5720|5721|5722|5723|5724|5725|5726|5727|5728|5729|5730|5731|5732|5733|5734|5735|5736|5737|5738|5739|5740|5741|5742|5743|5744|5745|5746|5747|5748|5749|5750|5751|5752|5753|5754|5755|5756|5757|5758|5759|5760 +5.625000000000,2048,128,45,5761|5762|5763|5764|5765|5766|5767|5768|5769|5770|5771|5772|5773|5774|5775|5776|5777|5778|5779|5780|5781|5782|5783|5784|5785|5786|5787|5788|5789|5790|5791|5792|5793|5794|5795|5796|5797|5798|5799|5800|5801|5802|5803|5804|5805|5806|5807|5808|5809|5810|5811|5812|5813|5814|5815|5816|5817|5818|5819|5820|5821|5822|5823|5824|5825|5826|5827|5828|5829|5830|5831|5832|5833|5834|5835|5836|5837|5838|5839|5840|5841|5842|5843|5844|5845|5846|5847|5848|5849|5850|5851|5852|5853|5854|5855|5856|5857|5858|5859|5860|5861|5862|5863|5864|5865|5866|5867|5868|5869|5870|5871|5872|5873|5874|5875|5876|5877|5878|5879|5880|5881|5882|5883|5884|5885|5886|5887|5888 +5.750000000000,2048,128,46,5889|5890|5891|5892|5893|5894|5895|5896|5897|5898|5899|5900|5901|5902|5903|5904|5905|5906|5907|5908|5909|5910|5911|5912|5913|5914|5915|5916|5917|5918|5919|5920|5921|5922|5923|5924|5925|5926|5927|5928|5929|5930|5931|5932|5933|5934|5935|5936|5937|5938|5939|5940|5941|5942|5943|5944|5945|5946|5947|5948|5949|5950|5951|5952|5953|5954|5955|5956|5957|5958|5959|5960|5961|5962|5963|5964|5965|5966|5967|5968|5969|5970|5971|5972|5973|5974|5975|5976|5977|5978|5979|5980|5981|5982|5983|5984|5985|5986|5987|5988|5989|5990|5991|5992|5993|5994|5995|5996|5997|5998|5999|6000|6001|6002|6003|6004|6005|6006|6007|6008|6009|6010|6011|6012|6013|6014|6015|6016 +5.875000000000,2048,128,47,6017|6018|6019|6020|6021|6022|6023|6024|6025|6026|6027|6028|6029|6030|6031|6032|6033|6034|6035|6036|6037|6038|6039|6040|6041|6042|6043|6044|6045|6046|6047|6048|6049|6050|6051|6052|6053|6054|6055|6056|6057|6058|6059|6060|6061|6062|6063|6064|6065|6066|6067|6068|6069|6070|6071|6072|6073|6074|6075|6076|6077|6078|6079|6080|6081|6082|6083|6084|6085|6086|6087|6088|6089|6090|6091|6092|6093|6094|6095|6096|6097|6098|6099|6100|6101|6102|6103|6104|6105|6106|6107|6108|6109|6110|6111|6112|6113|6114|6115|6116|6117|6118|6119|6120|6121|6122|6123|6124|6125|6126|6127|6128|6129|6130|6131|6132|6133|6134|6135|6136|6137|6138|6139|6140|6141|6142|6143|6144 +6.000000000000,2048,128,48,6145|6146|6147|6148|6149|6150|6151|6152|6153|6154|6155|6156|6157|6158|6159|6160|6161|6162|6163|6164|6165|6166|6167|6168|6169|6170|6171|6172|6173|6174|6175|6176|6177|6178|6179|6180|6181|6182|6183|6184|6185|6186|6187|6188|6189|6190|6191|6192|6193|6194|6195|6196|6197|6198|6199|6200|6201|6202|6203|6204|6205|6206|6207|6208|6209|6210|6211|6212|6213|6214|6215|6216|6217|6218|6219|6220|6221|6222|6223|6224|6225|6226|6227|6228|6229|6230|6231|6232|6233|6234|6235|6236|6237|6238|6239|6240|6241|6242|6243|6244|6245|6246|6247|6248|6249|6250|6251|6252|6253|6254|6255|6256|6257|6258|6259|6260|6261|6262|6263|6264|6265|6266|6267|6268|6269|6270|6271|6272 +6.125000000000,2048,128,49,6273|6274|6275|6276|6277|6278|6279|6280|6281|6282|6283|6284|6285|6286|6287|6288|6289|6290|6291|6292|6293|6294|6295|6296|6297|6298|6299|6300|6301|6302|6303|6304|6305|6306|6307|6308|6309|6310|6311|6312|6313|6314|6315|6316|6317|6318|6319|6320|6321|6322|6323|6324|6325|6326|6327|6328|6329|6330|6331|6332|6333|6334|6335|6336|6337|6338|6339|6340|6341|6342|6343|6344|6345|6346|6347|6348|6349|6350|6351|6352|6353|6354|6355|6356|6357|6358|6359|6360|6361|6362|6363|6364|6365|6366|6367|6368|6369|6370|6371|6372|6373|6374|6375|6376|6377|6378|6379|6380|6381|6382|6383|6384|6385|6386|6387|6388|6389|6390|6391|6392|6393|6394|6395|6396|6397|6398|6399|6400 +6.250000000000,2048,128,50,6401|6402|6403|6404|6405|6406|6407|6408|6409|6410|6411|6412|6413|6414|6415|6416|6417|6418|6419|6420|6421|6422|6423|6424|6425|6426|6427|6428|6429|6430|6431|6432|6433|6434|6435|6436|6437|6438|6439|6440|6441|6442|6443|6444|6445|6446|6447|6448|6449|6450|6451|6452|6453|6454|6455|6456|6457|6458|6459|6460|6461|6462|6463|6464|6465|6466|6467|6468|6469|6470|6471|6472|6473|6474|6475|6476|6477|6478|6479|6480|6481|6482|6483|6484|6485|6486|6487|6488|6489|6490|6491|6492|6493|6494|6495|6496|6497|6498|6499|6500|6501|6502|6503|6504|6505|6506|6507|6508|6509|6510|6511|6512|6513|6514|6515|6516|6517|6518|6519|6520|6521|6522|6523|6524|6525|6526|6527|6528 +6.375000000000,2048,128,51,6529|6530|6531|6532|6533|6534|6535|6536|6537|6538|6539|6540|6541|6542|6543|6544|6545|6546|6547|6548|6549|6550|6551|6552|6553|6554|6555|6556|6557|6558|6559|6560|6561|6562|6563|6564|6565|6566|6567|6568|6569|6570|6571|6572|6573|6574|6575|6576|6577|6578|6579|6580|6581|6582|6583|6584|6585|6586|6587|6588|6589|6590|6591|6592|6593|6594|6595|6596|6597|6598|6599|6600|6601|6602|6603|6604|6605|6606|6607|6608|6609|6610|6611|6612|6613|6614|6615|6616|6617|6618|6619|6620|6621|6622|6623|6624|6625|6626|6627|6628|6629|6630|6631|6632|6633|6634|6635|6636|6637|6638|6639|6640|6641|6642|6643|6644|6645|6646|6647|6648|6649|6650|6651|6652|6653|6654|6655|6656 +6.500000000000,2048,128,52,6657|6658|6659|6660|6661|6662|6663|6664|6665|6666|6667|6668|6669|6670|6671|6672|6673|6674|6675|6676|6677|6678|6679|6680|6681|6682|6683|6684|6685|6686|6687|6688|6689|6690|6691|6692|6693|6694|6695|6696|6697|6698|6699|6700|6701|6702|6703|6704|6705|6706|6707|6708|6709|6710|6711|6712|6713|6714|6715|6716|6717|6718|6719|6720|6721|6722|6723|6724|6725|6726|6727|6728|6729|6730|6731|6732|6733|6734|6735|6736|6737|6738|6739|6740|6741|6742|6743|6744|6745|6746|6747|6748|6749|6750|6751|6752|6753|6754|6755|6756|6757|6758|6759|6760|6761|6762|6763|6764|6765|6766|6767|6768|6769|6770|6771|6772|6773|6774|6775|6776|6777|6778|6779|6780|6781|6782|6783|6784 +6.625000000000,2048,128,53,6785|6786|6787|6788|6789|6790|6791|6792|6793|6794|6795|6796|6797|6798|6799|6800|6801|6802|6803|6804|6805|6806|6807|6808|6809|6810|6811|6812|6813|6814|6815|6816|6817|6818|6819|6820|6821|6822|6823|6824|6825|6826|6827|6828|6829|6830|6831|6832|6833|6834|6835|6836|6837|6838|6839|6840|6841|6842|6843|6844|6845|6846|6847|6848|6849|6850|6851|6852|6853|6854|6855|6856|6857|6858|6859|6860|6861|6862|6863|6864|6865|6866|6867|6868|6869|6870|6871|6872|6873|6874|6875|6876|6877|6878|6879|6880|6881|6882|6883|6884|6885|6886|6887|6888|6889|6890|6891|6892|6893|6894|6895|6896|6897|6898|6899|6900|6901|6902|6903|6904|6905|6906|6907|6908|6909|6910|6911|6912 +6.750000000000,2048,128,54,6913|6914|6915|6916|6917|6918|6919|6920|6921|6922|6923|6924|6925|6926|6927|6928|6929|6930|6931|6932|6933|6934|6935|6936|6937|6938|6939|6940|6941|6942|6943|6944|6945|6946|6947|6948|6949|6950|6951|6952|6953|6954|6955|6956|6957|6958|6959|6960|6961|6962|6963|6964|6965|6966|6967|6968|6969|6970|6971|6972|6973|6974|6975|6976|6977|6978|6979|6980|6981|6982|6983|6984|6985|6986|6987|6988|6989|6990|6991|6992|6993|6994|6995|6996|6997|6998|6999|7000|7001|7002|7003|7004|7005|7006|7007|7008|7009|7010|7011|7012|7013|7014|7015|7016|7017|7018|7019|7020|7021|7022|7023|7024|7025|7026|7027|7028|7029|7030|7031|7032|7033|7034|7035|7036|7037|7038|7039|7040 +6.875000000000,2048,128,55,7041|7042|7043|7044|7045|7046|7047|7048|7049|7050|7051|7052|7053|7054|7055|7056|7057|7058|7059|7060|7061|7062|7063|7064|7065|7066|7067|7068|7069|7070|7071|7072|7073|7074|7075|7076|7077|7078|7079|7080|7081|7082|7083|7084|7085|7086|7087|7088|7089|7090|7091|7092|7093|7094|7095|7096|7097|7098|7099|7100|7101|7102|7103|7104|7105|7106|7107|7108|7109|7110|7111|7112|7113|7114|7115|7116|7117|7118|7119|7120|7121|7122|7123|7124|7125|7126|7127|7128|7129|7130|7131|7132|7133|7134|7135|7136|7137|7138|7139|7140|7141|7142|7143|7144|7145|7146|7147|7148|7149|7150|7151|7152|7153|7154|7155|7156|7157|7158|7159|7160|7161|7162|7163|7164|7165|7166|7167|7168 +7.000000000000,2048,128,56,7169|7170|7171|7172|7173|7174|7175|7176|7177|7178|7179|7180|7181|7182|7183|7184|7185|7186|7187|7188|7189|7190|7191|7192|7193|7194|7195|7196|7197|7198|7199|7200|7201|7202|7203|7204|7205|7206|7207|7208|7209|7210|7211|7212|7213|7214|7215|7216|7217|7218|7219|7220|7221|7222|7223|7224|7225|7226|7227|7228|7229|7230|7231|7232|7233|7234|7235|7236|7237|7238|7239|7240|7241|7242|7243|7244|7245|7246|7247|7248|7249|7250|7251|7252|7253|7254|7255|7256|7257|7258|7259|7260|7261|7262|7263|7264|7265|7266|7267|7268|7269|7270|7271|7272|7273|7274|7275|7276|7277|7278|7279|7280|7281|7282|7283|7284|7285|7286|7287|7288|7289|7290|7291|7292|7293|7294|7295|7296 +7.125000000000,2048,128,57,7297|7298|7299|7300|7301|7302|7303|7304|7305|7306|7307|7308|7309|7310|7311|7312|7313|7314|7315|7316|7317|7318|7319|7320|7321|7322|7323|7324|7325|7326|7327|7328|7329|7330|7331|7332|7333|7334|7335|7336|7337|7338|7339|7340|7341|7342|7343|7344|7345|7346|7347|7348|7349|7350|7351|7352|7353|7354|7355|7356|7357|7358|7359|7360|7361|7362|7363|7364|7365|7366|7367|7368|7369|7370|7371|7372|7373|7374|7375|7376|7377|7378|7379|7380|7381|7382|7383|7384|7385|7386|7387|7388|7389|7390|7391|7392|7393|7394|7395|7396|7397|7398|7399|7400|7401|7402|7403|7404|7405|7406|7407|7408|7409|7410|7411|7412|7413|7414|7415|7416|7417|7418|7419|7420|7421|7422|7423|7424 +7.250000000000,2048,128,58,7425|7426|7427|7428|7429|7430|7431|7432|7433|7434|7435|7436|7437|7438|7439|7440|7441|7442|7443|7444|7445|7446|7447|7448|7449|7450|7451|7452|7453|7454|7455|7456|7457|7458|7459|7460|7461|7462|7463|7464|7465|7466|7467|7468|7469|7470|7471|7472|7473|7474|7475|7476|7477|7478|7479|7480|7481|7482|7483|7484|7485|7486|7487|7488|7489|7490|7491|7492|7493|7494|7495|7496|7497|7498|7499|7500|7501|7502|7503|7504|7505|7506|7507|7508|7509|7510|7511|7512|7513|7514|7515|7516|7517|7518|7519|7520|7521|7522|7523|7524|7525|7526|7527|7528|7529|7530|7531|7532|7533|7534|7535|7536|7537|7538|7539|7540|7541|7542|7543|7544|7545|7546|7547|7548|7549|7550|7551|7552 +7.375000000000,2048,128,59,7553|7554|7555|7556|7557|7558|7559|7560|7561|7562|7563|7564|7565|7566|7567|7568|7569|7570|7571|7572|7573|7574|7575|7576|7577|7578|7579|7580|7581|7582|7583|7584|7585|7586|7587|7588|7589|7590|7591|7592|7593|7594|7595|7596|7597|7598|7599|7600|7601|7602|7603|7604|7605|7606|7607|7608|7609|7610|7611|7612|7613|7614|7615|7616|7617|7618|7619|7620|7621|7622|7623|7624|7625|7626|7627|7628|7629|7630|7631|7632|7633|7634|7635|7636|7637|7638|7639|7640|7641|7642|7643|7644|7645|7646|7647|7648|7649|7650|7651|7652|7653|7654|7655|7656|7657|7658|7659|7660|7661|7662|7663|7664|7665|7666|7667|7668|7669|7670|7671|7672|7673|7674|7675|7676|7677|7678|7679|7680 +7.500000000000,2048,128,60,7681|7682|7683|7684|7685|7686|7687|7688|7689|7690|7691|7692|7693|7694|7695|7696|7697|7698|7699|7700|7701|7702|7703|7704|7705|7706|7707|7708|7709|7710|7711|7712|7713|7714|7715|7716|7717|7718|7719|7720|7721|7722|7723|7724|7725|7726|7727|7728|7729|7730|7731|7732|7733|7734|7735|7736|7737|7738|7739|7740|7741|7742|7743|7744|7745|7746|7747|7748|7749|7750|7751|7752|7753|7754|7755|7756|7757|7758|7759|7760|7761|7762|7763|7764|7765|7766|7767|7768|7769|7770|7771|7772|7773|7774|7775|7776|7777|7778|7779|7780|7781|7782|7783|7784|7785|7786|7787|7788|7789|7790|7791|7792|7793|7794|7795|7796|7797|7798|7799|7800|7801|7802|7803|7804|7805|7806|7807|7808 +7.625000000000,2048,128,61,7809|7810|7811|7812|7813|7814|7815|7816|7817|7818|7819|7820|7821|7822|7823|7824|7825|7826|7827|7828|7829|7830|7831|7832|7833|7834|7835|7836|7837|7838|7839|7840|7841|7842|7843|7844|7845|7846|7847|7848|7849|7850|7851|7852|7853|7854|7855|7856|7857|7858|7859|7860|7861|7862|7863|7864|7865|7866|7867|7868|7869|7870|7871|7872|7873|7874|7875|7876|7877|7878|7879|7880|7881|7882|7883|7884|7885|7886|7887|7888|7889|7890|7891|7892|7893|7894|7895|7896|7897|7898|7899|7900|7901|7902|7903|7904|7905|7906|7907|7908|7909|7910|7911|7912|7913|7914|7915|7916|7917|7918|7919|7920|7921|7922|7923|7924|7925|7926|7927|7928|7929|7930|7931|7932|7933|7934|7935|7936 +7.750000000000,2048,128,62,7937|7938|7939|7940|7941|7942|7943|7944|7945|7946|7947|7948|7949|7950|7951|7952|7953|7954|7955|7956|7957|7958|7959|7960|7961|7962|7963|7964|7965|7966|7967|7968|7969|7970|7971|7972|7973|7974|7975|7976|7977|7978|7979|7980|7981|7982|7983|7984|7985|7986|7987|7988|7989|7990|7991|7992|7993|7994|7995|7996|7997|7998|7999|8000|8001|8002|8003|8004|8005|8006|8007|8008|8009|8010|8011|8012|8013|8014|8015|8016|8017|8018|8019|8020|8021|8022|8023|8024|8025|8026|8027|8028|8029|8030|8031|8032|8033|8034|8035|8036|8037|8038|8039|8040|8041|8042|8043|8044|8045|8046|8047|8048|8049|8050|8051|8052|8053|8054|8055|8056|8057|8058|8059|8060|8061|8062|8063|8064 +7.875000000000,2048,128,63,8065|8066|8067|8068|8069|8070|8071|8072|8073|8074|8075|8076|8077|8078|8079|8080|8081|8082|8083|8084|8085|8086|8087|8088|8089|8090|8091|8092|8093|8094|8095|8096|8097|8098|8099|8100|8101|8102|8103|8104|8105|8106|8107|8108|8109|8110|8111|8112|8113|8114|8115|8116|8117|8118|8119|8120|8121|8122|8123|8124|8125|8126|8127|8128|8129|8130|8131|8132|8133|8134|8135|8136|8137|8138|8139|8140|8141|8142|8143|8144|8145|8146|8147|8148|8149|8150|8151|8152|8153|8154|8155|8156|8157|8158|8159|8160|8161|8162|8163|8164|8165|8166|8167|8168|8169|8170|8171|8172|8173|8174|8175|8176|8177|8178|8179|8180|8181|8182|8183|8184|8185|8186|8187|8188|8189|8190|8191|8192 diff --git a/runs/frontier-phase-factorial-v0/analyze_qwen30_prefill_fidelity.py b/runs/frontier-phase-factorial-v0/analyze_qwen30_prefill_fidelity.py index a7260bf..a7e01c7 100644 --- a/runs/frontier-phase-factorial-v0/analyze_qwen30_prefill_fidelity.py +++ b/runs/frontier-phase-factorial-v0/analyze_qwen30_prefill_fidelity.py @@ -222,7 +222,11 @@ def parse_simulator(manifests: list[Path]) -> tuple[dict[str, Any], list[dict[st sources = [] for path in manifests: payload = load_json(path) - if payload["status"] not in {"complete", "partial_not_decision_bearing"}: + if payload["status"] not in { + "complete", + "frozen_before_real", + "partial_not_decision_bearing", + }: raise RuntimeError(f"simulator manifest has invalid status: {path}") sources.append({"path": str(path.resolve()), "sha256": sha256(path)}) result_by_name = { @@ -305,12 +309,21 @@ def compare(real: dict[str, Any], simulated: dict[str, Any]) -> dict[str, Any]: confusion = {"real_pass_sim_pass": 0, "real_pass_sim_fail": 0, "real_fail_sim_pass": 0, "real_fail_sim_fail": 0} + anchor_grid_coverage = { + "shared": 0, + "real_only": 0, + "simulator_only": 0, + } for name in names: real_anchors = {row["rate"]: row for row in real[name]["anchors"]} sim_anchors = {row["rate"]: row for row in simulated[name]["anchors"]} - if set(real_anchors) != set(sim_anchors): - raise RuntimeError(f"anchor grids differ for {name}") - for rate in real_anchors: + shared = set(real_anchors) & set(sim_anchors) + if not shared: + raise RuntimeError(f"anchor grids do not overlap for {name}") + anchor_grid_coverage["shared"] += len(shared) + anchor_grid_coverage["real_only"] += len(set(real_anchors) - shared) + anchor_grid_coverage["simulator_only"] += len(set(sim_anchors) - shared) + for rate in shared: real_pass = real_anchors[rate]["conservative_feasible"] sim_pass = sim_anchors[rate]["feasible"] key = f"real_{'pass' if real_pass else 'fail'}_sim_{'pass' if sim_pass else 'fail'}" @@ -328,6 +341,7 @@ def compare(real: dict[str, Any], simulated: dict[str, Any]) -> dict[str, Any]: "kendall": tau, "pairwise_non_tied": pairwise, "anchor_confusion": confusion, + "anchor_grid_coverage": anchor_grid_coverage, } @@ -356,7 +370,7 @@ def plot(path: Path, rows: list[dict[str, Any]], metrics: dict[str, Any]) -> Non x + width / 2, [row["simulator"] for row in rows], width, - label="Frontier profile-only", + label="Frontier simulator", ) axis.set_xticks(x, labels, fontsize=8) axis.set_ylabel("Max tested SLO-feasible request rate / GPU") @@ -374,7 +388,7 @@ def plot(path: Path, rows: list[dict[str, Any]], metrics: dict[str, Any]) -> Non ) axis.text( 0.01, - 0.98, + 0.82, annotation, transform=axis.transAxes, va="top", diff --git a/runs/frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py b/runs/frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py index ecd0e74..e7344fc 100644 --- a/runs/frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py +++ b/runs/frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py @@ -138,6 +138,7 @@ def validate_profile(paths: dict[str, Path]) -> dict[str, Any]: if int(row["num_tensor_parallel_workers"]) == tp and row["is_prefill"].lower() == "true" and row.get("is_true_mixed_batch", "").lower() != "true" + and int(float(row["batch_size"])) == 1 and int(float(row["total_tokens"])) == 2048 ] if len(exact) != 1: From a2cf361ffe5d3f9eacf0a20f4194e93627b36a43 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 10:17:21 +0800 Subject: [PATCH 100/148] Generalize Qwen30 fixed-shape real runner --- .../qwen30_prefill_client.py | 61 +++++++++++++++---- .../run_qwen30_prefill_real_config.sh | 31 ++++++---- .../test_phase_factorial.py | 2 + 3 files changed, 71 insertions(+), 23 deletions(-) diff --git a/runs/frontier-phase-factorial-v0/qwen30_prefill_client.py b/runs/frontier-phase-factorial-v0/qwen30_prefill_client.py index de40676..cff055e 100644 --- a/runs/frontier-phase-factorial-v0/qwen30_prefill_client.py +++ b/runs/frontier-phase-factorial-v0/qwen30_prefill_client.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Open-loop fixed-shape prefill-only workload for one real offered-load anchor.""" +"""Open-loop fixed-shape workload for one real offered-load anchor.""" from __future__ import annotations @@ -14,7 +14,6 @@ from pathlib import Path from typing import Any -TTFT_SLO_MS = 1256.0 TARGET_PASS_RATE = 0.95 @@ -27,6 +26,9 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--rate", type=float, required=True) parser.add_argument("--requests", type=int, default=64) parser.add_argument("--input-tokens", type=int, default=2048) + parser.add_argument("--output-tokens", type=int, default=1) + parser.add_argument("--ttft-slo-ms", type=float) + parser.add_argument("--tpot-slo-ms", type=float, default=150.0) parser.add_argument("--timeout-seconds", type=float, default=900.0) parser.add_argument("--output", type=Path, required=True) return parser.parse_args() @@ -40,6 +42,10 @@ def percentile(values: list[float], fraction: float) -> float | None: return ordered[index] +def ttft_slo_ms(input_tokens: int) -> float: + return 1000.0 + 1000.0 * input_tokens / 8000.0 + + def run_request( *, request_index: int, @@ -63,8 +69,8 @@ def run_request( body = { "model": args.served_model, "prompt": prompt_ids, - "min_tokens": 1, - "max_tokens": 1, + "min_tokens": args.output_tokens, + "max_tokens": args.output_tokens, "ignore_eos": True, "temperature": 0, "stream": True, @@ -85,6 +91,7 @@ def run_request( f"HTTP {response.status}: {response.read().decode(errors='replace')}" ) first_token_at = None + last_token_at = None streamed_tokens = 0 usage = None while True: @@ -105,16 +112,23 @@ def run_request( token_ids = choice.get("token_ids") or [] emitted += len(token_ids) if token_ids else int(bool(choice.get("text"))) if emitted: - first_token_at = first_token_at or time.perf_counter() + now = time.perf_counter() + first_token_at = first_token_at or now + last_token_at = now streamed_tokens += emitted finished = time.perf_counter() if first_token_at is None or usage is None: raise RuntimeError("missing streaming token or usage") prompt_tokens = int(usage["prompt_tokens"]) completion_tokens = int(usage["completion_tokens"]) - if prompt_tokens != args.input_tokens or completion_tokens != 1: + if prompt_tokens != args.input_tokens or completion_tokens != args.output_tokens: raise RuntimeError(f"usage mismatch: {prompt_tokens}+{completion_tokens}") ttft = (first_token_at - started) * 1000.0 + tpot = ( + (last_token_at - first_token_at) * 1000.0 / (completion_tokens - 1) + if completion_tokens > 1 + else None + ) record.update( { "success": True, @@ -122,8 +136,10 @@ def run_request( "completion_tokens": completion_tokens, "streamed_token_count": streamed_tokens, "ttft_ms": ttft, + "tpot_ms": tpot, "e2e_ms": (finished - started) * 1000.0, - "slo_pass": ttft <= TTFT_SLO_MS, + "slo_pass": ttft <= args.ttft_slo_ms + and (tpot is None or tpot <= args.tpot_slo_ms), } ) except Exception as error: # Failed requests remain in the SLO denominator. @@ -136,8 +152,18 @@ def run_request( def main() -> None: args = parse_args() - if args.rate <= 0 or args.requests <= 0 or args.input_tokens <= 0: - raise ValueError("rate, requests, and input tokens must be positive") + if min( + args.rate, + args.requests, + args.input_tokens, + args.output_tokens, + args.tpot_slo_ms, + ) <= 0: + raise ValueError("rate, requests, tokens, and SLO must be positive") + if args.ttft_slo_ms is None: + args.ttft_slo_ms = ttft_slo_ms(args.input_tokens) + if args.ttft_slo_ms <= 0: + raise ValueError("TTFT SLO must be positive") from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True) @@ -174,14 +200,19 @@ def main() -> None: completed = [row for row in requests if row["success"]] passed = sum(bool(row["slo_pass"]) for row in requests) ttfts = [float(row["ttft_ms"]) for row in completed] + tpots = [ + float(row["tpot_ms"]) + for row in completed + if row["tpot_ms"] is not None + ] pass_rate = passed / len(requests) payload = { - "schema": "qwen30-prefill-rate-anchor-v1", + "schema": "qwen30-fixed-rate-anchor-v2", "workload": { "offered_request_rate": args.rate, "request_count": args.requests, "input_tokens": args.input_tokens, - "output_tokens": 1, + "output_tokens": args.output_tokens, "prefix_caching": False, "arrival": "open_loop_uniform", "last_scheduled_arrival_s": (args.requests - 1) / args.rate, @@ -193,11 +224,17 @@ def main() -> None: "ttft_p50_ms": percentile(ttfts, 0.50), "ttft_p95_ms": percentile(ttfts, 0.95), "ttft_max_ms": max(ttfts) if ttfts else None, + "tpot_p50_ms": percentile(tpots, 0.50), + "tpot_p95_ms": percentile(tpots, 0.95), + "tpot_max_ms": max(tpots) if tpots else None, "admission_lag_max_ms": max( float(row["admission_lag_ms"]) for row in requests ), "slo": { - "ttft_threshold_ms": TTFT_SLO_MS, + "ttft_threshold_ms": args.ttft_slo_ms, + "tpot_threshold_ms": ( + args.tpot_slo_ms if args.output_tokens > 1 else None + ), "passed": passed, "pass_rate": pass_rate, "feasible": pass_rate >= TARGET_PASS_RATE, diff --git a/runs/frontier-phase-factorial-v0/run_qwen30_prefill_real_config.sh b/runs/frontier-phase-factorial-v0/run_qwen30_prefill_real_config.sh index a406d51..2861c21 100644 --- a/runs/frontier-phase-factorial-v0/run_qwen30_prefill_real_config.sh +++ b/runs/frontier-phase-factorial-v0/run_qwen30_prefill_real_config.sh @@ -6,6 +6,10 @@ OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}" TP="${TP:?TP is required}" MNS="${MNS:?MNS is required}" RATES="${RATES:-4 8 16 32 64}" +INPUT_TOKENS="${INPUT_TOKENS:-2048}" +OUTPUT_TOKENS="${OUTPUT_TOKENS:-1}" +TPOT_SLO_MS="${TPOT_SLO_MS:-150}" +WARMUP_SECONDS="${WARMUP_SECONDS:-2}" SERVER_PORT="${SERVER_PORT:?SERVER_PORT is required}" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" @@ -35,18 +39,19 @@ if [[ "${#GPU_IDS[@]}" -ne "${TP}" ]]; then fi read -r -a RATE_ARRAY <<< "${RATES}" -echo "QWEN30_PREFILL_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 rates=${RATES// /,} rounds=2 requests=64 shape=ISL2048_OSL1 arrivals=uniform prefix=off cuda_graph=runtime_default isolation=fresh_server_per_anchor output=${OUTPUT_ROOT}" +echo "QWEN30_FIXED_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 rates=${RATES// /,} rounds=2 requests=64 shape=ISL${INPUT_TOKENS}_OSL${OUTPUT_TOKENS} ttft_slo=1000+1000*ISL/8000ms tpot_slo=${TPOT_SLO_MS}ms warmup_seconds=${WARMUP_SECONDS} arrivals=uniform prefix=off cuda_graph=runtime_default isolation=fresh_server_per_anchor output=${OUTPUT_ROOT}" date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" sha256sum run_qwen30_prefill_real_config.sh qwen30_prefill_client.py \ > "${OUTPUT_ROOT}/provenance/source.sha256" -"${VENV_ROOT}/bin/python" - "${TP}" "${MNS}" "${RATES}" \ +"${VENV_ROOT}/bin/python" - "${TP}" "${MNS}" "${RATES}" "${INPUT_TOKENS}" "${OUTPUT_TOKENS}" "${TPOT_SLO_MS}" "${WARMUP_SECONDS}" \ > "${OUTPUT_ROOT}/provenance/contract.json" <<'PY' import importlib.metadata as metadata import json import platform import sys -tp, mns, rates = sys.argv[1:] +tp, mns, rates, input_tokens, output_tokens, tpot_slo_ms, warmup_seconds = sys.argv[1:] +input_tokens = int(input_tokens) print(json.dumps({ "python": platform.python_version(), "torch": metadata.version("torch"), @@ -57,10 +62,11 @@ print(json.dumps({ "rounds": 2, "requests_per_anchor": 64, "anchor_isolation": "fresh_server_per_rate_per_round", - "target_rate_warmup_requests": "min(32, max(4, ceil(rate * 2)))", - "input_tokens": 2048, - "output_tokens": 1, - "ttft_slo_ms": 1256.0, + "target_rate_warmup_requests": f"min(32, max(4, ceil(rate * {warmup_seconds})))", + "input_tokens": input_tokens, + "output_tokens": int(output_tokens), + "ttft_slo_ms": 1000.0 + 1000.0 * input_tokens / 8000.0, + "tpot_slo_ms": float(tpot_slo_ms) if int(output_tokens) > 1 else None, "target_pass_rate": 0.95, }, indent=2, sort_keys=True)) PY @@ -112,19 +118,22 @@ for ROUND in 1 2; do exit 1 fi - WARMUP_REQUESTS="$("${VENV_ROOT}/bin/python" - "${RATE}" <<'PY' + WARMUP_REQUESTS="$("${VENV_ROOT}/bin/python" - "${RATE}" "${WARMUP_SECONDS}" <<'PY' import math import sys -print(min(32, max(4, math.ceil(float(sys.argv[1]) * 2.0)))) +print(min(32, max(4, math.ceil(float(sys.argv[1]) * float(sys.argv[2]))))) PY )" "${VENV_ROOT}/bin/python" qwen30_prefill_client.py --port "${SERVER_PORT}" \ --served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" --rate "${RATE}" \ - --requests "${WARMUP_REQUESTS}" \ + --requests "${WARMUP_REQUESTS}" --input-tokens "${INPUT_TOKENS}" \ + --output-tokens "${OUTPUT_TOKENS}" --tpot-slo-ms "${TPOT_SLO_MS}" \ --output "${ROUND_ROOT}/results/warmup_${KEY}.json" "${VENV_ROOT}/bin/python" qwen30_prefill_client.py --port "${SERVER_PORT}" \ --served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" --rate "${RATE}" \ - --requests 64 --output "${ROUND_ROOT}/results/${KEY}.json" + --requests 64 --input-tokens "${INPUT_TOKENS}" \ + --output-tokens "${OUTPUT_TOKENS}" --tpot-slo-ms "${TPOT_SLO_MS}" \ + --output "${ROUND_ROOT}/results/${KEY}.json" cleanup done done diff --git a/runs/frontier-phase-factorial-v0/test_phase_factorial.py b/runs/frontier-phase-factorial-v0/test_phase_factorial.py index 0ceb170..07293da 100644 --- a/runs/frontier-phase-factorial-v0/test_phase_factorial.py +++ b/runs/frontier-phase-factorial-v0/test_phase_factorial.py @@ -25,6 +25,8 @@ def test_percentile() -> None: assert client.percentile([4.0, 1.0, 3.0, 2.0], 0.50) == 2.0 assert client.percentile([4.0, 1.0, 3.0, 2.0], 0.95) == 4.0 assert client.percentile([], 0.95) is None + assert client.ttft_slo_ms(512) == 1064.0 + assert client.ttft_slo_ms(2048) == 1256.0 def test_grid_and_trace(tmp_path: Path) -> None: From 202ae718b3538d9b8e5c608af46ca9ac042c947b Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 10:22:56 +0800 Subject: [PATCH 101/148] Freeze batch-aware Frontier ablation --- runs/frontier-fidelity-envelope-v1/.gitignore | 3 + .../results/a3/capacity.csv | 13 + .../results/a3/comparison.json | 5638 +++++++++++++++++ .../results/a3/frontier_surface_frozen.json | 1736 +++++ .../results/a3/qwen30-prefill-ranking.png | Bin 0 -> 107863 bytes ...run_frontier_qwen30_exact_trace_surface.py | 13 +- .../test_fidelity_envelope.py | 1 + 7 files changed, 7400 insertions(+), 4 deletions(-) create mode 100644 runs/frontier-fidelity-envelope-v1/results/a3/capacity.csv create mode 100644 runs/frontier-fidelity-envelope-v1/results/a3/comparison.json create mode 100644 runs/frontier-fidelity-envelope-v1/results/a3/frontier_surface_frozen.json create mode 100644 runs/frontier-fidelity-envelope-v1/results/a3/qwen30-prefill-ranking.png diff --git a/runs/frontier-fidelity-envelope-v1/.gitignore b/runs/frontier-fidelity-envelope-v1/.gitignore index 71cf748..acd9559 100644 --- a/runs/frontier-fidelity-envelope-v1/.gitignore +++ b/runs/frontier-fidelity-envelope-v1/.gitignore @@ -4,6 +4,9 @@ a2-measured-fix-smoke/ simulator-a1/ simulator-a2/ simulator-a3/ +simulator-f1/ +simulator-f2/ +simulator-t1/ fleet-state/ fleet-artifacts/ profiles/profile-v3-batch/ diff --git a/runs/frontier-fidelity-envelope-v1/results/a3/capacity.csv b/runs/frontier-fidelity-envelope-v1/results/a3/capacity.csv new file mode 100644 index 0000000..da354c9 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/results/a3/capacity.csv @@ -0,0 +1,13 @@ +config,tp,mns,real,simulator +tp1_mns8,1,8,7.0,8.0 +tp1_mns16,1,16,7.0,8.0 +tp1_mns32,1,32,7.0,8.0 +tp1_mns64,1,64,7.0,8.0 +tp2_mns8,2,8,7.0,8.0 +tp2_mns16,2,16,7.0,8.0 +tp2_mns32,2,32,7.0,8.0 +tp2_mns64,2,64,7.0,8.0 +tp4_mns8,4,8,8.0,8.0 +tp4_mns16,4,16,8.0,8.0 +tp4_mns32,4,32,8.0,8.0 +tp4_mns64,4,64,8.0,8.0 diff --git a/runs/frontier-fidelity-envelope-v1/results/a3/comparison.json b/runs/frontier-fidelity-envelope-v1/results/a3/comparison.json new file mode 100644 index 0000000..b011206 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/results/a3/comparison.json @@ -0,0 +1,5638 @@ +{ + "contract": { + "input_tokens": 2048, + "model": "Qwen3-30B-A3B", + "output_tokens": 1, + "prefix_caching": false, + "real_anchor_merge": "both_fresh_server_rounds_must_pass", + "target_pass_rate": 0.95, + "ttft_slo_ms": 1256.0 + }, + "metrics": { + "anchor_confusion": { + "real_fail_sim_fail": 24, + "real_fail_sim_pass": 8, + "real_pass_sim_fail": 0, + "real_pass_sim_pass": 28 + }, + "anchor_grid_coverage": { + "real_only": 36, + "shared": 60, + "simulator_only": 0 + }, + "config_order": [ + "tp1_mns8", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp2_mns8", + "tp2_mns16", + "tp2_mns32", + "tp2_mns64", + "tp4_mns8", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "kendall": { + "both_ties": 34, + "concordant": 0, + "discordant": 0, + "kendall_tau_b": null, + "real_only_ties": 0, + "simulator_only_ties": 32 + }, + "pairwise_non_tied": { + "all": { + "accuracy": 0.0, + "comparable": 32, + "correct": 0 + }, + "within_tp": {} + }, + "real_best_capacity_per_gpu": 8.0, + "real_top_set": [ + "tp4_mns8", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "simulator_best_capacity_per_gpu": 8.0, + "simulator_top_set": [ + "tp1_mns8", + "tp1_mns16", + "tp1_mns32", + "tp1_mns64", + "tp2_mns8", + "tp2_mns16", + "tp2_mns32", + "tp2_mns64", + "tp4_mns8", + "tp4_mns16", + "tp4_mns32", + "tp4_mns64" + ], + "top1_regret_best": 0.0, + "top1_regret_worst": 0.125, + "top_set_exact_match": false, + "top_set_overlap": [ + "tp4_mns16", + "tp4_mns32", + "tp4_mns64", + "tp4_mns8" + ] + }, + "objective": "maximum_tested_slo_feasible_offered_request_rate_per_gpu", + "real": { + "tp1_mns16": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 153.15452707000077, + 152.64549700077623 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r4p00.json", + "sha256": "d82e386ec3effdf57c148171311928298405da0e4f896584bf05db2485bb6bd4", + "summary": { + "admission_lag_max_ms": 0.11588609777390957, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 158.6234939750284, + "ttft_p50_ms": 151.05855697765946, + "ttft_p95_ms": 153.15452707000077 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r4p00.json", + "sha256": "1565be89131e3c8a9c352c6f29694ef2793734b0623a2db1cbea7e0b0e8a2bf6", + "summary": { + "admission_lag_max_ms": 0.11451996397227049, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 153.18561100866646, + "ttft_p50_ms": 151.18564700242132, + "ttft_p95_ms": 152.64549700077623 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 5.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 152.8308589477092, + 152.43642206769437 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r5p00.json", + "sha256": "e4447d94a326c541412b40157cfe283a6350680e3285b5d10fed6b8de5c0cd93", + "summary": { + "admission_lag_max_ms": 0.17109804321080446, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 154.35138496104628, + "ttft_p50_ms": 150.91424097772688, + "ttft_p95_ms": 152.8308589477092 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r5p00.json", + "sha256": "572c17e309c6c9f23c490979e7acb885475f142bc530ecfee88916b048259ea2", + "summary": { + "admission_lag_max_ms": 0.1321269664913416, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 157.11930498946458, + "ttft_p50_ms": 150.50161199178547, + "ttft_p95_ms": 152.43642206769437 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 6.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 153.56597595382482, + 152.16859406791627 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r6p00.json", + "sha256": "68c8b39154d46f1ceda17f47e8cbd84a8c8a9f088c5e82d8f0ac25a7180dc14c", + "summary": { + "admission_lag_max_ms": 0.11641765013337135, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 156.50876995641738, + "ttft_p50_ms": 150.41159105021507, + "ttft_p95_ms": 153.56597595382482 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r6p00.json", + "sha256": "a970a2cdf7034026a56af3ea4da59856fd588da35d66ef4f9e15481a892edcc9", + "summary": { + "admission_lag_max_ms": 0.11551892384886742, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 168.65782206878066, + "ttft_p50_ms": 150.03317291848361, + "ttft_p95_ms": 152.16859406791627 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 7.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 300.90478900820017, + 293.0445579113439 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r7p00.json", + "sha256": "1aa74c18adbff53b36ae8a6f07b5b0865f6b68fb3e7862a70b36f67c18f0bba0", + "summary": { + "admission_lag_max_ms": 0.10505830869078636, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 304.15459000505507, + "ttft_p50_ms": 263.44375393819064, + "ttft_p95_ms": 300.90478900820017 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r7p00.json", + "sha256": "e14bc9dba35d453862839eeed77262a1ed9b8394404ca90c60ec8cb26a1aa73b", + "summary": { + "admission_lag_max_ms": 0.10963028762489557, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 295.1545650139451, + "ttft_p50_ms": 243.97043196950108, + "ttft_p95_ms": 293.0445579113439 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 8.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1323.4397460473701, + 1315.5151740647852 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r8p00.json", + "sha256": "9aeee7453ffe5098d203e02b84f5009ed347c478b736e89297bd3695dd81d477", + "summary": { + "admission_lag_max_ms": 0.1251589274033904, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.90625, + "passed": 58, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1477.699184906669, + "ttft_p50_ms": 776.2576789828017, + "ttft_p95_ms": 1323.4397460473701 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r8p00.json", + "sha256": "7630f4901cf4ec4ea955738b119a1c32aba53f9ca1edb8819556fc60ae1fda09", + "summary": { + "admission_lag_max_ms": 0.11316593736410141, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.921875, + "passed": 59, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1404.566578916274, + "ttft_p50_ms": 761.7035870207474, + "ttft_p95_ms": 1315.5151740647852 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4892.253074911423, + 4893.758205929771 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r16p00.json", + "sha256": "d3a2a85cf06a4f080edc9a3809f05493ac55b0aa1d890806ffbc551c8eca14f8", + "summary": { + "admission_lag_max_ms": 0.39830803871154785, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4971.215195022523, + "ttft_p50_ms": 2742.7029330283403, + "ttft_p95_ms": 4892.253074911423 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r16p00.json", + "sha256": "67110284c38a9a60228f00b1d2b0de9477d6dec660f1c91eb4f39e1b79e272a5", + "summary": { + "admission_lag_max_ms": 0.11306500528007746, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4972.818040987477, + "ttft_p50_ms": 2742.512951954268, + "ttft_p95_ms": 4893.758205929771 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 6650.756464921869, + 6751.845097984187 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r32p00.json", + "sha256": "affc62b27382400497bb82e9da31ed9891b60155318fc4f250dca0704ccc5e02", + "summary": { + "admission_lag_max_ms": 0.11010200250893831, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6920.621327939443, + "ttft_p50_ms": 3650.839229929261, + "ttft_p95_ms": 6650.756464921869 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r32p00.json", + "sha256": "df0c8d703b023692814be41f47da4db4ce1b7eb0c3423a742c10884b6447583f", + "summary": { + "admission_lag_max_ms": 0.1288279891014099, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.15625, + "passed": 10, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6897.979240980931, + "ttft_p50_ms": 3730.34580796957, + "ttft_p95_ms": 6751.845097984187 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 7732.2855240199715, + 7744.056030060165 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round1/results/r64p00.json", + "sha256": "fbfad78129c843c85905c3f94f278f468a0a019c0cdf0b62140d0fea8f0aaffc", + "summary": { + "admission_lag_max_ms": 0.11090096086263657, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7836.245508980937, + "ttft_p50_ms": 4328.587644966319, + "ttft_p95_ms": 7732.2855240199715 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z/artifacts/artifacts/real-tp1-mns16-v1/round2/results/r64p00.json", + "sha256": "0f005c642327708e1ceb058f71f806b4d46ea666f45695605552e8f234521608", + "summary": { + "admission_lag_max_ms": 0.10862003546208143, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7847.871358972043, + "ttft_p50_ms": 4332.9262170009315, + "ttft_p95_ms": 7744.056030060165 + } + } + ] + } + ], + "capacity": 7.0, + "capacity_per_gpu": 7.0, + "mns": 16, + "name": "tp1_mns16", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z", + "tp": 1 + }, + "tp1_mns32": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 153.44141807872802, + 152.2580359596759 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r4p00.json", + "sha256": "211fa276e4a99a061126882085b9850549b453520ac3414dd2d332c89e52b431", + "summary": { + "admission_lag_max_ms": 0.09491504170000553, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 169.56262104213238, + "ttft_p50_ms": 150.8563900133595, + "ttft_p95_ms": 153.44141807872802 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r4p00.json", + "sha256": "17428902a5b20e9084f20792963f0620a1af06cdc1afb76ee99fd15dd905b89e", + "summary": { + "admission_lag_max_ms": 0.12033001985400915, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 153.99700601119548, + "ttft_p50_ms": 150.5822929320857, + "ttft_p95_ms": 152.2580359596759 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 5.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 152.05012692604214, + 152.00478909537196 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r5p00.json", + "sha256": "b0eb19ff7ab8a26e34e91f213a983e58689c63071285e27b3719bda0a765ff32", + "summary": { + "admission_lag_max_ms": 0.11265103239566088, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 153.48999004345387, + "ttft_p50_ms": 150.21017810795456, + "ttft_p95_ms": 152.05012692604214 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r5p00.json", + "sha256": "3b857dff42dd16ebee95356ceacd6b58ff7bd0a9b91f3d6208d489b84985697f", + "summary": { + "admission_lag_max_ms": 0.11149293277412653, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 153.33813393954188, + "ttft_p50_ms": 150.45341395307332, + "ttft_p95_ms": 152.00478909537196 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 6.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 152.74227899499238, + 152.82704099081457 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r6p00.json", + "sha256": "7615fa664a99f02f9f6d4f1fb0f2d6cba11842772c086634b939bc08cd0f2f4d", + "summary": { + "admission_lag_max_ms": 0.1096969936043024, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 154.2669020127505, + "ttft_p50_ms": 150.6725640501827, + "ttft_p95_ms": 152.74227899499238 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r6p00.json", + "sha256": "345e3cd3f8cd4d7c7a5e9343f937dd9b1b5928500133830f3046b743da6df64e", + "summary": { + "admission_lag_max_ms": 0.12192933354526758, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 153.12304301187396, + "ttft_p50_ms": 150.68741200957447, + "ttft_p95_ms": 152.82704099081457 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 7.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 302.5522669777274, + 294.5978350471705 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r7p00.json", + "sha256": "3f54a3e3f2279a5325a173ff714d600fb1a2cdfcfba1e8e5c980f7b21b99be59", + "summary": { + "admission_lag_max_ms": 0.11122168507426977, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 304.24335307907313, + "ttft_p50_ms": 262.4604320153594, + "ttft_p95_ms": 302.5522669777274 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r7p00.json", + "sha256": "0832169cdf1bef49c5006464c6cbcd50f4567e0ab90bec98d722f64f7b476763", + "summary": { + "admission_lag_max_ms": 0.1059951027855277, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 297.1431679325178, + "ttft_p50_ms": 249.51713997870684, + "ttft_p95_ms": 294.5978350471705 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 8.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1321.9368209829554, + 1318.7565059633926 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r8p00.json", + "sha256": "a0f5900ac9d2c47dcc7d7ac08d3f3d23e1d6f529fdd0200989afa819dd3df055", + "summary": { + "admission_lag_max_ms": 0.17414195463061333, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.90625, + "passed": 58, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1475.8616799954325, + "ttft_p50_ms": 775.7647309917957, + "ttft_p95_ms": 1321.9368209829554 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r8p00.json", + "sha256": "c0fc3c85095ddacc66849e6f58be0851abab6d4022ee4eb7120c8cb8ea9e5d2c", + "summary": { + "admission_lag_max_ms": 2.749277977272868, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.921875, + "passed": 59, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1407.0930189918727, + "ttft_p50_ms": 763.8335369993001, + "ttft_p95_ms": 1318.7565059633926 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4893.3289990527555, + 4893.17379205022 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r16p00.json", + "sha256": "4e9ccaf5384f622b1da7c58d3de803c86099dd9deae91e363aabccc14aeabc05", + "summary": { + "admission_lag_max_ms": 0.11893699411302805, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4972.85822511185, + "ttft_p50_ms": 2742.8449960425496, + "ttft_p95_ms": 4893.3289990527555 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r16p00.json", + "sha256": "c207e415186aed1148e16085145bcdfd189314e7b92f74ea72fb2e4972ef4eae", + "summary": { + "admission_lag_max_ms": 0.09541702456772327, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4971.2590809213, + "ttft_p50_ms": 2740.232800017111, + "ttft_p95_ms": 4893.17379205022 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 6653.632597066462, + 6647.155530983582 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r32p00.json", + "sha256": "780f563a2739d892761905031f82ca8bd0ac4f4eb485c454390dcb2bcdfc1b81", + "summary": { + "admission_lag_max_ms": 0.105265062302351, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6922.717701992951, + "ttft_p50_ms": 3652.452490059659, + "ttft_p95_ms": 6653.632597066462 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r32p00.json", + "sha256": "2a5eb8fb37f03e759de149d24f9b23c812a26afe72ffaaffa030688904634e43", + "summary": { + "admission_lag_max_ms": 0.10375294368714094, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6917.111590038985, + "ttft_p50_ms": 3649.0042679943144, + "ttft_p95_ms": 6647.155530983582 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 7747.191094094887, + 7839.257381972857 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round1/results/r64p00.json", + "sha256": "b94a39a91842ab39d545c473fc4f872ec79eb1888943c3e03ab03c18961a9105", + "summary": { + "admission_lag_max_ms": 0.6591080455109477, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7851.370444986969, + "ttft_p50_ms": 4335.578468977474, + "ttft_p95_ms": 7747.191094094887 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z/artifacts/artifacts/real-tp1-mns32-v1/round2/results/r64p00.json", + "sha256": "69ab47305c4bb14f3b4ed5bc0b18a37b8a5e7c1cdbee9232e6d2b9616ed10f84", + "summary": { + "admission_lag_max_ms": 0.10753795504570007, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.125, + "passed": 8, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7885.349316056818, + "ttft_p50_ms": 4009.1016669757664, + "ttft_p95_ms": 7839.257381972857 + } + } + ] + } + ], + "capacity": 7.0, + "capacity_per_gpu": 7.0, + "mns": 32, + "name": "tp1_mns32", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z", + "tp": 1 + }, + "tp1_mns64": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 156.76817402709275, + 152.9719429090619 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r4p00.json", + "sha256": "df798dd7a1cde3cfaf698286f44bbac2b0730a2c806f125268bdf7226e2bb47c", + "summary": { + "admission_lag_max_ms": 0.1147780567407608, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 162.9874319769442, + "ttft_p50_ms": 152.67050196416676, + "ttft_p95_ms": 156.76817402709275 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r4p00.json", + "sha256": "df80057224d69ce2a79f937aa45c2093d66d131765f3c7bfcc2289a9aaff3d6a", + "summary": { + "admission_lag_max_ms": 0.2131829969584942, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 153.917470946908, + "ttft_p50_ms": 150.96506499685347, + "ttft_p95_ms": 152.9719429090619 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 5.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 153.4973110537976, + 155.23677493911237 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r5p00.json", + "sha256": "e614ffaf879859407fc6420847f0140a81bda9a94fcc438fb6782c9019bf5378", + "summary": { + "admission_lag_max_ms": 0.1283780438825488, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 155.08639602921903, + "ttft_p50_ms": 152.1085740532726, + "ttft_p95_ms": 153.4973110537976 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r5p00.json", + "sha256": "13e825eb5ced3d0cb2b5dd68d9243fdf1cee8779b90e9aad77be04053be22be2", + "summary": { + "admission_lag_max_ms": 0.1683390000835061, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 155.6691420264542, + "ttft_p50_ms": 152.42570801638067, + "ttft_p95_ms": 155.23677493911237 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 6.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 152.53963600844145, + 152.85827894695103 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r6p00.json", + "sha256": "ac530f98f57ec452538ebe2311291d0a7c83bbd5554e82f53e78a2311802e411", + "summary": { + "admission_lag_max_ms": 0.13490929268300533, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 157.18225098680705, + "ttft_p50_ms": 150.29372205026448, + "ttft_p95_ms": 152.53963600844145 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r6p00.json", + "sha256": "2f1445ee0890b0972252d10180ef6f00cbad45ee73bd174f21b370cdf950b654", + "summary": { + "admission_lag_max_ms": 0.16514293383806944, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 158.88926689513028, + "ttft_p50_ms": 150.2534201135859, + "ttft_p95_ms": 152.85827894695103 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 7.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 346.96662798523903, + 295.1050230767578 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r7p00.json", + "sha256": "c4765794d442daf7eeb64a2f587792701ebe7659c7702a3bbbfc8900801fff0e", + "summary": { + "admission_lag_max_ms": 0.16206549480557442, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 354.881492909044, + "ttft_p50_ms": 310.15722593292594, + "ttft_p95_ms": 346.96662798523903 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r7p00.json", + "sha256": "30addb463839e23a7356a5cf6d4060ec1eebfa2b9564fa9500be5bcb7aecb537", + "summary": { + "admission_lag_max_ms": 0.10446517262607813, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 296.6241450048983, + "ttft_p50_ms": 246.06937903445214, + "ttft_p95_ms": 295.1050230767578 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 8.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1376.107804942876, + 1321.1636180058122 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r8p00.json", + "sha256": "89e2975f9c7319e477789ea77015541d5235efa61d90fb8bd5b0420a58f7f186", + "summary": { + "admission_lag_max_ms": 0.1416490413248539, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.890625, + "passed": 57, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1501.330335973762, + "ttft_p50_ms": 802.2130889585242, + "ttft_p95_ms": 1376.107804942876 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r8p00.json", + "sha256": "9ca6935f1320033c3ec4b72a35c5d9909ccd7b0fff2e4b2c92c6a26f52887df1", + "summary": { + "admission_lag_max_ms": 0.12422900181263685, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.921875, + "passed": 59, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1410.047525074333, + "ttft_p50_ms": 766.8001820566133, + "ttft_p95_ms": 1321.1636180058122 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4950.155022088438, + 4892.750842031091 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r16p00.json", + "sha256": "4fce2749435bf2e8c77999e11be0a788cd85f459a647760f3db34f1dbc0aee85", + "summary": { + "admission_lag_max_ms": 0.10520406067371368, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.171875, + "passed": 11, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 5031.207721913233, + "ttft_p50_ms": 2775.8131299633533, + "ttft_p95_ms": 4950.155022088438 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r16p00.json", + "sha256": "33ded0192d8986dbcf0a7f5ab6329a440994243d4e528d6c1b278bb6b92720fd", + "summary": { + "admission_lag_max_ms": 0.14005706179887056, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4971.664474112913, + "ttft_p50_ms": 2743.070787983015, + "ttft_p95_ms": 4892.750842031091 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 6750.775643973611, + 6754.7571890754625 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r32p00.json", + "sha256": "44030c4d3374cecebe67b886b428ba8014c87d941c00a178d6049bc57c809f46", + "summary": { + "admission_lag_max_ms": 0.7358170114457607, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.15625, + "passed": 10, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6899.517056066543, + "ttft_p50_ms": 3728.6347220651805, + "ttft_p95_ms": 6750.775643973611 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r32p00.json", + "sha256": "f397bad36552c6df5bf5bcee702cfa968ddf3e20b20e9976554ddfbe838ef3b2", + "summary": { + "admission_lag_max_ms": 0.7476879982277751, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.15625, + "passed": 10, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6901.606322033331, + "ttft_p50_ms": 3729.3128239689395, + "ttft_p95_ms": 6754.7571890754625 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 7800.616211956367, + 7748.703246936202 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round1/results/r64p00.json", + "sha256": "61eadc2af60386d6972d69fe7b083874e705864162073d54d1bde2bc0e6de7e7", + "summary": { + "admission_lag_max_ms": 0.11387700214982033, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7905.77321103774, + "ttft_p50_ms": 4366.223473916762, + "ttft_p95_ms": 7800.616211956367 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z/artifacts/artifacts/real-tp1-mns64-v1/round2/results/r64p00.json", + "sha256": "db02e9a814154a9259cbe3fefdf15700923458f7e1208596e861b69b6a6ddd41", + "summary": { + "admission_lag_max_ms": 0.14900392852723598, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7852.748095989227, + "ttft_p50_ms": 4337.09563605953, + "ttft_p95_ms": 7748.703246936202 + } + } + ] + } + ], + "capacity": 7.0, + "capacity_per_gpu": 7.0, + "mns": 64, + "name": "tp1_mns64", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z", + "tp": 1 + }, + "tp1_mns8": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 154.07429495826364, + 153.62840006127954 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r4p00.json", + "sha256": "23d5ab491b70c02f51c2f06dec0407edc0627819a33b9bcef1349d974a4e20fe", + "summary": { + "admission_lag_max_ms": 0.14362391084432602, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 171.46877304185182, + "ttft_p50_ms": 151.62539994344115, + "ttft_p95_ms": 154.07429495826364 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r4p00.json", + "sha256": "563aa230e0f5e420cf453dfd9b829d7df493919192d638cbe76f85eb77b6d2e7", + "summary": { + "admission_lag_max_ms": 0.11323392391204834, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 155.62100906390697, + "ttft_p50_ms": 151.69239102397114, + "ttft_p95_ms": 153.62840006127954 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 5.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 154.20477103907615, + 153.81942794192582 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r5p00.json", + "sha256": "6f2a04a5c11eed5826e2a6e1ce6e6f8d98504b31d0fec30d9440df2cdb62e56d", + "summary": { + "admission_lag_max_ms": 0.10794296395033598, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 155.37187200970948, + "ttft_p50_ms": 152.59731002151966, + "ttft_p95_ms": 154.20477103907615 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r5p00.json", + "sha256": "27a0bfa65a66e5cc77e10a5cf39a846f0fd1dd266c10926dae9db7a80c581d6e", + "summary": { + "admission_lag_max_ms": 0.1204309519380331, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 155.40366806089878, + "ttft_p50_ms": 152.0398510619998, + "ttft_p95_ms": 153.81942794192582 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 6.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 153.00951700191945, + 153.5524509381503 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r6p00.json", + "sha256": "4562fae05d510cd50f2c9d7479916d056c8ac4d04851d2388b98f8410192c23a", + "summary": { + "admission_lag_max_ms": 0.10347133502364159, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 156.3970319693908, + "ttft_p50_ms": 150.414744974114, + "ttft_p95_ms": 153.00951700191945 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r6p00.json", + "sha256": "bd13283e2807150396b278f46ff630ae9820895e0c054a9efcfd8fc213947d47", + "summary": { + "admission_lag_max_ms": 0.13600767124444246, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 159.63539096992463, + "ttft_p50_ms": 151.83425601571798, + "ttft_p95_ms": 153.5524509381503 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 7.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 336.8800369789824, + 329.65395506471395 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r7p00.json", + "sha256": "fd5afe52238d6ddb72bdf7f7b142fddbeb3f62e160e4a1b3a639b308717964e6", + "summary": { + "admission_lag_max_ms": 0.14346837997436523, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 341.1952300230041, + "ttft_p50_ms": 294.81438896618783, + "ttft_p95_ms": 336.8800369789824 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r7p00.json", + "sha256": "b0fdc1ef617313261c490010adeb2f4ee93bbb053f1db10734ed5a63fd37fa76", + "summary": { + "admission_lag_max_ms": 0.11047185398638248, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 334.06115300022066, + "ttft_p50_ms": 281.1476809438318, + "ttft_p95_ms": 329.65395506471395 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 8.0, + "round_feasible": [ + false, + true + ], + "round_ttft_p95_ms": [ + 1289.2397560644895, + 1254.356418037787 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r8p00.json", + "sha256": "8877ed44b90f421afd22ec110fe6337581c67cd973b44378bcb7528fc8624de3", + "summary": { + "admission_lag_max_ms": 0.09899691212922335, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.9375, + "passed": 60, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1454.3529660440981, + "ttft_p50_ms": 813.0663179326802, + "ttft_p95_ms": 1289.2397560644895 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r8p00.json", + "sha256": "7ab24f5ab33bf166589af7e2f5c2a274ceef6a64f0029a96ec1879b42332123d", + "summary": { + "admission_lag_max_ms": 0.11694896966218948, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 0.953125, + "passed": 61, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1347.0055809011683, + "ttft_p50_ms": 744.0344559727237, + "ttft_p95_ms": 1254.356418037787 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4903.538785991259, + 4900.28300601989 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r16p00.json", + "sha256": "363d9b43164b5451bc3dd668caa0ce141d15114087cf444e7fd79d12b7563fd6", + "summary": { + "admission_lag_max_ms": 0.11420203372836113, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4982.48829995282, + "ttft_p50_ms": 2749.9175920384005, + "ttft_p95_ms": 4903.538785991259 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r16p00.json", + "sha256": "8aec70dfd6b69f4903bccc02c5d5474165ed07d9dd010c860ff9b447429da950", + "summary": { + "admission_lag_max_ms": 0.19414699636399746, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4979.007771937177, + "ttft_p50_ms": 2749.115635990165, + "ttft_p95_ms": 4900.28300601989 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 6760.187636013143, + 6534.733187989332 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r32p00.json", + "sha256": "7684ac0d2d6f851218b88fb37db3a5b54d877a6f1ead3bb604a6cb4d9a61cf3f", + "summary": { + "admission_lag_max_ms": 0.43534801807254553, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.15625, + "passed": 10, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6907.940509961918, + "ttft_p50_ms": 3735.540736000985, + "ttft_p95_ms": 6760.187636013143 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r32p00.json", + "sha256": "947974a9ae04e170b8c1aad32ddbda8c1000f3902a9b572c6338a7d0d4f23146", + "summary": { + "admission_lag_max_ms": 1.221106038428843, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 6798.48151106853, + "ttft_p50_ms": 3595.78589303419, + "ttft_p95_ms": 6534.733187989332 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 7754.704045015387, + 7756.428897031583 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round1/results/r64p00.json", + "sha256": "abfc2a158087f0ad75b560dd420604a9f81359ffdf39f0ce8e2ea25b451e6904", + "summary": { + "admission_lag_max_ms": 0.13654399663209915, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7857.916672946885, + "ttft_p50_ms": 4341.527819051407, + "ttft_p95_ms": 7754.704045015387 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z/artifacts/artifacts/real-tp1-mns8-v1/round2/results/r64p00.json", + "sha256": "192c0c850a52d2b64780bc6321cbfd10faddcc0ff10741ddad283b7d789b9def", + "summary": { + "admission_lag_max_ms": 0.11000398080796003, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.109375, + "passed": 7, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 7859.458995983005, + "ttft_p50_ms": 4341.894763987511, + "ttft_p95_ms": 7756.428897031583 + } + } + ] + } + ], + "capacity": 7.0, + "capacity_per_gpu": 7.0, + "mns": 8, + "name": "tp1_mns8", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z", + "tp": 1 + }, + "tp2_mns16": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 99.1211449727416, + 101.08128399588168 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r4p00.json", + "sha256": "f149882da390114f2a2a140a3bd2ca6f15b6f7761591ea7b49b5a961c9d18df5", + "summary": { + "admission_lag_max_ms": 0.10085199028253555, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 101.28417506348342, + "ttft_p50_ms": 95.90165293775499, + "ttft_p95_ms": 99.1211449727416 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r4p00.json", + "sha256": "154dfaf2729001a8ef5b7430290f4ca906df1d4758e867d5b350489410f36d5c", + "summary": { + "admission_lag_max_ms": 0.11863000690937042, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 105.76263803523034, + "ttft_p50_ms": 96.27402503974736, + "ttft_p95_ms": 101.08128399588168 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 97.1850419882685, + 97.03529300168157 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r8p00.json", + "sha256": "9338d5cb994b0c12a6420f030ae2a9335a04986b4a81e31f4781ac838aeef2d4", + "summary": { + "admission_lag_max_ms": 0.12352492194622755, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 102.26434096693993, + "ttft_p50_ms": 95.07436107378453, + "ttft_p95_ms": 97.1850419882685 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r8p00.json", + "sha256": "a35b67970948976482ee835e595ee5631fa921607276d5ea7928460514f0d06b", + "summary": { + "admission_lag_max_ms": 0.11170096695423126, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 101.72453604172915, + "ttft_p50_ms": 94.03157397173345, + "ttft_p95_ms": 97.03529300168157 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 10.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 96.10817302018404, + 94.37024802900851 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r10p00.json", + "sha256": "f67e0f4f6b41861fcbcbee04b4efbf0dee46e273239ec94ef22d650ba6afa22a", + "summary": { + "admission_lag_max_ms": 0.10012194979935884, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 102.40214702207595, + "ttft_p50_ms": 94.17163301259279, + "ttft_p95_ms": 96.10817302018404 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r10p00.json", + "sha256": "5a64b52f0298e9ecda2fddb8230cc27ab4b9c1db01575f68b5331816b324d1d7", + "summary": { + "admission_lag_max_ms": 0.10465399827808142, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 101.07159405015409, + "ttft_p50_ms": 93.24954205658287, + "ttft_p95_ms": 94.37024802900851 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 12.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 447.0475659472868, + 454.8899739747867 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r12p00.json", + "sha256": "c6e27b9078281a54feeb1fab09616385a70de819791ce14f46b001bf540a80ab", + "summary": { + "admission_lag_max_ms": 0.12051803059875965, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 455.56937297806144, + "ttft_p50_ms": 280.09015799034387, + "ttft_p95_ms": 447.0475659472868 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r12p00.json", + "sha256": "861f9c4186c33e9ee6365d19c53e98c10615c5ee529a84e69f84dc292f53ac4f", + "summary": { + "admission_lag_max_ms": 0.11322635691612959, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 457.4526828946546, + "ttft_p50_ms": 290.1579710887745, + "ttft_p95_ms": 454.8899739747867 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 14.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 1000.5082850111648, + 1004.6314439969137 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r14p00.json", + "sha256": "2dcaf1087edd1f12fbfe0618a6ab636b22484756a5729919bc0e51de153f8f4a", + "summary": { + "admission_lag_max_ms": 0.1881921198219061, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1089.2393708927557, + "ttft_p50_ms": 611.2175739835948, + "ttft_p95_ms": 1000.5082850111648 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r14p00.json", + "sha256": "16937b31f72298c7a50ec3c7b9c1b9c798f305d83469b3d3c058efc4d53135da", + "summary": { + "admission_lag_max_ms": 0.10659568943083286, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1096.6680150013417, + "ttft_p50_ms": 617.3943240428343, + "ttft_p95_ms": 1004.6314439969137 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1458.4544260287657, + 1471.3564389385283 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r16p00.json", + "sha256": "5576a7fd06977f51f1c43bb73c3c474cc21a93cd288b422ec3d09db671b3afe7", + "summary": { + "admission_lag_max_ms": 0.135264010168612, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.765625, + "passed": 49, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1583.2410319708288, + "ttft_p50_ms": 904.5663280412555, + "ttft_p95_ms": 1458.4544260287657 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r16p00.json", + "sha256": "2ccf16426046c412829442f72932268d5b27ce051a4312d135d32b240b48ec7c", + "summary": { + "admission_lag_max_ms": 0.11741300113499165, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.78125, + "passed": 50, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1537.9131329245865, + "ttft_p50_ms": 896.1611289996654, + "ttft_p95_ms": 1471.3564389385283 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 3348.027400090359, + 3348.575382027775 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r32p00.json", + "sha256": "d258760d15f1fdf2adefca72f7a3505c0b839036799dcb074450f7f526f76718", + "summary": { + "admission_lag_max_ms": 0.18610397819429636, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3442.3487790627405, + "ttft_p50_ms": 1801.6119640087709, + "ttft_p95_ms": 3348.027400090359 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r32p00.json", + "sha256": "edf5cccc1400639a94b5430897a04f20d2214bb1ffee7f4bfa62514150179a6a", + "summary": { + "admission_lag_max_ms": 0.1144439447671175, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3443.0311269825324, + "ttft_p50_ms": 1801.435066969134, + "ttft_p95_ms": 3348.575382027775 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4286.911493982188, + 4287.898648995906 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round1/results/r64p00.json", + "sha256": "2a6303efb826637c3c6448d6f766f7de4fc4d7b63db163a48f72c20df68394f4", + "summary": { + "admission_lag_max_ms": 0.10997895151376724, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4340.311317006126, + "ttft_p50_ms": 2396.4726959820837, + "ttft_p95_ms": 4286.911493982188 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z/artifacts/artifacts/real-tp2-mns16-v1/round2/results/r64p00.json", + "sha256": "6fafe3bf206583b8689be1b2b57d6b92cbd1ba54f7fb230ca05f6c68b3ab9757", + "summary": { + "admission_lag_max_ms": 0.12669397983700037, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4340.538130956702, + "ttft_p50_ms": 2394.112486974336, + "ttft_p95_ms": 4287.898648995906 + } + } + ] + } + ], + "capacity": 14.0, + "capacity_per_gpu": 7.0, + "mns": 16, + "name": "tp2_mns16", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z", + "tp": 2 + }, + "tp2_mns32": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 97.2925309324637, + 96.13083000294864 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r4p00.json", + "sha256": "812f8a108a9390fb759b46e5bd41e3ddad66bc144d47bf4024d834e42b47c47e", + "summary": { + "admission_lag_max_ms": 0.11386198457330465, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 100.76773096807301, + "ttft_p50_ms": 95.72782705072314, + "ttft_p95_ms": 97.2925309324637 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r4p00.json", + "sha256": "70f922eb7651d46d81d7d394b8453c5b778f921069cf48fb2ae2677afa94a842", + "summary": { + "admission_lag_max_ms": 0.30313804745674133, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 129.06646099872887, + "ttft_p50_ms": 94.53632694203407, + "ttft_p95_ms": 96.13083000294864 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 95.42673197574914, + 98.94769196398556 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r8p00.json", + "sha256": "24097d38b0ffb45f6cdbd93f0aad04dce3b7da41ff45e948eb3a794902b6547f", + "summary": { + "admission_lag_max_ms": 0.11571904178708792, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 100.04934400785714, + "ttft_p50_ms": 94.34181300457567, + "ttft_p95_ms": 95.42673197574914 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r8p00.json", + "sha256": "7251f076d34ba9484b08fa3fc4b898d2ba8d1199cb1eaeaaffc19930f8c889b9", + "summary": { + "admission_lag_max_ms": 0.1177559606730938, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 100.93720292206854, + "ttft_p50_ms": 94.77824496570975, + "ttft_p95_ms": 98.94769196398556 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 10.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 96.28185909241438, + 95.9081610199064 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r10p00.json", + "sha256": "85796457715a25c15fc909f6def32776fbd49956eb7890df1c4abcef8c5afe78", + "summary": { + "admission_lag_max_ms": 0.14786410611122847, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 102.46025596279651, + "ttft_p50_ms": 93.54631893802434, + "ttft_p95_ms": 96.28185909241438 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r10p00.json", + "sha256": "0bf4d4d006bf0b5bd8b989ceeeb4bacb0e3ee1b8ea66f30b3deb120a14871cf0", + "summary": { + "admission_lag_max_ms": 0.11162098962813616, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 100.54603894241154, + "ttft_p50_ms": 93.77408598084003, + "ttft_p95_ms": 95.9081610199064 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 12.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 439.0145930228755, + 429.9719400005415 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r12p00.json", + "sha256": "68de6e36bd7c5624cab107fe5cb21bb817711691e545439a958c926a27e3fe29", + "summary": { + "admission_lag_max_ms": 0.10475562885403633, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 445.90794993564487, + "ttft_p50_ms": 270.3119309153408, + "ttft_p95_ms": 439.0145930228755 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r12p00.json", + "sha256": "d8d3a44a07b46f1673817ece2948fd97c30f41bd451d93a2fa02edf074536c63", + "summary": { + "admission_lag_max_ms": 0.4323029424995184, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 434.6071509644389, + "ttft_p50_ms": 265.2625450864434, + "ttft_p95_ms": 429.9719400005415 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 14.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 1014.2896320903674, + 999.8802930349484 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r14p00.json", + "sha256": "10196c2f9a9b69c2e47f64d06bf5ee6863af4cd2b612301622f0e304949a942a", + "summary": { + "admission_lag_max_ms": 0.09005970787256956, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1105.1907669752836, + "ttft_p50_ms": 621.6445650206879, + "ttft_p95_ms": 1014.2896320903674 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r14p00.json", + "sha256": "c10cab85588a3563bed36fc4480462635d34d240938beaa0e5034f13186578a6", + "summary": { + "admission_lag_max_ms": 0.11327012907713652, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1090.6395060010254, + "ttft_p50_ms": 623.5586369875818, + "ttft_p95_ms": 999.8802930349484 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1488.3801550604403, + 1484.6198010491207 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r16p00.json", + "sha256": "ef5046c49c32178855bed6a0488ea21e8da22f786ddaabc503bea2d1025bb2f3", + "summary": { + "admission_lag_max_ms": 0.7892940193414688, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.765625, + "passed": 49, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1613.616122980602, + "ttft_p50_ms": 905.2404230460525, + "ttft_p95_ms": 1488.3801550604403 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r16p00.json", + "sha256": "de02c1d49414cffe7e20023abf4d502a4efc6d2e01fcb3e49e928081aaef77b6", + "summary": { + "admission_lag_max_ms": 0.10548799764364958, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.765625, + "passed": 49, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1550.9842830942944, + "ttft_p50_ms": 904.9503420246765, + "ttft_p95_ms": 1484.6198010491207 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 3391.811708919704, + 3362.9186518955976 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r32p00.json", + "sha256": "876c5444d17c4ce81eca907ae17d5cf7062ffa67fa266b491580e52f712614ec", + "summary": { + "admission_lag_max_ms": 0.12557301670312881, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3486.074637970887, + "ttft_p50_ms": 1840.204154024832, + "ttft_p95_ms": 3391.811708919704 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r32p00.json", + "sha256": "02b4cd93ce1f9816e822022fb608ba8d4c89265f673f6d27c8fa3d4f1523ffe0", + "summary": { + "admission_lag_max_ms": 0.11014204937964678, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3456.965696066618, + "ttft_p50_ms": 1812.343467026949, + "ttft_p95_ms": 3362.9186518955976 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4306.351027917117, + 4302.602957002819 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round1/results/r64p00.json", + "sha256": "13ba282f179b2d6ce13a0aa2fd04eb29f82e579b7995e3406cb0afd69a2d5ee2", + "summary": { + "admission_lag_max_ms": 0.3526840591803193, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4359.505696920678, + "ttft_p50_ms": 2411.1130589153618, + "ttft_p95_ms": 4306.351027917117 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z/artifacts/artifacts/real-tp2-mns32-v1/round2/results/r64p00.json", + "sha256": "767622d62f0757ef2d8c7651ea470bfa2e62edad0ed4e75f66f84a0b722e35da", + "summary": { + "admission_lag_max_ms": 0.10626309085637331, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4355.409953044727, + "ttft_p50_ms": 2409.5974090741947, + "ttft_p95_ms": 4302.602957002819 + } + } + ] + } + ], + "capacity": 14.0, + "capacity_per_gpu": 7.0, + "mns": 32, + "name": "tp2_mns32", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z", + "tp": 2 + }, + "tp2_mns64": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 96.54812794178724, + 95.08875198662281 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r4p00.json", + "sha256": "6e6a45d0c1010094c3395969815fd8cf9c35b919db202d772fb514744f981ebc", + "summary": { + "admission_lag_max_ms": 0.11306197848170996, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 99.27749610505998, + "ttft_p50_ms": 93.01548008807003, + "ttft_p95_ms": 96.54812794178724 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r4p00.json", + "sha256": "58aa71d694704cfea15c37cb29420dbac1dc505045c29a731e8be904846cca5f", + "summary": { + "admission_lag_max_ms": 0.11853000614792109, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 98.74943597242236, + "ttft_p50_ms": 93.57558900956064, + "ttft_p95_ms": 95.08875198662281 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 94.24941800534725, + 96.21043398510665 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r8p00.json", + "sha256": "311350af236cb0306aa635f05f8999b66bf5080bad7cc1f6703fffe0be31aee6", + "summary": { + "admission_lag_max_ms": 0.19771000370383263, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 100.52211803849787, + "ttft_p50_ms": 92.4453770276159, + "ttft_p95_ms": 94.24941800534725 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r8p00.json", + "sha256": "60fe989a36a526e76f6f5c98e217b9feaf081bc680b16b9d53650bf085047c06", + "summary": { + "admission_lag_max_ms": 0.3933720290660858, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 102.97230398282409, + "ttft_p50_ms": 93.26327103190124, + "ttft_p95_ms": 96.21043398510665 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 10.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 97.98661107197404, + 92.9473100695759 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r10p00.json", + "sha256": "d0a04444a0e41bf76f71ac6ea3815bfbe76f7a51f8ddf669976bc3a4896dd0c1", + "summary": { + "admission_lag_max_ms": 0.10270101483911276, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 103.49742788821459, + "ttft_p50_ms": 94.55623896792531, + "ttft_p95_ms": 97.98661107197404 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r10p00.json", + "sha256": "f330f26f45f6039c48150a0b6b146ca3dbb9ea8b2dff086a9efd5938ab981eb7", + "summary": { + "admission_lag_max_ms": 0.11239596642553806, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 97.4133670097217, + "ttft_p50_ms": 91.2688790122047, + "ttft_p95_ms": 92.9473100695759 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 12.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 260.68170997314155, + 319.04888805001974 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r12p00.json", + "sha256": "fed5eacf18018402b0c2675b8c5595f92693a9bddec30f354d989098f9fe4965", + "summary": { + "admission_lag_max_ms": 0.11784560047090054, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 345.19017208367586, + "ttft_p50_ms": 211.41935500781983, + "ttft_p95_ms": 260.68170997314155 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r12p00.json", + "sha256": "f2be27cbeff3e5c1a95050418f2dc3c712a4648cada8cdb3d36382cfffc9593a", + "summary": { + "admission_lag_max_ms": 0.12930459342896938, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 326.86293800361454, + "ttft_p50_ms": 229.8808820778504, + "ttft_p95_ms": 319.04888805001974 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 14.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 895.7287209341303, + 912.1733580250293 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r14p00.json", + "sha256": "c2e157808e0308cc443f3abdf5114898cef66c435688d24321f80bef2d966208", + "summary": { + "admission_lag_max_ms": 0.14902697876095772, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 962.3544370988384, + "ttft_p50_ms": 576.2168710352853, + "ttft_p95_ms": 895.7287209341303 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r14p00.json", + "sha256": "749ef7d31bf8782d42719b75ee1a2e4a443f097510af721d02cf53df191bc66e", + "summary": { + "admission_lag_max_ms": 0.42896345257759094, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 986.0872740391642, + "ttft_p50_ms": 569.7114520007744, + "ttft_p95_ms": 912.1733580250293 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1364.6171110449359, + 1366.6857269126922 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r16p00.json", + "sha256": "7545ea9210348019de88ede5d2cba7527913534274deee034fa6811c173a6e72", + "summary": { + "admission_lag_max_ms": 0.12612901628017426, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.859375, + "passed": 55, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1435.8308439841494, + "ttft_p50_ms": 848.9949750946835, + "ttft_p95_ms": 1364.6171110449359 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r16p00.json", + "sha256": "d75e27137f6a7811126d8c957bb5a081944fc1bb7643852cdd785111773f6010", + "summary": { + "admission_lag_max_ms": 0.1258660340681672, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.859375, + "passed": 55, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1436.2916270038113, + "ttft_p50_ms": 846.2129919789732, + "ttft_p95_ms": 1366.6857269126922 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 3243.9196659252048, + 3214.524411014281 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r32p00.json", + "sha256": "366ebddc167338ed096f373de5667740721422c13d2db1f887ed09186afafd19", + "summary": { + "admission_lag_max_ms": 0.09669095743447542, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3337.068362045102, + "ttft_p50_ms": 1751.3202880509198, + "ttft_p95_ms": 3243.9196659252048 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r32p00.json", + "sha256": "01b8f5081e36a64d5776285c8c5f423b53b1ad684c4b8f95a8caa5d997c6bf09", + "summary": { + "admission_lag_max_ms": 0.8310690755024552, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.328125, + "passed": 21, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3277.297410997562, + "ttft_p50_ms": 1790.5721209244803, + "ttft_p95_ms": 3214.524411014281 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4319.725521025248, + 4367.031275993213 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round1/results/r64p00.json", + "sha256": "92202709fd6bbd558ff6f1986df213848254110c4274447a7bd5879153290d46", + "summary": { + "admission_lag_max_ms": 0.10392803233116865, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4360.145599930547, + "ttft_p50_ms": 2403.5812759539112, + "ttft_p95_ms": 4319.725521025248 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z/artifacts/artifacts/real-tp2-mns64-v1/round2/results/r64p00.json", + "sha256": "9660d15d31bc70c5550ed29477f563313d4d81baf0e561126aebd3b7d097cfa3", + "summary": { + "admission_lag_max_ms": 0.17276196740567684, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.25, + "passed": 16, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4414.442001027055, + "ttft_p50_ms": 2260.2788450894877, + "ttft_p95_ms": 4367.031275993213 + } + } + ] + } + ], + "capacity": 14.0, + "capacity_per_gpu": 7.0, + "mns": 64, + "name": "tp2_mns64", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z", + "tp": 2 + }, + "tp2_mns8": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 97.17482794076204, + 98.03147299680859 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r4p00.json", + "sha256": "0e315f6d9e0087ad553e874b0c16c2e6de79eb6eca22eb118dc5deca52e6efe3", + "summary": { + "admission_lag_max_ms": 0.11708191595971584, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 101.92840604577214, + "ttft_p50_ms": 95.13642196543515, + "ttft_p95_ms": 97.17482794076204 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r4p00.json", + "sha256": "baa394a24d2a3db40a3a7ae6dae0c5ce49ff05499a1e17eaa0bca4b479ae33bf", + "summary": { + "admission_lag_max_ms": 0.10946101974695921, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 100.52333807107061, + "ttft_p50_ms": 93.19468098692596, + "ttft_p95_ms": 98.03147299680859 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 97.67980605829507, + 93.80627400241792 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r8p00.json", + "sha256": "303df27c6d4604d5af65f9056acacb7c4373fb6eb7edb6e5d42689eeaa384aaa", + "summary": { + "admission_lag_max_ms": 0.11111306957900524, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 113.2329449756071, + "ttft_p50_ms": 93.05417106952518, + "ttft_p95_ms": 97.67980605829507 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r8p00.json", + "sha256": "cdbb1bca17b24b67e18ef3dd4997494d6bd2a887b3b5d1264fbf411e688c53bc", + "summary": { + "admission_lag_max_ms": 0.12463307939469814, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 101.12245299387723, + "ttft_p50_ms": 92.26030297577381, + "ttft_p95_ms": 93.80627400241792 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 10.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 94.45723297540098, + 93.57301203999668 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r10p00.json", + "sha256": "fe17ef2417319307d651bc10b01a618234d61bab797ad87a7332cab20d78ecb2", + "summary": { + "admission_lag_max_ms": 0.3267280990257859, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 99.28593400400132, + "ttft_p50_ms": 92.22474799025804, + "ttft_p95_ms": 94.45723297540098 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r10p00.json", + "sha256": "550f693708a34624a82d77afd4da0ab4bc01e09f42242a3a79e4532897681a51", + "summary": { + "admission_lag_max_ms": 0.11238199658691883, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 97.77756198309362, + "ttft_p50_ms": 92.07128698471934, + "ttft_p95_ms": 93.57301203999668 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 12.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 320.49866404850036, + 336.9630330707878 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r12p00.json", + "sha256": "edab8169453151de732923c076b8f7dca4082ee47ccd2597e65e7ba68516bc2e", + "summary": { + "admission_lag_max_ms": 0.09914662223309278, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 327.930542989634, + "ttft_p50_ms": 231.4880370395258, + "ttft_p95_ms": 320.49866404850036 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r12p00.json", + "sha256": "a6dde4363d362ca10254409e3dc734d0f0e640e63156426f3f4fd7115db49e16", + "summary": { + "admission_lag_max_ms": 0.17016706988215446, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 428.6657409975305, + "ttft_p50_ms": 237.09688894450665, + "ttft_p95_ms": 336.9630330707878 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 14.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 911.7195709841326, + 910.7652489328757 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r14p00.json", + "sha256": "7eba81ee5c85c457a056f9aef28abd94d375fdaa116929cfe0a0c282a0738bd2", + "summary": { + "admission_lag_max_ms": 0.13163150288164616, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 999.2953530745581, + "ttft_p50_ms": 580.023527960293, + "ttft_p95_ms": 911.7195709841326 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r14p00.json", + "sha256": "6a113d3d876ff01a9c1101e8b75efa2fc71f96683d6bb430e26d5976426b5d23", + "summary": { + "admission_lag_max_ms": 0.11634768452495337, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 988.8870939612389, + "ttft_p50_ms": 578.9327350212261, + "ttft_p95_ms": 910.7652489328757 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 16.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 1389.8832038976252, + 1365.0342189939693 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r16p00.json", + "sha256": "d677251ee1a941c52469052a239c2e13be0a837285069aa8182a16e196445e48", + "summary": { + "admission_lag_max_ms": 0.11070503387600183, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.828125, + "passed": 53, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1460.5093500576913, + "ttft_p50_ms": 864.3530959961936, + "ttft_p95_ms": 1389.8832038976252 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r16p00.json", + "sha256": "f5177f53aa0f87a4e7291f470978d86ad8bc46bd1a8362b79f48cd128646dc10", + "summary": { + "admission_lag_max_ms": 0.3672370221465826, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.828125, + "passed": 53, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1491.050804965198, + "ttft_p50_ms": 841.7225609300658, + "ttft_p95_ms": 1365.0342189939693 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 32.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 3258.010295103304, + 3258.664960041642 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r32p00.json", + "sha256": "02a6ac2b9c2b9ab97854d6fcc13b64bbb36a079e4afdae84231e4cb187fac4ef", + "summary": { + "admission_lag_max_ms": 0.1296499976888299, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3352.278529899195, + "ttft_p50_ms": 1755.0864539807662, + "ttft_p95_ms": 3258.010295103304 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r32p00.json", + "sha256": "9ca2f44fc7e3cb3d38ada548af858903b31bbee5cd50f1723c29255ebd4224e9", + "summary": { + "admission_lag_max_ms": 1.1343059595674276, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 3352.7101600775495, + "ttft_p50_ms": 1755.3453800501302, + "ttft_p95_ms": 3258.664960041642 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 4190.255103982054, + 4189.1050330596045 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round1/results/r64p00.json", + "sha256": "53908d3a254e19691839831b4337014b09591ced9189c2a9b29e0d92e80fcb56", + "summary": { + "admission_lag_max_ms": 0.11591101065278053, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4242.029857938178, + "ttft_p50_ms": 2340.5529640149325, + "ttft_p95_ms": 4190.255103982054 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z/artifacts/artifacts/real-tp2-mns8-v1/round2/results/r64p00.json", + "sha256": "1c8c85c873aa96c01dc0200f848474a53c1173ef72f3828e8c23aeae70e92b11", + "summary": { + "admission_lag_max_ms": 0.284243025816977, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.234375, + "passed": 15, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 4240.770195960067, + "ttft_p50_ms": 2339.3583690049127, + "ttft_p95_ms": 4189.1050330596045 + } + } + ] + } + ], + "capacity": 14.0, + "capacity_per_gpu": 7.0, + "mns": 8, + "name": "tp2_mns8", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z", + "tp": 2 + }, + "tp4_mns16": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 74.66826005838811, + 66.5705680148676 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r4p00.json", + "sha256": "fb354fc8a097caa21844d4ea5515b98b03459f400816e5e5590a6cda81cf4815", + "summary": { + "admission_lag_max_ms": 0.1035250024870038, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 93.38432096410543, + "ttft_p50_ms": 65.75745099689811, + "ttft_p95_ms": 74.66826005838811 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r4p00.json", + "sha256": "ac8e8412f79a8a32ebf085445a94aa548dd2d646fce0b9f56b4378081dcc8960", + "summary": { + "admission_lag_max_ms": 0.12108997907489538, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 79.89856600761414, + "ttft_p50_ms": 62.386377016082406, + "ttft_p95_ms": 66.5705680148676 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 70.44799195136875, + 72.29184894822538 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r8p00.json", + "sha256": "05ebbd68bbf24e9c8b25684474411c7db122b92701cc1617f9d786cf19592751", + "summary": { + "admission_lag_max_ms": 0.1825150102376938, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 96.92095802165568, + "ttft_p50_ms": 60.598687967285514, + "ttft_p95_ms": 70.44799195136875 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r8p00.json", + "sha256": "d24f7daaf41073cc8aa030b74c06203783684c3e556770366883c37b826406f2", + "summary": { + "admission_lag_max_ms": 0.10628998279571533, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 75.13270596973598, + "ttft_p50_ms": 60.161596979014575, + "ttft_p95_ms": 72.29184894822538 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 16.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 66.02151494007558, + 107.58775402791798 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r16p00.json", + "sha256": "a29b2128983af630e1da8f970cbda802849036bc522eb506821bf2fe1a34172f", + "summary": { + "admission_lag_max_ms": 0.10468997061252594, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 72.93136999942362, + "ttft_p50_ms": 59.5597909996286, + "ttft_p95_ms": 66.02151494007558 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r16p00.json", + "sha256": "275ae35651cbe2200d16c9b06a68568efda527564065255a10dee6fe93346010", + "summary": { + "admission_lag_max_ms": 0.3144120564684272, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 119.78317995090038, + "ttft_p50_ms": 59.12985489703715, + "ttft_p95_ms": 107.58775402791798 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 20.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 193.8455159543082, + 187.52167106140405 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r20p00.json", + "sha256": "4cb485f30b81c004bab379e6522d94b811c0bdc0be6d8e42c9d8ae842d9600dc", + "summary": { + "admission_lag_max_ms": 0.10407401714473963, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 197.7908629924059, + "ttft_p50_ms": 142.73560303263366, + "ttft_p95_ms": 193.8455159543082 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r20p00.json", + "sha256": "63aa9dc03a08b781c5d6e8805f2b61698c75e71673608ecd38bd1132c62e6987", + "summary": { + "admission_lag_max_ms": 0.09386602323502302, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 190.29632199089974, + "ttft_p50_ms": 136.1723990412429, + "ttft_p95_ms": 187.52167106140405 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 24.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 568.2198900030926, + 573.4539540717378 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r24p00.json", + "sha256": "5e4d5b44850090d07fe1d3abc44eddea3bfdf2b682f59b12a45a866621e38f98", + "summary": { + "admission_lag_max_ms": 0.7511250441893935, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 613.2891150191426, + "ttft_p50_ms": 375.27791305910796, + "ttft_p95_ms": 568.2198900030926 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r24p00.json", + "sha256": "239e7789aed2e9b2bcf1d127215a6b122071eb856be2db64ccc184fda153aa48", + "summary": { + "admission_lag_max_ms": 0.11180061846971512, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 614.9205249967054, + "ttft_p50_ms": 375.5133739905432, + "ttft_p95_ms": 573.4539540717378 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 28.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 860.0242879474536, + 886.1482660286129 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r28p00.json", + "sha256": "8b249b52779e5b3cd9028aaee30ee73ef34b0b0a64dcd75b9aad57266c29fbc2", + "summary": { + "admission_lag_max_ms": 0.11425174307078123, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 930.8854410191998, + "ttft_p50_ms": 534.7980660153553, + "ttft_p95_ms": 860.0242879474536 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r28p00.json", + "sha256": "c9b28c8f3187542a343ecf1067783569ec2abfa64631d6827a94ba6876aa7b63", + "summary": { + "admission_lag_max_ms": 0.312068616040051, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 922.4315020255744, + "ttft_p50_ms": 549.7413279954344, + "ttft_p95_ms": 886.1482660286129 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 32.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 1147.1082780044526, + 1137.236590962857 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r32p00.json", + "sha256": "d27ff9f940a6cd7fba339d9a176de39cf4b5bd164dcc500f2af36bcb07cdef30", + "summary": { + "admission_lag_max_ms": 0.16146607231348753, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1201.626160996966, + "ttft_p50_ms": 696.4708579471335, + "ttft_p95_ms": 1147.1082780044526 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r32p00.json", + "sha256": "093044e87fffdaa222e5bad3807cbf4900aa64891d5b4ba27f187c7a411252d7", + "summary": { + "admission_lag_max_ms": 0.10430300608277321, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1201.045956928283, + "ttft_p50_ms": 684.3201500596479, + "ttft_p95_ms": 1137.236590962857 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 2064.763988018967, + 2039.6780440350994 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round1/results/r64p00.json", + "sha256": "fad7bdd80d5ca603790a4e5d19dadd4f494540b106ce583e3e8c1c05d2374a58", + "summary": { + "admission_lag_max_ms": 6.394341005943716, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.53125, + "passed": 34, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2144.80376499705, + "ttft_p50_ms": 1158.5425049997866, + "ttft_p95_ms": 2064.763988018967 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z/artifacts/artifacts/real-tp4-mns16-v1/round2/results/r64p00.json", + "sha256": "24fdb945706dbbdd29916e0f08e0eff807334db212fded94c729da2e1b46efd3", + "summary": { + "admission_lag_max_ms": 0.10294502135366201, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.546875, + "passed": 35, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2125.4275270039216, + "ttft_p50_ms": 1131.1213460285217, + "ttft_p95_ms": 2039.6780440350994 + } + } + ] + } + ], + "capacity": 32.0, + "capacity_per_gpu": 8.0, + "mns": 16, + "name": "tp4_mns16", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z", + "tp": 4 + }, + "tp4_mns32": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 67.80304794665426, + 64.59337309934199 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r4p00.json", + "sha256": "2e82a30f02c2702668e59661930431cf82323746a111d42bf38b7fdc0d083b3d", + "summary": { + "admission_lag_max_ms": 0.1332589890807867, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 72.74050498381257, + "ttft_p50_ms": 64.10286005120724, + "ttft_p95_ms": 67.80304794665426 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r4p00.json", + "sha256": "8294196c7dc52765a6820f5e1d8439c480490daed9505263f707810f73c95783", + "summary": { + "admission_lag_max_ms": 0.1283179735764861, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 70.6796320155263, + "ttft_p50_ms": 60.962425894103944, + "ttft_p95_ms": 64.59337309934199 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 70.46496798284352, + 68.22343391831964 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r8p00.json", + "sha256": "38bbfeab47e5eb6296fcccf20f04ef25dc24db658be92300c492734bbfe333cf", + "summary": { + "admission_lag_max_ms": 0.10555307380855083, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 87.7126749837771, + "ttft_p50_ms": 62.65173899009824, + "ttft_p95_ms": 70.46496798284352 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r8p00.json", + "sha256": "716f90177d6f203717c80a8e9b28697e53141f204af470493c299bc73a1ea3ee", + "summary": { + "admission_lag_max_ms": 0.24518708232790232, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 80.40577603969723, + "ttft_p50_ms": 60.29119901359081, + "ttft_p95_ms": 68.22343391831964 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 16.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 89.53388908412308, + 76.93439698778093 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r16p00.json", + "sha256": "063016efe822c31c2425e7d963eda4c054f3f7c0341df0a3cd73ab86a942e873", + "summary": { + "admission_lag_max_ms": 0.2996840048581362, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 97.41822991054505, + "ttft_p50_ms": 60.97789097111672, + "ttft_p95_ms": 89.53388908412308 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r16p00.json", + "sha256": "eecc6bdeff90a8bf195dfc5387c6bc2f4db20ed7c9aeb9b4ebe050efccc6051a", + "summary": { + "admission_lag_max_ms": 0.36267400719225407, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 79.42048599943519, + "ttft_p50_ms": 60.29245804529637, + "ttft_p95_ms": 76.93439698778093 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 20.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 206.2494510319084, + 201.13301498349756 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r20p00.json", + "sha256": "22a62729358fea7c38ca166a1a49f1235e98c7f07ecfe4948322110cda4b7178", + "summary": { + "admission_lag_max_ms": 0.11367700062692165, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 255.14698098413646, + "ttft_p50_ms": 155.46583198010921, + "ttft_p95_ms": 206.2494510319084 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r20p00.json", + "sha256": "be103a759100bd2ab6dd31ee4ce41b6f0da40d232a60ebe137e3d933194a487c", + "summary": { + "admission_lag_max_ms": 0.17853605095297098, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 209.22932599205524, + "ttft_p50_ms": 155.12292610947043, + "ttft_p95_ms": 201.13301498349756 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 24.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 580.5047769099474, + 577.3971510352567 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r24p00.json", + "sha256": "890a733a97f399deb4c5efcca661d9a442ed76080cd5d8b6a2c09368aef701b7", + "summary": { + "admission_lag_max_ms": 0.1084543764591217, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 621.5670789824799, + "ttft_p50_ms": 374.6346529806033, + "ttft_p95_ms": 580.5047769099474 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r24p00.json", + "sha256": "87e8543652b1743b1b9b8c3de0fc680401b0bf9a33d61fa6132120c944b2a6eb", + "summary": { + "admission_lag_max_ms": 0.11801638174802065, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 623.2662099646404, + "ttft_p50_ms": 382.11427396163344, + "ttft_p95_ms": 577.3971510352567 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 28.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 873.0683700414374, + 889.4930009264499 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r28p00.json", + "sha256": "f877781730a5437b4cbdb76948915f1c871574b19a6df27076d21b65bb18e678", + "summary": { + "admission_lag_max_ms": 0.6807943573221564, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 943.9258739585057, + "ttft_p50_ms": 542.9852310335264, + "ttft_p95_ms": 873.0683700414374 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r28p00.json", + "sha256": "82151dc539b35302489c17976cfe89b8f4c64944298954607fad8f5c9cb9e7d8", + "summary": { + "admission_lag_max_ms": 6.750226486474276, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 936.5667289821431, + "ttft_p50_ms": 546.0105559322983, + "ttft_p95_ms": 889.4930009264499 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 32.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 1135.3467140579596, + 1146.743991994299 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r32p00.json", + "sha256": "082cc4c47b2eec6e49f0edd14cff5a51b880bd998d98da8b9830bbf7615483ef", + "summary": { + "admission_lag_max_ms": 0.10443700011819601, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1199.8225760180503, + "ttft_p50_ms": 680.4794340860099, + "ttft_p95_ms": 1135.3467140579596 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r32p00.json", + "sha256": "517d3cbc1104b5ea1bdb0e381c033cf978181a93449b6498d30b52cd3dd3e770", + "summary": { + "admission_lag_max_ms": 0.4415180301293731, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1210.0602959981188, + "ttft_p50_ms": 693.0603379150853, + "ttft_p95_ms": 1146.743991994299 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 2084.9693010095507, + 2041.9401369290426 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round1/results/r64p00.json", + "sha256": "7e11771ed598826fb3c5594c7e05d0989e01030be2d8b36fa21813aff462496d", + "summary": { + "admission_lag_max_ms": 0.10768393985927105, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.5625, + "passed": 36, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2132.128676981665, + "ttft_p50_ms": 1113.0280290963128, + "ttft_p95_ms": 2084.9693010095507 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z/artifacts/artifacts/real-tp4-mns32-v1/round2/results/r64p00.json", + "sha256": "cf550423832a92cba942b7e1dc14a42000e8c9507403de8cd49aa56af9db6706", + "summary": { + "admission_lag_max_ms": 0.11262902989983559, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.546875, + "passed": 35, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2122.8132860269397, + "ttft_p50_ms": 1131.2310450011864, + "ttft_p95_ms": 2041.9401369290426 + } + } + ] + } + ], + "capacity": 32.0, + "capacity_per_gpu": 8.0, + "mns": 32, + "name": "tp4_mns32", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z", + "tp": 4 + }, + "tp4_mns64": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 66.89354998525232, + 83.23234610725194 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r4p00.json", + "sha256": "266cba4088f8a9483ceb41fdc02d04a2181cb7ffb6b52372b6b0687e62f2863e", + "summary": { + "admission_lag_max_ms": 0.10568101424723864, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 72.09669100120664, + "ttft_p50_ms": 62.014322029426694, + "ttft_p95_ms": 66.89354998525232 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r4p00.json", + "sha256": "26f3926c40504b96a37cb7b9dae1af5040c085c1a730e20feaf1d3e9ddbf8e54", + "summary": { + "admission_lag_max_ms": 0.108166947029531, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 96.42452490516007, + "ttft_p50_ms": 61.261708033271134, + "ttft_p95_ms": 83.23234610725194 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 67.26283801253885, + 67.34374002553523 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r8p00.json", + "sha256": "e84e6ba2c8822e012aceea7cc6f376ea1b6265fe4a3bb68b7badfc36b8813545", + "summary": { + "admission_lag_max_ms": 0.15694997273385525, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 86.54619695153087, + "ttft_p50_ms": 59.63437003083527, + "ttft_p95_ms": 67.26283801253885 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r8p00.json", + "sha256": "dae50073c9f79919181e8dcf274ca8929ba41c512f64363e40187de68839b87e", + "summary": { + "admission_lag_max_ms": 0.09962497279047966, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 69.00161597877741, + "ttft_p50_ms": 60.2562150452286, + "ttft_p95_ms": 67.34374002553523 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 16.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 71.37343694921583, + 64.9534990079701 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r16p00.json", + "sha256": "776c9956f65acf7c9b6c8df517a4f8c4d462289893a19e625a2a0ad7dcbf5506", + "summary": { + "admission_lag_max_ms": 0.1066719414666295, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 82.25136005785316, + "ttft_p50_ms": 58.52357891853899, + "ttft_p95_ms": 71.37343694921583 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r16p00.json", + "sha256": "90057204a5f6343ee4ee9cdd08323d87d83660d5bf8f02480882eb479d2e587e", + "summary": { + "admission_lag_max_ms": 0.10102998930960894, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 65.3582609957084, + "ttft_p50_ms": 58.98014397826046, + "ttft_p95_ms": 64.9534990079701 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 20.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 192.98715889453888, + 258.1990590551868 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r20p00.json", + "sha256": "6ce029388b206d0663acd41f4da1d67a6562389e1c84cf1826ce77c05d176d9a", + "summary": { + "admission_lag_max_ms": 0.12691703159362078, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 198.67739104665816, + "ttft_p50_ms": 142.75032398290932, + "ttft_p95_ms": 192.98715889453888 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r20p00.json", + "sha256": "0dbfb3f012574988ddebfb0af1532c735ac62d6d88863d24cd820aed50a92a59", + "summary": { + "admission_lag_max_ms": 0.3961039474233985, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 259.55891504418105, + "ttft_p50_ms": 162.6282810466364, + "ttft_p95_ms": 258.1990590551868 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 24.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 583.3089610096067, + 571.3486529421061 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r24p00.json", + "sha256": "0774ba2f285d81447c63343b32551d6fc33369b54010629381efa9a37df23a9c", + "summary": { + "admission_lag_max_ms": 0.09822729043662548, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 628.714332007803, + "ttft_p50_ms": 366.9840869260952, + "ttft_p95_ms": 583.3089610096067 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r24p00.json", + "sha256": "1986a05c366ebe73af0f89d2cd0e4efd3e577c800cf1e80997cd26ea0aaaec09", + "summary": { + "admission_lag_max_ms": 0.11414836626499891, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 617.9165309295058, + "ttft_p50_ms": 376.65714998729527, + "ttft_p95_ms": 571.3486529421061 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 28.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 869.708820944652, + 866.4831441128626 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r28p00.json", + "sha256": "ab987729fc833a59817eabaa18c320b0d0a2ecd402fd02bdb534b6c773c029d8", + "summary": { + "admission_lag_max_ms": 0.578614417463541, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 941.3277570856735, + "ttft_p50_ms": 540.4732850147411, + "ttft_p95_ms": 869.708820944652 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r28p00.json", + "sha256": "38f9c82c9ee30927cc0ba857e5e3fce2e79b8a24de03efaec77ce27258633a7e", + "summary": { + "admission_lag_max_ms": 0.12532551772892475, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 938.2033230504021, + "ttft_p50_ms": 541.139594046399, + "ttft_p95_ms": 866.4831441128626 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 32.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 1132.6635719742626, + 1144.196804962121 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r32p00.json", + "sha256": "358f5b8e1b72bff5752c076f2966b966d98f6f5767ce52d5a255b7f68f061e54", + "summary": { + "admission_lag_max_ms": 0.32093306072056293, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1197.7118930080906, + "ttft_p50_ms": 678.4568330040202, + "ttft_p95_ms": 1132.6635719742626 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r32p00.json", + "sha256": "3f6dbb760582fdd7470422d8edf6b62949d158ad0d437f04015441a0a4509003", + "summary": { + "admission_lag_max_ms": 1.1304179206490517, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1193.1529619032517, + "ttft_p50_ms": 692.5535890040919, + "ttft_p95_ms": 1144.196804962121 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 2042.27216495201, + 2043.943466967903 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round1/results/r64p00.json", + "sha256": "a24eae7db6eb4cd105ed903917988e8fe6eef5d4cc2549d4e05e9a03dfafb107", + "summary": { + "admission_lag_max_ms": 0.134881935082376, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.546875, + "passed": 35, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2132.23939598538, + "ttft_p50_ms": 1137.3268909519538, + "ttft_p95_ms": 2042.27216495201 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z/artifacts/artifacts/real-tp4-mns64-v1/round2/results/r64p00.json", + "sha256": "e85efaada0b64bb9dd686f59282171ccb256a1d8483c6f037589a68e3e1a104a", + "summary": { + "admission_lag_max_ms": 0.11892011389136314, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.546875, + "passed": 35, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2125.3299129894003, + "ttft_p50_ms": 1134.724210947752, + "ttft_p95_ms": 2043.943466967903 + } + } + ] + } + ], + "capacity": 32.0, + "capacity_per_gpu": 8.0, + "mns": 64, + "name": "tp4_mns64", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z", + "tp": 4 + }, + "tp4_mns8": { + "anchors": [ + { + "conservative_feasible": true, + "rate": 4.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 82.33969006687403, + 69.57727600820363 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r4p00.json", + "sha256": "e69b89ad2e07e6560ebd870773532b33e4415e9ca1515e585cbe465616678569", + "summary": { + "admission_lag_max_ms": 0.14186999760568142, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 86.54833806212991, + "ttft_p50_ms": 62.749656033702195, + "ttft_p95_ms": 82.33969006687403 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r4p00.json", + "sha256": "2c76dcf112a828a6720e079d29bb54a6a65d37d26abaaafdc6d5580751dbad65", + "summary": { + "admission_lag_max_ms": 0.11478399392217398, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 81.3983449479565, + "ttft_p50_ms": 62.220132909715176, + "ttft_p95_ms": 69.57727600820363 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 8.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 63.71171900536865, + 68.14328604377806 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r8p00.json", + "sha256": "0e5d687dfcdb4b6a0d1d060ceefe8a74faab94a5b241580d45b53c452f893b23", + "summary": { + "admission_lag_max_ms": 0.09818794205784798, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 69.63772000744939, + "ttft_p50_ms": 60.89008797425777, + "ttft_p95_ms": 63.71171900536865 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r8p00.json", + "sha256": "1bf016b603473d46fda4bc6d4daeaee5e1091ff4cda1cab7224e0c5edea512f8", + "summary": { + "admission_lag_max_ms": 0.11399004142731428, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 76.76690397784114, + "ttft_p50_ms": 60.9520060243085, + "ttft_p95_ms": 68.14328604377806 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 16.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 133.86176899075508, + 61.95738597307354 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r16p00.json", + "sha256": "197808b1b5fa8082e603a255f358826b5f8f48707223c7fd63abfe029674b817", + "summary": { + "admission_lag_max_ms": 0.10530801955610514, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 191.68931001331657, + "ttft_p50_ms": 61.92394997924566, + "ttft_p95_ms": 133.86176899075508 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r16p00.json", + "sha256": "53e809470daf9efcdabb254e85e8cc3b45d0ab8593c6c2f3aa3396e60d1656da", + "summary": { + "admission_lag_max_ms": 0.1288399798795581, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 70.40882098954171, + "ttft_p50_ms": 59.417470009066164, + "ttft_p95_ms": 61.95738597307354 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 20.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 201.29013701807708, + 200.0734859611839 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r20p00.json", + "sha256": "05b977824fb301517cdbe2d077b360b6053d57fee1c20acad3537321827d027a", + "summary": { + "admission_lag_max_ms": 0.3848400665447116, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 204.5107699232176, + "ttft_p50_ms": 150.0862929970026, + "ttft_p95_ms": 201.29013701807708 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r20p00.json", + "sha256": "baf422b7a3c95b0d3d376a31b0a30e075f69216c794ebc2ef51afafe1ec55229", + "summary": { + "admission_lag_max_ms": 0.24754798505455256, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 204.8192109214142, + "ttft_p50_ms": 148.99871195666492, + "ttft_p95_ms": 200.0734859611839 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 24.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 564.2021730309352, + 560.4996709153056 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r24p00.json", + "sha256": "029b2a9695322d536539c3dd7665725f933b21a8bb48fa94c0448c9f7f8bf0eb", + "summary": { + "admission_lag_max_ms": 0.14956865925341845, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 610.4218009859324, + "ttft_p50_ms": 368.3909500250593, + "ttft_p95_ms": 564.2021730309352 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r24p00.json", + "sha256": "94b46583af3a7a4bbb9f7256c4c4a5ce804b3092459abe3984ea0ce67f591f3c", + "summary": { + "admission_lag_max_ms": 0.11110503692179918, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 606.6705230623484, + "ttft_p50_ms": 367.50321893487126, + "ttft_p95_ms": 560.4996709153056 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 28.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 870.7952990662307, + 894.6108610834926 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r28p00.json", + "sha256": "07bfe263107bb3d2acaa356257343f90045e4295c799178bc0a0b5f1c1994284", + "summary": { + "admission_lag_max_ms": 0.6798935355618596, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 942.792383953929, + "ttft_p50_ms": 543.340316042304, + "ttft_p95_ms": 870.7952990662307 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r28p00.json", + "sha256": "861dd55076ef40a78c34824345c5279dff86262b604db120e9e38efa27b5be55", + "summary": { + "admission_lag_max_ms": 0.09912776295095682, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 930.2741150604561, + "ttft_p50_ms": 549.9696529004723, + "ttft_p95_ms": 894.6108610834926 + } + } + ] + }, + { + "conservative_feasible": true, + "rate": 32.0, + "round_feasible": [ + true, + true + ], + "round_ttft_p95_ms": [ + 1136.603599996306, + 1134.0074760373682 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r32p00.json", + "sha256": "332b805322b571087b018ee459dd91c443dfe0b2bb539b5bdfb4de902bba6da2", + "summary": { + "admission_lag_max_ms": 0.19067595712840557, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1201.8779760692269, + "ttft_p50_ms": 683.3873559953645, + "ttft_p95_ms": 1136.603599996306 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r32p00.json", + "sha256": "996a12a5ed962218dfbe6f30be688119030f10f6539647b6e4a9f25f032b22cb", + "summary": { + "admission_lag_max_ms": 0.11958007235080004, + "completed": 64, + "failed": 0, + "slo": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 1198.4891929896548, + "ttft_p50_ms": 682.7383439522237, + "ttft_p95_ms": 1134.0074760373682 + } + } + ] + }, + { + "conservative_feasible": false, + "rate": 64.0, + "round_feasible": [ + false, + false + ], + "round_ttft_p95_ms": [ + 2040.8174369949847, + 2082.3236129945144 + ], + "rounds": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round1/results/r64p00.json", + "sha256": "c5f17f625071e905b5f1bf8045cd233b2fea0fe6ca00044935dee9c663d1526b", + "summary": { + "admission_lag_max_ms": 0.7995429914444685, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.546875, + "passed": 35, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2131.160691031255, + "ttft_p50_ms": 1134.8516750149429, + "ttft_p95_ms": 2040.8174369949847 + } + }, + { + "path": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z/artifacts/artifacts/real-tp4-mns8-v1/round2/results/r64p00.json", + "sha256": "775db5b941347d9fd5d6df32d6ee037f8168bd47b9eae9a7bdb3dbd8c40aa990", + "summary": { + "admission_lag_max_ms": 0.17620902508497238, + "completed": 64, + "failed": 0, + "slo": { + "feasible": false, + "pass_rate": 0.5625, + "passed": 36, + "ttft_threshold_ms": 1256.0 + }, + "ttft_max_ms": 2129.518053960055, + "ttft_p50_ms": 1112.3740070033818, + "ttft_p95_ms": 2082.3236129945144 + } + } + ] + } + ], + "capacity": 32.0, + "capacity_per_gpu": 8.0, + "mns": 8, + "name": "tp4_mns8", + "source_run": "/home/gahow/phd/aituner/runs/frontier-phase-factorial-v0/fleet-artifacts-exclusive/qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z", + "tp": 4 + } + }, + "real_campaign_resources": { + "fresh_server_anchors": 192, + "gpu_hours": 12.074444444444445, + "measured_requests": 12288, + "runs": [ + { + "duration_seconds": 908.0, + "gpu_hours": 0.25222222222222224, + "run": "qwen30-prefill-real-tp1-mns16-20260717-v2-exclusive-20260716T164730291172Z", + "tp": 1 + }, + { + "duration_seconds": 540.0, + "gpu_hours": 0.15, + "run": "qwen30-prefill-real-tp1-mns16-20260717-v3-refine-20260716T181031481585Z", + "tp": 1 + }, + { + "duration_seconds": 937.0, + "gpu_hours": 0.2602777777777778, + "run": "qwen30-prefill-real-tp1-mns32-20260717-v2-exclusive-20260716T164731376588Z", + "tp": 1 + }, + { + "duration_seconds": 558.0, + "gpu_hours": 0.155, + "run": "qwen30-prefill-real-tp1-mns32-20260717-v3-refine-20260716T181032543571Z", + "tp": 1 + }, + { + "duration_seconds": 972.0, + "gpu_hours": 0.27, + "run": "qwen30-prefill-real-tp1-mns64-20260717-v2-exclusive-20260716T164732482666Z", + "tp": 1 + }, + { + "duration_seconds": 572.0, + "gpu_hours": 0.15888888888888889, + "run": "qwen30-prefill-real-tp1-mns64-20260717-v3-refine-20260716T181033562318Z", + "tp": 1 + }, + { + "duration_seconds": 886.0, + "gpu_hours": 0.2461111111111111, + "run": "qwen30-prefill-real-tp1-mns8-20260717-v2-exclusive-20260716T164729267199Z", + "tp": 1 + }, + { + "duration_seconds": 525.0, + "gpu_hours": 0.14583333333333334, + "run": "qwen30-prefill-real-tp1-mns8-20260717-v3-refine-20260716T181030504792Z", + "tp": 1 + }, + { + "duration_seconds": 983.0, + "gpu_hours": 0.5461111111111111, + "run": "qwen30-prefill-real-tp2-mns16-20260717-v2-exclusive-20260716T164734745039Z", + "tp": 2 + }, + { + "duration_seconds": 569.0, + "gpu_hours": 0.3161111111111111, + "run": "qwen30-prefill-real-tp2-mns16-20260717-v3-refine-20260716T181035686437Z", + "tp": 2 + }, + { + "duration_seconds": 1033.0, + "gpu_hours": 0.5738888888888889, + "run": "qwen30-prefill-real-tp2-mns32-20260717-v2-exclusive-20260716T170909930153Z", + "tp": 2 + }, + { + "duration_seconds": 584.0, + "gpu_hours": 0.3244444444444444, + "run": "qwen30-prefill-real-tp2-mns32-20260717-v3-refine-20260716T182308386197Z", + "tp": 2 + }, + { + "duration_seconds": 1063.0, + "gpu_hours": 0.5905555555555555, + "run": "qwen30-prefill-real-tp2-mns64-20260717-v2-exclusive-20260716T170910954319Z", + "tp": 2 + }, + { + "duration_seconds": 597.0, + "gpu_hours": 0.33166666666666667, + "run": "qwen30-prefill-real-tp2-mns64-20260717-v3-refine-20260716T182309391513Z", + "tp": 2 + }, + { + "duration_seconds": 957.0, + "gpu_hours": 0.5316666666666666, + "run": "qwen30-prefill-real-tp2-mns8-20260717-v2-exclusive-20260716T164733581881Z", + "tp": 2 + }, + { + "duration_seconds": 563.0, + "gpu_hours": 0.31277777777777777, + "run": "qwen30-prefill-real-tp2-mns8-20260717-v3-refine-20260716T181034592841Z", + "tp": 2 + }, + { + "duration_seconds": 1008.0, + "gpu_hours": 1.12, + "run": "qwen30-prefill-real-tp4-mns16-20260717-v2-exclusive-20260716T172922998830Z", + "tp": 4 + }, + { + "duration_seconds": 559.0, + "gpu_hours": 0.6211111111111111, + "run": "qwen30-prefill-real-tp4-mns16-20260717-v4-refine-20260716T183829231278Z", + "tp": 4 + }, + { + "duration_seconds": 1024.0, + "gpu_hours": 1.1377777777777778, + "run": "qwen30-prefill-real-tp4-mns32-20260717-v2-exclusive-20260716T172923955862Z", + "tp": 4 + }, + { + "duration_seconds": 556.0, + "gpu_hours": 0.6177777777777778, + "run": "qwen30-prefill-real-tp4-mns32-20260717-v4-refine-20260716T183830222875Z", + "tp": 4 + }, + { + "duration_seconds": 959.0, + "gpu_hours": 1.0655555555555556, + "run": "qwen30-prefill-real-tp4-mns64-20260717-v2b-exclusive-20260716T175248149074Z", + "tp": 4 + }, + { + "duration_seconds": 567.0, + "gpu_hours": 0.63, + "run": "qwen30-prefill-real-tp4-mns64-20260717-v4-refine-20260716T184826349294Z", + "tp": 4 + }, + { + "duration_seconds": 995.0, + "gpu_hours": 1.1055555555555556, + "run": "qwen30-prefill-real-tp4-mns8-20260717-v2-exclusive-20260716T170911977980Z", + "tp": 4 + }, + { + "duration_seconds": 550.0, + "gpu_hours": 0.6111111111111112, + "run": "qwen30-prefill-real-tp4-mns8-20260717-v3-refine-20260716T182310414444Z", + "tp": 4 + } + ], + "successful_fleet_jobs": 24, + "warmup_requests": 4512 + }, + "schema": "qwen30-prefill-fidelity-comparison-v1", + "simulator": { + "tp1_mns16": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 166.6461504479848 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 575.6631856791392 + }, + { + "feasible": false, + "pass_rate": 0.265625, + "rate": 16.0, + "ttft_p95_ms": 3529.1762077816857 + }, + { + "feasible": false, + "pass_rate": 0.1875, + "rate": 32.0, + "ttft_p95_ms": 5290.0301465092725 + }, + { + "feasible": false, + "pass_rate": 0.140625, + "rate": 64.0, + "ttft_p95_ms": 6180.6551465092725 + } + ], + "capacity": 8.0, + "capacity_per_gpu": 8.0, + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "tp1_mns32": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 166.6461504479848 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 575.6631856791392 + }, + { + "feasible": false, + "pass_rate": 0.265625, + "rate": 16.0, + "ttft_p95_ms": 3529.1762077816857 + }, + { + "feasible": false, + "pass_rate": 0.1875, + "rate": 32.0, + "ttft_p95_ms": 5290.0301465092725 + }, + { + "feasible": false, + "pass_rate": 0.140625, + "rate": 64.0, + "ttft_p95_ms": 6180.6551465092725 + } + ], + "capacity": 8.0, + "capacity_per_gpu": 8.0, + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "tp1_mns64": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 166.6461504479848 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 575.6631856791392 + }, + { + "feasible": false, + "pass_rate": 0.265625, + "rate": 16.0, + "ttft_p95_ms": 3529.1762077816857 + }, + { + "feasible": false, + "pass_rate": 0.1875, + "rate": 32.0, + "ttft_p95_ms": 5290.0301465092725 + }, + { + "feasible": false, + "pass_rate": 0.140625, + "rate": 64.0, + "ttft_p95_ms": 6180.6551465092725 + } + ], + "capacity": 8.0, + "capacity_per_gpu": 8.0, + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "tp1_mns8": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 166.6461504479848 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 575.6631856791392 + }, + { + "feasible": false, + "pass_rate": 0.265625, + "rate": 16.0, + "ttft_p95_ms": 3529.1762077816857 + }, + { + "feasible": false, + "pass_rate": 0.1875, + "rate": 32.0, + "ttft_p95_ms": 5290.0301465092725 + }, + { + "feasible": false, + "pass_rate": 0.140625, + "rate": 64.0, + "ttft_p95_ms": 6180.6551465092725 + } + ], + "capacity": 8.0, + "capacity_per_gpu": 8.0, + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "tp2_mns16": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 118.55152318709017 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 118.55152318708929 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 786.9875986853465 + }, + { + "feasible": false, + "pass_rate": 0.421875, + "rate": 32.0, + "ttft_p95_ms": 2566.908354521784 + }, + { + "feasible": false, + "pass_rate": 0.3125, + "rate": 64.0, + "ttft_p95_ms": 3439.824414996849 + } + ], + "capacity": 16.0, + "capacity_per_gpu": 8.0, + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "tp2_mns32": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 118.55152318709017 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 118.55152318708929 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 786.9875986853465 + }, + { + "feasible": false, + "pass_rate": 0.421875, + "rate": 32.0, + "ttft_p95_ms": 2566.908354521784 + }, + { + "feasible": false, + "pass_rate": 0.3125, + "rate": 64.0, + "ttft_p95_ms": 3439.824414996849 + } + ], + "capacity": 16.0, + "capacity_per_gpu": 8.0, + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "tp2_mns64": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 118.55152318709017 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 118.55152318708929 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 786.9875986853465 + }, + { + "feasible": false, + "pass_rate": 0.421875, + "rate": 32.0, + "ttft_p95_ms": 2566.908354521784 + }, + { + "feasible": false, + "pass_rate": 0.3125, + "rate": 64.0, + "ttft_p95_ms": 3439.824414996849 + } + ], + "capacity": 16.0, + "capacity_per_gpu": 8.0, + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "tp2_mns8": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 118.55152318709017 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 118.55152318708929 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 786.9875986853465 + }, + { + "feasible": false, + "pass_rate": 0.421875, + "rate": 32.0, + "ttft_p95_ms": 2566.908354521784 + }, + { + "feasible": false, + "pass_rate": 0.3125, + "rate": 64.0, + "ttft_p95_ms": 3439.824414996849 + } + ], + "capacity": 16.0, + "capacity_per_gpu": 8.0, + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "tp4_mns16": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 86.51189290936045 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 86.51189290936001 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 252.16908885537626 + }, + { + "feasible": true, + "pass_rate": 0.984375, + "rate": 32.0, + "ttft_p95_ms": 1219.2127732631457 + }, + { + "feasible": false, + "pass_rate": 0.515625, + "rate": 64.0, + "ttft_p95_ms": 2106.0636248827373 + } + ], + "capacity": 32.0, + "capacity_per_gpu": 8.0, + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "tp4_mns32": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 86.51189290936045 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 86.51189290936001 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 252.16908885537626 + }, + { + "feasible": true, + "pass_rate": 0.984375, + "rate": 32.0, + "ttft_p95_ms": 1219.2127732631457 + }, + { + "feasible": false, + "pass_rate": 0.515625, + "rate": 64.0, + "ttft_p95_ms": 2106.0636248827373 + } + ], + "capacity": 32.0, + "capacity_per_gpu": 8.0, + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "tp4_mns64": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 86.51189290936045 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 86.51189290936001 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 252.16908885537626 + }, + { + "feasible": true, + "pass_rate": 0.984375, + "rate": 32.0, + "ttft_p95_ms": 1219.2127732631457 + }, + { + "feasible": false, + "pass_rate": 0.515625, + "rate": 64.0, + "ttft_p95_ms": 2106.0636248827373 + } + ], + "capacity": 32.0, + "capacity_per_gpu": 8.0, + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "tp4_mns8": { + "anchors": [ + { + "feasible": true, + "pass_rate": 1.0, + "rate": 4.0, + "ttft_p95_ms": 86.51189290936045 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 8.0, + "ttft_p95_ms": 86.51189290936001 + }, + { + "feasible": true, + "pass_rate": 1.0, + "rate": 16.0, + "ttft_p95_ms": 252.16908885537626 + }, + { + "feasible": true, + "pass_rate": 0.984375, + "rate": 32.0, + "ttft_p95_ms": 1219.2127732631457 + }, + { + "feasible": false, + "pass_rate": 0.515625, + "rate": 64.0, + "ttft_p95_ms": 2106.0636248827373 + } + ], + "capacity": 32.0, + "capacity_per_gpu": 8.0, + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + } + }, + "simulator_sources": [ + { + "path": "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/simulator-a3/frontier_surface_frozen.json", + "sha256": "fe7392fbcd20f6a96562773b3644df9dac23c8562462356acc86585f7f1d40f0" + } + ] +} diff --git a/runs/frontier-fidelity-envelope-v1/results/a3/frontier_surface_frozen.json b/runs/frontier-fidelity-envelope-v1/results/a3/frontier_surface_frozen.json new file mode 100644 index 0000000..34009eb --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/results/a3/frontier_surface_frozen.json @@ -0,0 +1,1736 @@ +{ + "capacity": [ + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 8.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 8.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 8.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 8.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 16.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 16.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 16.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 16.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 32.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 32.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 32.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 32.0, + "maximum_tested_feasible_request_rate_per_gpu": 8.0, + "upper_censored": false + } + ], + "collective": { + "allreduce_csv": "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv", + "allreduce_csv_sha256": "9d693fd406616b599e57bcde66c980c7fc2831b3acf37d3eb633cec80ea0070d", + "backend": "vidur" + }, + "config_results": [ + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "loads": [ + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 157.35246539115906, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "705075942dea05ebacb4c83c2587a865e40e1c7af243d2894fa7a05409f8fe59", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.020947591286289, + "ttft_max_ms": 166.6461504479848, + "ttft_p50_ms": 166.64615044798398, + "ttft_p95_ms": 166.6461504479848 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 9.141019344329834, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "5dbdc7f1672445ef892ed104c0ea1bfb02baba3f87e8a3e707c523707861b747", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 7.75274759930899, + "ttft_max_ms": 586.937565208359, + "ttft_p50_ms": 436.98066306434936, + "ttft_p95_ms": 575.6631856791392 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 9.098863363265991, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "58835b434e8f85b91a8d14681bf4a6a0df5f38c69ead31bbc6e02c1c50f8a9c1", + "score": { + "feasible": false, + "pass_rate": 0.265625, + "passed": 17, + "throughput_requests_per_second": 8.523874834682788, + "ttft_max_ms": 3654.1762077816857, + "ttft_p50_ms": 1994.5136762864174, + "ttft_p95_ms": 3529.1762077816857 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 9.089977025985718, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "a83336424fd7682e60e1a02a62d5c1edbcded34f14919b6011c8bc76a3f24de1", + "score": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "throughput_requests_per_second": 8.660758725922316, + "ttft_max_ms": 5483.402803563629, + "ttft_p50_ms": 2911.6176150140045, + "ttft_p95_ms": 5290.0301465092725 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 9.088061094284058, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "000767c0b12d8ec4c9e83f978b3552c8c6b99a2b42414100fc1a7b3e8e5c721e", + "score": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "throughput_requests_per_second": 8.660758725922316, + "ttft_max_ms": 6436.527803563629, + "ttft_p50_ms": 3380.3676150140045, + "ttft_p95_ms": 6180.6551465092725 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "loads": [ + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 8.947673320770264, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "705075942dea05ebacb4c83c2587a865e40e1c7af243d2894fa7a05409f8fe59", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.020947591286289, + "ttft_max_ms": 166.6461504479848, + "ttft_p50_ms": 166.64615044798398, + "ttft_p95_ms": 166.6461504479848 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 9.042973518371582, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "5dbdc7f1672445ef892ed104c0ea1bfb02baba3f87e8a3e707c523707861b747", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 7.75274759930899, + "ttft_max_ms": 586.937565208359, + "ttft_p50_ms": 436.98066306434936, + "ttft_p95_ms": 575.6631856791392 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 9.092506170272827, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "58835b434e8f85b91a8d14681bf4a6a0df5f38c69ead31bbc6e02c1c50f8a9c1", + "score": { + "feasible": false, + "pass_rate": 0.265625, + "passed": 17, + "throughput_requests_per_second": 8.523874834682788, + "ttft_max_ms": 3654.1762077816857, + "ttft_p50_ms": 1994.5136762864174, + "ttft_p95_ms": 3529.1762077816857 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 9.044708013534546, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "a83336424fd7682e60e1a02a62d5c1edbcded34f14919b6011c8bc76a3f24de1", + "score": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "throughput_requests_per_second": 8.660758725922316, + "ttft_max_ms": 5483.402803563629, + "ttft_p50_ms": 2911.6176150140045, + "ttft_p95_ms": 5290.0301465092725 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 9.29175853729248, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "000767c0b12d8ec4c9e83f978b3552c8c6b99a2b42414100fc1a7b3e8e5c721e", + "score": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "throughput_requests_per_second": 8.660758725922316, + "ttft_max_ms": 6436.527803563629, + "ttft_p50_ms": 3380.3676150140045, + "ttft_p95_ms": 6180.6551465092725 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "loads": [ + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 8.991463661193848, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "705075942dea05ebacb4c83c2587a865e40e1c7af243d2894fa7a05409f8fe59", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.020947591286289, + "ttft_max_ms": 166.6461504479848, + "ttft_p50_ms": 166.64615044798398, + "ttft_p95_ms": 166.6461504479848 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 9.138724327087402, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "5dbdc7f1672445ef892ed104c0ea1bfb02baba3f87e8a3e707c523707861b747", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 7.75274759930899, + "ttft_max_ms": 586.937565208359, + "ttft_p50_ms": 436.98066306434936, + "ttft_p95_ms": 575.6631856791392 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 9.188970565795898, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "58835b434e8f85b91a8d14681bf4a6a0df5f38c69ead31bbc6e02c1c50f8a9c1", + "score": { + "feasible": false, + "pass_rate": 0.265625, + "passed": 17, + "throughput_requests_per_second": 8.523874834682788, + "ttft_max_ms": 3654.1762077816857, + "ttft_p50_ms": 1994.5136762864174, + "ttft_p95_ms": 3529.1762077816857 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 9.189127683639526, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "a83336424fd7682e60e1a02a62d5c1edbcded34f14919b6011c8bc76a3f24de1", + "score": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "throughput_requests_per_second": 8.660758725922316, + "ttft_max_ms": 5483.402803563629, + "ttft_p50_ms": 2911.6176150140045, + "ttft_p95_ms": 5290.0301465092725 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 9.189879894256592, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "000767c0b12d8ec4c9e83f978b3552c8c6b99a2b42414100fc1a7b3e8e5c721e", + "score": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "throughput_requests_per_second": 8.660758725922316, + "ttft_max_ms": 6436.527803563629, + "ttft_p50_ms": 3380.3676150140045, + "ttft_p95_ms": 6180.6551465092725 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "loads": [ + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 8.999529123306274, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "705075942dea05ebacb4c83c2587a865e40e1c7af243d2894fa7a05409f8fe59", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.020947591286289, + "ttft_max_ms": 166.6461504479848, + "ttft_p50_ms": 166.64615044798398, + "ttft_p95_ms": 166.6461504479848 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 9.192278385162354, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "5dbdc7f1672445ef892ed104c0ea1bfb02baba3f87e8a3e707c523707861b747", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 7.75274759930899, + "ttft_max_ms": 586.937565208359, + "ttft_p50_ms": 436.98066306434936, + "ttft_p95_ms": 575.6631856791392 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 9.040328979492188, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "58835b434e8f85b91a8d14681bf4a6a0df5f38c69ead31bbc6e02c1c50f8a9c1", + "score": { + "feasible": false, + "pass_rate": 0.265625, + "passed": 17, + "throughput_requests_per_second": 8.523874834682788, + "ttft_max_ms": 3654.1762077816857, + "ttft_p50_ms": 1994.5136762864174, + "ttft_p95_ms": 3529.1762077816857 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 8.990798473358154, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "a83336424fd7682e60e1a02a62d5c1edbcded34f14919b6011c8bc76a3f24de1", + "score": { + "feasible": false, + "pass_rate": 0.1875, + "passed": 12, + "throughput_requests_per_second": 8.660758725922316, + "ttft_max_ms": 5483.402803563629, + "ttft_p50_ms": 2911.6176150140045, + "ttft_p95_ms": 5290.0301465092725 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 9.192291736602783, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "000767c0b12d8ec4c9e83f978b3552c8c6b99a2b42414100fc1a7b3e8e5c721e", + "score": { + "feasible": false, + "pass_rate": 0.140625, + "passed": 9, + "throughput_requests_per_second": 8.660758725922316, + "ttft_max_ms": 6436.527803563629, + "ttft_p50_ms": 3380.3676150140045, + "ttft_p95_ms": 6180.6551465092725 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "loads": [ + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 9.093063116073608, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "195c2750784a03930b1ae369648ed6361fb8d1ea506f0828bc7c9818a4f55961", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.033134335322499, + "ttft_max_ms": 118.55152318709017, + "ttft_p50_ms": 118.55152318708929, + "ttft_p95_ms": 118.55152318709017 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 160.23531913757324, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "733c6a6bca1efc56f9a160dd89a9156446581d5a1c4c072b90f0a7659acc637e", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.006453678862723, + "ttft_max_ms": 118.55152318708929, + "ttft_p50_ms": 118.55152318708929, + "ttft_p95_ms": 118.55152318708929 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 9.289193630218506, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "37ba1e79ce560f7449049a3ef35429ee1601b35f249800f5e2e0b4d96121f0a4", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 13.760059242174107, + "ttft_max_ms": 849.4875986853465, + "ttft_p50_ms": 539.7284419614418, + "ttft_p95_ms": 786.9875986853465 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 9.239543676376343, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "1fb27df916c2d9002f5584c9a3ac3ca5fceed2f5fece4f40e69748dbad7aee3b", + "score": { + "feasible": false, + "pass_rate": 0.421875, + "passed": 27, + "throughput_requests_per_second": 14.110410219984, + "ttft_max_ms": 2660.658354521784, + "ttft_p50_ms": 1414.3128122232463, + "ttft_p95_ms": 2566.908354521784 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 9.141466856002808, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "ccefacf7e9b214037083892f6fa84c5e0fd288f041ebfe21880b9709c8b25727", + "score": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "throughput_requests_per_second": 14.110410219983999, + "ttft_max_ms": 3582.533354521785, + "ttft_p50_ms": 1896.1470654856275, + "ttft_p95_ms": 3439.824414996849 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "loads": [ + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 9.042891025543213, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "195c2750784a03930b1ae369648ed6361fb8d1ea506f0828bc7c9818a4f55961", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.033134335322499, + "ttft_max_ms": 118.55152318709017, + "ttft_p50_ms": 118.55152318708929, + "ttft_p95_ms": 118.55152318709017 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 9.136998653411865, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "733c6a6bca1efc56f9a160dd89a9156446581d5a1c4c072b90f0a7659acc637e", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.006453678862723, + "ttft_max_ms": 118.55152318708929, + "ttft_p50_ms": 118.55152318708929, + "ttft_p95_ms": 118.55152318708929 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 9.13957166671753, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "37ba1e79ce560f7449049a3ef35429ee1601b35f249800f5e2e0b4d96121f0a4", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 13.760059242174107, + "ttft_max_ms": 849.4875986853465, + "ttft_p50_ms": 539.7284419614418, + "ttft_p95_ms": 786.9875986853465 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 9.098120927810669, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "1fb27df916c2d9002f5584c9a3ac3ca5fceed2f5fece4f40e69748dbad7aee3b", + "score": { + "feasible": false, + "pass_rate": 0.421875, + "passed": 27, + "throughput_requests_per_second": 14.110410219984, + "ttft_max_ms": 2660.658354521784, + "ttft_p50_ms": 1414.3128122232463, + "ttft_p95_ms": 2566.908354521784 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 9.140004634857178, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "ccefacf7e9b214037083892f6fa84c5e0fd288f041ebfe21880b9709c8b25727", + "score": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "throughput_requests_per_second": 14.110410219983999, + "ttft_max_ms": 3582.533354521785, + "ttft_p50_ms": 1896.1470654856275, + "ttft_p95_ms": 3439.824414996849 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "loads": [ + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 9.090512752532959, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "195c2750784a03930b1ae369648ed6361fb8d1ea506f0828bc7c9818a4f55961", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.033134335322499, + "ttft_max_ms": 118.55152318709017, + "ttft_p50_ms": 118.55152318708929, + "ttft_p95_ms": 118.55152318709017 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 9.102084636688232, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "733c6a6bca1efc56f9a160dd89a9156446581d5a1c4c072b90f0a7659acc637e", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.006453678862723, + "ttft_max_ms": 118.55152318708929, + "ttft_p50_ms": 118.55152318708929, + "ttft_p95_ms": 118.55152318708929 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 9.09265947341919, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "37ba1e79ce560f7449049a3ef35429ee1601b35f249800f5e2e0b4d96121f0a4", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 13.760059242174107, + "ttft_max_ms": 849.4875986853465, + "ttft_p50_ms": 539.7284419614418, + "ttft_p95_ms": 786.9875986853465 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 9.14569616317749, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "1fb27df916c2d9002f5584c9a3ac3ca5fceed2f5fece4f40e69748dbad7aee3b", + "score": { + "feasible": false, + "pass_rate": 0.421875, + "passed": 27, + "throughput_requests_per_second": 14.110410219984, + "ttft_max_ms": 2660.658354521784, + "ttft_p50_ms": 1414.3128122232463, + "ttft_p95_ms": 2566.908354521784 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 9.191733598709106, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "ccefacf7e9b214037083892f6fa84c5e0fd288f041ebfe21880b9709c8b25727", + "score": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "throughput_requests_per_second": 14.110410219983999, + "ttft_max_ms": 3582.533354521785, + "ttft_p50_ms": 1896.1470654856275, + "ttft_p95_ms": 3439.824414996849 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "loads": [ + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 9.146432876586914, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "195c2750784a03930b1ae369648ed6361fb8d1ea506f0828bc7c9818a4f55961", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.033134335322499, + "ttft_max_ms": 118.55152318709017, + "ttft_p50_ms": 118.55152318708929, + "ttft_p95_ms": 118.55152318709017 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 9.068153619766235, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "733c6a6bca1efc56f9a160dd89a9156446581d5a1c4c072b90f0a7659acc637e", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.006453678862723, + "ttft_max_ms": 118.55152318708929, + "ttft_p50_ms": 118.55152318708929, + "ttft_p95_ms": 118.55152318708929 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 9.139062404632568, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "37ba1e79ce560f7449049a3ef35429ee1601b35f249800f5e2e0b4d96121f0a4", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 13.760059242174107, + "ttft_max_ms": 849.4875986853465, + "ttft_p50_ms": 539.7284419614418, + "ttft_p95_ms": 786.9875986853465 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 9.143300771713257, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "1fb27df916c2d9002f5584c9a3ac3ca5fceed2f5fece4f40e69748dbad7aee3b", + "score": { + "feasible": false, + "pass_rate": 0.421875, + "passed": 27, + "throughput_requests_per_second": 14.110410219984, + "ttft_max_ms": 2660.658354521784, + "ttft_p50_ms": 1414.3128122232463, + "ttft_p95_ms": 2566.908354521784 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 9.092010021209717, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "ccefacf7e9b214037083892f6fa84c5e0fd288f041ebfe21880b9709c8b25727", + "score": { + "feasible": false, + "pass_rate": 0.3125, + "passed": 20, + "throughput_requests_per_second": 14.110410219983999, + "ttft_max_ms": 3582.533354521785, + "ttft_p50_ms": 1896.1470654856275, + "ttft_p95_ms": 3439.824414996849 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "loads": [ + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 161.351567029953, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 1.0, + "request_count": 64, + "request_metrics_sha256": "d1d9086461911338e6c87a8303aed090f25e2bbbfd184341e2df531f64c23355", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.041293968822475, + "ttft_max_ms": 86.51189290936045, + "ttft_p50_ms": 86.51189290936001, + "ttft_p95_ms": 86.51189290936045 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 9.191051244735718, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "ee23707d601c6b4559c7f019b09020acdb589844eeffb79f6b46f2365805ac99", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.038674169035575, + "ttft_max_ms": 86.51189290936001, + "ttft_p50_ms": 86.51189290935957, + "ttft_p95_ms": 86.51189290936001 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 9.050537109375, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "63cce9144c69236a07bb94cdef18c7d09dddeaa1c0ba549a36ff29742c780c2a", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 15.535832140198398, + "ttft_max_ms": 276.9276050512883, + "ttft_p50_ms": 189.66908885537626, + "ttft_p95_ms": 252.16908885537626 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 9.04306149482727, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "568476977590c981958cc0acc4e07547c079a2ae00520b59f8ed7761acb67bfb", + "score": { + "feasible": true, + "pass_rate": 0.984375, + "passed": 63, + "throughput_requests_per_second": 19.925437356037587, + "ttft_max_ms": 1281.7127732631457, + "ttft_p50_ms": 729.6185162106785, + "ttft_p95_ms": 1219.2127732631457 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 9.14296841621399, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "f789fb967a981edf5c112d0e1a5d0a034456e2cf51f49d6e2e8597afe9df0f2f", + "score": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33, + "throughput_requests_per_second": 20.34996647854275, + "ttft_max_ms": 2191.8433254998167, + "ttft_p50_ms": 1169.8561499618286, + "ttft_p95_ms": 2106.0636248827373 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "loads": [ + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 9.14107608795166, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 1.0, + "request_count": 64, + "request_metrics_sha256": "d1d9086461911338e6c87a8303aed090f25e2bbbfd184341e2df531f64c23355", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.041293968822475, + "ttft_max_ms": 86.51189290936045, + "ttft_p50_ms": 86.51189290936001, + "ttft_p95_ms": 86.51189290936045 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 9.139281272888184, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "ee23707d601c6b4559c7f019b09020acdb589844eeffb79f6b46f2365805ac99", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.038674169035575, + "ttft_max_ms": 86.51189290936001, + "ttft_p50_ms": 86.51189290935957, + "ttft_p95_ms": 86.51189290936001 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 9.240875244140625, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "63cce9144c69236a07bb94cdef18c7d09dddeaa1c0ba549a36ff29742c780c2a", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 15.535832140198398, + "ttft_max_ms": 276.9276050512883, + "ttft_p50_ms": 189.66908885537626, + "ttft_p95_ms": 252.16908885537626 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 9.090924739837646, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "568476977590c981958cc0acc4e07547c079a2ae00520b59f8ed7761acb67bfb", + "score": { + "feasible": true, + "pass_rate": 0.984375, + "passed": 63, + "throughput_requests_per_second": 19.925437356037587, + "ttft_max_ms": 1281.7127732631457, + "ttft_p50_ms": 729.6185162106785, + "ttft_p95_ms": 1219.2127732631457 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 9.041213274002075, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "f789fb967a981edf5c112d0e1a5d0a034456e2cf51f49d6e2e8597afe9df0f2f", + "score": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33, + "throughput_requests_per_second": 20.34996647854275, + "ttft_max_ms": 2191.8433254998167, + "ttft_p50_ms": 1169.8561499618286, + "ttft_p95_ms": 2106.0636248827373 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "loads": [ + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.092233896255493, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 1.0, + "request_count": 64, + "request_metrics_sha256": "d1d9086461911338e6c87a8303aed090f25e2bbbfd184341e2df531f64c23355", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.041293968822475, + "ttft_max_ms": 86.51189290936045, + "ttft_p50_ms": 86.51189290936001, + "ttft_p95_ms": 86.51189290936045 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.093050956726074, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "ee23707d601c6b4559c7f019b09020acdb589844eeffb79f6b46f2365805ac99", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.038674169035575, + "ttft_max_ms": 86.51189290936001, + "ttft_p50_ms": 86.51189290935957, + "ttft_p95_ms": 86.51189290936001 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.242159605026245, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "63cce9144c69236a07bb94cdef18c7d09dddeaa1c0ba549a36ff29742c780c2a", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 15.535832140198398, + "ttft_max_ms": 276.9276050512883, + "ttft_p50_ms": 189.66908885537626, + "ttft_p95_ms": 252.16908885537626 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.19024658203125, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "568476977590c981958cc0acc4e07547c079a2ae00520b59f8ed7761acb67bfb", + "score": { + "feasible": true, + "pass_rate": 0.984375, + "passed": 63, + "throughput_requests_per_second": 19.925437356037587, + "ttft_max_ms": 1281.7127732631457, + "ttft_p50_ms": 729.6185162106785, + "ttft_p95_ms": 1219.2127732631457 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.18896198272705, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "f789fb967a981edf5c112d0e1a5d0a034456e2cf51f49d6e2e8597afe9df0f2f", + "score": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33, + "throughput_requests_per_second": 20.34996647854275, + "ttft_max_ms": 2191.8433254998167, + "ttft_p50_ms": 1169.8561499618286, + "ttft_p95_ms": 2106.0636248827373 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "loads": [ + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.088153839111328, + "offered_request_rate": 4.0, + "offered_request_rate_per_gpu": 1.0, + "request_count": 64, + "request_metrics_sha256": "d1d9086461911338e6c87a8303aed090f25e2bbbfd184341e2df531f64c23355", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 4.041293968822475, + "ttft_max_ms": 86.51189290936045, + "ttft_p50_ms": 86.51189290936001, + "ttft_p95_ms": 86.51189290936045 + }, + "status": "completed", + "trace_sha256": "0cef631f351f5afd880172d0467d62d9813de5ba36fd3795f91afe3256450c05" + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.048948764801025, + "offered_request_rate": 8.0, + "offered_request_rate_per_gpu": 2.0, + "request_count": 64, + "request_metrics_sha256": "ee23707d601c6b4559c7f019b09020acdb589844eeffb79f6b46f2365805ac99", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 8.038674169035575, + "ttft_max_ms": 86.51189290936001, + "ttft_p50_ms": 86.51189290935957, + "ttft_p95_ms": 86.51189290936001 + }, + "status": "completed", + "trace_sha256": "c49314cbe4e4b07a9335b57e861ec5b6541ea4cc061ca9abf923453276c419fd" + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.04839539527893, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "63cce9144c69236a07bb94cdef18c7d09dddeaa1c0ba549a36ff29742c780c2a", + "score": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64, + "throughput_requests_per_second": 15.535832140198398, + "ttft_max_ms": 276.9276050512883, + "ttft_p50_ms": 189.66908885537626, + "ttft_p95_ms": 252.16908885537626 + }, + "status": "completed", + "trace_sha256": "2504b6340f764e726110e48251f8017a2f9ba67b44b8af7d3a838bf674823d33" + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.139516353607178, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "568476977590c981958cc0acc4e07547c079a2ae00520b59f8ed7761acb67bfb", + "score": { + "feasible": true, + "pass_rate": 0.984375, + "passed": 63, + "throughput_requests_per_second": 19.925437356037587, + "ttft_max_ms": 1281.7127732631457, + "ttft_p50_ms": 729.6185162106785, + "ttft_p95_ms": 1219.2127732631457 + }, + "status": "completed", + "trace_sha256": "5508a3ce590f7d1b0b3c211a2d4f2131886a7ae2d85cafa0c436a8e3029675f1" + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.145836114883423, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "f789fb967a981edf5c112d0e1a5d0a034456e2cf51f49d6e2e8597afe9df0f2f", + "score": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33, + "throughput_requests_per_second": 20.34996647854275, + "ttft_max_ms": 2191.8433254998167, + "ttft_p50_ms": 1169.8561499618286, + "ttft_p95_ms": 2106.0636248827373 + }, + "status": "completed", + "trace_sha256": "8987c3ebd4a7db983dc0e99e44710cd44a9d5d21cdb8e6b58878b160918593d9" + } + ] + } + ], + "contract": { + "arrival": "open_loop_uniform", + "input_tokens": 2048, + "output_tokens": 1, + "prefix_caching": false, + "rates": [ + 4.0, + 8.0, + 16.0, + 32.0, + 64.0 + ], + "requests_per_anchor": 64, + "target_pass_rate": 0.95, + "ttft_slo_ms": 1256.0 + }, + "frontier": { + "git_head": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "git_status_short": " M frontier/cc_backend/backends/vidur_cc_backend.py\n M frontier/config/config.py\n M frontier/entities/request.py\n M frontier/events/cluster_schedule_event.py\n M frontier/execution_time_predictor/sklearn_execution_time_predictor.py\n M frontier/metrics/constants.py\n M frontier/metrics/metrics_store.py\n M frontier/profiling/common/layers/rotary_embedding.py\n M frontier/profiling/moe/moe_impl.py\n M frontier/profiling/moe/moe_vllm_kernel.py\n M frontier/scheduler/cluster_scheduler/__init__.py\n M frontier/scheduler/cluster_scheduler/base_cluster_scheduler.py\n M frontier/scheduler/cluster_scheduler/cluster_scheduler_registry.py\n M frontier/scheduler/cluster_scheduler/sticky_lor_cluster_scheduler.py\n M frontier/scheduler/replica_scheduler/base_replica_scheduler.py\n M frontier/scheduler/replica_scheduler/vllm_v1_engine_replica_scheduler.py\n M frontier/scheduler/replica_stage_scheduler/replica_stage_schduler.py\n M frontier/simulator.py\n M frontier/types/cluster_scheduler_type.py\n?? data/profiling/compute/h20/\n?? frontier/scheduler/cluster_scheduler/prefix_lor_cluster_scheduler.py\n?? runs/\n?? tests/unit/test_attn_prefill_prediction_fallback.py\n?? tests/unit/test_vidur_cc_large_payload.py\n", + "source": "/tmp/replayserve-frontier-rs1b" + }, + "profiles": { + "coverage": { + "attention": { + "1": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + }, + "2": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + }, + "4": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + } + }, + "manifest": { + "attention_tp_coverage": [ + 1, + 2, + 4 + ], + "environment_contract": { + "dtype": "bfloat16", + "frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "hardware": "NVIDIA H20", + "model": "Qwen3-30B-A3B", + "tensor_parallel_sizes": [ + 1, + 2, + 4 + ], + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + }, + "inputs": { + "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-composition-tp1-20260717-v1-20260717T014550833983Z/artifacts/artifacts/attention-composition-tp1-v1/raw/flashattn-composition-tp1.json": "ce461be92a5059e2d168b2cbf21b55d9a55ee71c3b0586a3e63cd8288938168d", + "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-composition-tp2-20260717-v1-20260717T014551831545Z/artifacts/artifacts/attention-composition-tp2-v1/raw/flashattn-composition-tp2.json": "997d67fb3a28b488c3aa4945ae7c6d4ecf8a08726ee3cb4652d16c221ff395ad", + "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-composition-tp4-20260717-v1-20260717T014552897206Z/artifacts/artifacts/attention-composition-tp4-v1/raw/flashattn-composition-tp4.json": "046cffd4c11d8a36513167ec25d3e1ca016324715679ea688a58d4a71dfe62b2", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931" + }, + "outputs": { + "allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3", + "attention.csv": "03180eec28087a301c9d8c185874c50bdb28b31e5e9af35fcce5a6a5a9e62147", + "attention_true_mixed_fused.csv": "cccf76e6cb6a9359f188b07d4dc2942442568a6e8cf3ff3d77ab3d35a052cecb", + "linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "moe.csv": "dd698113975e5ed77f51dfd8a69aebf764ed651c3f6f34b71914d25d6e001415" + }, + "profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-fused-mixed-total-conserving-pure-prefill-batch-composition", + "projection_contract": { + "allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity", + "attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target; true mixed rows use a total-conserving compatibility projection", + "attention_pure_prefill_batch_composition": "Direct FA3 measurements for 2/4 requests at query length 2048 and 2/4/8/16 requests at query length 512 for each TP; included only when batch-composition attention inputs are supplied", + "attention_true_mixed": "The directly measured fused total is preserved in diagnostics. Frontier's two targets are projected by the same-TP pure prefill/decode reference ratio, with projected prefill + decode exactly equal to the fused total; the split is a schema compatibility attribution, not an observation", + "linear": "Frontier profiler using vLLM 0.20 CUDA operators", + "moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero" + }, + "row_counts": { + "allreduce": 24, + "attention_frontier_compatible": 150, + "attention_true_mixed_fused_diagnostic": 30, + "linear": 36, + "moe": 72 + }, + "schema_version": "frontier_qwen30_vllm020_frozen_profile.v3" + } + }, + "root": "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final", + "sha256": { + "attention": "03180eec28087a301c9d8c185874c50bdb28b31e5e9af35fcce5a6a5a9e62147", + "linear": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "manifest": "70aecb6d70f9e509e9816b1a0929163b9505760c2ac30bbd4cd8e51badfb7635", + "moe": "dd698113975e5ed77f51dfd8a69aebf764ed651c3f6f34b71914d25d6e001415" + } + }, + "schema": "frontier-qwen30-prefill-surface-v1", + "status": "frozen_before_real" +} diff --git a/runs/frontier-fidelity-envelope-v1/results/a3/qwen30-prefill-ranking.png b/runs/frontier-fidelity-envelope-v1/results/a3/qwen30-prefill-ranking.png new file mode 100644 index 0000000000000000000000000000000000000000..ed8a22189147be87e8e5b641ec4062cc0de410b1 GIT binary patch literal 107863 zcmeFZhd-BX|37|eqam|IMo~yoDiI>1h?1F+m5S_yG80*$LaCH8LNc>wiHx?CSu)CA z+57uEx~}{4`#rw*U+{T6z8>AzeO>Z?d!Ofd9Iw}Ny^eqrYD#P9*y$*WT66S>{7H(U zwWO$}?yFYdJCsiCbNo}nQ9;*H!}hGBvx$Q_rE216XJzYXWns$gWbSa$!q#TbPSIUE z5A5JR@91cEQ9?w-`hR|6r>(;|5q`xlr|>Scc1KQKq^M147=BG!%7IURKj3 zs{gyQrlwB)is5tJs)r8IsK2tYYG9#PsIckhWKrqo|Gbr7RPpoq8*lEHZ)R~#3s}A> z(Rir1mRTWw(xX(yrORW_uHfB|e^00EykfCK zn%1iW&8ivSJsUHz5jXxdA&FC!o2_WC2SVM`d{yR>yD^;(0{${ zk++eF3jg)KapuSB{_9Wns@& z#(=4lU4MSPjXZdH_kA&oB^NGS=bilQJ;t;mu={iDN~fV8Yh)P$nJ*@&J?sog!eVaVJUgDI>%<6jqi~H<^!q;b7>Q9et?C$A#RaUm-Qg5B?$4644&VvTamM!Dnvq!^(tE8l) zJ!4{GV&_bV<>v?cZHGU~EWWI)T=~7w*{`^`c!TTF1+UJ50b%E0)(O7(k>o^?#?jv$ z9U4CeI@;T_7*&oQJ#ysGp+nT29jClsU0WG@_Wk`&pFgiRqg%7)YhGHr+gaAm$b!N` zw)4@~yuEi_{&S`-S!bQLw)U=l`>w~vZfVN0xVCZ6S?*oC(htqwGPSU<&@H~QSHgL4 zwWOpZxn&0V4(C6|Eccin*O6CJN*>rECYE(*@@8mgXwAZA8IK!y27x2u5)!8EIlaAYf^V^6I@?8I1WM_(U37Z+D{eY>B{^q*hDBO|Zys8uAH|5@lWBJ2Nd zj)g6hzleEzd%I7G`)p0N*7)rsTjfrkJo%(j$Xd#Ef*FgXXvnaOxZSf9VMvompEvdouW4(m*;l33=56jt$PEOwvWlj#xjQ4qmg@x607?ikOZYvL95}c)_r4{%j zCFVL|74L4CXD7Q0>mu(Vp8fDxk48jBl#J);n@6^K;~E(_I1D0R%PrF2&D$@Hi-t*G zQ~5JX!y@wNb#3LvL!_K&+FjxPdt<45t;Z6VdVPd+QYmU|e0*fOD>@dJmY$pI$Gyt` z#*Ok6{am}5F^eLPd3JAZ=6nfFivrKZk`10O&c6TI)^_#&-V0?|-D#(b%NFKm*E2Kw zjEyB&G-%>U$S-SK|a&of0mG5}(S0Tr~Z)==~ek`ZD zySrb0|IYbP;*wlNAj_?L_X5~NPX8Qkd!6Sxxzp6tbl0U`O@rbqELa-ePo6v}^GGwu z_YUEdFMc*PM&0xJABxF`j0u;p;A(ujg5_Ztwz>0n@USc zqX`ue~>!rA!_?mqvciK}PXV3CqU&|R77q=A!z_hn6DN!R~?atFh{`l+m_Yd|D zPfRGDIB~OfpfZ$eoh-wy3*UBVCY@BUF8WVD(>xS^cZ=fR<(SE>p(S_A{Oz59zK2m! zK1!jR1qOt+ZBtscdNp%VQIQX}pj_}IPojQq>Mo1gt!*P6mCDUK3=1?{)=5c8wN;00 zQ*K7>sW)x@@KBOIDJjXJFE%#z*E4H;&uDp|^Gh$9CDar3SVixVuC8|Y zs<6+h;c>d5on1PfFL#cl%dqiPQ=uE3ooAk4g|$a!TYhCsHOP<2n#Vfu*==6qKcE!M z9-o!+;>C-G56#Vzt@$@2A|tn3HDw*0`gPZcIdxH2&~o|m<+Mg_BfjZp-?O)&3CLZ! zQoy}x(No0Z$YQNg&h6VzzefGJol>T8(>*LDG5*=^<3~{&7@3%SvASqW7Z(ccCi+kD z?A)owuxvVT*ZoJ=+C9#)o?{7&isE?n^{W=e-QC+u=eaPQaP#5Alq`1_7nfU=b0QZ$ z$N0KDD#qP<_wJpW^~6B)HY|DnkIo9#%VZZ81Zwt7{CsuIRK|6p|CauK+aCw*zYjF$ zve<}NYowi);@Pq#_LKdmgix03?!&Jx>dv^V{abM9)Dm23i~s(sv`2HUE!$n4w9_eB zrps^Y--^G~_pPDelF+eZ$CP;wrsUbREm@f96RCTqrDE|QNh2Y~wzY)vdj5Pze#h26 zEGOM;OBSz|UP<(?>$h&*GT4L;YSoyz99^H*EY5krVHrN))sT?2WX;=WCkL^K z@Cpdf9U0i!qV=tnceiKUEB77k3`yxWC^vHESLh z+Sb*zZF7C_rv0YqrPEDPF2j`bU<(bU`An0x?_0`S=bdNJ`Cm0RZ`yP2!;YH4@B*xI z6V#&(`)xSx@3W>`y7XG@+5qJ$d~o~G->SZSxUK0IyVTrf2XcpphrN7#6`oWId^mU3 zUcb;u_(QbJI!d?5MU?;eqt%?8zZRPlP@wo2>l3FN1D(dEg z6LD68xhNe^3=5qoud1peQIgJT15@ZLk>d8el<}X=>hn!mNs9JM^J&kV+-q5PqA>?B zp*{2I7eQ8SUEK}XEEKnM!E5a8*WbQr4>;}IxieDAHD_w_`KcE-!@{=ID^BP++1t0R z8yu@XdhA&F)uk(kE7*)415epGIlZ~HMJY?##$I|)P#E_r-OMwz)MLGDOUr~0LTi8g z_<`nyKdaWPSx)u0mU0;uILi9cu^3xf-M|*=C>$zbN|+803=ORuDqRfWSH1rjD+fLB zF!~TS+I_c=FN$hwYtt>i3c6gmQugD==Iht5eZ)D3~Se} zC4k<|-TmvrweX@2)_Ch5qqJ0^(||XAdw+kw@%i(s0M+sOxAVE)g6it(QzMmJ>IrI` zk@xSH)zr|vzrXi&dqq%$nqNv)UwxV=nps|6-b?(=%*@Po{hS?^b%|GVteZ`7k$itb z&-_y`H^dv({v~yZafjY(`n-AbrhO(oeOF58r09VwV~H){*8|4}nsuyR*tSYX%+D{( z4w*I&mCRyScDxB@qE@j8X)`l2GA@&4@FA#DY`m@KOG1ScK8(hJ4@REzd^v_CZuw1l zf$d~}RQ6)0T+LBcRb@dH5VG;?ZxGENFLWMat-tKz!hGOcsCn;U^SBJpBN~}@y|2CT zH;>ZKf98Gl{ktI+=Uvw~tY?kr_KZJyDt7rSo!p2jo8w^10sEq8&xQOW?iR`u8|UVx zFPe0JdNTbXpnvarc6Py4-rNU0=5!OaQ|{O1%+F4;IXE~lhfL$go}khw=SH}rpsHM$ zl?I@;j{*fsw%WwRv?b>&+L*YZgT1OdX`6JE57vB_{@sTUg*R;2u!`~l5~xmTE^#lU z4y9*jvzVHh(My#GW`5-K^z_U(i=2!HCz+a>+CI=%5ya-5nwr{l;bmQ&W<#lGX{|@* zix&8q4 zlR{wDRTQBRJOTm&turX9uPQ1O%nh(8%0Mmx-8jyCJm=_`_35eVCf|>?r%-{I@Q@NF z8iF@V>Gd!_tGKj|UT(xaM7h%O;zcH#mOLKxCjp~_m!m&k`1btP?2wW|8puHMTnLY( zjjXAr{Dj-nr%&mMuZ*8$zx(8gNy(x|RUa0ncW`iUt#t3V6d|AM*PltPTwItMN`Fq^^)z%Ms6S>g(emrFHZ%#!OJ;G736R58KxwS)=4Lj^&e2h``RUOx ze!!n+K;lIIgV7F1>r%xw# z=%!PYv9&eoUfPmijT4~GjCak2QqPxFRjpdOV)aX0eML!uV-Ig?8Wv3+2M7b8GJ$3a zHzE_y@bU4ry}P$Z++$9h;865gH*3`5PbKcIjss0BC@edCx3hfdyeA+eA;Hcm;rJ5m zc|9B3N)$Sv%jKIlZ{8s3^t?qX#ow{!Fm?V%(E%*U^vk1Xr)Fl_!F-P>D|@G;2&3`% zq94VWRfhPs9$4-%)wZ&~Id|O}b}_HJciBMX)^FSxFy2?+ft$37p8hpxg8kI630e#8 z-x5+S{nm1x0B3J0y37RfB=6v`dt&f=5L$xiyGT(7GgLvmv}1a-i}Jd@W>X9)hkDt9 zpjhiSZ3+Uwebw5^m22C&^V|Ge6$T$UYb^d?zja*Cp3_}!D_bsh;oEB9mcy2of~e%` z0BDZ=4eJT2tVxmte4(dStzG-3qcZf@Q#9Yq8901q8nw`BDz$MCd_Qs9XXU?6A z#rinkS|UYSk;AVVo^9J!wzaj1yU&PycqA1Be7u@nY?Z96?0KjtB_0cNqsQmwCtE2> zMMcHs@@4Wg2}@E|rXwurR}IV5>}*$a&epWb46{m8Yy!$bW)al2t-XDTp`oEcsR!q# zeO51XTRk4w7N{tk)X*?7H(!4I_;E4II$Y;kR3{~Q`DHdgiUP0#polQhv+g(qA~gbd z|JGUt*il|V!Q}V5d+l}a!F}HCSNqZP+Nj9oD)>9EpdcMZHlX9s56+N~kg5+K-T)T< z?CT2xEuZ`IRs>3jx?aRvl)>lS*;viKS|N3(a&0(qn|3+%>lS$~ZU#GJD!ou+*^(#f z)zbAWAb=Ks`EzI}6yIbk4II|A$5M;{u_4PGXH8m%7zo+C4nppdX zgeYPmjSwb`4TN%h4Il)IxKmIo#jsF2s)?1lGS+<+eCJhdE#vWM>58YS4;UC3yYKL1>^!H6 zRaIBNLVvRB`oI|_afxnZWTff9hQGv-E2j26(-u<31P|PtW-APQ7cFCys)}p$1tn5? z{!_E1BPlE@Y6r(bmvr6J`)yjbYp(X@UInl(H_((_;eNoO^GL})fDi9Sj~=0_IgwA% zcn&16!3cyeJVyl$)biVN0gBX!7R9 z#xfvjU!adAp;{a)*_)-VXj;{e&Q1=|i{*$pY1h?^^XSxFnHqkQ?lNNf1UjMVcyv)3 zDi^Irf*K3f4LJ`zr@&Jt~uyUO6 z|5p<<5HCxpFKujIbxZCH~?wFm^ z+$Jfhf4(Iz!)sA%1X^8anG=|)X@xKzA8C-5mD-c@#~*rYn)QBqa*a9!M)B5pb!(}o zh;(V(*Yv2Us6XTOSfrv8kViZ9Wjy9w?A@Ut`Cy;ASxK;RbQT}hHX0j`M}MTnCDhIEIiTBb=;|c5Gv0c+)e&(RzMKgB2n zxyjt7nev* zMox})bHdXBPxR7}$}-3V)hR+pV(xHvE;LLxo{JWcQpAHR16c9x?mnOG&hqt3!to0G z;2SsSzE;Mo+%1=eC>Uw$ZfyLxtVb8RpD*w<9a-u`$|x@{PvlG%XUzVyaKtG5ZfU9K z)i>JXOHW!%pJ2=uyKKQ|WMXy{8y89}@3F{TvOVDjfZK9+cN+Qt;%K0Ye9N=FlogQ} zU%f}81_eLy#1rpbmUY|PASyCI>KBF>uSMwWiBb>u;OLIbXv5l0Z8WsDX4_Jh9CT_R z2)bCs;M)&&{~7%#BU0;8BYmy!YjT3vG*ASm!|an@KRxuU3_}-}yq1E8lFosBYdOSE z#5Dvd@4Niw)hpSZBLZnaR@DQG!0r)g!D90 zXC4KBPU<%(dHCt$-rWAQO}_bT%Dq>@RmU2#FWjwlT3A?c=qlJwd+glDM=Z0A1(%Y` z_ANhJ*97d`FoBzKqHzNIsXg3^iZTtN6FUP8F*uW@k-GSbSaEwM$Qyd)?bpv~rptbJ$G%oZ4(pwL@PCP$h&%mMp z!$x{>?&9;)#q3a|+h?_swJYL|hS`mEtHWjKz;hs8*fRj%E3YgMeL45E4qImU&o33L zhIAT@r^jA-(JcM>>(_D^D=o#YY{}XwjJd5vm-k>br=~|km$#qB-A+7RECG1o2f2dt z@b%TN{MT=RnYCisiWNlJr|deb*7yG?F4UcHZOOC8z0t@^1%_$QOikShAcbcFJI(*? zIxgvO{d1ns?V+Ob9Y9V3=+IY9 zaQWz?t3h=90Lo1P>{ijz-g=Pf44Dl7roN$hoy?pUczxFc-VC6Fh8O47tysN*qR_g_ z(aX43E%8{GKBt+Y$8u<}tGeFEa}8h<@ul#?8xA;Zg@Qq6=cCzImlP(h8q16Ax7YE~ zC7SJzw2y0~=w&nBg@1A1ws`FRm5Kg)HA#ic0s(!Z!oq8y&@H>^x-c_2l$MnOhml?E z{D~bMHJfwM=$Z3N;XN#&-o1Z60+lP#O8>=KRWaKi98|gc;)%S%!l1W-PY*0}NLUy| z>kyAO`Qb!E1Alwz(xv(5won{WSD1TxdMGb2`wutdh8)4#0ixUA}Z zv7^7bBE}9XZ)J2FeZOCDRuD2nfztpTx3lHx9uJmqY#jZk$L_^A%tL5{K=UD4f)zT` zUp4O#gAcFNM>-x4)`Nv-CR<{&28^lL>Ea0m-2 zlOquj`eQv87l1yOP;)bVX?FJZ0=F`tm5W{(JAcO9{0VfgNHL3(w`TCrImy$;+J6k| zP&eO!4@G1x5OR5CWyjp~Xr!d`&cT-aYY_W3qq`adH0EyLazScb&AELv6;!ScxZ@6rt(k;B-usmOhn6?XF! zy2*M5hR3^KDQmzd(8>RScK{;T2&xf^qRcBTeY)+^=+9S6t$q}31Q&$1x3RsW1Muld z`bTvCU#=g&d|`r^|0-4~6e{#eBAi1!x@NFDe>b1K)Q|j&O2@b_Ub>_;`>-6w93Hb) z^AK=jy7^nysDqaoQBmZrte&?_E;$t#qAv^6lZG<3wGG8SNwgMGtp()=Kej9xP)Aa6|8yP&2@N`W2aA>X1YRxY?0?MPR+aB<$cU}c z#fm2s5%w4dYP`$TFb`OZQQZ86=A4w8@!HYz78XhBwc%n1gkbjJPEGZ{`1mJVpjYbo z8gu}`u(H--*KPc%5|Wb1^C5M>a-vrY#TmKwe$6yjiRk0x;6PP=>lU3&*$p)9wLkTJ zRPXy0^z9w`IcPTt$GPUFIzov+RW`K!FUwgJh}Zx0k9631=&W*(_Z67tWEq&0`Otc! z(;66E?D`-)F`}&huY2`o5jY~*%BV$yYo|fpi72_>fK7C>pB#H%!=ken+=aMAyhcJi z)FbHFf(Z$NJNZP%pX@$$>J&lBKBO?K?pe{PCOQch)~%Vjvd%C5D7$2W^gBMu<&9{e z^n0D`DJq&kbD6@3c`7L)p%PMba`ApoG znex@B+iTOJYhVO5HDVqDPk;t@FAOZ)XWu}TiH*-~nhshNqH#U5+BW-Keayh<#o1u) zTSq!mZ{eQNL+WBgYJvBl+f@3A*^{~ew@grkm`@i{%TP*9!0F_mC{#=$>Y?8~awd8f zI<}yr3?33O!|_HwFJB%SnMY?gpt#8v`f||E&yPNseJ>-m3R_~wSY+nMhk(n8)~(4p zA{p~URr&Ud0Oy3`Y>tkO;hSfRL=;-T!ngOKV-c*%-h+IBUapPsv1n;Sar<_kn^^K< ziq=MT`knb*0R?w6d%lQ0oY^HRs@{>oTnu1RH|VU+ z?ATNU!HAAp29?`hA`kMR=xG@l8Ciw}Pr8$hL+3KTRL`THP0!pZ{>|df_O=gux@NzP zZ1tl`5P=Bml#~>D_svLbbj}E-0!9UT_P!=)$jW(k!a8FBF4)%z>}a7JXM1~f2^;(4 z?unM{Z;Px)9h{t4>yxgmP(G!n_w3+Gbp2{)=?isFxiT}uY97DWi@{MARii1OW2dVBxw-DS1$ zDix)RxOCp&}u=Tq29z*tB2m`2=@L?@jn03PG$a z;;b=c5L3#~F&LoX5v2hY7nX>xizk9NCRi(f0|=UlUMQh$DDiS&98F!vjtacn^MV@~ zu~X0tqPDovrpB)GfQXtC=PWG)LAz|kB#yyHr9>Q2{xE1TDpU}sc5@=J=$+c4rXt|x0 z+aa^CNq7h}CZ&hpQsZ1%bv$i#r*^V_*7^0g1&@=HgP;TDo# zCY!NbWZm2jQjhIB-b6~fOOTKiOvQjoRi~yGF=|QMIFr-EwR8l9QGY2ftkevN+gbPT zWZk}FRfxtAprP_s?b>!j5fvvvyGvb+Rr7->=bOt;>Ksko-nDz^@(wKJO?8}pwbIA$ z3)Xa{jNMCyIYe?g17A7)ATzW972HX#n(A-dLwXO|iz2MTU1zEoiY|{n{UpjnEK+wX z>5%EA|FB%{Z{NN(B{h##O+BJ8I$D6V5}O0Pf4b1aDjkYVB4=+Oe@Q z;%580SE&sFj4b+V#1QcoVh>wACqy0tf(L8niA#qjW@RXa3uR+0L;g4ii1RgUsGt=1N z{@G6!F%h6G=(8K~vxIO*!B+!7^NWt=iiv^#!XHRLRO!Nu5Ndc^Pfq~BLFg$+{)mj^ z(a*lyu^k=5{s5Xq>kL+Z8#(~>*uhd(pmCpqYTR}Uq>FF|4?8-Fq2G7`AGWpRU(9nF zV8_z48);X_viPzB7v@t~Sop0YBRjhSe@!xN@z$HA-L~)C4}as6Sx*~ETnr4N z6lo7;f{6#C!N2_WjTPXKxwRCbf@QK^^1QOxxW97j?d^#zOfU;siEw^=f2^d~`68?N z1*-Wa2)=sdy_|ALxG^?4sdGOn@u;^f(TTAEO?ztN$(AE>B{~9JU{;7jE^`paN?^`!( zAA4!}qbT3y14K09opOtct_Q`2e|bIflll4ciFSG|Bsuc-e$>*&Wr~f~*%j~Jt;Ti0 z9nEZ+obnbNbj^#4_;$HQvLa*O(V=E6&@*SxK9gG48*)va_Z5sw0-u+yT)Ab}E(S!F z${|yYVCh4qH|zTF5K?hDT$s+MviyYb&S?G{3rJ)j87t7(K$(&f*tHFnxKzUuE*1I< z<^JnoylOS_@7_%c_G?rT8Vag2i9sSjr)WkT>*J3O(oi6Lhs|TY!3g=46ULXK$Fs_N z>Trkz`W)~sILAUB_k%<}SDQLM!=qanBOg2nl6kgu+qMgQ>hu)YeT94q3L#N?*=--7 zxze^2Zjf-?PSz375{IPI zYAjq-+x;YDpxxY5=qy4R<7tA^TLx6<)}1>moA^v_6$ST(;qKm}MYpYEkDAu|RW&uS z13k!r+ffuM0x4Ba{S8;&y*oyrs6+JL_+QYaO=CoFhcnBNcme6U0&--+uHg(~~d3Mm4o> z3LV)UfFPKGfx*xAc5mB~=@lC{ZX{YcNfSUTM-pT+__^%-%s7ipT5umBwe1c-a!nJ@ zn$Da#^Al-9$_qlgplaeZ_Wn#s8mHFhy>t7hT}8WNhhIlE&4fHyoRI3z?n@=)qA5?IZ))FLF1vID_D zr&^I~^Sum8@W|{?so6+-g+gF7DZbriFO${FFdZS2$Dl(4apozma2nhAE{^RTL(CAYK&YsekH}g>WhuQQ#Ma z%}|aHX>6H?QSToRKyy?7sM16O+nWSO;V%jB@>dG$Wxb3#aM5nOS4&<|(e&*d;SF#E z4C0e+KVPlI@$r~yJR*ZYnC-2B9z{d7@j-cR9dFhEp`z8<_x1G&JzP6s)`8p~3U|;^ zDHsV;+)w#_WLbpe9fV`TJAR2k`Dn=gbOx$@m0X_kXU=TJq8CLU^0f}vvZ;}rH&#EA zTq?i>90JFWrmFadc^GYLiM!BFon&oQbD7?fphQz+V=p3lfYY_Rf=0318gbOef;302 zzPK=VIh*BD9LoddfRGT@gdpLN4*uEa8e+n6J1myx{QPMge0xm!wxdklz_oBO8u}_# zak{l@SrcMx>Qf&Y=v%aFLqLMD!^i((`~3+9uNIlz;?V1A#9^?lhtJNPs~q-B{(d`e z3P;qPJKAvqCscYjf4hTw@*3ITLpBk|8y9U($md98_8+)&o4@7RtRG`-(moY$tEk>b z^$IEF&4^eK*>q0-is3&^xA)(?`j||n3`ZgpJ9{9get>5^(qIi8_6`m(QY!wO)KV4V z=U)N$mLd7tI83bO9P4YTsWw;J2filXjU0Y~lvR=Iq?$<~IDXLLm(~v-cD9D!l9rZc zANT^^Ty71;X={l{3M((4eg-m?hJuqOiwpBmgO*WsX+~U#{@fpM;~`*QrU@9wcxt%K z(zBn&!87sR-@p=D(WusNKv0mztjMT2LTB;@-`-@eKJRgQ@8yj9-#qp69exqz8-ebn z=sU0BxQSl;eJopMiuG>$m9v(Xs6M8Qdf*TzP+{DxVHj5=WudPn?tXFxK(GB`Dy%by zFUjux*qkRaOch3KEliayF3(Mg61SAfmw6MqanI9F?zGhnS6OUIqaNPiTGq+r#3 zN&4Tj&SkdN11iMl4Bg)PJ=FWyRHQ`zEN;ley1O}7#H+>i3i_~eGao;H;1?dD z``WBii$mW{OiTom+sPJI@Xs?o(=kV4L_@1D(|SMtJSz7v=HR)%r9)c#-+_mjXnaHy zxY(8nPcM)6vOq)Hwr~hG@<~(LB^AQb(wjLYotQma`WjzsLxzb@`--4HV4Vq4=ahqI z_c$V{dV2Q_9-z#lb{RZ3*oCK>vY01^+?M@iIHwIc)(^^@AVox|{6PvNQoCb`1Jf>O zCG?cbpHAAAmKM#^p3pR8q4-xk~<*|8yj(%`5XpVyY^sDPYZ@=QO3|KrVzHSDD|A_*%oiz_5Q)OozTC8qGATI8XVk7 zhI%<|zNU3+2`3TSPDONMW5SJP^=j#fK(&a?c7Yt9@MrEjA_%x_`72i>C}Z5@?K-Ij zB~C|=9*sP3@o9}k3Sx1)5Vp@KxPg}R^muft{d16h5_3d{4d>shoV<)?nrR8*ecb$+ zN0PU&C}?Z5LM=-i9_>N)XMIBPZc))!KD11o-}YNKX&OM-E?v*acsnf=V*C?iJOzbL z6ki@)(dsdqQZ^6|Kuz)%Berm;?emEp*JA)2cmsR6>jEkQovp2H#JsMq?ql#TAhl>| z-29S#_`E~_Y3hxoIt52?O&aEi^v)2?kaA#}22i~aQQZksoj{id21Xolv+JR!i^!^P zY&>J%Gmb@Y+g9Q=-ZS9q&$xjb!NH$n6zF~D@1SerF)g|Iz)l9brRYYLZ};|`BhQ`6 zul6FJS<#+N3zlyAvUnoJ1ANeRgw@p4l;Jt1JN#14pA72Tyi*nJ`f5goC}oVdMb5~x zWFMv=hX4Fw6fz%2hO20$d9^kx5&c=TogeJIzzWgS(4Z~(SAYKv%pOpv%}BPHmXsfq zWf(?Q;;q)4<{2oQe+%+O0oAi%A^+&GMkj_B!;buwWCLods$L>gK-756w9}kGyXPCy zd5|Yv344<0bRY#~7_BJh`VA)0xzLrlyOv9{&H}Klq<4K&UAzjL{r%_AER3se@`xjG zR$~pJtDJ7#<@b-IHf>XjdI{=aKR0DBX4A3(>4U4+t}T5e;~4@x87Mfdbln>W4Cu1%0PZC}^>;ll@*5}9QSNZX0RYNztQ zyiB}rd4v-h2jL(fiWShChzFB({__&x&%@YD>L`&wy}aV$8xS%m1OF_yUb%)n?oX2| zwmBfrbr=<~upQ1fX7WALNm~nJjzs=Zn2{CAYgxAgbi^{@H~io`Z|X+@npNENviVf* zaPQlv-Bv){*bE({&wHfbY75fCT$p8B)I8{Y?HU3pA%LIAqggOk(_t7_!WpWDgD!gN3! zcR@ZTRS{7#CET^Y8CXPOh*<%_Is@`%#_l(4Ms7U&&D;G30nQ@#oLdVee+fyvAov5q zhY188G6R7X`4ie{dua8Zv+qtc9Jl{ok01`AbqG3#K;er;nS&pw2<$;*dT`;u45CZ* z$!h<*PLh{t0$Ys5Lj+LyQ9{crmTp_~xIoogW zKwwq5v*XomK9x_;PwgSfG*O*FQrB~Fg`$ucW4V}?nE^BWy4@JqcSSP+SQ5hq*VEEO zz--BjkmHO6ZAY%>6{t&weTO1~C?t_e+ze`a)b&fKpHryGG@bm%*KkTOTQ_CN-rH?% zhY1Oi6T^!HLmyZf)cTZ=+%RlO#NwF4h9D;GJRK_`A%OxQ18@(0>?$&v#t5IFcF9CN ze29j|qwP=#nhDkA8iG=!^3zp1IXhPZ(jxV+O7m{qQyZB$ErnU!6`YH@Z`s$cED(S~8$OV1VfHs95!=yUU1r7vFzw(6 ztZ2~@qq_+S48}4zA z3llv=2!z#1Qgo1LDzIox%*?J058L%OWz$1TuY4?fO?m5ObxRE8oUe_e#T5?2V(5Sn zvq9QT9E-&V`X>QgXkW63#goWVyvBa~E>?s-u0MRpNd}cr4vCe4kLNN!vlni^H?{~> zm2wQNDT}dLiE!YUlYD=b%XuY=!6*=kB08AnsSls&!0hz}T(l5-^fZkGYAuF>L=?dl zBHXY0_^bk&JK3m&^5cW39B+j*g4r3RfN@Dy4{qP)MZC!(%M*&096~L%B?N)yT!MmB zD}AJ*vXY)8tO51v-st35oiYJDlv7g+sa+t6PWadt+|>9O1Abya586}G_b%_Ex%v5^ zUsKFJv9V@XO*JpAHE;Iy@i{O2p=^Sakuhe(UEl)rx^^dA`OdnA3YNE+c%i>!EpR2N ziLJ5A_o;|N;OM=c{{H8mC$1pwj^*GBTAN+D2XPisew#jK3TQlCg+6~jiZ}jDXi~ui zUHHTOFZhQpey^-litUKUcA1-KwzV`9=yi7omC4Qs7tm0I4YinU_ z=iEc~0MvB+{QTbXyz3$MJkGR6#Nh;)Q!)c1qeamW{?w3(@DAu_84T+1!323;^~*9! z3JSDX**XQq$lVRU6)9bXA5e~72Q!*Bm7Lq0~FM@i&j1^w@;{`5D4wxu>BDQ<3*kfJ~QN?1#N>v>l9rD7mjPETi z;nLga1R=L4o_g`LRy=&XHT!~2z7vVQG}z$jI1G49|6^X>9#ocSA*Xk;;O4W{oO_Q_ zvHbAi!}0D28XCsVW^#9Fgyn$I{mtg){mth3`X(J*32U6)a2paz-qF#~O?m$yT$bnm z4Em1Apv0A<&_I+t3Jbm+?h1w?phX>ACVL1q>L+G;;Hbv}dv!x{c=EJxrVhKI4XB_! zYwxUFa7Xn5iIwiK(wQC?`onV#xtya*$vjR72xWk7bcH)oe6E zY8^VuRhZ*1k#4;!V)Od`r< z;!uz7nu?7HG~rWXnkJMeiufrq=?0AvP}SJn+!rO_CE5@gagEMrg^hqga{$pqEdqS? z^75iGYJAagfYV762CUJ2Q23NZW`2wb69e|Y-)$rlevr0aL9QhufMnPKkP$iKmyx1o z;o1)bKg`bHb6q35V}2e2_Zo-?@H#g`LPpG;on$5db?nV$AlMEgG=lJw9TFWyE+b12 ziG&z&6$^Z-u`x+w8>+^uy1J}G_Go;*5d9b_G)IP>I^^8hN~e28FaSEry{8 z%pyDf8(o0s{5TuT{y^ArKk>M67qE=hH04+arIpMO2TwUSJ16H2mQ4|a^+SrY4K#>M50QL0I31fD9qwc{TrySUpD?u$>Ni= zj@j+xg&eTz)uxhyV7;kwn6PV%8EkXdB9gwH~>c#Mcfv6UbOPsY)(0}f+! z9Q)to%$a52WMqN{H?LVpPrJ%78FiK(n8pZms z2u5O9tsN>IPvpFzklL(M`cNhbYR|?P+#s< z;z;XL#y}g#YG-^NO1raDBr*Rk>cETeSe}^nm$Ub!wa88Z44|Qa45QE7W~DHdwiZvB z@>)*My6fm)3*6sdTKPXG9@Kt<(>5~rT@u6cCO?WU$8)Z=xFz<)3;~OZeHDD`BX<6% z9}^{ma~S!LcXUU%7%5(a3|CV`n^hLfV%%2jhXzsJ>q1BtzPL6O*RJ*Ydj;KQq()FL z!5`G|oKA*Z&*DRT9H>_oiqP=;{jK3n;_{lSU5vg|_M-ZdY>|PX;pZG{eKMtl8jD9m zHf6@SkDHN7p(rx$1qgk|p}c9!{ATpq(g=sGe}ApKyl>@yj)zzSn6U=Bk zL9#TOOc>Nf+uc>UUceE6TcI80acaG@volHU7ZeomZ8%k306=Hqcj%wD(>(K^dqf_< z3^m^Uc_~QRYe;W|Y#|`e1JMm%n5cnKDom1QST?a)hxy10@U`X$CXJLOSN-!}|G$66 z9KWY#vGU*F^Y4cz|2LNO|BkZ2ZTjDR@&<}7`TBq7nvlEjzu*{_&nvg$M1s=!iB04X z66he9@;=0S3x5c~rU=O>VPj`Uo5DPneQ|DXZkc`E_=o^>GnUEE_4T<~uctp2LTe_| zB_KbTCk^QO4dHdgm%&+KP4c}8bZ%&^aVmS zeSCd^;ptHLNL4{v;uCN-W*XTbJI8bOt`vgB2+&;#Mahm}DS3Hr{5Svr8O;SWAr9HE zU%ud zm;@mT!o!CTYf1=EM8q(@4$f%@jJjZpt7NO@(5}}O!2|g z7$BbsJp+sb)F2#WxsFTkxm|3`P70V?TdWdjMQJ;?K)i zk%KN^|FNV71qB_(c?={52^9DkB%Y)LA@oUC+2cQgahuxF3>}76!W)KgMnZePAJ`5< zo@J95#6SRIf4qo_ldDJK)fcI5GswfAJ_{_wyYZYPW?QJRnB6XRa zNGZDns6x*EfHWoQx!|_rROT`~0%6DJkYCAh9j8m&0-!XtBWPQJh42ZKPi}Hr08(vm zWSkaB4?f=^plPCJQNv?n^04bn&YrzaazyY4j;N`<0d+Dtf8PJ{=Yh5w3)fPih-4Vc6rLpqon@q==hG%yOHAPYp&*{%!$muNZG&HD&50qn%$MPTts zlcnL}a%f{qDMUqqNT0w{rfZ&_nQ;)5nK^Gz=(GmwekFBdvlKfyJEF`9CoN4gCilE9 zJGYK(aPp=8`N*;bqZ3%YGv!2I2E-$-CPf0Ws2>W*O8?1zkJ&7;o=AH1FH(buz6RlO z_kcHZPw3aI@x{ievD6}<31SKZ3(Il}@LK1VP&jU{H}_+l>ymeQl%FE!7lG}3sK1+> zECi16I5zegXjv>&|LK|A4`)t%7rO>^kmS)3S6N-+I=L3ptk0Z_C?xStz;D5fC2l{7 z@(^GzsFQl*=1qp++rKvBa1#(kRK)GpFN|L;fnT(gB2(yK>j;JazOYSl{0Zjuu+v#4 zFln$3sEUMPs9o+ej`3f=ET~{T`Xb{>lqTCBMFNg8fT@Wm5tAZVQHI@C37ZZW{3ALgel0o41r-pBt8M-y!a$m}IAA%*y z`~NwBhZKs%#P(=sf}s<`gOR^GR_kDI{|b6H3%>=87l!*v=KiQaMN>I;^4@`q{J`?s zrXwD!F?&>}CeV%H1^z$OONb9LiT0tGe&M&s29?LL2qzorF@ua-H^LIf8?hT_x+sB~ zP!ysFc~Eq3&4&Mi%dP!*Bh)(-xn&fXVI{Ww-riW;RzXr7h^vhPNOh`7Ze(WG^hu;a z=$3eUbNcsN#*OYidbF0J3;XFWtzFy4{xkcpgYkCe>XCRQC4#yc5fPDAOXLpF6Z|13Px-oIEE|*c%2o+hO+`~vE=n+>f7#WQ9OZS$rd9`q{8mS@VPQ@l2N#!7=RzD{u@Emr zrzbG8aSTRep7c<-zk*gvSFU(y5Du6?XhxSs@W6qMfYox?>tuRWLqmhaXo&QH!-X^_ zA|`)SfA~N|YGee!yeS-96aO_qElQjQh8P6grRFu^S20)$Lbw9of%YTV@}|CCheHIE zkYZqER%ug!6m7|!A zAR}1T*tik82RXR{ToZ8PDv7c{pHN&Z*O_JK1O-C^}(C{t5C3DFRpmWB)KvoZ#3ZJoT1fKUf$WF>N5 zA-I8MI1~;n8d)|a@LVEb#l^=baY%A=AL7t&@&vp88`MlS+=P)dXjk$WjUh*eU`T=G zicCC)tFEF1VAUN$kOashOh0X~F*YR<@|r*P#w(=L0B~?t2suy4g{w+h>A>DCq=1qy z_fN~nu0DPQk%Z;c5{kIb%bWILdIG~UIMo4SG_-pR2jf@JQa-rU?JWyvh3yTL480z$GI*y7olK0t6Lxjzm1jA=8T*>C-!p&^k}=ZhCLHbo~SBs46bov{yu z@xwh5o{;pIb3s&EdiEH_O$I^{Q>)l_8nvW+1b#+rdT{@xuA!l{%u?h;Gpx~Uf|wuR zpe5=M@khurG<)1_j_C(<8&wOV%vl_%B8(g$zY!OfpF=YVb(}{wrN))jYZpRoICSW+ zO*4_$zpwQo=Lpz8fVSp8{lfP=@}dRHn0Jx9Aa zR04)`*-f6qVg>y8;4BlOc(E){-LD+zvrrtfYQ@hCCE@1LBf_TLq=OkHUyNA@4IKWe zGI6@Lvw+2?T`gB;z!*Z*JCgQu_nAS4br>_CRq51CVZ zKsdb=QjbW2u0(qO@9AKeV#k_IYRSdsZiDGVtQ|5{137vNkUlvvs$=ZWpE4jdWK&~D zM8n40E9i;P-rnjC2TN0?X#*19jKE+N1vZjBi>$JSJzWLj0LwIl=;k2oPEXIE)F_Q2Tr{z$L)}*hJcm(LnT&BrzO#11j`uoVbH892J+^ zfKIjo0;~%(Ca)F&G2-I^f*eMrz-sYBfyeGAmOY9|wU(O&*1K-oc^F#+q~Nyr;rUiq z)B@d^9#Cv@z$haDw73R}1kz1_7?2Fz&AVj_(U73o;#@*@Qnvx$+Xe?yGe6?>Ca7vC z*Iv1~`-uXGHA1X)kV0?T3Cy5u*`d5HhrQ@Zdo^ocaJ}DG)3mBt%cnxIA%!0pBLW10Z#n-(P~D(td)} zNEQdDNuacPUA=lWZvd9^&D*!Vu}j+#o?doS|MQ6ngCvRHftUbcz>Q!_K8wStRw1vN zh#cmR689i{bIbPas|bO*L#L5RJb66jE$GMimOPjap^B&`;^N}|1IWJd+&Tiv!h{LT z7&4AO4NUVI7XJtiQY0Eaj-Gl2l7O&&k^}5aThdr?7$VWsL02*C?~nUuhWrj815np? zc`nS!sW$f^l8Jguj?^Pq2Tn7H%gLV}fs()f=hTa#mFVmUBOHP#PCQY#nZpCQt>knN zg}{iv5+s3v<>6=VY{1S|wVo%qWGiqTD9>K+-w2;VI6lkR!8Y`!ZRwhf4z`(JZE$ zgmK`Y)fa?neC?BxGhZMh68vl!D@Lv;;g7bb(1$=h;sVc-;TjB_hUZ9K{C$G~<2RE( zP+}{@aWIoFuJBj$P6-J;81PBrAF!2@(-(^-!(-@fq5{Z~7}7p+bV9@3r(Z+F@@>qc zl@R#Tbb0uz_dCo(pOZH*y zPN!;V|IgvVG3)+0?6^CO%+HM|6#$d4Jqrb~33MmXcN;^TPnHLd+uorMYD)RCWSAMW zIFdwX%IrMGyv2R%(Go_3Wz1`sX%-*UbfF{*ntjK7(T0!cP@^e#RrjgyTN8`fb!>Mr z%Wc{4RP(vy&5iILY#4G$$_XvoUXzEu#@dV*=nUxgHg(PO=zUc|r?zu;W?Mj%ig&>C zG&+UW^p;W#cIBP^TBmopn%}VVdnl$~Xwh3W^WA@J-KHf1E5`&NN)Ghv>ZFWFBUi7X zfx#a>i=$*H4s6)88O=Gl#Sf>;B_69SukDX z)p)XJgl*Y$k^=T|v~97~?O?zX1}3K1^k_ViCrBM4au)Rf1UW6ZVA&JcC&VnKxU+1& z>sh=*nulSwvCuiKrJ!B8j*gz5iAe_s{rKX|*z!VQ#sy`N(?u-EnM%X}I(D$;!7$&d zKa(H|vEdUqB3K}MAW%rX4^O0h2J^kNpeM?+w~IrA_NZUL=#KB8kO=M(P1VCC>bHub z_Y&v3i98Lh9F04P**KbV#5Dkk6R@hK5u$8j>dD(h-%1NH^Hm9L3j6=gvL564LJp7m zyp5NiU(rB&zwgc6na$M`D?8PKw?Cv49TM}y{gs!KTY^wxR-6X=Z)<0SmOE#8 ztS4C%8~`QXy|9oFgP8Bg^stIU>^NUWX>IJe74WGl5?!%1eJ7A%g6JAupE&vV?WU%D zsHWzA@%kWcTQ7l=?m{<&U1aOqYu6cYu2dD>SZV3fiMv+lkbr~uWE8-e&Y&(fi~XWk z&_k@;x^?e?QD0(&J#GCdOzWL-n$P|Fw)e-piS-UVME4$vf2sJROEZq_4v8Onc}~$+mt}3Q!!HAyS_vm7s2v_7Ad$PBPj>ar-Dc`A zVo7^}div5(;tFKHu!!0=H~R0c&QlC)(LBcSA#?I9&^C0Vc?es*V^b%-MB3>$u(WnJ)UGT1aTZ&zBtJMP z=pv4?A3$D7jswcHH+tglaWa{$u-IKP{ypu;7!YusY~4T$1_QpOu;Xk2e_1yxo#lVG zqOx-Ck|kHD>gw6ZH>I;c1x|)8E=e{GkdOmKgtr_A)!cP@{)d7C1k2VbAk!5RY)rvD z21z2&$OCH)%D3r+{m*i98{6&^tqx7&J)%ayd>ZidrP^R*@W4-XvcG@~PEMW-Sn_bf zMB(P0aDWb=a-?PQfrSC;6Ti54#+!C|ZHh&~-vqa`2M%q>-b7YhD3%xH$iW2+DL)aZ zmdnVHTpB!dy|<`uCxg?XJSy_qmHfhx@IOPgJBfir=?M}bovWq`6ugGPSo<)v9+4@K zJ<2^x&hm3^9F7q!>tIqcTE)h0ekmvT`WmzO`Y+J+N6X zfUEr{0wa0cflYGD!L(0YQc?lQQ90xkGZ(C9Af}uE>s8vR<2207E_r#EqJ~zq%6N)% z-WSCQu`r0m0hrnzLf@c}#M?OuKc#y+m_3pkT zDjZLgWafAT-;uGld>F=JFKUI)BDeKSwLF&rW#hG~lOy|=Fy!&TtFwb`S3a;c0RSPg zI%d*@$1+aW*%AV-g(AwDffQv83=MflMo+KkikP7Vsnvq@`%c#@AION&-5eigbgsF9=ci(K1WO2Tg|#A4Wmq>n*L&(0&Z|AQ&T2hR zd7GP2C~Zx6bFJE22s{Nu*vlR&@xN7P)?c&HQUcoxNd`~$YOh1{uNmncMlwVuj!5st zj{YeRFZin<)y(D}8a6f7ZRYk9>_U}iEunDOB%?CZ+F@B=#FPWdT}v!ipt541H4$Il z{va{YDqYYz(RG;;FvB^(%%@+wApxLij)cwIKhULtOJfWl3~NI$KS!GDh4-8WEev<4+GVfU%HtCUz0!Yq?&s03m^n za+)IT<;@wz9_&R-S<&NZ>!zNL()05>VhX~@?FT&aPOI5+&ejqwA-3OneFx{Rkrj)$t;GskFlf(l* z*;lp5oU&|}=c;fF2n>up>hd8^B7fQCN;J2Mn4-{DP^pS_QGC4FZ3bM1$qWz;B2qhQ zCThjb^PR(>c^G2bz5_UeznXukMScbUjT5?W+NE)p|NJWV{A+AUe+e#-NR;1mGP|zD zfmX9}{2nI*b+!Im^-v2QG-GkRiOr?Z>Y$y(Y5=Ke7D^(sEeuXB35$3gcKGCO^Od9L zJergR5p_kl^fM(>(M>f-7m|@Nw$)8R5qSB<02sYm#_~sW{W1=`@b-A2-bn_meIYGs zm)PNhclgCOY)D6$x!rbkY}bPMK{f&c&Xs>yQ|8$daF%>ogR9k=6R!3bxD9;{SM*xR zM-w9gYh!IBMfs(?c%G{T=;+6){xOyA49=EyT~5J8G*AN_45XJSwnvYy2U3RnJ*nkc zU5=e;$}@f=o<5>&Hkr80KHYd;)7heddq8%TaEAT#88hHjyLhqD?dnVkfzhpI(CvWp z7*>(3z^WJJ1h@y5^#Py2pkZj&Ng%;Rk zyD#WTJzIdz`$l`s!Oh>7htJT3p}NT07fr?9-1^mpXI;=_zGsXMUThDtyQ{&Bs zY9fK$K5*A<^qAOi6r7LXq`G?)Z4$ptKJC~%u-Q2T+wkRjRWMvD|= zY#x7<7^2_tVJH_lA=#)aH%y;8l{`rWcozK?N&r^*{pUE|41a$`5Dg|Vu%^*M*O8v3 zqpm(1LHq9Ju_pFv@C9?7^2I#BO2a zRzJ$Nsf#kxaEvz^MN=wsWJ)V4v~)OEXUR392w8-#=DZ0^nUQgF;+$~4d#|{sJlZql zJQ>fD{9?GC%+HY9!N+aydn|R%rG-_9oMLD;=57L?g0ni~EXCjbi2(gU#|zU*&Y;!w_!yYc!Ee<^BE*sPJ;keiMU{hced$WYD-Uth{hVL} z2$V!oNFk|6#U>zp81`Bsqid2tU_CW4HUz83jf6}fI6nrskCYw*Sva8xBd9AqV*vv| z&Hd+c+vgA6!+Z*4ALqbQfc#)Rm?Bhd0J2>#DOm)dsuwB2<&xjY#h1#cjjt0DR-s5# z`n+}9AAtWZ1D@E2M*`EYX$-PAKoxsIdKt;(kx^iZz}2fqot>*sG%Vk7aUY~oqA&j9M9Kj_9xY<3*k>-2S&*HbO$OKKKlTC&V#uSqV_XAEL_Xap2qa)lb?qlvqfJO4h15u&s%5Vb2Nh>1>fAFAzZwX*~fL@93e(L zA>tTn9GZ|oU_Le#$pCjga?sf;_GyD)MJRwUgo1N?-et9gRvAHX@Bo@ehL28N zRU8!l1760GneYD>#acke|8NE-;*nAf4=W{aydHX~Q-~zKdU7l1NEgvG`*_2y;0$y) z4TT^@H1=DS&WlA~2rP@IrLt&C?m&>y8GMKIl_=LJv=opZfo!%?uw&Kbd}h)uG$-GK zA56N@ij6>{D7Xc!-D_yWv7l)45dcuc=SMeUD$TmU{-=$TyAhiOt3=ztUd0c^tEGgS z38EIF#p~@FuB(hV4x_)dXm7k0hz3yFP60IpnCUsVZ{5T_bFf7NB`~dC&jf&dG0rFA zDEmAc!w#>`F0ZaGLXt~_G@lWjU4Zz&#!$c-p`ySTj+>@8zzq3_PmKPVL;4jM)Lezn z3-L@)geihahtpd~O4q^!n_yK?x*U@T?C6`mL98M8gplG*)p1}tz$N5X#{GR(VjwA^ z!5?zx@c#XNUrfpcQ5_QefRFnWytiwBeQ@lWH{)ct-0w^50_F)&Kqbm@G0~XvmlHn*SUVN** z!tWu10XBHWJv;EGa;L62;AMz9AE4ox5X)FAv+cAEspE+J7z^?dcJSBnF;}dnN8s0B zqbZO;0(c2c<|0I?*hCZLe?si&e1Lf9PXZ|bfFRmZw++QY6#j;&Ei`ZntO>U_AF+BS zp=J0GxC9yVlRXZ)m=xX+rpZ-n{GO=+)*{BL5Hpp~6_AeTmSZ-!U5+LYSgtwGYeONI zK|6nUvYsgVi9`W|?Wv33$jcXC9l<85h?sI@S`UII&QVZCs6wNrAI6F#85B!MNfpx} zMsW)_j;smMw1d=j4%Qkvr9{QT@t^>#=lAeCL$@434MW;~2agq8kOkzCh?NbM)^4~^ zk;6UOMFGi^2X1x3WGW;<(y2oe0|)~;zT?AF_Dk+ZxGECekbsh`_V@3nDQ;1ye;2?Y zrNZI~3TW!>8r(fNi&U(@anX&Q0u$&RBpZOe32tNj&Y|Q@7%M{0$^c`J#)sSa;8Z~> zFRV%udy)VP9FmpjmNUG^-1XyunG?K3LQ!aEK4Od08I36P2EOmAVVmXx+IogxR*_ii zY~i0pPEc430WKLq62pVgp#0Pq!5j1Z`Ke$p(C3BHQJe%8XjghS1oZ!7G5_YpKj!R5 zwV2#NoT=Y&dyz;T1F4FXMX^hG=JH$waSLra($l?p^QL09r>q%ZMdGNTtG^40puzn~ zyKjLntP!R@0PP~pQBdBzq9X2}k{H3mf2Y6YGgeQ?a8TkiF&DS#G2+l92JS6Ixu!x7 z)6(5Yk*!WnjOTB1GepF=9y2ia=hU172XxnUR{jC26m-Pr%O8O z(~ym2BgVqmjH184es}ju1j&tHg3Tj;gNAHTBEX@;Ob*8g=oHcr2p9~s2zG)@_{E4S zLHZiG*%|oP+}7AvSaxzhzV5K{g#EL?CuF1xHao;Lx`G4t_MZJp|E;n|af#mnSq7#` zZQS?_z9G2(9B`_AjDyl1Orcm?ut5nZZp!&@^7`n%>Xxx~#PeI|IgmICz90tm@2_z} zf&K&q4V~@y6@FykNMV1@0;;*~1by4JfB)yd56J5O7fPJ#&;ooF#_Y}5rIY{r(XhU~xTfC9 z-0_pyIsfYw82yQn{xfKR<)<=jYERN%r{v-S5BN4a3Tk)d{mbkN;j$ z+WM;%{nsO>fB#>#&Ch%Bzu^?&@)?iO#KEBqm0|2s&be;Tm&~e_PMTYh7u{M5vmG9b_mO16-S^JUqmR6pcYqooI#|65(ojOHEa;Ls5 zUa;UHeBXcw??Nk93>tn=*bWO^^aZp=;hx5=!Qvt4?XjPXaO8Xs0rOgo;$Jiw5Di-f zLrO{t^W7|Ensz-Zj>=rQYME?reWzZDTgoN=-weHO-75k-4}61h8jZe1&@{&`n&Y-0 z#_K*v#(W3nE&lac@u$SWNTn(i3)~$V`!jw4JJjZ%6=Oh z)C)~GfB|l3;{@)r5O@OQ6ZjM=T7<>EZGI+PXzl(}B8X))GS#g@n~(GCX`JHGka$pr zfb`=QT#KOV1!an8WB4`96f(5(R5pa8s&D};meK#X2*i3a1Cml_0y$%sy!W^v!Lc8y zzeocs*;&l$3}blIu)JHlGVi1#iGI;=S=*!Ot&3x)_j|87+HG+V&-`42J=+Mktvj!NV%O@kg#470#brh=(J+%C1x%kI!ZLcSQP7g$L%h@vCo3$}T$J~M z{5BiMDKX4JPm+hpV1<2*InAvV6Pt_EdnYV-_S@LZ1nYqEM|7G?a>CIqhFR*0@x8cX z>Fcm7k?Py;m6t#Ly+9mPD}a_fVDo{|`ST3J*$9Oc8SyCg7DA?hRj>na<$g0Wtv^R! zpheowYBp>iryb913W+XBMlsYsV*t*F|&IY zN`4w&m1~=Q`kd>E1VzUNj z*!#~OZ8{VNa!{k>t&UARt@j6@_s2cHe5&ccT_-wl687~&z-$Yz?OBY6;J&-K3?J7; zp-D0r6vwMqk9lI^-CDr+?M5CzB>(_SK{h{`*pNgy$DbeLF%rl@vmt;8nHSii_`Hhl z3F9w2bMKxg zT)c20=7{p4n^H_5Dtd%AFc?F*qbvOkk%xvRl9~!=FWhsssjAMrx@@aAhOoSc6WCAF zvq3C$%y^p1C1q=41B>C+IVMy>(BNgTIT`#?9K|T}Km;uUL69Eh6+XFTNILcE=?yV{ zj32@xy~c^N;HjoIj#-(RO$JPc4ilmcjd}vIAE+W273 z6hJf+n?w8ePr{rP5;`$R35eN)m%3jBt`?F~v^UN{@FpV6t+Flu3g|qILVI_s7>afy z5+Hvp&5=cA_|vOb0!jzwj%clx_VyY4Qn$8cC(VNC81)Cy9_PdsATuHw(}@Cx-V<|a zHo*NKE%Ym=c=j^L*8&ngfN?bS4aRYPfEVD85RJ4GYx5zv>fX*CF<$zpTjgOc^LQih z6`DH-NsaFKQPT$w4 z5R2aDCYhsuGEm71j+2=Tq-oykth8dK@m#R*Hqg>Q2%)G0@hVUc@*f4@wjWRx#Ett9 z9F#||V@3;hElKe0Cn9A|M(v)7mG#q)fQE$Qf%NgcUTBP;so}zT4== zwz5cH)b3Pp;thT>D4e^dGQNL)@C7p<&S9sJ>E0z@-(BFV(a8encg2=1i-}-4Vq6u3 z0bKh~Dbb8FI^qCJ;pm~=M}#RXDlv#7a&2Lka0baj?-#(4&-Zd=)T-F;&B4LGrvIeS z-v=4}H|p_l3|AKXUK#v3hR?yGB_P569oh=(A(FvEipDuj4z^YoSOQECX=qKO7@5>E z9&!Z6JGBSl?o95)7iTU!i$fq9H{UFfEb-0pFbrVv4a!|^Up*wDm_mPs8VX+)E?96A zTX_>2hv;I_xKp>D~oV4zlW_FZVg$Cp=M(^ ztd>y2XtA)#C(w)LI`)tfkf13 z0)>`%It>e&5 zb7c_#N{T2N;Lv0+kTWJrJi|$`AA5oTm1VHgmbxm9`uHiVWM~cuw)cw^BoEv}xA=>T zQ5)k_ECQ-ShFj6_v~lPLxb&CP1zLv@X?9jw*E2ISC;J#x^~8+UfxYoERPuKa=bs== z+lMWR*lY}70XLDbC}^kAC}y-Qi`VFB$`I~hc(YiIwH1l)xldxfV&&1PULXbYKY)H- zwZ6A;)J}OmcKZ&PMhhmMdU1(CXSoEUHZVefQ*JdHxL6$ic3|eX!Ww@CgpO zk$D(GRjXd<7=H& z8J&N4J83Dc`!+sz!@SBTs+_0f4!`Pqd9t+bP{am~;|H3@WGcN3+5K2g$rdGSAfLfd zJ165KI~pO&MFVsVPPSI;-YL=NB>`Nmd-M$}rV5xtYe&a|_8&;0UzIF@7Uq)N`lsuf z5*N!kX+}J@pHM0sez{U~OP_pk>Wog4edo`v{9yOM`EY|xiMeB=e@@x6A46|fl(-(s z6JvFTd?`4lY?w2mKTS7x&3xAzAs3f*XQH31prBBK(NM79R;teHTnKbya}L^JMszElb6dRPwGnTlq=uaMPdL zUY4B)YdSonN_jx}sm6)K&qEJd@`WCCw@Avok!1O1yRBTP_oLzK`cKUtdGceH%;so* z7<+oE`DlZjF;+@2_czzI*(4FBpl%9k$Jjqas0ysd$Y ztuu)P+mRnW>Tfb@!vmH)KtI2-qeoE2v zn#=8B?ftJRuMVgSUAM>=sF8VR5hXc;!C+jI+=^v^t<;6yKBiv8A&zxmTkE06Bf!Fm zUJWQR3oekhZ|`hFUZjf>MN1;BgnMF6s)qMiw0){^nR57&7k6t8aXlCa?EfNtT~+WM zh7}^#mt!$yf%95@dyh6?@5>toOOmzAC=xt@NDybeH{w_rtgufSqn6JC%>vHr+rQ)wMZnpf zWw~(E(VMv%HjjhC!}pH3Bl&w9f7*jkXEl_*n$J3#B%thn4n(H>24Y-ZTLPupd zr_7_Kv3jsMa%lfhns9LZ-n?ASoy|I_#T@!04&p`384aF|x~atzf_t~l9+x_^Vb@oI z)(nkJ<)tfInj?jBBQEun_N@BW3m)^lC8MTjopQJn7uCw-0pLw5Ojd#^&}HKP!EZoB z@UutKduIqv&;S#+TfX?kt5<4TFU!tNm~1(g4-7xT(m0(1zt^gqsbMos&DC{ulbc!2 z?kh_p8;2TKKCS)XXrk=Wb9SYw^_a??Y}2vhZl7#pmZ(p!8>{Tue~C4>oV~0WAC^OZ z`{)-*^LOq`el9c(y@NE#y=UIx@gF7DhhMwMh2PF@Vk^}C zu<{5tov9>KCwI5B_4}!`;Yzbtl`O$Mjx}4tE!YD=2H%|gy=snBHPDGPjWbge<^5%fB z6U|ab@28a_hWXMe!>Rm|v4kvP(-1?9r8Zu3sE?FygS3Q&J=jwVm!8$#xwUsk3zC4|Y%MjP_^~UXV?`l*vqX0T#&gNY znUj%OTB0U4d~hsCR+wUkRl_I&+i%uFai+RM5GaIXZv_3Y{e_&VO^Zi%+V+4wH zcBBWT2AYY~*gTjQdt;JV?Vfu_Z!zg{Bw@sAb?R6q>^aECI99{lUKe;4Jy+;^Zb8&aLo<ftcl95z$f z^+=?pN&F(bv3Cr=90r{P3v`-mrDuyw6Po`&r3PCPJa z2&jI28 zBdr$aqtpb)!2MyzAW_=Wb3dp8QJ1BKV-o4f2Ipz(E!;p(ZR#&ei`-8P81^_^GG|`xFvTsE;=>x> z`bc)vF0SX0vqRyG)Dea54^N(Q9W&><{4|2Yb@7hUhz3z3W|58P9vpNQQPcl>AFi21 z)a?|F7CTU_h?L%3_L!sNU2E0nOOO!Z%obu*9Ex;{yL2gpa}}rvZAt3X+Dl8bV}FyG z=`|Yck(w+1Ni*b$QyOeoT<(qZo9#b0WpK7@AlU7dESt)(gs2_gzmlxk0ngMFb7 z>LLl_sxb)+n^*@Lh}LT(@e2rYC<(dt~2c^PcB!c#ZF_6%wOI?>&w-8-L?I zYM&|lc0m%$^+{l5$$g{vq9YzB`i^8O+WN45gSW)>K9FNRy#gu==gz%_BiI@{*?T-bpuA+uZtHQ!JE`BVx9nT#<5Bl0 zs`ho8<$C?6p~u&3_bX4-s2RC)HD9DnOhiIL2lT59qJeEa@%3TjX9(N@J0;-2PBJYl zC^GY?K;lf}b5vA1@$Z&Y<_?LN^hOA|l!9yi;Vd`&2BUTu{B zT+XXX->)lu&GotJMWf(ox(%wW6)2uC*GpDy=-0D)C~dPrnsFF&9ke24-TKkg?n1)g z0$rD+RRuE;7z-P%N?|%VaAE{9^C!kfGt$!=v-O@gis&TV1oO-iy~EJ;`_E!rqcGk9 z5nBVEFT0^^b%Kx#Ei*O6=-sHIv<>#1KD%N(Jet*euk3}V^ODRDHZkM(`BK~qJ*^+! z*Ua6bG=7WC(kkpa#_1E##^-x2E1=9g+Iskkbl>sJ+3&oEpSUUKY&FiX@B%A|Z4n6< znoa9*Y&0jPPaYPu*dFG%1s(N44Qs9n76B;hf>(J zYc)Ur!^Rxsn7qEGY`{BNu%))exL>W8UraEqbgN6hQk)0739ihT@-|W`Qmc_nq;PZ@ zE8##;Wp%;gP(auxvzm9WObi?9_RMOZAtYDsamAxy%P9&rgt9JH2jE?AgzX!~G6^0nZ(~J656e z`WCun)25$^BhJFgK^j&1pRYUKhq9xBxV1Z@(Wzv|Ksk~`MDSzBf{>fRcIP&l5h`ry zF`juvoaRkm`=`lKD3jb0t; zNjNaUih15<`KHNat;@tTSDse~ula1tkX%%CC&7oe@O=!Q;g;AJZqB3eNkPoq2M>}_ z&UQNN&k}Li*)5WOKt4* zGio2#F40Z9;}g~sQ^;2=!zDHOgWh-Vp~m`pku_^nk#BZkPy$^yFy#2oWq&^=2E)Mj z5w}s_hlNyU!O*P>RX`0O0cQ+;S~zn$IGVcvWody*B3dBe|7hQO)5T@i(w8kbQ$qkMD%Hv=`DGHU8dhzdk`IREv+79zaK36ii1;~v61*c!yc zR!kj6G1Lvz!@4rg2ZTI9#8{OJQ~&jQ*UuKs%L4om5oZs|-EJVTuz=nyYH&v$?Sgb2 zrGNeKHDLpr?kAZ^Mmdn}g);q1VnCQW=(OoyKeW$&VMKxe3n0(6NBiCY@!25F{jVRm zb5GST0dm|_w9z^d92PG9_oWyZ$0=2@V=i6#t**ZQpc95{|M$z{5+Kdz`_|*14xmp9 zm#W9T@9D7}HwJvXK|0K$=vn=p{X3Fhnj(5)gY$?QR*8I@awh_B=nV76t?oYz0A;@F z(ERT!GXBiXM6(S1U-)O~XreSBMbHWbZz9nx+Uk;t=&U7ChPFjD5@O(FJJHD~^%wF5 z8kdX{TzJ*0=cUIdG346Hp3&7`+0NsDiZM%_{mWI(J zS$JCrXidPIRB?(xZU?*E9YB)39LG^W(j%c#aN&l7R2c#pWXIaU1X*?64z4Uf-~Dg` zwnX^$WBbCrV~kNHv`xFQ&UXtNwmZL~9Em$tuX^*i7Wg3MhHiGd#3-?f}F- zEKAdJJ~FLX>~hO0OpwcT?$(2io;7+wXyXMSLA*A&n8Dz`%-X~wJqw>F?1MhQ2lAK9 zA8K;N7{Iikb*5`^S5+=d{`HnhXG<(9P|*mma8kNpic+OaEeL{e{t#nn0iqc|(ZPIM zh64Oc-nXm6W1yE-oKj;!;ZqI8L_COF3AphN&dz}_@`KVg8qdla&$@4DNW+E zcR&JeH#-K8<}f%CRBwY@-O|<;hg`f96Hzq5756rSGfx!+YqL^k)(30x(_|a?s7v(ij78 zLkkjI*d$ar%+2mt!);}&HZK1HMB4>>fCRQklV0F+HZ*Ke=w-G5_pe1(DS`nrc!eaG zAGvg=lO&L?{Wac@8blm%~=+Kv~3I4B*kh+9DQe4~7LjN=E`2 z_-LOdBoO}jo(Pl4Xda1}Hxpyc6D&5wy|hhZJ44Y?)g%#-Dmxb&AHD&JP-a+T?2BQpLoDI!Q77jtm5xCs z*ojim{xu(_xDv36r9+FBAor5ZO{c!8!8%CTqyY)C6XG3(7c7?=WNBRCLhyXSK~se% z!PowS6lW;BDYk@SARQda!#kD$WiCIQme0HHy)e=|jDe^^H*MhE@n)Lmv}m(~MbtML zD`*=i`$G)Us*)3i(60;XZeS#MQ9!34ic5;_$=GRns`mnxu7A`pmU9dapDy$V?O#s} zxfenXgf{fL@{{27O5#4dVX}iazAAbD_+y{@GB67g(KJYaxIj4X;{q^9D=3!645pa znMM>ojwJ+Yz4d@)VzGugQPdiyn_WQ8J%DHs|Ks49HBS-Per@N|x0fG#v4d>#gHSX{ zg8dJt#%?f>T_9seWuA;!kpM8&8pTGGBcRsOQ<(8Ez)Uj&bp<)VDvTIGc=ao>U_aWY zUCmA07LGKcYWI9|WU(T_Igb^j0oa^NXOZ-SAVXuJne zS%9KeAjD!Y1|oL@lK7$B0Q(B5z*W1qLoi264*m@L)LZDFhYEnp?i7Fw6PHc_Ld@7; zF36&0Rv~N?(H>CL98uE(V{jCJvIfvC0*{nwUVrzHFPhK%kgyJXerJUD3`G9{)r*jZ zMj5kMum$T%Qpl>Vp@B{zOw@|Fq8JVB)lF>mE{}=PNIc8nPsRv4+E99MFIln%kS-Q~ zGE9C&G;f~90z~+b1sI`8tp_YgV2v+zqarL&dpuigXNh{6&BVK7RjSh+qezPJA>lBx z$8M~^9b@k;p12PNM;J2&>c~VK=cSQyg}4H$SC-1T8+-id-Ic-fejXPOPb^TCQp~Mk z!}4(>T|N>qZ4Ba}>J1LB7d$DP2O57Tyx3F*>#Y>BDB$YSV&KE?DBaWTJTg zeit$#3Jutk^V<(`eO;4b7^R2X2qGfAGJ>OFxA;F6pAHNHt_{D%crXOL^+L{1aZCgv zg8+s=uK`Sb7^UD8s%_^D zaE+cVmcws-tA{nIM&ZQx14Q(fJ?Ljb` zpn-s6+d{l(R6VW=o~FP?tby(qy3M%+V?#w^y zNOOU$?C09FNUSnM- zJ%CRHLt+G2kja!B;o5^C8e>49IbIVZ1!RAg_k#h8+t|j}xy-?0M~2q7iDj4$QAZq^ z%4oej=7C$RE-uKzw`#P7tXPyEy(&fusBveJ5k83|BBVw82u8^`gV4zmGuj}S&9mQY zp1{jU6-`aeJP|F>H){~9(RBE(yoR+}4|`A-*JSb0RqInwG`s)xmM_fuvVH3l;NF}V zYQ#oBmxT2aOU%m;@=M=fomDIZSL70RUw$e4{^7DkyL$J~N2J^-*-7e?LL1!f@T z@1^2!eh%q9GEQ_FsvzkUb!Pp#$?}V!$S6MfrKczSPu}y`>J{)-#lr;RpIjWY2$^s# zT0=VM)U>|5u8r;rila7tG;UwH*)`ttYcYmv<24^2wWuB>a_jlzT>4k3H(-~);#fts zALOf*4qNQ=ur^rP#m~&euYb@L+;a5K)5G_0n|zfS?-sS#&dU z6G=W?KsP#>h8blCtz!=MOe`*gsDrC#NTM0I?Hv5Niq^})>uLFep(N&u5;^|xuHbv= z%T`}2HRM=qq#cPw)x@6Pm;;)F8qPgJ2wHo3Qt-}(T^Yab;AtK2l$%Tzl0I$ZKwLcy zm}~@UmJZHtX95(7se%$Tf3j@PJ!JEh4wy54P^`p^;`kJ1f^PK>|rTeLBkFkyiCrsH{uG z;$|J06B3Yl+ikn~M7LGMF5!cR-=I7va64>$`-cxvD8IKMI8fH016##~KD_T1%zo~) zX7D`Urla!H&uwdV8`>Mj6cvv9@g8#nkS=_%u=_{(RzZ#Hmn3b}$Vh{!rE>6EQ5N#k z3Veqvyc^G^#AFo52gg-4A3X8v{?z0oR!Jv1K(+9U$TBP99K)cU5ws&Dkbr&*iOKHw z@7Lj2p&@u?>-Bqs5)pzCLOB6W_V%po34bx6tfj5~MsK(J2<5yu<*@qUQ>ODA9L-i& zKNj+Y-%h{EF%0=B0^}0(Gn!6)#@+-%T7Z;21#d=Sf+eUf@XYY=L?(w0mN?~tl+Jx~ zPsU;Er{@Z)bjbo3Hr`u3#*TMu@_o zuNyHTwk7Ou7LgNz&jLhrl|dQ~jyb_Hp5R%y&?;5774vT6IMpCAtVXyJU_k{|h4hZ= zo%PoeW&N6#&!3OTx{GrNN6YLD>;& z3#4I_T0#0uRXHVPum`|ELxPDY`=U|lKFhY&C#5qM7Mby6LEI4t5$(>qtIS{g85>h2 z@N1Jibb;ZLg{(SDAxdcUhg3SD?QI_sCxkYa;nuzkOhM8hbnel$s1uR!pxlU~fmMM5 zj7zP89fN8z)@3|%6%y286KwYzYim`N3r-EcEQaZt5}vkOUwfHdU`RxNizJkgAvO~+ zBQn=N4Ykx3JtHX#sj1>nA+c<GOi$NYhJ^av3_EE!IFmvwSrof@ohd6Gy{#w=#9plgkY5Um1P6i)|Kt2{51vRn- z-gt}$X>0q4SR}By7F+36F-lY=`#cR7hxtP5)}=8=1?T(5g8RJ#fy1APt*DH{h85f` z7>5L>?IT#`D@zJs-HGSr$F04a5GOq!H8%3MHGmvh6<8Ji?}A`7oZ^*@mw54`ZUQUX zK6{;N3M!i$zO}da__v*mP((<7+2Eoqo~EW8i#XWOUXtEfg3K4(@;8g^@|Lr}E2+fJ z+rZ>~Sg%y|>*7wgtNN`_&_QdSRe>B6h+f3b(bqSmG!WR>F$mfvMMVPYuE1Vi@tmgl zk+@;ZO&X}VC{g=1iuyE8_!wCA-+ICEp4WqH$~t_L{R*{OrOT9EvbmQoO`+D(yVM)k zaLpx;9&KYwpb~rswRCv_!f*cO+(c|O^cK@yPNCFtX${n87BQQM0^l_W!_gfX)hVdz z)(Rh7T!d;w{FOauORj290}zP(_iK&$kMF&0AAy^8 zpm^#Xfs@C2^@?)Tm|S6pk!ZJlgx@m3*6OaX90!BOLKd57bd!-_oFYz=SH&1Rjf2?t z!$4i3MJc(V=hOv;hin-ImA# zJmH2}@oXlD?zrtQt;NP_;f81_>kb|jFtPL#ZuCU3&xN+Y7~z~wjM7|I1ts9fqgI17 zT#i^cJ_7HjhocgUU3wYlafkaf4gxV~)tAxh;8(=DFD5R4RDO}pakLzSv>niNr z32W-ThIjUz$qlX9@+FV!(2J)+R1@I2R)Ba;5eW&Y8qg_i0FDFjz985g-YnOc)Py#5 z8_*5?2tIo@YU5OdpKgRQ5s(r_hQ6fX_g6ESJE=8`KMJty@&|%|w+6Hwj=cz#2}}}_ zKrp4AK2E_3sKk<>U#ESUf!l%5?~DLC2D~5%hGfV|4`!X90ta2*E*vTKxsLG&aq-A- z5^)bX(N93}i@a6K@$sAZNcYgLn49st0fNqgKM@#{RahSV7{Z+N|ElAu1O<_)647g3 z_zMgVhR{R{B?dRFnNUqpdzR<|I;qB+kspa*I0B^J1UzS&7tcmXU0i3jz6MU_EdVr7 zICKCKNHWYyGI8yZg!Q^6b^#UK2ruzC47!00*B7`w*vwiZfJ7+{vWDj%gyL`XVeg8q zln62>(jFC}6(U-ql32D_+Q~9Wu3?Vc4^2ewKwn>9nlgZ+2e6q5jamdoML_+lNI6sR z9oS|C4OG$%IBS(+K4IjkJ}=p^er*J6ee`iK)iIU;SeONO0s0z2bngH9eg{jexdCUk zk?1o1mFWJg&JN)JwTAc)lRCfx#`Hk?pQF^KI_XX`^4OnZj&?ffXHGvy)jq@)9mp_z zYm4DQ4W5z?P~3_oj-Mon+y;BIZd}tMsUyZ=vS?9+ikV-$qG*0!z?vmivWwtF6vPD1 z5kUB;Y#aNE;^iZ!d3B(SXsvwG-sqpS)iP$m@dQ`=cR($9GB5L=Q?b6~$hZ`a|RAvdei;B}En!vZDf z8bB7W32<)!`?wo%DHQsJv+1Nrq*`)7QfuX4M08XXBqwXJ@K%=>V7GaHo5?6!bXmX} zOQB3ld(TemBu834$Y*lQBz`@MYr_kC6bX$aXj%f&pPskrJCM4vnLMjBvEYy-Vc z?5vEeEM3(3RhTm{Yr%SXI#qLWhO|vrXPKOMKaI*r2$gEF-JKDdK>0C6(u(;Agt=qr zdMeht(N;0&?E&Y`4{P)xX<;8v7iozk|z3?`jI_c5J!->Ce>@cEv`Ty1QU8dND1_J|ok z$p!F4G#aX*utK00)peAkvme{C8lj9Zww}7XyKuVr!O@hHd(};HbViY$gfJDF=BGu^ z2PXz-6oLqF8blPt0w&xL=Q^OBx5m4BVuMXGTx8@G#Y0pDwH0UVpM|^1%7(#C$cr{` zgk(xo>#Mxg=D8}$)wkCg_+GinKk#hikkiyXi-EK4M-bR!`%3gq6SmVeZvTYw#uDpp z(cYyVhW`t5y6gDKa=-2f>7f2&)#)lkoxa}T`Z_e;Up1rc_HpSFX18d1Tc;pb%E^OI zn-#GDPq!c?*E{G4u#?$8RS6FowKC#))*xz2DN0IgHj zt)eyb*G|BnB21B1bT#EafIce`mHN}e-_V}|v_5a?7EHZZy4m3=`#n=6J2c4Cb64K^ ze?XS_kZG_NDJk!R9Feyo2rxYjE*~gQ|Isi2v}TI*0JVnJi>YZ3lovK>>rd zb7oE7;XOiRx!11f7vL<8=?r~7)g&aed~QsWoYAQjJ{b=~8>UU9*yy`Y3_bO5|I8m~ z{KCR%YGQFH(UaSFjlWEcZ)wfFR$XKCVas@?!MAn|JtN1O(N5x_h*y8JgauZO$m`tF15Ups&R-T$J~!k5z3)Wmji?Pn zhep~yxh-6}JxMhEZ_>gU(Pku^aH&LFWZ^id349X#^%B90-r z!t4%SyEj8;69=`2(GFNA+*6&&EU^$5s9Sd9683%SruMP1F*tiHk1k&YF}#H7#Gd{0 z=LVK9kMTbDNDLgCtqm`^9QBOsgdNjPDjCE??U-_ak0-&{cVZ;xy`0=}@rojbIWE9D zcgciuf%ZW#dIHzv{O6tuWEp&MdVKsx;+-}PPf3j#w}-T$G7{~5@I!d(m&0_Yo!Bh! zd;V9I!5Y!6-m?XQ?Y_y}!5u})(enPipXDe{N*qifmJdLnfIMOikNR1&WGgC;P<>~2 zWwAeiABrMo<^%VG_Cov;`1FB!7kA+6c==kmBPpyONt99>O5^mle^7~a(Fupe8 z*-_UKt%iBFXt_DDKb|{#7KgQY;?WNrbLP~-K4cAxG8h75lOY-Z$JcY~CCRhXzlYu0 zRhacXY)@cO*YbA2Qt19*q~Nxs;wHp()J&zwZAX?l1_|l>_ zJ7Qxn)3&)_YqfcfZ3FM2*@rK^8@z*)g)}bJssD-q7Od&#T20sAHHgv!)(s{WWfD5R z;Tvs3kx3v!u)(K?jJFy|9g~Gl75j|mKi&@vDd~)?s+|N~!fvo1Q~<<_-6Kfwv?Pw; zPxYt+AZta|HwVKyi{S?qgTxA44a1&Azg|b2cN$vfWsChwEb2IB@g@(SpK9=_@)fRD z(RZ0Bs3SniJ2Acs%~a2MJs!9vFp4rnGJ(wd4b`+uHambma2GF)V_V}VQ+E|wDZGWh z=*#qOOseK%Lb?nn(__B?r9#lZQ;_%UyV;XeSL6c_M{V^H`VP)^k!BIr<6uP0+or; z8@dUa+c3U57XAj#(5gCW>D_*taTrq^h&J*QA@qqbgi{-ezJo$h3yEl!HJevl2!mUEpmQy#O+DbW>UvW;SG>gF zH(|qp(Th&GO*rzha6}T11ywW~a~1Hc(ZTU+xkq^;8P%g6Yi(^60LL}=-n|5*T-4Fc zvv?KYlA^oeq{}9OktBl*%d;pIPbCb^7|^bZnx-RRcPz`QNomG-)}wMju0jYJ~j-N=Oq9V=S7m%D`^qq7$rqdfAn=v8b-Pg z2i`y4nQ$z{$Z*UBgqL!ZXV<#yk*@h8|7Mmvd9s7xSR^Qvi$nQMtt7)tvn&z)l%TkK zr2(tg9X~#FqWy0B^MzlG(Sp05X_p-BIY(Ikvy6yNOv3EX6WUAb^P-w3-rn*KOn%yF zhb_?eWGFz$&SchN)`x$PL;l?Stj1ICK-NxC#vWgI)X_Z$2%d-_jo#Ab0-$&n#c1Jn zAeQI>NY(|yu+T@WXykRITpD#^k=kgbrP`IY|7njlNUU@!hkt?AyODLV318Y~bvyNc zT`p-|Z`@S4#vfFUpf!fLv4hq6{<5`s3NJqNo8}!Yn7#CE8lSr%|9t&go#KBk?Q8Qu z8C>6u-aihA1MVKcU$}inVel5X;n0sRJ%IB+>$-UY${lS3-Sz1MbFysSJu~YSw3v`H zS~AU+~9;x;iDJULy9w!pZ#JjFy!M7tv~2A zwM;_xooXp?d8AqZxbFi;3(E$V_CK_%iSaKJAAD>6<)E2>(4h^xVrU6ZyqrPfeV`}&IE*3S1a z+Co=vj1G?uTUE42ChmS`C=wwT?A`hzF>c`S;hHOU+{>0lbo^Lco1)vm>!T~ywvX;IFD}>lMGbo2>9J;U2o1rm3=Z##5Uen)po5n~d_C9dmj9Yoo0}#gopkV`NT#d$u`JDlLBUXku%beWDmXtKNM9?EF=U}QMqXXgm<@WsR9^vM^o)NJoFpyZJUrE8A)W|o2<7|C zL?yo!eSLj|iS-kW0{|FOcYo9&LHlHA>mkrG(U_vnKU#Zj*+?iYK|*Gmu^j+=r%>oX zB}+8{RVLIR#pn&91yEK9|IR>ReG|B|w4HQlFY!+*>v+(qE$RNUxb*t!9BPp<@ z?k>i1DDqyliMexh_L7>WG0)7C-Rfg~m8uKJ>%L?i%o54Sxm$ZdB{)#%_}ajJk%JM* zX1U^tBe%s3fdMj0I=$KjC3%1JM;A;SyZ?Cmj=rRfCJyb!8>uiGw?g&+v9}NMs#~+UVy$c!9REkM92f?C(43oX=psi0e zFLq(od6sG7k+I;4AH%Al>(xzFucv;_Z}uy<)Bykpyn_dA^JU>%I@){|chyJdDEcmp z|FqdbiG9R)sQPr%yR5Q9If5SJ*T?M|r+zynbaiR<`E)U!(r0dBEVq$4VAAPeB$MEk zF*%<_qYCWa#(Tp#c^-R`Og$o^Ya8-busAWNFeN4BNONv-?fw3mm2z@p&6kEG9Xj?o zVz5u;q3Xn49-_m}Std!NQPcZ7*Rl)?_GRbI7u7AB#FucOQn7D*zsuR$cXv+S5L6wO zet*;E6PHg8+T|nYxcKJ@sn8(G*O9)7F#~2j_e=WK-*yGsCqGFsxNo3cxFjw2_uvHO ztS&MZNYnDjKmRdKBpj z(whzSP!%|IL$k#a_XV3$Tqm$yyq5_&223hPCe)j7WA z^kVH$;oRnXte_@!NU)g7zoS!}<^oQ}v$Z*=&TZUnbRm3*WoqP>PzSpoPohqD)!D4s zO8$ut>fI-kALLnqnmX@rh^9*^tFNEcS%Mh_A>`3()6Q+C#VHHZIyh)45tF+_6T9zZ z(bCRdbie6tn9AUGIc#>9J47T%MFZoI-?c!vY%Vg}tgKS;?$9;$lvdNx)SPfGz>}<+oNFaH$CDKz)bi z$L72HA9Zq335Y^5#E}oWja=P=c(S~9-T`~pg`hfmR8IbsY+4JSjw>x*L;IN5o`IL? z`?|+OvrT;|HhoR|o>>&;6$P5ew&YVn0Jc8;rHrf*6)a~G&V=mz3+~Nu5#WP2^zPymcB=KLr=IXPBFdPUf-e^{&_X&#f7x_ zwU?D^-@L>jOq`|BdI-*ubk4X$Kx{oEq!I?afdp%#T5Qg`lz5(*Q|&B{gVpB=o5f;w zk<2XJET)633GC_9xGx$Q9ph+sB%*OjRAr?6uS<~IFXK=BGT+D-yPv8fH-8)lf!~F2 znQc4}IuAiN$D3D*cUWT)@YGH_V^`DWrr5KM+-|vhP7jX47RPyXnKzN0EAG zI&Z#RShaCKRnWXe<=KxjJ_T^a!tggjKq4ClXruCzGxs}JMW4+cY~=5}evxHy@q(#V zj*5=UluHPs4iKaHRBv>S)3}G=WTt|I;@P*lasl(c6vNpEw`U#uDg34*9vV3=asv%1 z^@2KbP$1nY`n9^@5+;0q|LbZ`a7o4$vg->co4!KU(J?nJZORBEhe1Fcxig`cPoMt$K{w~-98Y9*9Eg(xnGA^n zu9>zLsM8;CsB)vRB8wJ*fAy7_cGKG>TRsl6QbjE0uVh%=`0Foe`n7AjgBiJbx%=NY zRMzna+J|fw8}1o4muSsjI-7CWeSWi&$|TMeqv6k zvnO;gD2&1>Q#vRI67`Gcla39qrCx zr?Mn=G;wHW-&!U+_)Y1;zzMKsS3XC)GbX{Bd#TdYkuP7~f|qp!y|hpX7Y%T- z>Va>PitrxBJcCt&j<4)Qfgw8A>~3z7pGE#qj@}fX=leE5&FiP1y6$sn7D)UFo~fw+ zrW6IrgB5ML6as%#`UI>G&M|!cAa%3Z<8%dUEl@HhP!VVx*-9noG#WBI;3kH<&Qj57 z$ty|yotkZH+uafO`nBHNX(=hg&YQy=Zh!tbd*S6l>bsJ$p3dRn*ai`6Ue?Vl_fGL& zsXf_hb-AzEU4~sJ`!TJ3SuVlE*mg(%T;beMzDDNddvN_uAbJ3>fh_Ip8aPr5BA{RF z^;JDjRoe7+ECt{!w0psXu>=CGEE%{5K(X>OK}N~RgoBEZACWQC`w%q%l?)gJ?{HJ| z;up|aEoNif!HgyojvpOgL4aw}+X22dj-~O6S}ScbASuJ9ot?B{K3AcqDvW9o;deUQ zkgs6Z0H2QL?E24V1!`F&p#SIi*N*@VUcr+EJ=;0C` z#snH-%sl6{fWll(xTJ~QLmFtLDOELtdO|Kl=p0=cn3Rwys!D&#=ds4n1mkGshAo99{ z7#`X+FG$QJw&G*WpaoEjU&Qbsw!YW^_(*2pA&-kzu!l z8iJtK>!ToX4s=oI&P9oi6o10M`t@FMHi5(Ce^>ILn>!awB1e7MH>ZOpcLUt?;=$O7 z1EC6{p8(75ig;pUgo&;I`)knwg=0@Z!h7@-V?lZ`LZ5yVgH+IR+`__=fL=E{+Rvy3 zXE8^+<8B|8Cusu>$0*eO)z)y(uC)Nr0n6yKL;ObXIB5+Ex1iJA%B2&XBl5T=k3aFT z>t`e8DfUMujhWzW9aZJgQAE*z8ghq$Rx~?}-?2i-WwPw_to_TIvzPnzm=CI*#h>vj zE)GB7Id|rTva3t`^PE4crWPPE-*&}y>+&+>PP$PPztXPk|M91DDRbj^g)Uc4P>DQ?+Gpv8@DfAh;y}ZE5E`bPeCe-CnuQLFe>_U2kpLj76;aqwWd_u&Xqq)hT#(*!3snJlRG{53B;; z#$6mv1die8HvRhNx$)i_ReW-ro2d3SMz|8{wK}&iPW`QOyQp4@J9(DveD&STX-VuY zo3g_63=Cq?`%Hi^AqZ}@WV7QP5TgLlg`zM*mj+nc3geilO3T0xs0U zPz9m~4?%oLyRi-imX=L|nqr9bF21Q>{;N>_RB}73bo5KpCGRa0R?w0c9~(|+duXS` z?{-HeH9;y-wIg&&eJqm_f92D0i>hs?TTrO51&N;a9Ax3^KXGsS#6^z|wof$K@@*CJ z7E7@}`l`2x%PjdeTkWOV{djz{m|JgW?#~Q=ZY34nS8n(F)xO@2_M8{xL#k$~M1OrE zLuq-(ynq~>;4Mhtj73mxtU7^^`=dG1qD?LTl(&; z^W3!TB2U-6pM9Ias~xe``~HT-T!lUB`dK%6nXqdVJc)>mu=zBvvu6EzUtkxP-NdO5 zW1UsKhn8CVCgS5Hj>>sLKvqvkl94Zd zd{_6}!HPodC<=%w`swCNLn}Kw)!X#w2ZE^vr*14m3Yacjo;aBd>N2`er%`LXgED9f zBw?Vb1b#rrlT0cB7Xb>1#->kVd}zpyBd{6)qa$fGQtc@82K24fj^^dtf4WAAzt|Bo z(sbS-D;fVcb-cOktsbp^sp77pUw4uSxCk$hWZF=_Qp&!BudqSxj&vL!BMc6!D#-g8 zm(MV6fQ(O2KE!Pr%9{yvY~p}wZQr%k`}gA~j#H?fp7Pswu3q+VjLyWBK(CANrKNkypn8Nabq!Q>Q z5?LrwumBn$gdWOp&tPCp(0qw~Vbh>KjZI0*-7+_^K=oo_%ZSbA=4^hOJ@PEFiXJ8AsA@=xc%IB~65fkjrl>Bs^&vv;bfeprx&4gz13tyCwZh3Km^mgT`kJ zjFfZyJbD<>mu4&z3>TV$_&b8_879c>$dVr@YM@ZF*nJPD*3qRPh4^Rn%ZH>p*$&gO z70w&i;gwP)7pAYX)FnqpIAkVjjU4Z9OAK~s1s(FnmHr&bIF%Qwo#)aXhm3AfozYm& zfBCuIij@-b*3IFoHY9cpn6qg<;w5)#6I77Eoam}BX|*h({HFcuT|c*H6-LZiI0a&$ z#37Wddi=ZI5f-8_!KkfQ17t=s@wY*(lBpuZxPkReScczZQy{_-Nyqys@O8c#>F5-5 z1Nb6ZA8b(YGu{$OHQ0^B2?norl}q zBm*M_CS zgFaD3xk%8=XsUIgsnD*tC~!RRukqmp<*5sDuq`HRJ(&DHMCw`&CwtJX-vTBlEK9;P z8+ql7*>NybNu>~?x-w7}!b~Lq%AR$5e0#J0k1hr8e|0GW!W-FXi5zZ=KXy+fXk`U~ z_D`Yaj&)K;#5W9mkSXTYjEE@lNCcXkXTKPtvXG~RTA;d2JQy82PoagSBA3&ZIs*|t zv5YrBncUYhd*8{-tkNk+ve+DJ*wwVYsmKi8G^e+q9R4D3)qIY zdJd_Am&Af|#2Z>1JVep0JMz%Z&SiGwAirA|jhS1KQWvf+ZQa+Tt(|APoJ_+|&}i56 aLkDkhsSf*ggoYB$Yp{AZpO&`f6*(E`< zYZSw(YdVyL{PU7kOZ)2>Wserz*quB>H1ZvSjf(!qCv9pfIk_v08>K^Zts z3+7g!=cnYgDB>`i)RblR$eh(n71-x~DzaJS;b(h{Ci3K1#y?%*nh4Y1Z8uk?J+X88 zScP4b4v$>mm$Krv`+a;zEGv5AJ(dFmiHCmW6}h%$1XUk;>@$TB?~=6vOJ`#BfKaRg zk{*$|Ai%N;-9+C9XvpTmQzq%Z%4BG`q@%y_wZuwLre>$&><6L^tSO0E-cJzbyMzJ= zq=`razUn5(W5bpuPV~g5i8M<{w-Fcd2eF4}!o9t{?K|KaOlS_|Kcl#jhcLjK0=9>Q z{1V|c0%IVCOhpicS!HnnF2$BefGp(pt68J`hK~J)RMML_>@3yYvVD7eT`{+ewX@3E z6j`d=+-QwZc=?jVuSqJE!3TMH6_tuZnPZFHjy@!vj=I$Ar#;oh(0Q=Df?25D6fUac z=d6a>7`Yvg7pg0G$jyU)8!k+hp_6K?WiY(6d+7d0o2^S?I{rM`%0As({4)c?6;utn zGTI>o${su% zj%ZRmA!nQ**POK?17}(_diKm79wa=jM664DWTz!{> zv(R}bWEj{+$@V=uXe3iTRA)E^)D6CTe6~6TF^p=iZ9g}x88lOQWMZI^_F2a?UZ8SE zyxiESpoIlHIy(Cb6k&OQ5X$;!nkVImhs-lMt6 zsjAvw-?Q0qD%f-zRyFe{kM*zFG5RDQGMG{ibnI9 zezU!+(X3DnZfLz_SEO+-h#n=Qno=TMY-uV>zvRUo+HShUM&;(NGMXKg@$mGlPZ6K1 z>=B$#*H)|uVx8XA7wk-xT9%ysk1n|RgekRh-LDUUrw22(!G43739?>Uz@WxFq_3yk zw|muPOK?If1Y9x-a5X!|f>S5a33!M>G0LDPQRc)Mt&pn`lTM&N(E0_E0v~4_>w992 zM~?zpX9$RV^`tPALKNZ-G14Uc;Nbm_0q<>`=cg?~#=w^Li4tOd687M|19jH4L~;Et z-m||ydv<%$Vf@R*2vhZAjqmoOOOqm7G4sK02rPW`aLoF)t9V3p!_V{6HKbFd5~c6o zT-2keziH7kT-*vS&~-0)j*JBxuAa=bhSNpgux5_pR_8Ta*i<;5jpbUFw(6V>8Dmvg z+E1OeSlr8{lO^yhy8X##vKx@*z`Wn+YTbbV|#$-FLy!S=PD zRPDlqsg>h2VA!)8b|y`R=EYq}Z@09u@~OV^fKj!}s4XUX@7{g%La*_9oXZO?bMgga z62Yoy9I)DM zr8QQ0?X2N%+rifFG`xu8aCKngsqxNU0`iry1OSAja$YG$ZW(^p`^p}>OOY@#Qfw)2;f5T2%{5yTltv=DYc_K0( z`%J5I1@lXn2yH{K>m4fW3}2QCG4bC0{nPUU>Xpy}Rg{wKHjM!ZDWGHsv^)s ztU-pt^+Ccyj2UoT8jieA3R)=$ejic(ll~J9S75>HpmIVpN3;Tj*#Yqp9GOUJX#Cfx zY#E+8k>cPv_9A8(L5Lc1GKAPPp$Vml&#CKx%Yi$^@YUVztFzr;m9#p{&f0^G&yMF$ z4^3CE-G+TUl9~sf*OrCtyGda?^t5g7SY=2@=*OAi%S#lhQR|b7A!F+Y8mNa>bLKYG zDXnVPeo@YG;22`%v?JIw|M+IyTs`Ruv%HTSlwLPptXgAu@ql6_Z&iES$ zB)4AqAWc6q5E~BmOGvu~SJ06Ml*;L9QFvt-EGx;buG2rHvB)f%^~}4od=s)%Vwb!C z`vyyEYuQ5v`(d2K!gu*D={&uCdsf_ens}2CClrU$rzBim-zbrGMphxiu`mPV`vKI@ z5OjnQgDYbBi33v!LV9A;OpavI4YU1~5kjClEI3khBVCGx^;++z7XisX&Y=Q_^ntA(0+!-t>;4qZt4}6wKlu1$n^A)= z#eor+yt&jfh|yMHq1_^yqx#2N-8(P+HQ+hsLqU7ux>UEQ%Vfb{F5`wF41LtJy;Juz zNv8YgdNMj=@Ul8X$5)DiiajT&jzkuSVj7z?Y^Q1<@j63yo&L0nC>*=TkS93ixb0an zJ-1&d#h4+%K174F*ylRYmbQ15J4V7AB?V01&Rw5XKhj!j)R5=iUzZYn$3Hc$Ylrzj z>X-$`O_Vl9b1CBe?~KX$SkW^#HFN-VVW^mk%AUJBH&2OrUOr$19u2Wb3Q11~dL;^Q11cSo5)vCAfhA}SJqm=m19_2Z8qwvi=okvw{W6J2 z?4dn6#A*hbRn+PKFhDBjAtezJAwYLhhbd!7`H6i2azRx@jS0Zw2_WFHqpAYWAsOx2 zG8nK!WLJ;(&b8?@WWhvb1b*oitN_wp0V|TY>yUCDpt=6F4vlQH$RM7AgXmUg+2@A! zHzY;8nmB(pSz6zwdz{T>zR#Y#sussdKfsc%>b=Q8`4W*f%Z#B&eqY0T2JwXyt1GzvQ z{S_nYcRjW*Nf6RB=pL&`%x#mqH{)3`Kc)U!HN)dCxB0-Ld28#Ht<1|}+R~w4IhST{ zmgz~4PCSM-=yJ+Kc+4%@K(rRXSN}?k57NV zSy-;|Nh}k7N0I37oIt|_XXwpZSj?afjwd265X*{Y`#I2L(}ZhU9fZh)Q(hN$?gV)D zHGrbHww?wGgQA)nmO5JZ@81^z(+Cz!gq#N>gCoxl6CqZHW%BHLmc0z;8S9BR`YpwF zs-IhXA6AM^CYa24Y`w$7k2*Il+u_a&ROOFJU>`@Ly8$O?}>!#hIWm&EW zEw)zox_(VL)s`z+TSRAgqhsl{`=vb}KR(3zBC;sFL^MhY;=X+OgNSYj?Gxo2r)wp~ z=#-=QCSV~B$chhEDPR)upSM3;ZBc@+T4GR4YHg!h$755&Ej8);ZrUsjY12|Pln*_vv_gH6cJ%dTj^(D7DH-%B`se{HCGFMV;@8$6TIUw^ay2r(7H=pFhI&~}Jl zvtd{wo<6xBN*|V`-+vet7cx#tYy1c|6#P0CDO*gO8@#Wr;T(eyp|LN-nmOOR+fA|w z$Sw>T&%pH#EZ8@<-tzj2{_uzhXE4FzP4E9GGPrb3oxb1)jnA8AW7od=6DwvU>E$ig zM@5cIw*8g9xP5Dl8}qJQZiq&*5Qp@MkhoZ#^Pt!PO# zHe*Y2faD8v4~YJ&b(3+J8EB5qeGuqOcsJyniX>rvRTr|Q4lvky-m!eqvG5TV_tU49xBss z*TvFl7gY{Hv}LuQOJ=_zn(b6D!u+==*}tJD1;zQY96NSQrSY$2`UfVarxg znBEr~C1q{zLLQ{LQ6@=Tpt1bQkpwi2{Z>!z>C5~@kCz-tIBGdiy8M@u=ND(*U%8et zuHhI6@Og}_leh&XI6>V(1n0tGCj+&w!0_TWi0g@4H3VR_uqHY{Mbj`G(4%Sf6E2l= z#%KeP_D_fv%6n)cRU>o(yF%*o*lEfPNk51iZHk|k-!YT@D-75)LRCb+lyII<3II6C zK3+mo@(7;7*c>p-e$2z7lh_ZPA{(+l((aQW6i&f5XWfzaTo6uydI^vj9)dRdWCDEh)f~g1)Ta0mQ8;H70WNh z^xF$Hu5-qSio{X^m|ikK;WPA=8+JnQ#)=guLqFm;eucOr0pos4-bY7=Zt@?zzOXvY zwtui?>sLA48{HEpiM%uW+4fz3s(%;6LA>*&7FAH{6Y)Eaw~VZXsdr$k zqwC|;o%0hWPP*Y~#LF&@q~bf%RtkGoBFLsrtonBSJ1N5dT$#!6B(A(Ynco5)c7)dg zvc?!GJ0L#kLxap>Rssxp6e_4Ia#a3pGVFa3&l9{ZHpKgW6lcj1V9INQfB=uz{qk*I zFgBzs=UE#XyDwv9Vy&{RKIjS12ja>Ovzf{s zl4RlR9Y6{27U`um*07Z(uo+@enfNjTvxSH?8*2Ca6rzMd9trclTD(=VY!EZ$al7en zCYHS#rmGcI7asoX$B(ze0S`Zv66XuX5ZqKCqa&hN(i8;AgRtFzh3NSTfzS~N)+rmf zJA|nSYp-ZTRD?Lf`yq;H{Hp8Ir;EM=|9Wg!vGeJQFnuFN9xBE1uq#B35reHjAp17h zj)oBP6ynngHrQjw^k6GOJlTc6h`FXz7^ z>Tj8RzpmTr_DM%2F^;&Wp=E_BE$RcC%KjF%z{&$9S4G;tmxWkclFHXdfigCye!Prq zZX|IODvk|*m?lD$PuzUUu#{1a)xnUK9A)}ycbC7rqnD523zID+!&4XqQX0I6o@w}> zA^uOWs^z8r;m;n#=HlR-vJ>j4q=}KUKJkQ@_mKP z#cq;v>5@FsPfi;^9&FiGBuj!&VDI({=+V==74GmWuwqzOnWeGWvkBKGx7f7Sz8k~S znBEn>!8mM#+Hfqr3$kXr870To{~ztL+WHk~NpH}7L~AVpbUb4y2#L7Fb?&oZ3#{^5 z*j0hslnRGojw-a=Ja|V`Md0IEL>!r-rA{ZX6bCOaulA*XEvGNe=^9d8;H->|K%-s< zw;8=sXv6R9m%(8i=!IYv52M?X0bO)CabBZ^MSmQ(Pg)h`hPPm@)R$du>`5~< zm}B4|LMOPPcI9I92$&gi<`(cZIs4KZC(PmUnIyUF}E9>h6`Oz**7_8FYg zv;0;4&*gvC{``GWU2TGELd%;KBcG1a$`F|YiU*Sj7jOY|CAcB9bG)$}?qbpt!I7+5 z|KW)QGnrgMB`ODOZRS*Q5JO29ZA&KY67k666I=daOpMzGBIgp}k%*+EkP@#DE-?$S z1RTBChygVX0g2dzQi-*m-VuqKJ^#uByoIhapFfn@RDUzj9XpJydi1(TKMaN#-lza! zGo&}fhXx?rHpPF6Ttc1v4{AQ?Jo58Jn>R&g3gh_<` z-yusKfG{rcz+n$op^4{ExYMht4XN z2O&ix-lb^1y%OFAUp{EGafLHd6P&$wdH2;D7@q z+jMFnX5StA8FBv|MrpzMk3t0}4^B(Im~o3<*0N{T_j=3rgo8^Q9c&Hr178QPnbUj zZiZ)0YOEZR)Z+gBIu3{{&ViX!`=}c2z2>GPJRs3LqdXHxiL1MBXlYqA(H6j zEKU_K5t29L;N-_}=8}Fh&LEJ;RI*f-O=h@iy~)-nEUMQ{;a3yN^>ZHoQM7)^EVWAO z+yl*vjm*ZsU$5q_QUgr;dWR@%rp7mS=q?tRuG#3#CU*3b7x(J}cKVIXasC_Zei9Rf zEGnj3j-|h;E|E^bk&Y~~@Aan>B9_zMev27r1_aG%$RYRy&g%V#9WT3DWAiXPXy_z& z66%4yK_uGvP@x`0ec1quhT0+*Cma#RXg4n35wDNnXgdf(517T5tq)#hmMZ@kayom( zj@LzlA1srabv%7%%Zf}?8#@o~sM+^c^_K>_mpiskeBnU{wEBh2q<$M+x62iOP+?L( z?{)8a&X~5s4v*3Nkn@5+>p5VD8MO`3wDTQlVKbwKEw3cUpGuzD_|vtl3)P!&!t}jQ zkL<5o_OI|~zh>iI zK7rY4nD9MoJz*;qT-NLTxoyO=QC~GFcirR=J?dcb9hIWZ9~(!;uFZfz5k)Z^d8bns zXR(CyOT8Uyji!*Oj=puNzrPGClPE^~VPT;m;_#KeA1XJ)faRtz55De<#bh)Oh%P zpByvSR;6^a>gOx^Sa$HP+LV=4ZgiW_$j~+R2e1(S0g@in^h9F3{hkHSXUv>WKKp$y zez}_uj>rr%%q#)l^@ZBC-lw1_J2PA~XSVY6e}3D2{qtt?hz@}TC9d)T2ZZn^YR*fyNLq@M#o zKMQ*Iv0RV~@OH@z5Q|wDs+FVOL7d52R)yGmZUH|5tSA*|H7R65LJeBtd_wF@3J4iB z&L)3UY>%xxpt^!eXKU8^MOBbPq++1C`)64V}!RpI`6&ooU!Rnz$L82fo z--gY`j$rY3kLvrQv)_<)g#3&S^4yPq+(2(n<549DW`#P~}JdVQ-L6?UaYYV#nc<_ZRF@Oy{qgv<> zKDQN1R*!&jS53ynMM$kBtUNMz!p4S%dYrh_m&0zbY=B2QPZ=?sP_f`d8jB2y$b}UtlT9ARWjoHOv_~ zc;LhQ1j1nTqW>0G+xy487>d0k>T8w;wn<4A?FX!&r;~7foxtRFi?v7n#xQ@aqI70# z8y@@Ok4xDLE(0nN3rtaA?l~((nV6z-s3(spn~rL~yWIQs5%FRpjWP6|&(N`6$+iUH z_*Gv%Qu&`bO#e9z6<*+M@q!Ood)6K(u`o?70Xyz@D#FZtWJv@IAw4q?4v4M>brG|< z2|7H4tVSi<6FN^!gV{$^T*$d$4wggg&(Vz4>}n-vH`qu0XhWjP@yDMgtT&ts#(=Pi zfj0g%)xHBspcgDGFAa| zuWQjP2Rh6_rsp&ppk+roS~G?C-neIS|DaR$qQX84Afpyi#&Q>-*R}Xhd!2L6+ zs*o)%yppl%`k$NB>h+LL;lb3PKZZNu@nuswg(-9q7$xu!tNT?S$$JP}2-Hj1qhu^g z$DUeY__`eZ(qgY#l3KcusoT;*4)IGvYoixAZUZVoLia^Qhw9}tR8LE+C_>;0s{vw) zhoBwQ`b7DFAVUgdlpJRvq6Y~h0ss>sR&-xmS^ovy_I?}~goR8fCd0$2kj;=`6(EJE zz~#&u*}i=T0E7rg^?Csu5$OjI2Ij3>WAKkebX5`dREa%343-T-0FB|&tjKENvEC2l zj+obYkwoMeXv=tr|#4SX2-He$9Zr_-GBD`&4@jr zO4Bo49*b_1L!z>Y{mnbR42M3l+w#M+l=PRaXKF9(ufG!UN_Um08!8{ zf8d6As=_ES*$=)Xowr%T4tDAVA|vd9{Y9ni2UAr$^W?1hH$QC@^Tslf z1OjO*4@OBd*3U$1{$NJ=<-kPZR)nJv@|hx&$JM7Cfc#j&>9hOlPaEQPjMXYB?583K z_$tW@N4L%aH?YnIKh~spVRB+f65lXBIk^%9!7x-3Pg*lk7xp-{&9(lx(AqniJ{>GE zT2`@hOqw1=%^8j^d)F5B`N{UkFzL@vmZEA4P6W9aPd!K|nyYUa&xG=2*ZE82PAhAo#%42jsfG!2@s$>YQR0Uln-%U&=6R416qQ$# z7w)w*c>O^vBe(bG>E?`b+r&9BSm6tM?%*FSHz>Cq5Gb{HdX3|8WW6c@rM3&0s2Ld< zWpw#vO6val_ev`A6lOk$(IcX;ySIT(_uTLzT=2Zn)1Z`bYCr7Od!Kd@lLutb1#7X> zF)Nzs7F(Jhn*lUq@Zpi(%Ui#U@h@F#!usf$Ny^R^z%0!&fe$81!!wdgGYl=ag$al5 zKm*=OFK;z&J-1*tMAjL6lV2AvPNa?_N4~k+@k(}!?FV(w2lELeXya_1K^J~Bbl92o z^lk?|vR?S>)jEuA1`qm|Te+mHeH~#u`_oA`T;fm4q9YE%TcG)$j4;2{y{lwJHc3BGJe_2^s+xd-@ zQ}X%~tl=Jru*Db#(MtW57Yl#F&=^@24FhX+xSl6qSpyC7*S2fn&uOvM#6+kdh?Sy& zdy*#$q)$*{Z3i>-!SFYNz2syCICHRbLZNW}@H{`^z~qWnl)JwbQENu)1+_ z78&X?nEkED%(>Lq}!xgIKwsv9#?z#q(M%g9EE>nBM|q zqMuorbA~Q9`u2L}E+uaNvDf=xACNu3@8({?sV*I@Si)1JeC1I+8QG+dmBPRtR-39E zadWe($HzU!aC$GX(KAC&Iz0Dw{QVr$ZgBWYtJ^l}*W{SDHQO|3v-wI3cRf%VMMSIH zF;y2ihoQMz!v^YBdD*Rw19^5SAWt+~mUG2p18p}5xew22Q7o5A^e*^tcHiACE+V-& z;^kA3zgGE#W9R+TmY>|W9iT4a2op>#7=6Q;t04601|^e0`lSX~*SY-(i5ckF8z4?G zASDU;V3yA%Ia}?OQKzD*|NO@4iPk=xuH=0If&=m^F;kyY;j0cI6oK<_{mBPD-ZPUB7(Y{Mz$1Sz*{c^X=P;&j72o zggHHOzQdW1uPr?PVb#trONqRD*x3IU5N_ly{LgJ>ijwnW!354!F^^ zVpHKu^F5TZi#K=qWmbuBBqnETBqa1N^_t>P(czn!57e}%Q4I&So;x(QE!Xb-#hY5L zGhONip*D)z+Ssk_hJB#WEK!w%})D| zf98(7+A&tIX|9h5k^Ax5+IQ$!AhyB&){l zh7~?{DijTd2s+5d6g%m>33pjE$SUI^7Tn=r{#2O5eEOeM ziQz2{cyv7o7Gadei>09%mDwNoFDtW9tzNueOvVy(167@p&0F~Md*}=S4=cme!uTH2w#jbbsCLUcz5iLHn+b)pM(HUVoIJ-72381SaRDBU3luu zZ_taJO&$wfHsH|j#j|y(N37L;^o^CPsPHP9o-t?RlVG85-?KijWBNd-V3O+Vx~LD` zYzn)=n{K~Q61A|8DHIlCTUd&CKI$*!Puo@G+t0mY+lb87{fcCXlxk1u-5fUexj0Ja|q|Pv>p)$~A`Sf0UuiGKO2|)5SZ#k!r zoIoQ5V=c>Yic2&I!|^%*ds)b?)xm&lg#P~iBs8g^ICQbBI9wW~p7~}^qNbYq2*lNa z6ye$4$R6QrQXqksRhaqo?`;3V#biXuM#Lr`N>7NDe2hYPSm)RV1xVd>=$+Tra=*rc z>JedJCl9!^0P3ccAv-woVTAl)e1j)v8+*EtlDvk`!sT2^7=UOwkCfouhY(}LJcF?} zE_F23u9hXY&N{bXR2*7kB;ru}ZMM#d45#VExmjNp4m|9TqUz8+?l|m^!?TZF@=T4d z?HO9WuTPArL^vxoD4R-~qt#edx{%lioZN=kuX^yU$-+6j^ipBO}1YofDdN3NR?a#&<}3l5Qqt56h7##A(%oG6Q)HIz zcG5#aBrhjdY0|a_$1%1@VTet=x&wxBBdqelu?#lNLad?IibZoR!`&EOl59Wpe6a)% z*6STWIt=PIEP92eml(2mj_>RDtmIrHLy_Hg6oIJ=QsdNow^oN9GP(^3wk%HM%x34Y z$_bA|2}~kucIXLhbeU?&1UOJ1Y*o6;PL{5P10uIO$hhCNicBU8`Q!QOTnxC%#5|F0 z5dWh$KiO=>XtpTb6?jy7>PUiV5Il#F_*%Fp#D};vQF&3p zg<#zXcL(Tp2h8CHs+Q&%OP!cOL^4!mJ2wz)zgrF(N9 zKWm8ff5vOKxMc~mYA4{6GVlTUu)EiRl0>o;I-|I)??cY8NgHx)?J+!O{YHKN%6(p9 zbaJ=+{QNTri6MAJ-8xINsvCqpUiIkVPC85S$$0!_S3eaZ8+3#X&D3&587`Xg{|@~u zdW4V`O3q^{U1yNsDBfaat@fy%#W?9c8qD3`?uPZe=PAl?Wf|LE*mPsFT<6kObR)%nNA6+`^aw+Fq6qz!!yscp{pTC8ix%^A!k0YnsT zzqQZu-_8QxsldVu`dTkC%nL?dHQRUG%XAR{XMcr;u{=VqKb++y(Y07~F0MhM zk-OMh+1Hqmm=dJxDSV(5*^qN8_(M56!B)uRV)=<_t2m9eBI~Kq1(YIjbMtjIOsEP+ zJAD)mQ1KmO8;dmAeZQ`8tfvZ4yR@pcP$r6COAta{>WU7JOi=;K#d8N;m)Oi zdSn5d$`_8k;y6OTCudAdhqm1*uf91pS{E^m=#eutR*aD#dHa$Zu=jQw9Bs)=RY~?O@3O`SSGR=$of_=ki1xozY5e_O)Df?Q zWMLag?X!q5i30-QQ!|p%U$NLQ@Qr9wxqwKlLOK%3uY9BZG!phUt{I5)rR*?CJBFc^ z(K}I}l78x&F3UnPvRSzI;dNg7FCQI?*}i?o(HHE#(#*vyil(NfB+9|bR*=<$1xGSsZEoy2~2IjVVaEzictp{)9Kw+!P$W4Rfu!>TO zuY9}yg%QB8#!%R?p@$YV;f~HPeh_0h#c(QTiUD8wFowWzfC{gC?MIKpIg;G#pm3FF z=m8kd!0ru32opd_i+M5*q#XTrHZd_>;AZ(4_1r@Uby>g1V-lB1W-zv31~dH8;I{b# zD6FKvW(L^3jb{0u6_N3vE5e8J*PgD2@b-paN7c>l%toeJbt+>!Gf8Mb8CZ^_#D|7|D$P|MI$C>qC-#i zAt;;amVRGiTaS^m>P^?^aYhZ21Cs0sBj&Pu8!d8fa@W9hQ8H}DJ?ozS_A^(_qkd$$ zFv5J#b^Ic|0P73s0~ITRMfCd&_@gEKj;E6m^6;+5QR=+es%cXF-s0OT{kS7ODq7{y z8jjkp(9Ene2-^-AIZd(&99u*@T0Gyy^DPYH$12=Kk}ol<#Gx?)c{qf_TgXO(IYeXHGLF7eSZPonE@T99W94Wo`(X z-68}HJ@@?)Ud{bu{35vwhQyrm6l%YPCpPK;Q~^njzayhTP4}Kj{^xAr$@jV>*C1!h zcBm*zJY{9+I0p{HultKwsrcyYeFQ@&`Dd?AC!nx*ok-Ln2tzLNFax%Cvk4#w_aG&! zyc}%mhXYa&b*M{8CXg%rKz+Z!4lq)KRh{L&-J@E_l{i`wuCp88h@^3CGgWro!jqzH zGx7u#!o5a$bC&ZIJ0m_89`F9{?-AgSA0jot5kuaknuhxNZUlm5+wah}{ajgVXtdgw zAm~;h1t$1%rPUzGz=DE;a(65I=i7Hkd%~V{))+fnI#;Xe(aqgKu=+Ok>}UnlwbSc2t3i5;oah!Zz z{uiEJMQL`y&jP`&oGV9b2CzVy;A!lx1V$gyV$yf79h1ppRvS0L=QWGGi~9b4O6n&kn_UIHrS$OFS}$yU zB)*F2Y6RI%E5TjEx`5M58Q_Q=Mr95fII{W|HwKd?`o7q2;hD)#*?pT4NSze&63KiP z^g8u?X)5tka3`0PLjh)mIB^-&dZ~ZTfyji+<%s${6v6HQHDmM*hv56XZ_n|mpbWz0dE`_i=8(7JDL(lg zsKJw=fM-kffGH8-k)Hj;tXZtZ%@~^j))PXBPe^#P(e=x9kL2%x`cAN7{OC3v?BW5s z40jpc?5I!pJjGwzR$ZX!N~E%?D#LaVr?)@F0YEBy_9CPK??~x^?8X7_Y+y|espl&A z7+=qMz}vnQRLJIJSug*X9jEtfO-zg2h}}B^m?XrpKYncI2jt@fqaj$D6d&m+I4sWe zzy}gzEReJDA)7=^lx0@&Y}KBSHq>s++TVlz$3zm{fxVNVe_ei>W<{^yZOx8kCiVR+ z;mi-aPUVY%Jm&!8jWINPd&p}6?~6Q1a#}HdzqymnmHj^AlJU1p>}k97Pf6qeghXvj z7QG%U!7$Sb;?e>(F2U;YIN?l95H73n>eH$w^50&_)zrv@xZA3zOJ&0)#F4MoigG7T)0Y)GLT0_WntlD2}CFgu!U5~_a%Sj2(Gc83YF_9dEaE2 zV)61ps*lA+kavPS;2Z#+6-rtXs0`KbpZorVau1TqVOr5AVw;*>iw)l&AD5n?U_sNppFPL;nZ zrGhL2fVE0OV&Yh4_(D?L{_mf^#zOh)Sq}N45^OR8fyr&n`cAfcdg&2fAWqzcyE_EA zF|Uq#PUhQ>GBGQDdxR6nBhd6g4lLuL@on8OT}VIszWOH83DRY z>0J5uds+Kv3s;(lGBPM21;G@8ka8DIw`-yLQb(X6wK~~zlc9?751`c8%KY8O>ft#o z0+N|I>M%lt`G_rOE8slgA|=a1-*57W2Ya;W*GGjn#A#&o?G7G3Kcdd=cbx0nudiOi zD*v8gaY`JiWmlo;eqV4*YVZ2~`Oi_(GReo6{rc>oU0L5RgrLd!&o7zQ#6*72`Ml|V z8t#h!Tt{d&?SiMo_a}ZZ!1H@Tcr@X0ExYn3cMU*jxj2wV-=5g0kETVSn1dyWfq7r6 z6sU*8@mz4c;epQZ)DF`JB)JDgBv@YyxhBbKkR0OW#>V~gq4#3a zjwcg*8X4X(yvlr1X3g{|xCO!AtIBqez@iifY}4iGHFBSX=@S1nqO)I+lF8xX?%U1> zBqZcETu(-oZiuXn-0Z*hZgfh*S(F!!h_yAC$PNI1<=9R@6YoGkMEBs)Ki}!;riU>x z2R4d*Ia-Cn2xsqZ(!w4Pu?HR3JY{$ie=idGbD>9aDIRFDh*4dAeGHoQV|CAyw73mW z@sP#1Joi=8@?>{LJ2ZpDirSr|^Z@|KU)m!Vq+FDgfCoUh2?Qo3TmvMp44XlKFyZk^ zv6n#;_}*ASL3g7U5IH$e@&;~fmVXVdT+iGB$r->TVdws6f~B2>-0kbd&`$G4=hG%d zmmxBcgu{Y>S3Lhzhk#M2Cz46y7y&uiZuoJHym8c#u~ckdFXWTtbOGS(fcR`6+6~~Q z0z9hlRlDtfgO09T!FAMU+4GgT3TW%t^!#UvVZuPC}~G0|xQgs3250ZS|s1QF>{j8YV^fru!@#Au>KjFYYc7LX1C zB7z7eib}UoL8&v>4@4N&Qk zmMV>?jKw&H!pQuxl%rs~0^k}kTDBFx7PevnunTjm;IMZWHH!ERkw9^>Pfz@L^&!o` zK$Z?$s3swKaefdfBnQoshBaaDJLBRl95{iHPK+8aa}$L4#(g~2u{eI6X{M#h*4I`4 z1|z_R4)@60Vg&I(F>l4EBW+Mx>eDOcdRMSX`)f&}03TKlu4{ixnQGTvq#OV0$6Zs# zd37Xi^HRb!;@^9BT$^Bzn|$^0#j}C;5@#g)F9zIGXPVn(%m#k=>IErhoHSMrjD>#{ zm){1!bvOOrePAIB@sk%`^U-&a)FK0{x6 zk_pMdeKBtC`7>hE$g>q-rmPNahAtO`US5N&Oo!O;NUCmhul33}<7ygd5D(x{Vg6ae zfe)mX9XGhV@7ui!)5{)!W>)&3O5XlAnb@pSz{@Vn8k1I@t^gCII8)Y2-Ch~=MoW)f zYca2|JZ=NhRw%;s;eZh_{#lo?LYVApPpyYAi&+k9-LVa)2%*Sms*X_GY*;-X7`pad z43p$)M$g>vU3CxWc?V$DL}kNq{jMON#5Z1?k(%qWc?0r%-;7MR6BcF81U}|C-&bxY zLfBZD+v`?ciA9?awkD)b>}NrV4=6u>0VIX1prxDb-QB^aMQCVj9LTjFebPlIZ?*=` zC_q9%5Lz{!J*uT6qO#zGbE=T-%)!Ibbs$&t`v#-vw)6x5@oeNg%pA2 z>+8dnCfI8kD`s!2N3$?NS&z~nq+8G=Gt|gjNIpGXcNVze3=r)2a zfh5>MS5kl}IvhIKC_vY4r4&@*A4oQk957S~ZmyAa0b@+wPr*3Q0k5KBS-Rv;o?cw< ze#J~hv%@`V?fp_P2uRuO@;0GL8}i z>ri5g(9_?=$&u*?OP(Y0Cjv0Zrb&;Ds~XoA`pip9nRHb{C%R{=^NY_G9T>4oZGv!c zcTY9+T`YHRT55g5H{uX&;4&d!`}b!sK3#vo>+9VwxA_#}&>`0eZdWU9d?hHoaTt%d zaQPJ<)LqvYS6z&DO9Nt}`K$gp{~ zbSdxCjQMXi_0IoCYh)Zb(BquntrUpX<*r83S&8l?%1E)?O8O12Hm;ap()C1F;dzYP zt#4qiBUaJfWSSf2zHKqzI_9^)U)0vIKPkak_eX=Q0}BZ|Ji@UW90wp2^NmN;n+B)n zTryP&ZtUzG_4KfE8TCYbS~ys2!OIllRI$1>Px^z;&vlt=mBMF&s4V;-Wso^(kJcG= zzjDtl#Mv5%PDdLGh~ont;i=z^63$k1KXOurVX&A?9d}$Ns9#qoV>EUNq~F^t(rCsK zz0jcVP$DhL#$){KL|abA)D|e!<`=Ra>8eK2mo_nOI92Ugy#Rm#dafS8C)Xb{D!>0x z(DoySG3uBhNM;dQV-=dv{rFQ&OixNVB}m$#crF@aS0;S%N^`bt)_RzIWc2KF8=O=Q znK+p~1Qu{B#NKJnuAe^1_^fBN<)gRrG58mTYy8*V91Q*olOKDlzG$ER_VN!M^K%Sd zbpDajD1X&kH6U+m?(n`K)yQpK?-7KN#};PP5)lEUpb@fcQ|U)?`|{R)e_*>0&NfhN zVGQ;NazNSI2I=0VM>s{u=kox!jqrLKyZaz0R(q^BP+2x)C=Zv)V(N7d_mA0UD_CX( zRoGfh)f3b<8Bh%+$Om$J42>fnR3=V9lJX5dYj=OYtfG$sEnMNOO_GMwg^6v6DmvUu zt-do)O~67bX1x+MXb&Hme9+^63>h(sO;9xYzOVTuoI$HvJ!V#zwzbr13td1;gYf3PNa5L&f5~9O-y*LrNrdrZzNmEff*UnCKJ%Dh~b4YSMtV;8%;lXEs@($A}+0|MFhxlKe~WDar|fYcR4% z*FIBoWtdjS*Whkf!YGu6pI=$sf(DfH0OZ*CzQaK13tiB!kfTnsM5EzdeP8ue7zQ>4 zb6K|I<6MPZQ`TcJo{cEEva+&B_=$ufvUWldDy!$kmTvd&Wr)P>Z{v1e#bX+uPBLGR zk7>iiJsA!#qM*>!HXRwS?d=J5_HSs@Ol)6535}zt^z}TTC(Sp!ceG#ziNEzg`b)|6 zC7$=Q;A&$5Y@DjyDdg$TIUB#+dZr=ubw>2XYP^NeT3_VZysq|xjvMD3KwQb7$pMBq zD^bNHpa)ny?2*ynrR^-qV!!opv+O?B^+E%)(IA7zs={}x??l#@1RP*LGI7uO6efD zZ<%@ElEfwd!3LZj7%KPWEuC)Rs1UmtS~%cFI7x=HF#<%EhTo;i4-&#mP!MA$$|A}a z?VoJQF=Y{@x*PGD+;Cx^p1y!D1x{zCd!kD*Q% z%H1?jIrb39c>oe7AO-EsbW~H_HL?~(Zj7<1=j)J5T=;T<#FZCH;Ds}vfREHCu?G{=xd$7U$IT#2uP7I6}h z(JSxBfx`+r3*F=g2ff;Ha1AC}eAxG`8NS4nWD*C1 zS2Y~Tv4N&i<3`2@s-0Psx=@HlxdeNPf$y*{Ge$8`(ff7Olx2F^Bxqo3NqaN_jN(N+ z{zm}yNhaJw$$x0NY>oem5pPr22mfGoF0>HAlrCmZ|26H zH(xX}@K}}VQaeI#%YlWx| zQ=A@Fov)vCf5~+R$8Cx=j*Oc)wJ*2tU9N-(0VoQcMRQVe6Y3%@8eoeA?vHk?I3^>% zn2bI^Wg z&hs+UuNp;AB956y1fZF?jD@*{sN!~Zx-C#g$;%kRszG+2VYjSEO4mV0mT>3 zuAz)gL($g5(bwSQ2~LdT=HH+DJVyr9BZZ=EWsRS4D0>557+jAHQK6oDhZRsMBpL73fu9%^y+@zc8T{3HMbJ@LdJ8v(hnumD( z#Iv%q$92tnz|*a)YY89>S6ma=cvvav7Ht9ipfGR}X`TbU4hFoqXR?!B?nvCk0~X{& zf-D?kEivr5wevn8z$|o`;ZSxNA_10CK5C%&`ta4Xzubc8{}>L+seq2+jp_sn zia*7JL&2(Y$i#u7G;^W!i5Z+X#CwXLMxhev5-4Fc!7&#q>z!-h4AS}gFz??0bqk!5 zGVQ~CdOKGk_ZwZKIfXM4i5H|DE`az=7jUSbkzQuVPsD19Qk?VUT3Hd0kGYf@SHO&N z;5X}9YQV3~v1ze}a%D8`UFTNL?RmLPqQw$O5iiP&IcmF7qDD~~XnpIHM+7+_*`ONF z9$tcCoT&a-WC9^X_a~utSG{cssZ-K-vfh9F=SEgmJ?XVQXM#4f<%PQ?j`AwtcGjFM z$(1Igm2|<%l!5sr4BZ=B`c^(pFi;7Xwb7svzZWkAjsF2OeB@Nz)w6>~eZonI%SFJ( z!hSvuyg8g6zk-^aN2ss01)FBF|;dvdZL@r%gnj?U!qd-9fQtCF8xaIM|}h2Mz%H(^ekqxu7F{ zRspHa5X&A{I9TJ}J;FnwvZsp29dIXXqjM9RO^t13$oIZw_pyqawZKR@`aONGA4)ik z&9n{N!7@D=3}keF!Ex&u0SBLjq{k?MNY}@8>!Da&_6HP9Ha&}y&bRwCLn>eYPm{GtjZvj$g&wsnaryl?f|u|CdCV zfNp_y2N&GfU5`O~H83_j+~!D>cF4k)V!-X}HI=uHaf4!P&Kr^R0bgAZptJ1l)wl#c zr_(hpAjqk;A~Y2Ex!Q!kHw#8NKaa2}|N4Lh9Z2(cIg4y#qkYo8dr?|Tl9azwbGIyGEqDX#D_ zcOqyZ?7@QvIQ6rdDgFj#6AIW~mFA;!t3nN*%YM9^@2`iy@|p{kJ_Rytp3!cs#b^)R z{4|cT5az=nVY!szV4C|dg*BiAx^sZ`@}!aJC<1boD;j8MMt-g^zia!g-@eI&3XC77Iq9+r z3Sad&-wPt!s6dkfxEdpHqQDkUmB{E>k%qEAy$=tDJgNdDCAGOe__SQvONVOO=G4vy zR&kF|OQGEdW6Na2Jcx7N6W+wZhIGGzbly}$2J+ngE6$LF&~YQjGl>3+L>;Mu0(oRd zHIHi+Z)9&M4zw#~a(;@+CVBH5(Z)J}t!bLisZ3q1Uek^6| zz+#E$1oo?LvFeof8RF91sKW}5U~4RdA{58*n z6A8>og7+v)70!Ycg;+ur2C`Iq2z%%_$tuu=;6QRC8M>*?0$G(m|E(ejdyLd@_~Z%! z`~uq+1B;wb4IUK|<~MS%BWX@ZtFUmY2Slk0=8X&vPgcVnA^`!OyO>L<6Ze2L09D7I zXbe6f$KTdMo8DPY2`S6Ny}O}d%yMqrc{7C>=$Z8CJZoQR9x|OUJSE|M+c z9aXR)qfy~e%Yh!PJa3Kv^0_(c@IsrpZ?iwT-*O=J{!8~KwXFx<=+%l3*p-gGH8Hgj zfTQxncVYL%gVH{G35mk_-CZ|EI5N2aRxHWe1&0CuB*sZLwdUU+P#B{*Ui#eCk_b+; z9hqLX2bHZ!%LjZW%a%Carbm$rMisJ;lPCn@g`o7po5ia0w&I09ce3j_*HQXY9DajE z^U`;pO_`*8Y1y;V$4@TS9$IK)_BQ*6`(l1vk5Ah7exJKKrB?oW?dt>h?fe#QlY6`O zwDPWZVI3Ks-+0|?m;BzTzr4uo+4{BbEj)zg^vmNbZg(nn%y2NwR-BF5|B<`)tHdpJEf}LXOF`b zJ&kQIemQS8eRWvC=(D|np%bozKUn>9wB_t6(<{%FWp9a_mlQBXY0I%G)f(2PG^4NV z@Hv<3vZA^&u|4bb-4yem0?tI492=S2r*UF1FNx1-ztS>UNT_SMe*eThWGe_xeYk|+ zd(WwO2bBR^2G)Zp0LuXA@E3(bqvkza{OR}rhz#}*MFxF%N}W=0VW(cMe{0&c-l=`N zpM`tA`TZAqiH#KP-u&{XM|W>-U8I(Ff$^x5HTTzipJM3R@;Y=*$FNY%?$Cg#emVv9 zkw#&yb*pE)mgUZBi>-Ai&{h?C+$ifjp1kYHr=h(Msy~UgU-MpGQu_>S>vK2*MRjP_ zS5KPye&?J@AzqfW=y|*;rrmp(jXAGSB|&oLhM&a_Qo-c>DFDJR(r+jhq1?>br53b) zSZ!P5BcFAMfiHdOUM%mkcS`%v(LsBE{-Ut0T|2P&V)aa6Yw(GrlHo^dYD3n1|JpD- zG%m_Sn0937s=_dpmCXy2h0P0-bPB_K!lo#-uM!oF>(M!vv;AInv|^=wlh|jaV|29R zb7Noqp|zjYb}UIm7o{PjPi?vW`^|Xu%wyjrQ4q)xM36YU0f{sc7QwA?;eakKq)aj? zf?)O-APiNZ3*OZ^)CNeJN>EdY->SaphCsuaQ!2TPAqz+82CZnxoFTlswZ%+w`&&U7yY?9`eK8BW{LV!?W5=3`u9cdInZ)0t8m`38}|P)RD7(RQx<7Bd<*&* zn|IX{d$oswYB#SO z&vu``ANMM|!ZJd`Jk!BGd;T<`;k)O1V;$4qINvPye!Jt`mDb=$^Al>d>uXo_m2M35 z8(g~*4GY3(4bX~1fc|mr{EQ88$<36y9QA&4cAQ(;nogX{5oqNyw}$aXNCZaQf~)8R zI8UD1Q%`PRlGo3Xo$pwODkU*GQT9PWFwXNCMA5)a-n)Niq=lZIo=Ev$xg$ra7OG@y zSpG%N>Kd=7=|w=Pf;J?Mz8BpXreUooO2~QJb>v_F3R)S}lswL9T~gT7OR@X6t+LnG z)%B`5Ya`6WE^d#bkOuuCDZvq&8z?%d@CghR)rwfnEUWRC(NsQHP^I924Gl!JmN5ea?xO zkB~*a(>16O5i=X;Q2Q02myom=g8}I9#vtH|2BV&8 z5AU~W5yh>qC^n_WPe|)Gl>qF5tYO>D?s($M2SgDd1uhDm=~zEgC`e?l3Y4H<^&Y#o z27_5vDG2%PgKlua?6 zIl|QL)?%P@{pSf!ehyj3!iqya3Bd)m>ie>*Dh( zT5>d>W@toSyBp!|dt#oQ%Q^kd?$SF{eMU8;dCu9qWaILmCNIylJP})J<=+~xLuhYhPFsRYVr2@!A%V0&LVR?JqL zjVO@OT&VO<)uhoBLER_1Xz{&>UN!DDR<1o8xm(ODwp1jTQ-BAM|64{HP;NCfeD|5( z=bM6Uw@Z6OGY=mT&lWbw*3gd9zNh(kUsd1Z3B3;k^3^jtM%}D$8CDZkURm{ZwDtC7 zChMH41^GU`Jvv)2y~tj$r}U*qZ^Y61AGIGHGp;bZd~Eo&CtDu)`VBA++SSKImB$bU^nA6GsNYHj{p6d^h`TcJtQSl*A&ha_Tp zsbC^*2}aov$^6O5rvd*Iqu~wPPA7ix0D)#(+1kdr+t9rG3s?#Xhjx}=G&9HVKyp%2r&kFYPUW|g?ydOQMS670kyrB|6 zP-y)XjLk)r-GOhRS({C}3?*8uO!&D~${WWhjrENzPLMM6!AgY-OZ1a$qw(V@&MVXsNKAWHGm%-?^ExnCQ?Dvwjh)dXI!Y}CWF z1ra!FXiWpAqe{#@4eNU0@F}ZqabnWx*H-UKuJ?_wgK&l9F7ljK!~A{$*o8LmiLLtc zo=5I^x`O^q5AXSQUg@6^8vAtI_C>T3XYGX%jwURMLk0N=1D_4drE26#g_kfO7vpJ~ zwS&!dyW&g+1b|(|*Hyjm8h`E6Gfm99_Dg$Be$nWNQC>UOc}HpbHRn80gz53S)nB7SbZ5yUeFW%)wniDuF;WN9 zIjHaxsQV4Dck>^oN1qhIoSsl~3E*gjS2*eudYJ{ui5WFnuflM9j25tVU1+{tZ;W=R zBkzkkOwyhYlx-wF%~OF`#VGCqT~SFZn|%;!CZ@Aj^b&&9O##lzIdM8Ry&+xKhSfwo zy|h}pGv<|Re8rZ*#sAbj9;@1uX?@HjO?{VR-FJ%&fA^EHVkqRhAlI$Z%Kh`;Sm2$l zS6aMOA+AffoQwBnkCdN6bXgI^-K(HrRFsmniW|`1mM}O55PM_DM!$S`v`(Vj3xuFr zns4Bu3}v7VH$E<=R##K#f*_W9z_Z6+hG4kyDXtsR=<7^_Ikm>0H!n zqgR#u!~Kg}CE8lwmg^zgWe=Ddx0{nLZmImHFfrR=<97%0?qXkC_FcM`xs%yciK|aE zi#I3tsNb{6oiDt*Cwc5hErG9c&xOOrDSj3t#Np5(0B9c0kCTrEMdSP1` z-XPB{DNJ_i4ACI86OBO9O5yEAz{5Q~jl}tbsCc$|?1AWSvELE^5`Q{JtVR+7ViR6e zuFU?cAuthIh`Zl^(Gca~8&tKUw8vl2=h0JtvN!ir+h^;N=?wzZ>Cs+X*;+WMy|}q= z4|KVETP8IPp7cZJiMOp=Y({QQ%q{g)*!+=vG-?rxBS~z`KwSu*Z;H64Co6nnSE6j5D-WlXs`wZ7T*V%k95^0=Y$J1ZBUj zCy8Fs3)Pf^07@@O2H5eJ9n=Vdc1(DoYOhtS^pGqwmU_lFlTv<~`{Me&*)I1lkM6#{ z(arRpeX+mbUbJ4~*oPhUkZzW7M(3=}=4addrk=Au(?>SB^oAG|r$FG3~ZX`P)v8C1SpnP(!=ZE{-AxP9@tO`p+eG4G_^|uH9GGc z^$uVI1QtghYnusKP^?!tncar!c8_Z~5RFV^TeK3F1VG3H3(gu;f`jpeY`YleQ`7j2 z`vyF@U>KbarH8hp^1yVl>gS>3$=a1wRM-Qp$|vOjS!>4cX(=5Ls#LlBY?W3&^zxit zBg`fmrN0WO*jsII>rrt<@0#O@J!h9R*xxa{`~EejL(?)<*wx-*)6$DRy%o$dqVwLq z(0u3)lr4Dgp`Ow3`Beq}7B1 zl6q)DBs!&SInWc=#YTr0Kz$GqXksxP!Yxo+LZ)s}2E>S#F>fl{jj^AKDPbeHiBo$I zj(XZ7fHT&yoMP%d`|w|K%8TDK<#>H$#OXxS#C4}M_Sms7@wEmq%!I--|Z^V-k`3Bn%qI8jbK#0D-q+knbQc$N9~41rDghp8O>^FeD$EH{G>>PBGqymJq7R zJ|sed;i>@9O||jsCZ; zpSd09*MFccAF*nM%ssS7H}S{+ogXLwYS@4QV~^Y?_twY&^Of*zEFnRUSccf^YJjS! z+fo{Ea(~-bwpk$_0s_3jjqd>qQZy zw<0V9FWd}9Xnxlp)q^;M&)x8Y8bE=DH(sJ_<_hkF-}jw38lO9jiIEMI-~ec(ORsbn z({Tf5I~5P zy1wA%#6MA1@qxr6=xdt1mjhJ484uOhk?|?vu=iZScumPPEXVUpQ>d5$o5`Z~H~q`q zC*=lCDQmGF!Ryia22V{`BQ!xbsbod4&FMwej32Mzz=@Rw;I0YQ5B5h z1I0KO3@;<2)2L=ggecR$4qJ6N(RNPGg=j6x4?8J3jZG{zM>(hDjg+_u5#L~k!Ke^z z2)_oYrQbqG>+^xeYw}9KRU8;0P-I3L(VBf9{Ufz_&O;EjL-Qfk8#8~yLfRhliHlfb zpk&w=I54UdXmSL4xs(4JKY%5aY90#vY)42}+jj zF^X3i9QKoPEON>p4aBJ$qvN5`b?LB$0YD%pajPU^S%#xhCF|K4f@aKUMMH6d{gQm@+w2j0-h3ndbr&tA zVfrpkR6#!2nTNMF=ahZ@e)wgp-yc8P3UTx4_ZldyY3TQuqqXIi0qwZY^z)}jPQPT} zgjGn=ycIqcb>OT;I#G9##DbRJn>8{UsDU{upYN|SwVU$qe>Y9L99L!905FR*m|N

wi}y)U1=0k&m%F&P?{b%?6o@7#%>sA z8$t9@c_G4RI-8>#tl-)YQ=l1g5Y+)4`rEggrtp_d6x)09H;|@;u{liI&Uub9JH`$@ z0u&+?Vf3*-qFxcR`(QFn)7voUfi$5>6e_QWmN&_hlSIT}w?=yYvoQUI`?4FsEUp$& zgaZExu^rY0- z3WSu&dieEJw|L{aLr_oyFYdki^XG9cE3tMNi}fpv;(kw)kXa}piSc{?YsI(^pWIg3 zPyNqKsexE575Gd8^XWJzKpw3MqzutO;AOdZNjXxYfG&!zyymR!vb`rCADIfXCW{5s z;U{?_UETK)MU&01CIDv_iclo##IM#-0yAKY=BHgTVTuqN*#+CRk*kK$N(qQdWGEVO zM&iYsL>u*1niVE%&+NqL`W z9y$dqQvd)3IBKj~Y9O}aHZ2ub<0<0QVDb6jkFcpaZGz@I^$!S2y%qvDB7}^cbK>cI z)Gj(P*-5c!g;Z#?yWtN6ncIgV_wRzCuJU;b_qe%9_Pf|s|1x^ zP1t(%AHz7f962s?OPZy^mFi^8wM)4!O9%esNk$bb*gRzf4*1*Ig*|Ni87hp8Cn6*V z)MSDy%BdYJK*_sh>&X(<;qHHemS-deJ_K4m z2t#RV;AF~%1^D+GcORUwdJv5sw&H z8`Z}e*L|jp*wt$P&HtPvVK99eAKU2a9a{$frOY8}3T?YyFMx`4Kp#Z_~;*DwNY&zlvi5X>5 zkf4r&Y6-@n|HzOVVkL8beLelWK{KOINf%{^FHGk9D_7Jb7ZC{hSDS^kZX7RJ}ebP zW5rkvbj}h%^A?0O_@p}tEq`lSbxhs`AxI*d&(8*L+US-XdSVmPfG3@sE650 zS^1i&?*;UI6T>b@zj*x|HCQ4~AFw#`GGMJfG{9NtdR_kbe5#tuMCsR-G&++LP8*1x; z9~6VAYI=o<@A4}#HrsKO36|2%ceQ=tkQEr&R%0#iiO|M@oasj!^)gx1*1o(o`G5X= zKoR-(-1sDIJE4GP3|oNxy$Olvz^gW$J<^Iss1g4^`Tm3cs5VaQ z*ENQ*SSo&6&iEE|p5dgz7>R;Fst;DCSg8*f&s`E}f%a1aJjJWeu4m%tTnF27TK<$1 z|IrjeG>GNrjl&_@QY9MbjO)U?e*xq#peBh@K%6rl%%_sv7>Q97`!b9i36BR?9GLVG z2t~|rXD~cv67&X)hE0wFGJBY+KeV)IM18E%q=qnW6r=_aXr;VCf!~e>p}iNum{rgS zVUFpTH%^>0O2x%A@BiObQ8dhS!c<1ukzqn35XhcNVuJwjMnetYY$F!>4@WnybR2Uz zw-Hi%5?hR_A?D|yp;Sa-*D5?##?XwsvOxNi7M;aN)f|rq^dQ{Xrd~H7@AaIv`3K3% zV3IMTc8c3Oa4u;8l(zbC#b8R9fIz0Rj2GZ-6rEr7F>VJA=)XNv`8{v`d<2y7T%^w! z=?_;M95p|WCYeuW;&AjtgUw8jflxuYm`hNh<0hRYfJ{tVYG7m#<3ZvrB5?+76&pDK zmbbv3qyhLd9+>^#>p!V|W;E5FcI-ZXbPJnvYEA$yJ!-#2K2c5I*k<7cl|hd9a=;`+$2vgicm}SzJ*<&F}3)0RH4Y4Hw;!b zu^;G*PeQkvjyA{o_i|=M5Zr_ZIiXDE@>sN3=Z<0gN`-}8f5oG3HYzKVKBh$nB-QDN zOl&8&UTA7IfN!mTeH!V3I|tP4A4b*e3$pkS(>nDbNH%?Qtnst~$Ze5M(}ltH&z4Gm z?)nru2)Wis>;c;IM@WYQaLrDxgL5sD{GmPpu}R8yql!jGnuFJ(2?MUs{}d2D<>@ z_!hoI^gted<6&7O#!HFWKrOiX<$Ane((+=^HvgX)*U+G?#x4P)a126sB?e5e6NKT% z>*7;DLrLQpj0lxFPAu8fwf=JrZ9Re6&On`r_8lolZ*T9uvT>xZP&bi?Is^1Xl8|so zm1M(;4?^wv3sj5Fhqb*yNJp}VB+IGjk{e8kZfHHCfv|Z%Q=f|8ryhetTstE#iQl(E z!ZR1OvKV-G==*L>`cAHX`55_@IwGbUyDhV@Ne3fk%ODdG#p8?9ikb1i=su=MG6kBG zubAHt(4Q?t?;_Z_)R~LX@2wrT=h7(XQyxXIN01MSSM8n6ir4^l z$ih~L;Y=4?tHcpwBBB*TA{zt7-QTFPrrl%27rrS$J@mF0CHXR#4+kd6eR+>)PDW9f z(wRMHvwqI*IcWTT9Y6o#;?!bY?@5n z48g1VdV3n*X^fUW?GF1BAjGR1p&2UP@yIm7t1avXt_(3jm%url7#LMFU@ambYSbVg z#Z`sm$4L|*Lp^0~`-SwItsjbB@2pMjKQVI4Bl3Y)rC_YdhE7Qmtw^i_$!aQe>5H*s z<^WeMB^{dp`X>`x@bF<7wxIkGo96aRcxw6$J8I`}##JB>j3IK2I8Fl8A!%a}!BH(n zDVv9-;>^o|JOT7d!q|b^$Y4--fHY#y4pvsgy53QSGbXFlb$(t;d}&UX{zk%=K(+{7 ziP_(z$T>k$RZQ>OB)_>6(MNC$``}c_d(Xc3sEVDK1w zlb*6Uq*5)&Z=Ljc$=TqtvUtLy(E0-ox=0-JD=b$*K02S;^;Hc~XjGJvE=?pb*7BMz zJ_3~=G!(k~{@-$it4~!mFc&L%BYFp?7cx`e)rkoZ#BmL?V+Op!oCBc9wo^yl;psDW*u=odk(F&^isNmF;m~x_j;SCL`^8FKPcg<%{ZrlPAW1t>|cwu*T@0 zL&hxq^_yKkn4O7FIX%cfW>eAflAVsN3Q{Rdv*WAJ)5qG2vrHp=S4 zIbHMVM|9jCGwugaiI*f{8^7#o?7g+hBk#d+InL|N9ZLNPd~QeyfxBWVRh}4f*2TH> zFOQYH1t;2I8Xa&>StB0zP5b#BtG~n*h;u3$zE6A*&8NT4cJz7#g5aFNH08F(5ekX> zT?|vFI$HH7LZ(5eYYRy|Yc{2ZXiXa+^VJs>g)f|sn6@31Er5xem<#8Ru5tgHUbKIW z@<_=&VIWV6A!Ff?8i{^{fj11|qP>K;4LMhM6ce7ioIayg_fAZJhz>r15O5NCh;#I( zx{p79F2LEK8O_b=(+0PEx^Mbm+HeegNXsbaB}C~4V_KX7?nVdCN!PglY+Y3K4%30e zKSD1;d@!=Q1sM9E7ju^cWRW9fn@f=q4m!&XeZ1)a#xyQ(Q`h?2f2qHDCq@BVE#gT) zdOdjlFvH>a9(2rxjUU!@RwVCu1^*sT6Tea8mxUmFGRiLU+AK2dR0a)_2JGehftPfG zMQbzi9x1K>{;YkAx@pv6JKV&|032m)FC64Z()*7zuwv@ZfwM)n~yGZU>#KL1(zF_`wG_q49?hg=S-eqA#a5kG57$$ zxSg?X2r<&Z8zpw;P;!zNb1-)szdztp?T4ZO!R^=Nvu*5S9sUrlycopIK0>+91qKLxgWx$O0+)6=Vlr_*-LYee|{ zDs$eZOS*}GeE+G9*a=oDmmdGvv<1=ip}0ZO4z)zZSQ?NikA7$^(=7t)?lW*0o98(1 zd_d!b#mGPdF@b)j-!i5Se?AI@o9H4RFuDh_q2y+X`_++XK)!^d9OZN%Qpj7fk z2j-!XabajJHJEFsjcCPBy=+w9UpfP>!08;@wV5&YC16~1UBV%EYrrEch7fo5-nTEB zk}I+1vth04r2K?uJlPq1mRb0(MaR{ZBq2i>;PG@PT{|cr|GLu#M=u{Lt{Qa2?-axs zID#HkrQShHPX9(m5JjoR@ExWgE#eAnEcvaNkZoTY9!AX}Q6xS*Vix%47Q=V^s!s4k zX$&=l93)g@n>BkZG&9OQJZU9YOk0t-C4mpt z8tXO_H~~FY96s7ofdE|IQxzOs+3?j*ItHnW6M13X*F=+st(Ts6fNpP^jh=Qy3zo^8+AC=_O37^Q)8F3vg!62JytTf~z_AQ=TFzkLT_ zwH1|mO6QZ|r%C%axly16@fB}0U}@HU(T-_isu24y>jg?q`OyGBRzsfFag@`e?tC{vfTv<@!l0nM^ z*Yudm^Vy$z@k6pZ=Bs3_3-t52z4+6~LTWN_3$2OjHgq{-)VrQkHeyHi>P$$6#3DX> znEbk^M-l4j*?a#>aasPL^dNK|B))H9<8bDQX9c^XpXq956v2UqWG>a`^?}$+Em|B< z=R4Jjmj?@5u^m*?y)lqWuDimskOXi-F>5$&1pZ>I{8S;ZDt|0rqZaa>yctR8a`mH6 zB{!Gx$rhc^Xi;h=zZfdu#S`2;Jh!1(Es3-FR(?4SMkPak{JwdP6Qt0>skeAXCpb%J zJlejL%{GS@f?*8m5J-YmN$OzFS=DdE%GncIctUM+s?Nir2yLIh(lA~(^I9jd4@)h` zObZ@grZQ#X%}OG<5sbli!zqR}kElc2#lH5KC{EA%wTVKR+%_Ta-D`m;DNo}&&^MK#OV{3BElvPrE zaQg8?p@M)B*-X1uwzx2jkR6RyjHZMs_5PRdE;v<6?5R}oG2)8M54dN4-k`z9au1nB zBsy$nRIctZ`Mlj>OA9K$o=J9o0b9JLy1CE`CHOz>uk1!~T=$rBaaSN_ezR-k30VpY z4>`W>f|xRQ4-XR}$I)n1snNes6zu}je4EnIhfwU%NLz)gao?MNNxn&fY4Ky&-8Z0i z$M*jFavp`yOzRbj;HPBNlq5zNwXP0rhc~uC#AGcOzXFq2?TxKvc?&<2w+fS&M0p=r za@yqPv=M{N&#zxR-nd@?VGxQY-?*m@<6`rIg7hG!iN0ZGH#$43N(hiQA)Bd&5;dwahTl0pe zP+dwW2|Y?e;nK=m#NM$Ox9bJ>6JZ8Z={q9I4a3MkNC?^CYgZwDNF)$9mNE{OKY9wK z(jrbN@=lg0pK z1k+C(GD@Y08of^KA6kIX`ot8yoDPLXZo( zCf=sX9+Q$NVsQ9IcG$&*UXj&Jw}Erexm50h=5x1rMc4E29qCv6PVwLHg`uM~);=AN|HC9VW@Z?6hr~GG__YjwP=o5$^nRo)Wy{Jj z6MbbU@GO2G)=y^_ZKK2J#vynkITD?xgfwlD$o2j{XRP>Or#B% zKugJke==mGL>EcF0i+Ac?URPOQ)h%`h$$Ol9F+|ehf`RFp_l=>bpKJisQUV!$&}!s zZ`{Th3(5mIa*zwfUgZ^pO!XviM~jZF_N1m@Zm{Tck`31L3}UuN-*T))pp)c2Jl@Vn z#Rq#=lyH4eSCU&Y&)28m|hkCcL!-V$~TBQ#On6HemOa_vrTW#4l}i-Ko7Ah zJ<67sz6Fx()=(OeJ8!j(!{7sd=xk|6K1<*wQENqJ@fmfiKNLG-s)C%delj*aYQ4Wc z$bF*tfvD4QqS#{k<)Y?qTORZ-Tvii*Xm7TOt@Zp_Hu7~=N_N5SXM!sYqa3W)&9c#J z5;nA2CKG zP6Ts2i;edylsZB>c^hWFe3((n!<>nhP^LsJ@XAMjnQWQcacCVCJWbQ0oqNRoBA-_K zFpG@N=Kq7g!lI>f(Af|gV8Ncerg6UoX%O0MdAk!K=oGV6IKWAY+uuI5n6xV6kh8R; z1BTq9E5;7{EqE~odg^1?2x^dG>7D zVpHFe(+0uP=~7OvYzTiCIE9Mx)5!D~J)1?csb$m*ivR)IOMS+tHr2D=f|k)_ub1>Y zFS=Iw_|tR>LbJh6>(0Q$Qphx4Qy*>MrN55TbWt~viF%9P>FNf0p@AltVus#p?85YoOzw?6YH)vXkzKlt`O^=FF{*}ee%5sJ`2DIrHhC{lAQ zo5mn7fC;s@Iugn-{9yYd<+7wDHD`HmsKse#<>?UHMhK_(Wo_>aA%&~S6*8^NzJfdJ zKTYSr4Dd=($SdRRwgNU|+bR%3%W~y`V}CM>oSdTWesCUW0SPb)@|h&_=;g>{T;c7A zXx1Ey|NKhS_qbd*e2xTPt`4sB@|Ym~#r9{iKNskbiwcmHFYoLr2N{M$now4b+C;v{-*DjaBRQ9{nW&NZaE5ip?}NtL;?Y>oNSC!6flkkA7vsKO;1QxaDD5R_0#OyUP-F4R50L<< z_6x^>>%d~h;T1b!fY8J7$KYs$bPI$~G6&EG#V8OFrKRAK+K($MNFV&BtAjD?T{`DG zEeb7KG!X+XW>u{(cVHUelqj-gD+?9ZU5pOuW|P~jP8+Oj&AVx5I2w(wv5X#}1?EBvZ>B%i#PTy;OD{EfT7COqF60a&ifq<$Djw z{-d1J@&CCq>CR(?!3T`HsO%NwBa1B9q7s7^CU8Xq|FU!I zseajqR^VtV&`7Xzw#@00H!avOD|rF`0-qC;IR%IOv~&>7+hgt6Z}Qq0j)TDM*KM$Z zzcRQZ2jfe5A+In2SE+FwOkoowGc%@4*9SA4`H(3QA(3;z_D&A+!PJsRQ#k$w z+QO!KW_`W}jx@alMe&d;K(pKt!B-VU9(Kpv>yLTV$2J#%d^634g7GjRuMH3J2}k-M zz-om52quhHwAq~($yTP$YxfM*vB>eBDSS-{<4oZWY{KMgO|hSc@~wRpiVdAGZi3fm z@iEZhyx(>Ja1yO+AmpMJ0dQR2am6EF9+Q3obCU*Y!KqDzUD^B47JU1FAu5-?7S2My zSZ*=ZZK>=9d-%Aod%@rxsKtYb+Cm&?Rb9X4)n~2!O9RUHZ~wlx#&A z=Vt@F6N;wjc{=XS6iI%6d<=8z8np1BG9^Z}a%0oPLwBeBYv9)j{Nu9d{9(`}fjBZI z3z>YQZYdwXzeve9n{TBw&fXjEW3rI9-HR%gZI;zQPvaV{FkUkRo+}!^CN3)MIzH{# zyeVR?kx8y{a;HT9F8fiTBLc>1n)M`*jwFT;RX!CQX(Lh$tkUwizRzxMFVVXlhK4K zZ3l=`2Tc!%N^sF&6${p^7$1lUIT8=m>C!wM{DEbXCm($-l!ibmCNP{u;AEcxJgu`y zG1THW*+UG?Zp42hd%&j-89o}1frQu2WDj0+3udVFoVC-OAsBu;L%ZN&2Yzkh!@xyQP!EX1Y( zNason1=!0@SpuDv{fkJx7Th{i0cj+MTzfd}flDGskJ}oo6Vum>8CXkW6xrYZ)pr6+ zGB+=_PyXAHCr3K9^Pj7haxMXsHD@6xX`>@FIuVh)w5{| zeJDR}?rqgKW91~pTv8U~#({>R^rR9cd98K|+3jvxGKJo!YLO52-v%|2NIya_*RUsK#5|D>| zD{*+S)1>hQ{(HlBymgFjR>G$|lmO5WfT0i}@xicDKPcX`oPg+OWJAn_&mQTw3I14T zfao_yCACAOCApLhL|jH!+ifIof&-telPG@dhDx)ZA!D+Sa6?&_{j_K;->OEXsM)E)FyDVu z>TB931@rGYA=YS30%q^GQ2UG#EI6!zm0c=oW5dak2ZSu)FEfapwvJS>ymYs*mqs7>)le=LY&<8pz z7Dx1XMwQn77Pu~n7Gbh8)TvR*Zql@XkT=U2;4fb6$E%TDV1AO2`M@h_jSfu8{AOB4 zkq?^vI^usQ9DzOvMJc75gHMx=1cnWgCe(HIK@-|Q@L9X}^-*$-ykzq&(mZ$@2cuhE z@%)+zho9buQF4)LhEGeOBF~8G!l9E zwn=}iKK;c9Y?l#p50wE^Cp@Z?#BA9Ho}6J>8Fn32D`$G zW@Uv=XQd0hvQYQzBkLi1It%zqEc42~STNJ|O7<@P9hGp?XA9Q&4th$07G1a)0jir1 z)sy{z4o8fiG6{5fo9S0i(ekavXhI1}y_UthLW0^@=2O|P-vydzw^j1hXg1?}y>X`S zdfbcmH~0e8cRREt1Mv`)C-OomwtfoXxnBn8CJCA65e3>CIQ{FvrN;#D6czKTZSE#+ z;xXzLUE2AmQ&zDX)dlqdc&Np}nJ=lJ7T9zb-cx;7;-2G=is2Odl0Sp?d!Yytwxshl z2P03mYQe@%#z?{rAXeG%oo9K^9{S#}c|Zl!&1!KgXykLta><`_r?7d6W_{w($|EiX zma3|WE*>+U)x)9YqW9)?mZ5%VbG_Tyld~#u=j*L7^KgAk3^Gc}+b_7!Z=04DGP)rSZv6KWc}a7`s> zv=F@M8SW7{S(pJu)E4pwMt75FZ_0w`hOY2yMmHjpTX0?`#)igy{6G93UH*ickr72BhYQ;?2r9G)aiOdKN) zp0RAT_RLMn?d__7b+Tg&!BGb>bO$rXS&deV9rqy_@N@m4RS~<@3A(__J^Xo;KJcHCm z3`*Z*z~w{^fzZ525`jf6%vryAir9}zZoh(stEXc0MOB(BtZP2?7a`W-dZ7l6BC0|t z8zni=WY?Imk7G$0IZye9?|9i4XGz|lg$_uQHV}r7QA`OnR{=Kn#imiU`vP^7;+sIU zV6uPuC+*y*^A6y_=%h=c`iws~d0E%?Uuif7xjpDUM~2~Li4$#*k; z{mi4%bK(jnR~knhRkeA4jB8?9V_23yuz1FplMI!@{@KM#$0)32J_2i!&dp7rX6bJs zaDaVe9_w`x1)Lj(O&l+3XhcI-2)Hu{i+L*hq_$nW`G=eucRqze7m;AZDUbV|>y}S} zQ4eU-F6*dk>}*?Dd3+PBT%t$}1z2&#Up7l5$LP$=;aDLUYY`T08Mv_l3MhR}J8Wzq zlD|cjAhx)3=0i#tJHh>hLXI+8_@|D5FhlYi)+?@hC29 zw{%)Eq0jMs9cWwl{*0dXh2x#aa z>}~MX{^~5%D3s{4k!S1{v(t)MmN57lC&iwQZSiPN$=THo5V+aB9i|Egm28@DM~f}EGV5EY$}F#B=1v&ZMnDfM6lRS5fb-f#yq|Bag}H+beRzcAU_q0 zzU7w%qB*iZa~oD#-ZR?o7^OVHlPvIW&u3jm{_v5q{+2`FP~cj@Z9&Y zPkVq7d$6|Uf$ST>1!jnxw_-a8&Rn4#*|{^ z#GCRz)vKFExUzA{!-w7DvAGOOOV+rQp=aC)`>eiS8wRiitRcOa$W+p|g6p6+6k^Cm zTR?)PU@`m&Pw_J0jmWrV;2cA1Xd-IzVs0dF$i{_uw_pSdz^Ed?xw4r{1V~t4K0F&P zh_a-<*3HmosbPexo{0T@m;5QHQzi>ZfAChjJtx3&oA_MmSksJ0Z8~(}o7>}1Lm7Y6 ziVC<8()9I|v{Q-%jhdTFaea)IgBGN8j28t5vJTvcB)A9?MpK5T@~Wu*gCzAkZ~8Tu zZUhE^tqZeEg5#pt#K6jHHMP=mD_)DRH7%k?8J-9m$qqoprs|H@vT^Jo{LkbSI(zZE zM8%h)aYQ|Hy@&y?5`VmFHy`rwE1qs~_)k}|8=72`4ry5rZqvj7qK3C{NYdqeK>&@Z zN4)T%&{J%Wu-^ire5}Qc&mrz)iV5}x#uKM(|!Msqx#b#p99;MTEBcB?h5Lv-UDJZF+Lr~| zzzLq~dHpGZ^;gB=rPIU3YWIA4@lAHpZ#>%lYJ{|)xU#A;XGaaT-G@yGywpe!i9kV literal 0 HcmV?d00001 diff --git a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py index 3d8aebc..f461645 100644 --- a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py +++ b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py @@ -148,6 +148,14 @@ def find_request_metrics(run_dir: Path) -> Path: return matches[0] +def trace_manifest_entry(trace: dict[str, Any]) -> dict[str, Any]: + return { + key: str(value) if isinstance(value, Path) else value + for key, value in trace.items() + if key != "shapes" + } + + def score(path: Path, expected_shapes: list[tuple[int, int]]) -> dict[str, Any]: with path.open(newline="") as source: rows = list(csv.DictReader(source)) @@ -409,10 +417,7 @@ def main() -> None: "allreduce_csv": str(args.allreduce_csv) if args.allreduce_csv else None, "allreduce_csv_sha256": BASE.sha256(args.allreduce_csv) if args.allreduce_csv else None, }, - "traces": [ - {key: value for key, value in trace.items() if key != "shapes"} - for trace in traces - ], + "traces": [trace_manifest_entry(trace) for trace in traces], "config_results": config_results, "rankings": rankings, } diff --git a/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py b/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py index 3f9813f..20f4e32 100644 --- a/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py +++ b/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py @@ -143,6 +143,7 @@ class FidelityEnvelopeTest(unittest.TestCase): parsed = module.parse_trace(f"u0p01={trace}") self.assertEqual(parsed["requests"], 2) self.assertEqual(parsed["shapes"], [(512, 1), (2048, 2)]) + json.dumps(module.trace_manifest_entry(parsed)) uniform = module.parse_trace( f"r1={trace}", rate_contract="uniform-spacing" ) From b6ef6eeae72b2071d0a819fa51933d831d616e70 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 10:26:21 +0800 Subject: [PATCH 102/148] Add long-context attention profile closure --- .../jobs_attention_long_context.toml | 43 ++++++++++++ .../run_flashattn_long_context.sh | 65 +++++++++++++++++++ ...run_frontier_qwen30_exact_trace_surface.py | 1 + 3 files changed, 109 insertions(+) create mode 100644 runs/frontier-fidelity-envelope-v1/jobs_attention_long_context.toml create mode 100755 runs/frontier-fidelity-envelope-v1/run_flashattn_long_context.sh diff --git a/runs/frontier-fidelity-envelope-v1/jobs_attention_long_context.toml b/runs/frontier-fidelity-envelope-v1/jobs_attention_long_context.toml new file mode 100644 index 0000000..ff94406 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/jobs_attention_long_context.toml @@ -0,0 +1,43 @@ +version = 1 + +[[jobs]] +name = "qwen30-attention-long-context-tp1-20260717-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-fidelity-envelope-v1 && timeout --signal=TERM --kill-after=30s 900 bash run_flashattn_long_context.sh" +artifacts = ["artifacts/attention-long-context-tp1-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +TP = "1" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-fidelity-envelope-v1/artifacts/attention-long-context-tp1-v1" + +[[jobs]] +name = "qwen30-attention-long-context-tp2-20260717-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-fidelity-envelope-v1 && timeout --signal=TERM --kill-after=30s 900 bash run_flashattn_long_context.sh" +artifacts = ["artifacts/attention-long-context-tp2-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +TP = "2" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-fidelity-envelope-v1/artifacts/attention-long-context-tp2-v1" + +[[jobs]] +name = "qwen30-attention-long-context-tp4-20260717-v1" +gpus = 1 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-fidelity-envelope-v1 && timeout --signal=TERM --kill-after=30s 900 bash run_flashattn_long_context.sh" +artifacts = ["artifacts/attention-long-context-tp4-v1"] + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +TP = "4" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/gpu-fleet-fidelity-envelope-v1/artifacts/attention-long-context-tp4-v1" diff --git a/runs/frontier-fidelity-envelope-v1/run_flashattn_long_context.sh b/runs/frontier-fidelity-envelope-v1/run_flashattn_long_context.sh new file mode 100755 index 0000000..5230e16 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_flashattn_long_context.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -euo pipefail + +TP="${TP:?TP must be set to 1, 2, or 4}" +case "${TP}" in + 1|2|4) ;; + *) echo "ERROR: invalid TP=${TP}" >&2; exit 1 ;; +esac + +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +MODEL="${MODEL:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +CAMPAIGN_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROFILE_SCRIPT="${CAMPAIGN_ROOT}/../frontier-qwen30-vllm020-profile-v1/profile_vllm020_flashattn.py" +BATCH_SPECS=( + q1s16k 8q1s16k 16q1s16k 32q1s16k 64q1s16k + q1s32k 8q1s32k 16q1s32k 32q1s32k 64q1s32k + q1s40k 8q1s40k 16q1s40k 32q1s40k 64q1s40k + q512_8q1s16k q1k_16q1s16k q2k_32q1s16k + q4k_16q1s32k q2k_32q1s32k q1k_64q1s32k +) + +mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/raw" +exec > >(tee -a "${OUTPUT_ROOT}/logs/long-context.log") 2>&1 + +IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?fleet GPU is required}" +if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then + echo "ERROR: expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2 + exit 1 +fi + +echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} model=${MODEL} runtime=vLLM-0.20.0+cu129 operator=FlashAttention3 tp=${TP} batch_specs=${BATCH_SPECS[*]} profile_script=${PROFILE_SCRIPT} output=${OUTPUT_ROOT} expected_wall=1-5m hard_wall=900s hard_gpu_cap=0.25_H20h" +date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" +nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader + +test -x "${VENV_ROOT}/bin/python" +test -f "${VLLM_SOURCE}/benchmarks/attention_benchmarks/runner.py" +test -f "${MODEL}/config.json" +test -f "${PROFILE_SCRIPT}" + +git -C "${CAMPAIGN_ROOT}/../.." rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit" +git -C "${VLLM_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/vllm-source.commit" +sha256sum "${PROFILE_SCRIPT}" "${BASH_SOURCE[0]}" > "${OUTPUT_ROOT}/provenance/source.sha256" +uv pip freeze --python "${VENV_ROOT}/bin/python" > "${OUTPUT_ROOT}/provenance/pip-freeze.txt" +nvidia-smi --query-gpu=index,uuid,name,driver_version,memory.total --format=csv,noheader > "${OUTPUT_ROOT}/provenance/gpus.csv" +printf '%s\n' "${BATCH_SPECS[@]}" > "${OUTPUT_ROOT}/provenance/batch-specs.txt" + +timeout --signal=TERM --kill-after=30s 780 \ + "${VENV_ROOT}/bin/python" "${PROFILE_SCRIPT}" \ + --vllm-source "${VLLM_SOURCE}" \ + --model "${MODEL}" \ + --output "${OUTPUT_ROOT}/raw/flashattn-long-context-tp${TP}.json" \ + --tp "${TP}" \ + --batch-specs "${BATCH_SPECS[@]}" \ + --warmup-iters 5 \ + --repeats 10 \ + --profile-kv-update + +test -s "${OUTPUT_ROOT}/raw/flashattn-long-context-tp${TP}.json" +sha256sum "${OUTPUT_ROOT}/raw/flashattn-long-context-tp${TP}.json" "${OUTPUT_ROOT}/provenance"/* > "${OUTPUT_ROOT}/artifacts.sha256" +nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader +date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ" +echo "FLASHATTN_LONG_CONTEXT_COMPLETE tp=${TP} cases=${#BATCH_SPECS[@]}" diff --git a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py index f461645..02e30a9 100644 --- a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py +++ b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py @@ -276,6 +276,7 @@ def main() -> None: loads = [] config_knobs = BASE.knobs(config, paths, args.output_root / "cache") config_knobs["enable_prefix_caching"] = args.prefix_caching + config_knobs["prediction_max_tokens_per_request"] = 40960 for trace in traces: run_dir = args.output_root / "runs" / config.name / trace["label"] result_path = run_dir / "result.json" From 10afe4e2e80f2fbb5915a377b55ef836ddfb7d38 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 10:30:37 +0800 Subject: [PATCH 103/148] Freeze long-context Frontier profile --- .../profile-v4-trace-final/allreduce.json | 824 ++++++++++++++++++ .../profile-v4-trace-final/attention.csv | 214 +++++ .../attention_true_mixed_fused.csv | 49 ++ .../profile-v4-trace-final/linear_op.csv | 37 + .../profile-v4-trace-final/manifest.json | 129 +++ .../profiles/profile-v4-trace-final/moe.csv | 73 ++ .../freeze_frontier_profiles.py | 49 +- 7 files changed, 1372 insertions(+), 3 deletions(-) create mode 100644 runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/allreduce.json create mode 100644 runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/attention.csv create mode 100644 runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/attention_true_mixed_fused.csv create mode 100644 runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/linear_op.csv create mode 100644 runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/manifest.json create mode 100644 runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/moe.csv diff --git a/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/allreduce.json b/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/allreduce.json new file mode 100644 index 0000000..aae1f0a --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/allreduce.json @@ -0,0 +1,824 @@ +{ + "environment": [ + { + "backend_env": { + "VLLM_ALLREDUCE_USE_FLASHINFER": "1", + "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm" + }, + "gpu": "NVIDIA H20", + "model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B", + "torch_cuda": "12.9", + "torch_version": "2.11.0+cu129", + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + }, + { + "backend_env": { + "VLLM_ALLREDUCE_USE_FLASHINFER": "1", + "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm" + }, + "gpu": "NVIDIA H20", + "model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B", + "torch_cuda": "12.9", + "torch_version": "2.11.0+cu129", + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + } + ], + "frontier_consumption": "diagnostic_only_in_base_profile_only_run; measured lookup requires a separate CC-backend injection ablation", + "rows": [ + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08288000151515007, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 1, + "payload_bytes": 4096, + "per_rank_time_ms": [ + { + "max": 0.4872959852218628, + "mean": 0.1310015980154276, + "median": 0.07679999992251396, + "min": 0.06217600032687187, + "std": 0.12790721677293843 + }, + { + "max": 0.4402880072593689, + "mean": 0.12842560112476348, + "median": 0.08288000151515007, + "min": 0.0655680000782013, + "std": 0.11220176524616535 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.0793600007891655, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 8, + "payload_bytes": 32768, + "per_rank_time_ms": [ + { + "max": 0.12716799974441528, + "mean": 0.07871360033750534, + "median": 0.0759200006723404, + "min": 0.06032000109553337, + "std": 0.019331314939874535 + }, + { + "max": 0.12380799651145935, + "mean": 0.08059840016067028, + "median": 0.0793600007891655, + "min": 0.06217600032687187, + "std": 0.01687088356254092 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.0713919997215271, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 16, + "payload_bytes": 65536, + "per_rank_time_ms": [ + { + "max": 0.12697599828243256, + "mean": 0.0767391998320818, + "median": 0.07078400254249573, + "min": 0.05910399928689003, + "std": 0.018775178979463278 + }, + { + "max": 0.11430399864912033, + "mean": 0.07594559974968433, + "median": 0.0713919997215271, + "min": 0.06124800071120262, + "std": 0.015719922150631036 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08056000247597694, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 32, + "payload_bytes": 131072, + "per_rank_time_ms": [ + { + "max": 0.1037760004401207, + "mean": 0.07954559996724128, + "median": 0.08056000247597694, + "min": 0.05955199897289276, + "std": 0.0135697420393132 + }, + { + "max": 0.10608000308275223, + "mean": 0.07971520014107228, + "median": 0.07593599706888199, + "min": 0.06028800085186958, + "std": 0.015319849772775456 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.0865279994904995, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 64, + "payload_bytes": 262144, + "per_rank_time_ms": [ + { + "max": 0.14470399916172028, + "mean": 0.09121599942445754, + "median": 0.0865279994904995, + "min": 0.06441599875688553, + "std": 0.024893837894277456 + }, + { + "max": 0.12438400089740753, + "mean": 0.08531199917197227, + "median": 0.08031999692320824, + "min": 0.06364800035953522, + "std": 0.01878029830059533 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.07135999947786331, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 128, + "payload_bytes": 524288, + "per_rank_time_ms": [ + { + "max": 0.11753600090742111, + "mean": 0.07606079950928687, + "median": 0.07135999947786331, + "min": 0.05843200162053108, + "std": 0.01755519771639284 + }, + { + "max": 0.1103999987244606, + "mean": 0.07607359997928143, + "median": 0.07073600217700005, + "min": 0.05721599981188774, + "std": 0.016904445220949783 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.07321599870920181, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 256, + "payload_bytes": 1048576, + "per_rank_time_ms": [ + { + "max": 0.11740799993276596, + "mean": 0.07749119997024537, + "median": 0.07203200086951256, + "min": 0.05862399935722351, + "std": 0.017701381594822835 + }, + { + "max": 0.11382400244474411, + "mean": 0.07733759954571724, + "median": 0.07321599870920181, + "min": 0.059039998799562454, + "std": 0.017188890882557036 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.09025600180029869, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 512, + "payload_bytes": 2097152, + "per_rank_time_ms": [ + { + "max": 0.13980799913406372, + "mean": 0.0950367994606495, + "median": 0.09025600180029869, + "min": 0.06815999746322632, + "std": 0.022829835127539378 + }, + { + "max": 0.14764800667762756, + "mean": 0.09710080176591873, + "median": 0.08720000088214874, + "min": 0.07152000069618225, + "std": 0.02500574954063789 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08083200082182884, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 1024, + "payload_bytes": 4194304, + "per_rank_time_ms": [ + { + "max": 0.15574400126934052, + "mean": 0.08760640025138855, + "median": 0.07846399769186974, + "min": 0.07097599655389786, + "std": 0.024487311423551025 + }, + { + "max": 0.16284799575805664, + "mean": 0.08963519930839539, + "median": 0.08083200082182884, + "min": 0.07100799679756165, + "std": 0.026047320562445356 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.10891199856996536, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 2048, + "payload_bytes": 8388608, + "per_rank_time_ms": [ + { + "max": 0.1582079976797104, + "mean": 0.11541439890861512, + "median": 0.10836799815297127, + "min": 0.09062399715185165, + "std": 0.018285136316576037 + }, + { + "max": 0.1578879952430725, + "mean": 0.11537599861621857, + "median": 0.10891199856996536, + "min": 0.0960640013217926, + "std": 0.018246538626977286 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.1703840047121048, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 4096, + "payload_bytes": 16777216, + "per_rank_time_ms": [ + { + "max": 0.19327999651432037, + "mean": 0.1707327976822853, + "median": 0.1703840047121048, + "min": 0.14815999567508698, + "std": 0.014211056022719618 + }, + { + "max": 0.19276799261569977, + "mean": 0.1658592015504837, + "median": 0.16379200667142868, + "min": 0.14560000598430634, + "std": 0.013840249648693638 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.25539200007915497, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 8192, + "payload_bytes": 33554432, + "per_rank_time_ms": [ + { + "max": 0.2807359993457794, + "mean": 0.25750079900026324, + "median": 0.25539200007915497, + "min": 0.24624000489711761, + "std": 0.008925204570802302 + }, + { + "max": 0.2863999903202057, + "mean": 0.2585055992007256, + "median": 0.255280002951622, + "min": 0.24371199309825897, + "std": 0.012059738582656496 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 2 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.1021759994328022, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 1, + "payload_bytes": 4096, + "per_rank_time_ms": [ + { + "max": 0.9770879745483398, + "mean": 0.1974783968180418, + "median": 0.10099200159311295, + "min": 0.05913599953055382, + "std": 0.2660581738745569 + }, + { + "max": 0.892799973487854, + "mean": 0.18164799660444259, + "median": 0.1021759994328022, + "min": 0.06435199826955795, + "std": 0.23947520188197013 + }, + { + "max": 0.6467199921607971, + "mean": 0.15839359983801843, + "median": 0.10100800171494484, + "min": 0.06800000369548798, + "std": 0.16617013141866102 + }, + { + "max": 0.6725760102272034, + "mean": 0.15686400160193442, + "median": 0.10044800117611885, + "min": 0.06063999980688095, + "std": 0.17523222161300497 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.12694399803876877, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 8, + "payload_bytes": 32768, + "per_rank_time_ms": [ + { + "max": 0.6659520268440247, + "mean": 0.1927648030221462, + "median": 0.12694399803876877, + "min": 0.07609599828720093, + "std": 0.16697784531907736 + }, + { + "max": 0.695360004901886, + "mean": 0.19356480240821838, + "median": 0.11726400256156921, + "min": 0.0796160027384758, + "std": 0.17588096678867862 + }, + { + "max": 0.5939840078353882, + "mean": 0.1868800014257431, + "median": 0.12379200011491776, + "min": 0.07427199929952621, + "std": 0.1463231714943902 + }, + { + "max": 0.6635839939117432, + "mean": 0.1874335989356041, + "median": 0.12014400213956833, + "min": 0.07526399940252304, + "std": 0.16680959677760304 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.09161599725484848, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 16, + "payload_bytes": 65536, + "per_rank_time_ms": [ + { + "max": 0.12665599584579468, + "mean": 0.09443839862942696, + "median": 0.09124799817800522, + "min": 0.06431999802589417, + "std": 0.0205690155775906 + }, + { + "max": 0.1303039938211441, + "mean": 0.09712959825992584, + "median": 0.09161599725484848, + "min": 0.07648000121116638, + "std": 0.019072048129173236 + }, + { + "max": 0.13836799561977386, + "mean": 0.09821119979023933, + "median": 0.09148800000548363, + "min": 0.0727040022611618, + "std": 0.021314066545189116 + }, + { + "max": 0.12992000579833984, + "mean": 0.09272959977388381, + "median": 0.08931199833750725, + "min": 0.06406400352716446, + "std": 0.021360803830354765 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08580800145864487, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 32, + "payload_bytes": 131072, + "per_rank_time_ms": [ + { + "max": 0.13449600338935852, + "mean": 0.08960640132427215, + "median": 0.08299200236797333, + "min": 0.06796800345182419, + "std": 0.020633597898445998 + }, + { + "max": 0.14735999703407288, + "mean": 0.09248319901525974, + "median": 0.08580800145864487, + "min": 0.05913599953055382, + "std": 0.025960234928829564 + }, + { + "max": 0.13705599308013916, + "mean": 0.08947199806571007, + "median": 0.08460799977183342, + "min": 0.0634239986538887, + "std": 0.02122838946992339 + }, + { + "max": 0.13846400380134583, + "mean": 0.0859104000031948, + "median": 0.08308799937367439, + "min": 0.05974400043487549, + "std": 0.022373631424433445 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.09867199882864952, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 64, + "payload_bytes": 262144, + "per_rank_time_ms": [ + { + "max": 0.1279039978981018, + "mean": 0.09748799875378608, + "median": 0.09678399935364723, + "min": 0.0655359998345375, + "std": 0.02315719144614622 + }, + { + "max": 0.1356479972600937, + "mean": 0.10018239840865135, + "median": 0.09532799944281578, + "min": 0.06185600161552429, + "std": 0.02330700253650042 + }, + { + "max": 0.13142399489879608, + "mean": 0.09778879955410957, + "median": 0.09492799639701843, + "min": 0.06560000032186508, + "std": 0.02307579212430045 + }, + { + "max": 0.1276479959487915, + "mean": 0.09611519873142242, + "median": 0.09867199882864952, + "min": 0.0642239972949028, + "std": 0.02273612181973431 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.09646400064229965, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 128, + "payload_bytes": 524288, + "per_rank_time_ms": [ + { + "max": 0.14060799777507782, + "mean": 0.09770880043506622, + "median": 0.09115200117230415, + "min": 0.06950400024652481, + "std": 0.024752645089849798 + }, + { + "max": 0.14377599954605103, + "mean": 0.09824960008263588, + "median": 0.08999999985098839, + "min": 0.07103999704122543, + "std": 0.025403407389046027 + }, + { + "max": 0.13680000603199005, + "mean": 0.09993600100278854, + "median": 0.09646400064229965, + "min": 0.06790400296449661, + "std": 0.022297424273985882 + }, + { + "max": 0.1391039937734604, + "mean": 0.09769919961690902, + "median": 0.09601600095629692, + "min": 0.06835199892520905, + "std": 0.023899922100804445 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08377600088715553, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 256, + "payload_bytes": 1048576, + "per_rank_time_ms": [ + { + "max": 0.1464959979057312, + "mean": 0.09276160076260567, + "median": 0.08377600088715553, + "min": 0.06777600198984146, + "std": 0.024751086465295658 + }, + { + "max": 0.14319999516010284, + "mean": 0.09080640003085136, + "median": 0.080400001257658, + "min": 0.06796800345182419, + "std": 0.023365718881708488 + }, + { + "max": 0.1382399946451187, + "mean": 0.09063360020518303, + "median": 0.08193599805235863, + "min": 0.06627199798822403, + "std": 0.02372618650854302 + }, + { + "max": 0.14313599467277527, + "mean": 0.09044799953699112, + "median": 0.08128000050783157, + "min": 0.06652799993753433, + "std": 0.02462486862033686 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.1128000020980835, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 512, + "payload_bytes": 2097152, + "per_rank_time_ms": [ + { + "max": 0.1600320041179657, + "mean": 0.11426239982247352, + "median": 0.1128000020980835, + "min": 0.07657600194215775, + "std": 0.030240087702350687 + }, + { + "max": 0.15881599485874176, + "mean": 0.11206399947404862, + "median": 0.10628800094127655, + "min": 0.0772159993648529, + "std": 0.029933135345483627 + }, + { + "max": 0.15612800419330597, + "mean": 0.10761600062251091, + "median": 0.09860799834132195, + "min": 0.07689599692821503, + "std": 0.02634237020678647 + }, + { + "max": 0.15865600109100342, + "mean": 0.11094079986214637, + "median": 0.10979199782013893, + "min": 0.07583999633789062, + "std": 0.028619217028542445 + } + ], + "selected_backend": "flashinfer_trtllm", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.08755199983716011, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 1024, + "payload_bytes": 4194304, + "per_rank_time_ms": [ + { + "max": 0.11999999731779099, + "mean": 0.0843871995806694, + "median": 0.08755199983716011, + "min": 0.06332799792289734, + "std": 0.016638543890716024 + }, + { + "max": 0.1218239963054657, + "mean": 0.08518079966306687, + "median": 0.08032000064849854, + "min": 0.06393600255250931, + "std": 0.019900899429956945 + }, + { + "max": 0.11849600076675415, + "mean": 0.0843968003988266, + "median": 0.08702399954199791, + "min": 0.06297600269317627, + "std": 0.017031908773433545 + }, + { + "max": 0.12300799787044525, + "mean": 0.0846304003149271, + "median": 0.08139199763536453, + "min": 0.06195199862122536, + "std": 0.020106523698622265 + } + ], + "selected_backend": "nccl_fallback", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.12361599877476692, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 2048, + "payload_bytes": 8388608, + "per_rank_time_ms": [ + { + "max": 0.17155200242996216, + "mean": 0.12168959975242614, + "median": 0.12055999785661697, + "min": 0.09609600156545639, + "std": 0.022565485532483928 + }, + { + "max": 0.9246399998664856, + "mean": 0.19978560134768486, + "median": 0.12361599877476692, + "min": 0.09715200215578079, + "std": 0.24230694305662265 + }, + { + "max": 0.9317439794540405, + "mean": 0.20037759989500045, + "median": 0.12327999994158745, + "min": 0.09603200107812881, + "std": 0.24450903278000383 + }, + { + "max": 0.9321280121803284, + "mean": 0.19875840097665787, + "median": 0.12230399996042252, + "min": 0.0950080007314682, + "std": 0.24519252220785093 + } + ], + "selected_backend": "nccl_fallback", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.20030399411916733, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 4096, + "payload_bytes": 16777216, + "per_rank_time_ms": [ + { + "max": 0.3261440098285675, + "mean": 0.20826880186796187, + "median": 0.19974400103092194, + "min": 0.1409280002117157, + "std": 0.051397264855718945 + }, + { + "max": 0.3248000144958496, + "mean": 0.20548800230026246, + "median": 0.1979840025305748, + "min": 0.141184002161026, + "std": 0.04980002399656717 + }, + { + "max": 0.32547199726104736, + "mean": 0.21280319690704347, + "median": 0.20030399411916733, + "min": 0.14127999544143677, + "std": 0.05304243085685509 + }, + { + "max": 0.26047998666763306, + "mean": 0.1969312012195587, + "median": 0.18079999834299088, + "min": 0.14057600498199463, + "std": 0.042174123638424224 + } + ], + "selected_backend": "nccl_fallback", + "tensor_parallel_size": 4 + }, + { + "communicator": "vllm.tensor_model_parallel_all_reduce", + "critical_path_median_ms": 0.2924960106611252, + "dtype": "bfloat16", + "hidden_dim": 2048, + "num_tokens": 8192, + "payload_bytes": 33554432, + "per_rank_time_ms": [ + { + "max": 0.45052799582481384, + "mean": 0.3013375997543335, + "median": 0.28273600339889526, + "min": 0.21334399282932281, + "std": 0.07214223764527236 + }, + { + "max": 0.4466240108013153, + "mean": 0.2968191936612129, + "median": 0.27796798944473267, + "min": 0.21241599321365356, + "std": 0.07400678240849741 + }, + { + "max": 0.3830080032348633, + "mean": 0.2946112036705017, + "median": 0.2924960106611252, + "min": 0.21084800362586975, + "std": 0.05323627615746332 + }, + { + "max": 0.4609600007534027, + "mean": 0.3054272010922432, + "median": 0.289792001247406, + "min": 0.21062399446964264, + "std": 0.07608482904865672 + } + ], + "selected_backend": "nccl_fallback", + "tensor_parallel_size": 4 + } + ], + "schema_version": "qwen30_vllm020_allreduce_frozen.v1" +} diff --git a/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/attention.csv b/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/attention.csv new file mode 100644 index 0000000..835d3e5 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/attention.csv @@ -0,0 +1,214 @@ +time_stats.attn_input_reshape.min,time_stats.attn_input_reshape.max,time_stats.attn_input_reshape.mean,time_stats.attn_input_reshape.median,time_stats.attn_input_reshape.std,time_stats.attn_kv_cache_save.min,time_stats.attn_kv_cache_save.max,time_stats.attn_kv_cache_save.mean,time_stats.attn_kv_cache_save.median,time_stats.attn_kv_cache_save.std,time_stats.attn_prefill.min,time_stats.attn_prefill.max,time_stats.attn_prefill.mean,time_stats.attn_prefill.median,time_stats.attn_prefill.std,time_stats.attn_decode.min,time_stats.attn_decode.max,time_stats.attn_decode.mean,time_stats.attn_decode.median,time_stats.attn_decode.std,time_stats.attn_output_reshape.min,time_stats.attn_output_reshape.max,time_stats.attn_output_reshape.mean,time_stats.attn_output_reshape.median,time_stats.attn_output_reshape.std,n_embd,n_q_head,n_kv_head,block_size,num_tensor_parallel_workers,max_model_len,batch_size,prefill_chunk_size,kv_cache_size,is_prefill,attention_backend,is_mixed_batch,mode,seq_lens,total_tokens,max_seq_len,min_seq_len,avg_seq_len,equal_seq_len,seq_len_variance,seq_len_std,seq_len_cv,is_chunked_prefill_sample,chunk_start_token,chunk_end_token,total_prefill_tokens,profiling_precision,model_arch,quant_signature,measurement_type,is_true_mixed_batch,prefill_seq_lens,prefill_kv_cache_sizes,decode_kv_cache_sizes,num_prefill_seqs,num_decode_seqs,decode_batch_size,total_batch_size,total_decode_tokens,decode_avg_kv_cache_size,batch_composition_ratio,batch_spec,projection_policy +0.0,0.0,0.0,0.0,0.0,0.01414399966597557,0.028863999992609024,0.019705599918961526,0.01771199982613325,0.005157200849836681,0.047968000173568726,0.07046400010585785,0.05810240097343922,0.05810240097343922,0.007477463486041561,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.04947200044989586,0.020412799902260303,0.01635199971497059,0.010107497379722417,0.046560000628232956,0.08323200047016144,0.05587520003318787,0.05587520003318787,0.011126758739503428,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.022207999601960182,0.017033600155264138,0.015312000177800655,0.002819235991970241,0.05104000121355057,0.07692799717187881,0.056396800279617305,0.056396800279617305,0.007481982178637539,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015072000212967396,0.022272000089287758,0.01706880023702979,0.01616000011563301,0.002460889579319197,0.06931199878454208,0.0838719978928566,0.07432000041007995,0.07432000041007995,0.004777766433175866,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.018592000007629395,0.028543999418616295,0.02095999978482723,0.019183999858796597,0.003198175496053494,0.12179200351238251,0.15408000349998474,0.1307712011039257,0.1307712011039257,0.00858807797538298,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.027775999158620834,0.03385600075125694,0.030131200328469276,0.029680000618100166,0.0021152558103575215,0.32678401470184326,0.3450239896774292,0.33396480381488797,0.33396480381488797,0.0045872424917606375,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.04438399896025658,0.05084799975156784,0.046540799736976626,0.04531199857592583,0.002277905811237223,1.0959680080413818,1.1151360273361206,1.0999775886535645,1.0999775886535645,0.005694403246120485,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.078015998005867,0.08691199868917465,0.08114239946007729,0.08019199967384338,0.00292795706334475,4.070400238037109,4.113152027130127,4.087088012695312,4.087088012695312,0.013660567012509554,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.05167999863624573,0.022115200012922286,0.017583999782800674,0.010340094822340818,0.05196800082921982,0.09011200070381165,0.06328320093452933,0.06328320093452933,0.012557341255467452,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01583999954164028,0.026623999699950218,0.018927999772131443,0.017376000061631203,0.003514650316260619,0.06681600213050842,0.07993599772453308,0.0725280001759529,0.0725280001759529,0.004343502558613716,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01648000068962574,0.030880000442266464,0.01945280022919178,0.017967999912798405,0.004096211183007485,0.1311360001564026,0.1546880006790161,0.13908160030841826,0.13908160030841826,0.007511906874366178,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.03558399900794029,0.020851199887692927,0.018559999763965607,0.005235911594130716,0.32950401306152344,0.350271999835968,0.33912960588932034,0.33912960588932034,0.006027400986663648,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.019328000023961067,0.040608000010252,0.022790400311350822,0.020704000256955624,0.006113051965778337,1.1415679454803467,1.1518720388412476,1.144483208656311,1.144483208656311,0.0032332311374389127,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015807999297976494,0.030688000842928886,0.019596799835562707,0.01774400006979704,0.004343384771033462,0.0,0.0,0.0,0.0,0.0,0.049056001007556915,0.07580800354480743,0.05948160067200661,0.05948160067200661,0.009031541471446955,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.02486399933695793,0.01923839971423149,0.018079999834299088,0.0032282528537266424,0.0,0.0,0.0,0.0,0.0,0.05142400041222572,0.07353600114583969,0.059328000620007516,0.059328000620007516,0.0073307807735143084,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016543999314308167,0.03977600112557411,0.021379199624061585,0.018511999398469925,0.006593576176246171,0.0,0.0,0.0,0.0,0.0,0.0488319993019104,0.06435199826955795,0.05479039996862411,0.05479039996862411,0.005672522998491864,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01635199971497059,0.02844800055027008,0.019267200119793416,0.017952000722289085,0.0035068687666949577,0.0,0.0,0.0,0.0,0.0,0.049855999648571014,0.07798399776220322,0.05986879989504815,0.05986879989504815,0.01043914754878828,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016383999958634377,0.026079999282956123,0.01923519968986511,0.017791999503970146,0.0032161974331284568,0.0,0.0,0.0,0.0,0.0,0.058111999183893204,0.1045759990811348,0.06708480007946492,0.06708480007946492,0.013479022462646494,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.04057599976658821,0.019100800156593323,0.015455999877303839,0.007512281243011577,0.0,0.0,0.0,0.0,0.0,0.05363199859857559,0.07782399654388428,0.06090559959411622,0.06090559959411622,0.007544620176348091,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.02191999927163124,0.016684799920767546,0.01563199982047081,0.0024293118621811216,0.0,0.0,0.0,0.0,0.0,0.0629120022058487,0.07891199737787247,0.06891520097851753,0.06891520097851753,0.005472695665695425,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.038943998515605927,0.018313600029796363,0.01561600062996149,0.007127270260115769,0.0,0.0,0.0,0.0,0.0,0.08675199747085571,0.10662399977445602,0.09391999915242194,0.09391999915242194,0.006988099589086635,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.054687999188899994,0.021439999900758268,0.01539199985563755,0.012052764849597775,0.0,0.0,0.0,0.0,0.0,0.13488000631332397,0.1528639942407608,0.1431359991431236,0.1431359991431236,0.005436271464033599,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.041919998824596405,0.01899839974939823,0.015343999955803156,0.007989843526623287,0.0,0.0,0.0,0.0,0.0,0.06176000088453293,0.08374399691820145,0.06747519969940186,0.06747519969940186,0.0066067747128778,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016256000846624374,0.11353600025177002,0.042761600017547606,0.028960000723600388,0.029104301538020762,0.0,0.0,0.0,0.0,0.0,0.09734400361776352,0.14422400295734406,0.11392960175871848,0.11392960175871848,0.013198594600417867,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.034143999218940735,0.018918400071561335,0.01643200032413006,0.005500943993080684,0.0,0.0,0.0,0.0,0.0,0.12918399274349213,0.15087999403476715,0.13807999789714814,0.13807999789714814,0.007658330538677587,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.016063999384641647,0.03641600161790848,0.0198208000510931,0.01780799962580204,0.0057264128169845765,0.0,0.0,0.0,0.0,0.0,0.22099199891090393,0.23904000222682953,0.2293503984808922,0.2293503984808922,0.004861342907006028,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.035840000957250595,0.018908800091594458,0.015792000107467175,0.0064374817924757475,0.0,0.0,0.0,0.0,0.0,0.10134399682283401,0.12201599776744843,0.10896319895982742,0.10896319895982742,0.006336330809165179,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.03417599946260452,0.017846399918198586,0.014800000004470348,0.006495007539635255,0.0,0.0,0.0,0.0,0.0,0.13126400113105774,0.15561600029468536,0.1389280006289482,0.1389280006289482,0.008381472811075022,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03519999980926514,0.019168000388890504,0.01600000075995922,0.005995522477654695,0.0,0.0,0.0,0.0,0.0,0.21728000044822693,0.2343679964542389,0.2231455981731415,0.2231455981731415,0.004720730646739123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.03670400008559227,0.018441599886864425,0.015023999847471714,0.006596535162793127,0.0,0.0,0.0,0.0,0.0,0.39321601390838623,0.4524799883365631,0.4058080047369003,0.4058080047369003,0.01578349755088941,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.022143999114632607,0.016912000067532063,0.015168000012636185,0.0028156089295136347,0.0,0.0,0.0,0.0,0.0,0.15587200224399567,0.3079040050506592,0.17838079929351805,0.17838079929351805,0.04355865575265927,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.02112000063061714,0.015516800060868263,0.01488000014796853,0.001940870731593904,0.0,0.0,0.0,0.0,0.0,0.21587200462818146,0.23561599850654602,0.22250880002975468,0.22250880002975468,0.006181951170646666,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.039264000952243805,0.023609600123018028,0.02131200022995472,0.007236979625711548,0.0,0.0,0.0,0.0,0.0,0.408735990524292,0.470335990190506,0.4336863994598388,0.4336863994598388,0.01844662383160074,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.025407999753952026,0.016227199975401164,0.014960000291466713,0.0031617375441736185,0.0,0.0,0.0,0.0,0.0,0.7412800192832947,0.7627840042114258,0.7464000046253203,0.7464000046253203,0.006112167448837547,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01462399959564209,0.02812799997627735,0.020652799773961304,0.021359999664127827,0.004308706957613102,0.028383498565450627,0.039859687970646644,0.032492258074592426,0.032492258074592426,0.00453597266208597,0.029312501176103633,0.04116431058787463,0.03355574193327539,0.03355574193327539,0.004684436757701648,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.034143999218940735,0.023171199765056372,0.024255999363958836,0.005908614918096971,0.03333159243114438,0.038935341782478095,0.03580428402241854,0.03580428402241854,0.002082270297044095,0.03633240903369937,0.04244065945634484,0.03902771507087562,0.03902771507087562,0.0022697354261490147,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.02611199952661991,0.019635199941694735,0.018112000077962875,0.0038298050749257795,0.04189529417991216,0.057484239920526384,0.04744885718421094,0.04744885718421094,0.004779830748455743,0.051672703037266184,0.07089975418195164,0.05852234134479412,0.05852234134479412,0.005895334539786378,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.029311999678611755,0.018662399891763926,0.01673599984496832,0.0044017112162725355,0.04322973959325901,0.05049827064705393,0.04535414343408448,0.04535414343408448,0.0022944156000240697,0.08733025617719538,0.10201372836398578,0.09162185574241775,0.09162185574241775,0.004635047631846023,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014592000283300877,0.026367999613285065,0.016336000058799982,0.01515199989080429,0.003415353455946402,0.06031842775160765,0.06618323188375198,0.06274415549817247,0.06274415549817247,0.001985647289644255,0.14768157653992678,0.16204076249052324,0.15362064543185072,0.15362064543185072,0.004861590945216247,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.02454400062561035,0.017167999967932702,0.016191999427974224,0.0028685016454498436,0.09128700688359712,0.09689150775996329,0.09360396051475776,0.09360396051475776,0.0013477542744916764,0.2740889887523892,0.2909164873209846,0.2810456356995366,0.2810456356995366,0.004046628526808561,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.01881599985063076,0.03097599931061268,0.024598400108516216,0.024848000146448612,0.0037539437391565975,0.1035249255866932,0.10603132147437412,0.10399797220840973,0.10399797220840973,0.0007025109429056814,0.5652750707893444,0.5789606938874117,0.5678580377517648,0.5678580377517648,0.003835906384194897,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.018624000251293182,0.043487999588251114,0.024460799992084503,0.019600000232458115,0.008922081629781394,0.196169204945307,0.2076140047945799,0.2008444429250931,0.2008444429250931,0.00394350953292805,1.1196708009293268,1.1849940417370974,1.1463555573610091,1.1463555573610091,0.022508285530529783,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.027135999873280525,0.03667199984192848,0.02945920005440712,0.028256000019609928,0.0029446678408169553,0.37757279619664613,0.3898113624476372,0.38075520430942184,0.38075520430942184,0.0036600203831042254,0.2522831942132049,0.2604606493092597,0.25440958703617433,0.25440958703617433,0.0024455194930253126,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.0435199998319149,0.049695998430252075,0.04502719938755036,0.04395199939608574,0.002035785660169308,1.1048984388245497,1.1189053886476108,1.1112371236754128,1.1112371236754128,0.0050220696724624985,0.1395495077239122,0.14131859607071193,0.14035008841031085,0.14035008841031085,0.0006342911944856293,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.018783999606966972,0.024288000538945198,0.020627199858427047,0.019567999988794327,0.0019457052717059358,0.09455999732017517,0.12185599654912949,0.10207359939813614,0.10207359939813614,0.00753346544014261,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,2,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512]",1024,512,512,512.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.027295999228954315,0.034752000123262405,0.02943360023200512,0.028528000228106976,0.0023159160681123767,0.14416000247001648,0.15904000401496887,0.14979200065135959,0.14979200065135959,0.00480624675866005,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,4,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512]",2048,512,512,512.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.043455999344587326,0.05084799975156784,0.045500800386071204,0.04411200061440468,0.002490556062831049,0.24454399943351746,0.25865599513053894,0.2504959970712662,0.2504959970712662,0.003778494140301353,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512]",4096,512,512,512.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.07673600316047668,0.08374399691820145,0.07970559895038605,0.07873599976301193,0.0022907976135004057,0.445248007774353,0.4758400022983551,0.45409599840641024,0.45409599840641024,0.009133230340383306,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512]",8192,512,512,512.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.043296001851558685,0.051072001457214355,0.044972800090909,0.04399999976158142,0.002451216824441962,0.6147199869155884,0.6290879845619202,0.6204927921295166,0.6204927921295166,0.004344846739788608,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,2,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048]",4096,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.07737600058317184,0.09123200178146362,0.08094720020890236,0.07980800047516823,0.004065264798368611,1.1744320392608643,1.1887680292129517,1.178323209285736,1.178323209285736,0.004395876469319665,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,4,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048, 2048, 2048]",8192,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.03254399821162224,0.01810879958793521,0.01611199975013733,0.005058478889747962,0.0,0.0,0.0,0.0,0.0,0.06339199841022491,0.0841279998421669,0.07019519805908202,0.07019519805908202,0.006070126182471763,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.030271999537944794,0.018124799989163876,0.015711999498307705,0.004696400814632773,0.0,0.0,0.0,0.0,0.0,0.2739199995994568,0.2922239899635315,0.28281279802322384,0.28281279802322384,0.00633037978599564,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.022431999444961548,0.016844799742102623,0.015471999999135733,0.0027624803153841917,0.0,0.0,0.0,0.0,0.0,0.3909119963645935,0.4160960018634796,0.39996159672737125,0.39996159672737125,0.007037174636804898,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014240000396966934,0.035071998834609985,0.018764800019562246,0.015583999920636415,0.006305594700523817,0.0,0.0,0.0,0.0,0.0,0.7383679747581482,0.7597119808197021,0.7430047929286957,0.7430047929286957,0.006192645960911466,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.028991999104619026,0.019401599932461978,0.01665600063279271,0.005434953793780546,0.0,0.0,0.0,0.0,0.0,1.427008032798767,1.4517120122909546,1.4361984014511109,1.4361984014511109,0.008802755821028187,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.02844800055027008,0.01820160010829568,0.014944000169634819,0.004942002036909087,0.0,0.0,0.0,0.0,0.0,0.08361600339412689,0.11027199774980545,0.09058240056037903,0.09058240056037903,0.00872938604423622,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.023104000836610794,0.01648960020393133,0.015632000286132097,0.0027628828340349578,0.0,0.0,0.0,0.0,0.0,0.5063040256500244,0.5497599840164185,0.5168287932872773,0.5168287932872773,0.013557229606313293,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01616000011563301,0.029055999591946602,0.018780799955129622,0.017215999774634838,0.003794434947431941,0.0,0.0,0.0,0.0,0.0,0.7439360022544861,0.7719680070877075,0.7502080142498017,0.7502080142498017,0.00839205598262567,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015968000516295433,0.03046399913728237,0.018822400271892546,0.01726400014013052,0.004212536831927695,0.0,0.0,0.0,0.0,0.0,1.4256000518798828,1.449504017829895,1.4325888037681578,1.4325888037681578,0.0065448028108711165,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.027712000533938408,0.01633920017629862,0.014944000169634819,0.0038488220071983326,0.0,0.0,0.0,0.0,0.0,2.798719882965088,3.0184640884399414,2.8293471813201903,2.8293471813201903,0.06375662767704417,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.021407999098300934,0.01637439979240298,0.014992000069469213,0.0024463594774459265,0.0,0.0,0.0,0.0,0.0,0.09347199648618698,0.10473600029945374,0.09820479974150656,0.09820479974150656,0.004061668684999473,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,1,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.04790399968624115,0.020179199893027543,0.016447999514639378,0.009635010283506967,0.0,0.0,0.0,0.0,0.0,0.6221439838409424,0.6444799900054932,0.6277984082698822,0.6277984082698822,0.007234140179397069,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,8,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.03276799991726875,0.01921279989182949,0.015232000034302473,0.0065506148511265076,0.0,0.0,0.0,0.0,0.0,0.9111359715461731,0.9307839870452881,0.9151648044586183,0.9151648044586183,0.005528712062927265,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,16,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.02707199938595295,0.016512000095099212,0.014688000082969666,0.0038989080209321414,0.0,0.0,0.0,0.0,0.0,1.7645119428634644,1.7849279642105103,1.7684095859527589,1.7684095859527589,0.005876963340184193,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,32,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014240000396966934,0.029184000566601753,0.016825600154697896,0.014752000104635954,0.00457910514148248,0.0,0.0,0.0,0.0,0.0,3.4781761169433594,3.5388801097869873,3.490892815589905,3.490892815589905,0.01818910600692522,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,64,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014976000413298607,0.03139200061559677,0.01898880014196038,0.016848000697791576,0.005083715524393418,0.13752702814163098,0.14117629917511842,0.13848196486571557,0.13848196486571557,0.0009880564964713527,0.523336967410432,0.5372236808930884,0.5269708253945994,0.5269708253945994,0.00375988994658546,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,520,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]",1,8,8,9,8,16384.0,0.1111111111111111,q512_8q1s16k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.01833599992096424,0.02707199938595295,0.020995199866592883,0.01976000051945448,0.0028392656352050185,0.17817885890237703,0.18622915300900206,0.18109068484526028,0.18109068484526028,0.0027040006080457624,0.5449571488834201,0.5695788427872015,0.5538629212834322,0.5538629212834322,0.008270141985514729,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1040,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]",1,16,16,17,16,16384.0,0.058823529411764705,q1k_16q1s16k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.02703999914228916,0.03308799862861633,0.028883199393749236,0.02759999968111515,0.002285758578932753,0.4688266550410815,0.48301665772804186,0.4727750380198454,0.4727750380198454,0.004396396389258256,1.0430453981052807,1.0746153117238624,1.051829759343436,1.051829759343436,0.009781101336187197,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]",1,32,32,33,32,16384.0,0.030303030303030304,q2k_32q1s16k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.04368000105023384,0.059328000992536545,0.04809600040316582,0.04617599956691265,0.005158792915002645,1.3509461459747514,1.3759066409666496,1.3589365122072,1.3589365122072,0.008008877716183384,0.9213738861449998,0.9383974724214122,0.9268234851606594,0.9268234851606594,0.005462224239661061,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768]",1,16,16,17,16,32768.0,0.058823529411764705,q4k_16q1s32k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.026944000273942947,0.040511999279260635,0.030291200056672095,0.02817599941045046,0.004386386489305873,0.5071830964059985,0.5142792136001758,0.5096392092471257,0.5096392092471257,0.0022059272685869546,2.175632932188972,2.2060727208328075,2.186168772243963,2.186168772243963,0.009462633998570079,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768]",1,32,32,33,32,32768.0,0.030303030303030304,q2k_32q1s32k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.018303999677300453,0.03017600066959858,0.02039040010422468,0.018943999893963337,0.0034612051983613614,0.21341429693945616,0.21552776300295579,0.21393496609766816,0.21393496609766816,0.0005736773262009222,4.617401495144284,4.663128147488988,4.628666619290515,4.628666619290515,0.012412001359412127,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,1,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768]",1,64,64,65,64,32768.0,0.015384615384615385,q1k_64q1s32k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.02364799939095974,0.01809599995613098,0.016352000646293163,0.0035481127058959038,0.04822399839758873,0.08566399663686752,0.05961279980838299,0.05961279980838299,0.011445665413968877,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.020864000543951988,0.01668160008266568,0.015664000064134598,0.0025769063833097584,0.049695998430252075,0.08057600259780884,0.05882879942655563,0.05882879942655563,0.009515126108519331,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.028704000636935234,0.017430400010198355,0.015056000091135502,0.004349294613335555,0.049855999648571014,0.07366400212049484,0.05459520071744919,0.05459520071744919,0.0069610920757925574,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.03161599859595299,0.017788799852132796,0.015711999963968992,0.005005385723318659,0.06102399900555611,0.08179199695587158,0.06650560013949873,0.06650560013949873,0.006947995595801105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.04416000097990036,0.019200000166893005,0.015488000120967627,0.008561241323364038,0.08054400235414505,0.09196799993515015,0.08607039973139763,0.08607039973139763,0.004145329035483754,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.017855999991297722,0.023391999304294586,0.019667199812829494,0.018463999964296818,0.0022577669687832585,0.18729600310325623,0.20585599541664124,0.19546559900045393,0.19546559900045393,0.006339068824303663,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.026240000501275063,0.03446400165557861,0.028297600522637367,0.027088000439107418,0.0025148441522922374,0.5754240155220032,0.5889919996261597,0.5800191938877105,0.5800191938877105,0.003858869273829596,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.04182400181889534,0.047200001776218414,0.043036799877882004,0.0423360001295805,0.0017073405772076728,2.063199996948242,2.0787200927734375,2.067151999473572,2.067151999473572,0.004959651271127835,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.05648000165820122,0.02270399993285537,0.017935999669134617,0.012377915150727689,0.049536000937223434,0.07196799665689468,0.056015999615192415,0.056015999615192415,0.0070476637552742884,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01408000010997057,0.021344000473618507,0.01611520005390048,0.014928000047802925,0.0025499884993961702,0.07072000205516815,0.2642880082130432,0.1588256008923054,0.1588256008923054,0.053220347086102376,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01360000018030405,0.03014400042593479,0.016975999902933837,0.015056000091135502,0.004715159697364111,0.08393599838018417,0.11036799848079681,0.09160000011324881,0.09160000011324881,0.007911669434472792,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.020767999812960625,0.016233599931001663,0.015392000321298838,0.002202615907995427,0.1998399943113327,0.22070400416851044,0.20855360180139543,0.20855360180139543,0.00689307200230667,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.033440001308918,0.018684800155460833,0.016048000194132328,0.00553991005639174,0.6043199896812439,0.635807991027832,0.6126143991947173,0.6126143991947173,0.008745933953408096,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013824000023305416,0.03359999880194664,0.017811199743300678,0.014944000169634819,0.005926140483565472,0.0,0.0,0.0,0.0,0.0,0.045471999794244766,0.07539200037717819,0.054758400097489356,0.054758400097489356,0.010253548506101549,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02409599907696247,0.01641279999166727,0.01473599998280406,0.003347158638713987,0.0,0.0,0.0,0.0,0.0,0.0461760014295578,0.07529599964618683,0.05460800044238568,0.05460800044238568,0.009748937798340135,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.028416000306606293,0.018611199874430894,0.01598400017246604,0.005209875093863647,0.0,0.0,0.0,0.0,0.0,0.048128001391887665,0.07897599786520004,0.061353600397706036,0.061353600397706036,0.010488153655157845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.025728000327944756,0.016092800162732603,0.01512000011280179,0.003300888123052605,0.0,0.0,0.0,0.0,0.0,0.04864000156521797,0.07648000121116638,0.05810560062527656,0.05810560062527656,0.009473544218404408,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.03551999852061272,0.018588799890130757,0.016064000315964222,0.006071057437992447,0.0,0.0,0.0,0.0,0.0,0.04822399839758873,0.07862400263547897,0.05660480037331582,0.05660480037331582,0.009565394213730401,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.028831999748945236,0.01699519995599985,0.015024000313133001,0.004285000744868188,0.0,0.0,0.0,0.0,0.0,0.04854400083422661,0.06719999760389328,0.05778240002691746,0.05778240002691746,0.006852125554679805,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.0144640002399683,0.024927999824285507,0.0161183999851346,0.01521599991247058,0.0029774808524673907,0.0,0.0,0.0,0.0,0.0,0.05379199981689453,0.08966399729251862,0.06270079985260964,0.06270079985260964,0.009983591277092696,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.03001599945127964,0.017648000083863736,0.01547200046479702,0.004585126309484848,0.0,0.0,0.0,0.0,0.0,0.061664000153541565,0.07692799717187881,0.06704320013523103,0.06704320013523103,0.005500191798490629,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.026367999613285065,0.016947199776768684,0.015039999969303608,0.0037951566103550205,0.0,0.0,0.0,0.0,0.0,0.08799999952316284,0.111455999314785,0.0964031994342804,0.0964031994342804,0.007397541615558088,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01369599997997284,0.021247999742627144,0.015359999984502793,0.014640000183135271,0.0020942770950814317,0.0,0.0,0.0,0.0,0.0,0.051711998879909515,0.07065600156784058,0.058054400235414506,0.058054400235414506,0.006633034815910025,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.0307839997112751,0.017148799914866685,0.015039999969303608,0.004882374686312284,0.0,0.0,0.0,0.0,0.0,0.061919998377561576,0.07843200117349625,0.06628479920327664,0.06628479920327664,0.004962852689801192,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013887999579310417,0.02969600073993206,0.017500799987465142,0.015232000034302473,0.00467273319705314,0.0,0.0,0.0,0.0,0.0,0.08819200098514557,0.11097600311040878,0.09493440166115762,0.09493440166115762,0.007509235042985577,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.022112000733613968,0.01751680001616478,0.016047999262809753,0.0030306621792915785,0.0,0.0,0.0,0.0,0.0,0.13065600395202637,0.15110400319099426,0.13857279866933822,0.13857279866933822,0.00750137841249771,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.04012800008058548,0.01892479993402958,0.01550400024279952,0.007459545267816961,0.0,0.0,0.0,0.0,0.0,0.06278400123119354,0.08259200304746628,0.0704512007534504,0.0704512007534504,0.005979055984382744,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.02236800082027912,0.016332800220698118,0.014928000047802925,0.002784044363186731,0.0,0.0,0.0,0.0,0.0,0.1003199964761734,0.1279360055923462,0.10921279862523078,0.10921279862523078,0.00862769617046716,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0225600004196167,0.016128000058233737,0.014800000004470348,0.002958953332547708,0.0,0.0,0.0,0.0,0.0,0.13116799294948578,0.14812800288200378,0.13783999979496003,0.13783999979496003,0.005696384361148053,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014208000153303146,0.029983999207615852,0.017468800116330386,0.01508800033479929,0.0047379550962483065,0.0,0.0,0.0,0.0,0.0,0.217631995677948,0.2447360008955002,0.22715839892625808,0.22715839892625808,0.008831828221847138,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014047999866306782,0.02304000034928322,0.016512000095099212,0.014512000139802694,0.0035026774828624254,0.0,0.0,0.0,0.0,0.0,0.11020799726247787,0.12307199835777283,0.11600959971547126,0.11600959971547126,0.004667637266950902,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014112000353634357,0.042080000042915344,0.017510399967432023,0.014607999939471483,0.00823117883530167,0.0,0.0,0.0,0.0,0.0,0.15702399611473083,0.17587199807167053,0.16399359852075576,0.16399359852075576,0.006588299074676393,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.013919999822974205,0.0208320003002882,0.015299199987202883,0.01462399959564209,0.001916768086505386,0.0,0.0,0.0,0.0,0.0,0.21779200434684753,0.2415360063314438,0.2260768011212349,0.2260768011212349,0.007251352080685236,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014015999622642994,0.021088000386953354,0.015619200188666582,0.01473599998280406,0.002013227452302141,0.0,0.0,0.0,0.0,0.0,0.3959999978542328,0.4152640104293823,0.40332479774951924,0.40332479774951924,0.006942401431914052,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014175999909639359,0.03379200026392937,0.020595200080424547,0.019600000232458115,0.006548754248881344,0.026192623739694512,0.040006882507168426,0.029155180178492036,0.029155180178492036,0.00408828748438241,0.024591375524545756,0.037561119537986146,0.027372820355088746,0.027372820355088746,0.0038383559348575944,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.020896000787615776,0.015318400040268899,0.014431999996304512,0.0019640312960926966,0.029349018208693862,0.06236262941356679,0.03714313592014963,0.03714313592014963,0.009618313791872569,0.028826981462526914,0.06125337308649041,0.036482463672774496,0.036482463672774496,0.009447230957011863,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.032127998769283295,0.017286399938166143,0.014479999896138906,0.005536495446636193,0.03273085874558354,0.04394578491937082,0.03563837490653034,0.03563837490653034,0.003429601730256567,0.03488514202593899,0.04683821345079978,0.037984025407085474,0.037984025407085474,0.0036553316361902684,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.013856000266969204,0.021503999829292297,0.015078400075435639,0.01425600005313754,0.002238435975478849,0.042100813549974185,0.051784144690147756,0.045378692890289625,0.045378692890289625,0.003184109940650555,0.05111518843748309,0.06287185663927425,0.05509490773570219,0.05509490773570219,0.0038658725544299132,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.02687999978661537,0.017667199857532977,0.01566399959847331,0.003864811846387173,0.048475323773821306,0.05599956978723733,0.05123499252968345,0.05123499252968345,0.002427379820423941,0.08429268135885387,0.09737642835214408,0.0890914090616175,0.0890914090616175,0.0042209177332077005,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.01462399959564209,0.03359999880194664,0.019804799742996693,0.016032000072300434,0.006750519908145474,0.07121508474579985,0.07292308367136396,0.07194410845270183,0.07194410845270183,0.0005500065915943844,0.14760091249712767,0.15114092373010238,0.14911189243564577,0.14911189243564577,0.0011399477384397005,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.021856000646948814,0.016835200227797033,0.015263999812304974,0.00283854448975065,0.08146338272142935,0.08560865714384096,0.0834932625520734,0.0834932625520734,0.0012259635905347874,0.2782486219401307,0.29240733788179385,0.2851819365989658,0.2851819365989658,0.004187435731481665,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.04495999962091446,0.02143679987639189,0.015856000129133463,0.009709987872683342,0.1194459208702178,0.12302524755181463,0.12053885718696096,0.12053885718696096,0.001005118119341339,0.5597220650459199,0.5764947443228802,0.5648435507167102,0.5648435507167102,0.004709970715400788,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.018112000077962875,0.026496000587940216,0.019318400137126445,0.018432000651955605,0.0024249605112359905,0.19770253574610402,0.222811797868348,0.2054811520619412,0.2054811520619412,0.008000944254868043,0.13941746080159492,0.157124211777114,0.14490284788179197,0.14490284788179197,0.005642170080515992,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.02595200017094612,0.03155200183391571,0.02727359998971224,0.026575999334454536,0.0016260948764843166,0.6014684881116659,0.6095472988212,0.6046757700946376,0.6046757700946376,0.0023537435563177303,0.11325152264578045,0.11477269562841545,0.11385542721485654,0.11385542721485654,0.0004431903698023628,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.03359999880194664,0.01814719969406724,0.015375999733805656,0.005678506740662529,0.06774400174617767,0.07891199737787247,0.07312640026211739,0.07312640026211739,0.003826583051422731,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,2,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512]",1024,512,512,512.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01788800023496151,0.028543999418616295,0.021340799890458582,0.019504000432789326,0.003718627037219618,0.09548799693584442,0.1327359974384308,0.10823359936475753,0.10823359936475753,0.013230635292043864,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,4,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512]",2048,512,512,512.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.02627200074493885,0.030368000268936157,0.027184000052511693,0.02643200010061264,0.0013545679205210022,0.14364799857139587,0.1597760021686554,0.15008639842271806,0.15008639842271806,0.005367723415171222,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512]",4096,512,512,512.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.04150399938225746,0.04726399853825569,0.042950399965047834,0.04224000126123428,0.001720147501765378,0.2433920055627823,0.28963199257850647,0.2549152016639709,0.2549152016639709,0.013450991019687407,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512]",8192,512,512,512.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.025887999683618546,0.03407999873161316,0.028303999826312064,0.026367999613285065,0.0028877495368841042,0.325439989566803,0.3441599905490875,0.33442879617214205,0.33442879617214205,0.005693597811441922,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,2,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048]",4096,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.04137599840760231,0.05084799975156784,0.044828799366950986,0.043087998405098915,0.003560363865797613,0.6110399961471558,0.6421759724617004,0.6204223990440368,0.6204223990440368,0.011214490370794758,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,4,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048, 2048, 2048]",8192,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014816000126302242,0.02364799939095974,0.016668799985200166,0.01592000015079975,0.0025515238179941247,0.0,0.0,0.0,0.0,0.0,0.0544000007212162,0.09014400094747543,0.06511679962277411,0.06511679962277411,0.013005726711295521,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.0208320003002882,0.016604799963533878,0.01566399959847331,0.0021353594266203244,0.0,0.0,0.0,0.0,0.0,0.17871999740600586,0.1961279958486557,0.18568639904260634,0.18568639904260634,0.004566142885274747,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014431999996304512,0.02006400004029274,0.015619200002402068,0.01508800033479929,0.001572984783715635,0.0,0.0,0.0,0.0,0.0,0.2730880081653595,0.29721599817276,0.2815328001976013,0.2815328001976013,0.007016534727616923,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.02755199931561947,0.018678399827331306,0.015919999685138464,0.004323555372382858,0.0,0.0,0.0,0.0,0.0,0.3893119990825653,0.44041600823402405,0.40225600004196166,0.40225600004196166,0.013442998165884852,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014303999952971935,0.02659199945628643,0.016748800035566093,0.015343999955803156,0.0036412341868394082,0.0,0.0,0.0,0.0,0.0,0.7404800057411194,0.9689919948577881,0.8451807916164398,0.8451807916164398,0.08637455920036795,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.02160000056028366,0.016752000153064727,0.01521599991247058,0.0025167650350367246,0.0,0.0,0.0,0.0,0.0,0.06412799656391144,0.08956799656152725,0.07242240011692047,0.07242240011692047,0.009797120588901621,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.022911999374628067,0.016137600038200618,0.015343999955803156,0.002330911555470513,0.0,0.0,0.0,0.0,0.0,0.3128319978713989,0.3282879889011383,0.3203647971153259,0.3203647971153259,0.004598568238907996,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.03574400022625923,0.01919359974563122,0.01583999954164028,0.006642521913489215,0.0,0.0,0.0,0.0,0.0,0.5050879716873169,0.5311999917030334,0.511932796239853,0.511932796239853,0.007150929647317677,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014655999839305878,0.02595200017094612,0.016992000117897987,0.015456000342965126,0.003390731937723684,0.0,0.0,0.0,0.0,0.0,0.7385600209236145,0.7681919932365417,0.7483008027076722,0.7483008027076722,0.010214373356208012,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.038975998759269714,0.01952639976516366,0.015327999833971262,0.007695927285621061,0.0,0.0,0.0,0.0,0.0,1.4228800535202026,1.5237760543823242,1.4435008168220522,1.4435008168220522,0.0342955291725742,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.03907199949026108,0.01793599994853139,0.015248000156134367,0.007167103363234667,0.0,0.0,0.0,0.0,0.0,0.06947200000286102,0.08857599645853043,0.07469440028071403,0.07469440028071403,0.00583249952929747,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,1,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.02175999991595745,0.016710399929434062,0.015327999833971262,0.002705383977079183,0.0,0.0,0.0,0.0,0.0,0.388480007648468,0.4073280096054077,0.3966591984033584,0.3966591984033584,0.004487315516095964,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,8,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.0144640002399683,0.022048000246286392,0.017011200170964004,0.015392000321298838,0.002776899649845722,0.0,0.0,0.0,0.0,0.0,0.622048020362854,0.6347839832305908,0.6262047946453094,0.6262047946453094,0.004593360122958751,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,16,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014592000283300877,0.02937600016593933,0.01847040019929409,0.015344000421464443,0.005043809142122341,0.0,0.0,0.0,0.0,0.0,0.9105280041694641,0.9304640293121338,0.914108806848526,0.914108806848526,0.00563919268816644,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,32,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01462399959564209,0.039903998374938965,0.01791359977796674,0.015440000221133232,0.007367547649297217,0.0,0.0,0.0,0.0,0.0,1.764799952507019,1.7965760231018066,1.770739197731018,1.770739197731018,0.009082122770320404,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,64,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.03187200054526329,0.018553599901497363,0.016127999871969223,0.0050628774268006264,0.16963527081512533,0.1737871320906266,0.17088926838108623,0.17088926838108623,0.0011703114258121128,0.47362872483230506,0.48522089403242025,0.47712993814280913,0.47712993814280913,0.0032675581298664976,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,520,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]",1,8,8,9,8,16384.0,0.1111111111111111,q512_8q1s16k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.03868800029158592,0.01921919994056225,0.016959999687969685,0.006691309533030663,0.1589600576212269,0.16117782913137854,0.15969057520605917,0.15969057520605917,0.0006212984448748182,0.5199519263456005,0.5272061673552852,0.5223414198520004,0.5223414198520004,0.002032242112153396,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1040,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]",1,16,16,17,16,16384.0,0.058823529411764705,q1k_16q1s16k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.018400000408291817,0.024320000782608986,0.019910399988293647,0.0191040001809597,0.001841532763984352,0.25650752966102763,0.27066608538463055,0.259736894547936,0.259736894547936,0.00462927315947332,0.5278764825612624,0.5570139063374764,0.5345223138928448,0.5345223138928448,0.009526755161797178,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]",1,32,32,33,32,16384.0,0.030303030303030304,q2k_32q1s16k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.026048000901937485,0.03728000074625015,0.028636799938976765,0.02705600019544363,0.003313953649011259,0.9270006318443208,0.9712965120641314,0.93480767601568,0.93480767601568,0.012612551450284608,0.8181833128578277,0.8572794566782395,0.8250739157811953,0.8250739157811953,0.01113200873299586,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768]",1,16,16,17,16,32768.0,0.058823529411764705,q4k_16q1s32k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.018079999834299088,0.037696000188589096,0.022092800214886667,0.019024000503122807,0.005986278786633071,0.2839857165542317,0.2876110048757805,0.2849506939696605,0.2849506939696605,0.0009995178425916847,1.0871823008331585,1.101060989333509,1.0908765231323903,1.0908765231323903,0.0038264533900426207,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768]",1,32,32,33,32,32768.0,0.030303030303030304,q2k_32q1s32k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.015072000212967396,0.025567999109625816,0.01726400014013052,0.015728000551462173,0.0031324465940990903,0.137270464802061,0.13799613818579368,0.13751499486424038,0.13751499486424038,0.00025221761530472904,2.3021855212208884,2.314355908780759,2.3062865750744197,2.3062865750744197,0.004229983070201486,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,2,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768]",1,64,64,65,64,32768.0,0.015384615384615385,q1k_64q1s32k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.022784000262618065,0.016435200069099664,0.015519999898970127,0.0023688301421469523,0.04879999905824661,0.09139200299978256,0.06054079942405224,0.06054079942405224,0.012152608702448775,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,False,0.0,64.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014816000126302242,0.02844800055027008,0.017008000146597625,0.015696000307798386,0.003941466294662679,0.047807998955249786,0.07356800138950348,0.05626560002565384,0.05626560002565384,0.00842179125412236,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,False,0.0,128.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.053408000618219376,0.019353600218892097,0.01532800029963255,0.01138302400841626,0.048448000103235245,0.0785600021481514,0.0556256003677845,0.0556256003677845,0.009348274502616908,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,False,0.0,256.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.03139200061559677,0.01823679991066456,0.016159999649971724,0.004860071238302512,0.055424001067876816,0.08505599945783615,0.0640383992344141,0.0640383992344141,0.00921448636178623,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,False,0.0,512.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.026528000831604004,0.017324799951165915,0.01536000007763505,0.0036004822686428305,0.07660800218582153,0.0942080020904541,0.08209280073642732,0.08209280073642732,0.00515895587669752,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.021247999742627144,0.016403199825435876,0.01532800029963255,0.001995897022647934,0.11395200341939926,0.15113599598407745,0.12431039959192276,0.12431039959192276,0.011164431123683732,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[2048],2048,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.017184000462293625,0.028672000393271446,0.019865600019693376,0.017823999747633934,0.0035798491315929977,0.3171840012073517,0.3341119885444641,0.3261695951223373,0.3261695951223373,0.005111046452878918,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[4096],4096,4096,4096,4096.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.024639999493956566,0.030400000512599945,0.02656640000641346,0.02556800004094839,0.0020189846603237303,1.0648640394210815,1.0828479528427124,1.0715327858924866,1.0715327858924866,0.005558639263049851,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,[8192],8192,8192,8192,8192.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014976000413298607,0.021695999428629875,0.017305599898099898,0.01593599934130907,0.0025718046517268054,0.04956800118088722,0.07932800054550171,0.06228480041027069,0.06228480041027069,0.01027160349757827,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,64,448.0,True,FLASH_ATTN,False,vllm020_batch_spec,[64],64,64,64,64.0,True,0.0,0.0,0.0,True,448.0,512.0,64,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q64s512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01539199985563755,0.03580800071358681,0.019686400331556796,0.017136000096797943,0.005918387811304003,0.05158400163054466,0.10713600367307663,0.06364160068333148,0.06364160068333148,0.015832957809696766,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,128,896.0,True,FLASH_ATTN,False,vllm020_batch_spec,[128],128,128,128,128.0,True,0.0,0.0,0.0,True,896.0,1024.0,128,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q128s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02223999984562397,0.016672000009566545,0.015887999907135963,0.0020934945946034563,0.06521599739789963,0.08902399986982346,0.07520959973335266,0.07520959973335266,0.007913840684102929,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,256,1792.0,True,FLASH_ATTN,False,vllm020_batch_spec,[256],256,256,256,256.0,True,0.0,0.0,0.0,True,1792.0,2048.0,256,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q256s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015231999568641186,0.04028800129890442,0.019971200078725816,0.01646399963647127,0.007351305886577286,0.14467200636863708,0.16844800114631653,0.15363519936800005,0.15363519936800005,0.008174435899956223,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,512,3584.0,True,FLASH_ATTN,False,vllm020_batch_spec,[512],512,512,512,512.0,True,0.0,0.0,0.0,True,3584.0,4096.0,512,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q512s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015519999898970127,0.02304000034928322,0.01775679988786578,0.016784000210464,0.0025077243712082584,0.33740800619125366,0.35343998670578003,0.3445120006799698,0.3445120006799698,0.00445648463590358,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,1024,7168.0,True,FLASH_ATTN,False,vllm020_batch_spec,[1024],1024,1024,1024,1024.0,True,0.0,0.0,0.0,True,7168.0,8192.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1ks8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.02393599972128868,0.01809599995613098,0.01654400024563074,0.002998393102466254,0.0,0.0,0.0,0.0,0.0,0.0504320003092289,0.0843840017914772,0.060083200410008426,0.060083200410008426,0.00986959318572296,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01603199914097786,0.030047999694943428,0.019510399922728537,0.017680000513792038,0.004065185265963332,0.0,0.0,0.0,0.0,0.0,0.05004800111055374,0.07036799937486649,0.059315200522542,0.059315200522542,0.006537768821329647,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.02393599972128868,0.018035199772566558,0.016512000001966953,0.0030667689116777724,0.0,0.0,0.0,0.0,0.0,0.05100800096988678,0.06735999882221222,0.058387200161814694,0.058387200161814694,0.005832787739241381,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.026208000257611275,0.017500799987465142,0.01646399963647127,0.0030666103306165714,0.0,0.0,0.0,0.0,0.0,0.05023999884724617,0.07254400104284286,0.057254400476813315,0.057254400476813315,0.006890853653068151,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.027327999472618103,0.0179776000790298,0.01648000068962574,0.003783417447531392,0.0,0.0,0.0,0.0,0.0,0.04819199815392494,0.07100799679756165,0.05621119923889638,0.05621119923889638,0.007824391484124725,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,128.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s128,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015647999942302704,0.036448001861572266,0.019136000238358975,0.016704000532627106,0.006076530095624803,0.0,0.0,0.0,0.0,0.0,0.047488000243902206,0.08441600203514099,0.0588383998721838,0.0588383998721838,0.011275940726332522,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015135999768972397,0.033952001482248306,0.02119360016658902,0.018000000156462193,0.007136646614305304,0.0,0.0,0.0,0.0,0.0,0.04931199923157692,0.06992000341415405,0.05755840018391609,0.05755840018391609,0.007297587694315226,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.034272000193595886,0.0204415999352932,0.017311999574303627,0.00695404701803013,0.0,0.0,0.0,0.0,0.0,0.05215999856591225,0.0735040009021759,0.06117440015077591,0.06117440015077591,0.007381118436894922,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03743999823927879,0.02012479966506362,0.01775999926030636,0.006181045509079057,0.0,0.0,0.0,0.0,0.0,0.06355199962854385,0.08508799970149994,0.07019200026988984,0.07019200026988984,0.0062246580442117845,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,1024.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s1k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015552000142633915,0.032607998698949814,0.01959999995306134,0.017487999983131886,0.004882475068460749,0.0,0.0,0.0,0.0,0.0,0.04918399825692177,0.0865280032157898,0.05973760038614274,0.05973760038614274,0.011546847942113974,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015168000012636185,0.02675200067460537,0.017430400010198355,0.016560000367462635,0.003243135094239819,0.0,0.0,0.0,0.0,0.0,0.052671998739242554,0.08367999643087387,0.06238719932734965,0.06238719932734965,0.011207824780630104,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.03612799942493439,0.019327999837696553,0.01688000001013279,0.006058251425153085,0.0,0.0,0.0,0.0,0.0,0.06364800035953522,0.07878399640321732,0.06935679838061332,0.06935679838061332,0.004515588328677643,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.03711999952793121,0.019670399930328132,0.017152000218629837,0.006165994646522264,0.0,0.0,0.0,0.0,0.0,0.09071999788284302,0.11507199704647064,0.10252480059862136,0.10252480059862136,0.008782544051535657,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,2048.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.026528000831604004,0.018396800104528665,0.01601599995046854,0.004279788048986283,0.0,0.0,0.0,0.0,0.0,0.05331199988722801,0.07977599650621414,0.06076480001211167,0.06076480001211167,0.008761579122069606,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.02643200010061264,0.01726400004699826,0.015855999663472176,0.003210008417242029,0.0,0.0,0.0,0.0,0.0,0.062431998550891876,0.08246400207281113,0.06970879957079888,0.06970879957079888,0.007016341676068233,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.02223999984562397,0.0166015999391675,0.015887999907135963,0.00204913969129354,0.0,0.0,0.0,0.0,0.0,0.10127999633550644,0.1141119971871376,0.10621120035648347,0.10621120035648347,0.004029318311754605,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.03363199904561043,0.019305599946528675,0.016671999357640743,0.0055445582307981234,0.0,0.0,0.0,0.0,0.0,0.1319040060043335,0.15839999914169312,0.14040640145540234,0.14040640145540234,0.007998041073596942,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,4096.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s4k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014368000440299511,0.04447999969124794,0.018908800091594458,0.0157279996201396,0.0086814937461721,0.0,0.0,0.0,0.0,0.0,0.06428799778223038,0.08982399851083755,0.07349760085344315,0.07349760085344315,0.008605284512197258,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.028672000393271446,0.01890560006722808,0.016080000437796116,0.004797584627815021,0.0,0.0,0.0,0.0,0.0,0.11123199760913849,0.14115199446678162,0.11942399889230729,0.11942399889230729,0.008513394706791027,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014271999709308147,0.024927999824285507,0.016915200091898442,0.015216000378131866,0.003374147499442635,0.0,0.0,0.0,0.0,0.0,0.15887999534606934,0.18111999332904816,0.16934399753808976,0.16934399753808976,0.007415181260761123,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014336000196635723,0.026944000273942947,0.017737600207328796,0.015199999790638685,0.00415598714375255,0.0,0.0,0.0,0.0,0.0,0.2192319929599762,0.23472000658512115,0.22809920012950893,0.22809920012950893,0.004730841376327335,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,8192.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s8k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.05052800104022026,0.020320000313222408,0.015520000364631414,0.010604522177924678,0.024270629882498958,0.03340247625954076,0.028446344104128624,0.028446344104128624,0.0032330369099793834,0.023697370291069768,0.03261352722995356,0.027774456325453972,0.027774456325453972,0.0031566742680923386,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,72,,,,,,,,False,,,64,BF16,generic,none,CUDA_EVENT,True,[64],[0],"[512, 512, 512, 512, 512, 512, 512, 512]",1,8,8,9,8,512.0,0.1111111111111111,q64_8q1s512,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.015359999611973763,0.033663999289274216,0.019987199828028678,0.018240000121295452,0.005522929139253732,0.0259194055660947,0.03719755183990719,0.029916029687899703,0.029916029687899703,0.003966666590554318,0.027104595853449518,0.03889844644729374,0.03128396954022015,0.03128396954022015,0.004148046317967881,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,136,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,8,8,9,8,1024.0,0.1111111111111111,q128_8q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014592000283300877,0.03299200162291527,0.019840000104159115,0.016207999549806118,0.006546343008726814,0.02587869595769926,0.03763167265431482,0.030174939058162802,0.030174939058162802,0.0037303581782277364,0.026473304070195713,0.03849632587654989,0.03086826083864964,0.03086826083864964,0.003816069654529222,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,144,,,,,,,,False,,,128,BF16,generic,none,CUDA_EVENT,True,[128],[0],"[1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024]",1,16,16,17,16,1024.0,0.058823529411764705,q128_16q1s1k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.013824000023305416,0.021023999899625778,0.016304000187665223,0.015584000386297703,0.0023236165806545476,0.031810621525966996,0.04156949936878106,0.0346749350032807,0.0346749350032807,0.003130209181143985,0.035677378270900374,0.04662250161636451,0.038889864871740294,0.038889864871740294,0.0035107033960828727,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,272,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,16,16,17,16,2048.0,0.058823529411764705,q256_16q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.02175999991595745,0.01569600012153387,0.015008000191301107,0.0020882666168036863,0.038211712107062853,0.07212229256520057,0.04364509673334097,0.04364509673334097,0.009671953074025085,0.0476442859917874,0.08992570455184198,0.05441890342616107,0.05441890342616107,0.01205947791784038,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,288,,,,,,,,False,,,256,BF16,generic,none,CUDA_EVENT,True,[256],[0],"[2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048, 2048]",1,32,32,33,32,2048.0,0.030303030303030304,q256_32q1s2k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014368000440299511,0.033824000507593155,0.017628800217062236,0.014864000026136637,0.005791495403857738,0.05214261250030033,0.06266261508706669,0.05677791295527661,0.05677791295527661,0.0030298223079651514,0.08648138506878382,0.10392938682791132,0.09416928531647478,0.09416928531647478,0.005025126612204489,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,544,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q512_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.031199999153614044,0.017900799959897996,0.015343999955803156,0.004974475661316249,0.06411958891421111,0.07405276123263956,0.06793047918211377,0.06793047918211377,0.0033918658556700006,0.14058441263169497,0.16236323591492058,0.1489399211274489,0.1489399211274489,0.00743678300375353,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,576,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,64,64,65,64,4096.0,0.015384615384615385,q512_64q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.03855999931693077,0.018495999928563833,0.015647999942302704,0.006918530056761627,0.09872985549401277,0.10683454583043753,0.10185316839884552,0.10185316839884552,0.0020802254037042074,0.2743261390166379,0.2968454508984596,0.2830044295482752,0.2830044295482752,0.005780016596065092,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192]",1,64,64,65,64,8192.0,0.015384615384615385,q1k_64q1s8k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.030400000512599945,0.01977920001372695,0.01726400014013052,0.005350883464206169,0.13918872472233365,0.16279522855335207,0.1501912864839173,0.1501912864839173,0.008992185529011513,0.11892328861766266,0.13909276049083735,0.1283239123428725,0.1283239123428725,0.007682951884956939,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,32,32,33,32,4096.0,0.030303030303030304,q2k_32q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.01727999933063984,0.02223999984562397,0.01819519978016615,0.017680000513792038,0.0014397298270620873,0.3728044181625443,0.38295504353701676,0.3761264483787333,0.3761264483787333,0.0033660272332490977,0.07967557017401883,0.08184495665371805,0.08038555277807365,0.08038555277807365,0.0007193856241088455,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096]",1,16,16,17,16,4096.0,0.058823529411764705,q4k_16q1s4k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.034591998904943466,0.01941439984366298,0.01756799966096878,0.005541380584373771,0.05951999872922897,0.08268799632787704,0.06715519949793816,0.06715519949793816,0.006657802116288364,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,2,1024,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512]",1024,512,512,512.0,True,0.0,0.0,0.0,False,0.0,1024.0,1024,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015104000456631184,0.03283200040459633,0.019180799927562477,0.016543999314308167,0.0052187911233635975,0.07199999690055847,0.08675199747085571,0.07749439924955369,0.07749439924955369,0.004849874741156772,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,4,2048,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512]",2048,512,512,512.0,True,0.0,0.0,0.0,False,0.0,2048.0,2048,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01724799908697605,0.02284800074994564,0.01928640007972717,0.018400000408291817,0.0020641390157565697,0.09676799923181534,0.12310399860143663,0.10618879944086074,0.10618879944086074,0.008982738207839057,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512]",4096,512,512,512.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.024224000051617622,0.03315199911594391,0.027436799928545953,0.027328000403940678,0.002679154874546328,0.14716799557209015,0.16412800550460815,0.15470399856567385,0.15470399856567385,0.005423454433987419,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512]",8192,512,512,512.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q512,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01833599992096424,0.04806400090456009,0.026252799853682517,0.022672000341117382,0.00878438440429033,0.1844799965620041,0.2072959989309311,0.19359359890222552,0.19359359890222552,0.007663539820967659,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,2,4096,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048]",4096,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,4096.0,4096,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,2q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.024480000138282776,0.04979199916124344,0.029081599973142146,0.025679999962449074,0.007227422044689197,0.34147199988365173,0.37968000769615173,0.3583200007677078,0.3583200007677078,0.011804422100726231,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,4,8192,0.0,True,FLASH_ATTN,False,vllm020_batch_spec,"[2048, 2048, 2048, 2048]",8192,2048,2048,2048.0,True,0.0,0.0,0.0,False,0.0,8192.0,8192,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,4q2k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014720000326633453,0.023744000121951103,0.017753600236028434,0.016368000768125057,0.003061040281616705,0.0,0.0,0.0,0.0,0.0,0.05686400085687637,0.1090880036354065,0.06629760004580021,0.06629760004580021,0.014803727026812366,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.021088000386953354,0.01647359998896718,0.015840000472962856,0.0017964402433206,0.0,0.0,0.0,0.0,0.0,0.08540800213813782,0.09961599856615067,0.09146559983491898,0.09146559983491898,0.00447423706371901,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014560000039637089,0.04156799986958504,0.018512000143527985,0.015584000386297703,0.007849701681877904,0.0,0.0,0.0,0.0,0.0,0.17948800325393677,0.194815993309021,0.1866239994764328,0.1866239994764328,0.004156328241374654,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.029632000252604485,0.019449600111693145,0.016736000776290894,0.005126811425136928,0.0,0.0,0.0,0.0,0.0,0.27529600262641907,0.2898879945278168,0.2825664013624191,0.2825664013624191,0.004542801609674093,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014399999752640724,0.031328000128269196,0.019759999960660933,0.016367999836802483,0.006266303740589241,0.0,0.0,0.0,0.0,0.0,0.39190399646759033,0.4079039990901947,0.3987520009279252,0.3987520009279252,0.0037367727509362725,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,16384.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s16k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01539199985563755,0.040031999349594116,0.0212032001465559,0.01649599988013506,0.008068547381446682,0.0,0.0,0.0,0.0,0.0,0.06588800251483917,0.07897599786520004,0.0703904002904892,0.0703904002904892,0.004214826869769361,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014495999552309513,0.022495999932289124,0.016006399970501663,0.01515199989080429,0.002265581884342885,0.0,0.0,0.0,0.0,0.0,0.1231679990887642,0.13526399433612823,0.1288223996758461,0.1288223996758461,0.004328976434897094,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014592000283300877,0.023231999948620796,0.01705600004643202,0.016048000194132328,0.002810904312746391,0.0,0.0,0.0,0.0,0.0,0.3158079981803894,0.33129599690437317,0.32348800003528594,0.32348800003528594,0.00417599076136664,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014527999795973301,0.021888000890612602,0.01708160014823079,0.01609600055962801,0.0025289234621475062,0.0,0.0,0.0,0.0,0.0,0.5103679895401001,0.5200319886207581,0.5132320046424866,0.5132320046424866,0.0035759433157749533,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014688000082969666,0.032416000962257385,0.019353600032627583,0.016176000237464905,0.0061458798960684425,0.0,0.0,0.0,0.0,0.0,0.7400320172309875,0.7516480088233948,0.7449311971664427,0.7449311971664427,0.004573461776168607,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,32768.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s32k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.0144640002399683,0.02006400004029274,0.01573119992390275,0.014992000069469213,0.001654446341262991,0.0,0.0,0.0,0.0,0.0,0.07097599655389786,0.09932799637317657,0.07828159928321837,0.07828159928321837,0.009464602895232642,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,1,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,[1],1,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.02319999970495701,0.016883199848234654,0.015775999519973993,0.0026693906156048403,0.0,0.0,0.0,0.0,0.0,0.14176000654697418,0.1598079949617386,0.15063679963350293,0.15063679963350293,0.005741662969679433,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,8,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1]",8,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,8q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.01484800036996603,0.03203200176358223,0.022224000189453363,0.02247999981045723,0.005996176183124981,0.0,0.0,0.0,0.0,0.0,0.39180800318717957,0.41046398878097534,0.4003200054168701,0.4003200054168701,0.005552433764307601,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,16,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",16,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,16q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.015072000212967396,0.028192000463604927,0.017152000125497578,0.015536000020802021,0.003847486121602065,0.0,0.0,0.0,0.0,0.0,0.6239359974861145,0.6367359757423401,0.6285343945026398,0.6285343945026398,0.004006084286606002,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,32,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",32,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,32q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014879999682307243,0.026688000187277794,0.01718079997226596,0.015488000120967627,0.0037132536813398722,0.0,0.0,0.0,0.0,0.0,0.9141119718551636,0.9721279740333557,0.927455997467041,0.927455997467041,0.02148767779232235,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,64,0,40960.0,False,FLASH_ATTN,False,vllm020_batch_spec,"[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]",64,1,1,1.0,True,0.0,0.0,0.0,False,0,0,0,BF16,generic,none,CUDA_EVENT,False,,,,,,,,,,,64q1s40k,measured_FA3_core_plus_measured_KV;reshape_assumed_zero;mean_as_median +0.0,0.0,0.0,0.0,0.0,0.014751999638974667,0.047520000487565994,0.028710400220006704,0.0266720000654459,0.010774688401598903,0.06094816381288764,0.06816969726785131,0.0644060660218149,0.0644060660218149,0.002203106589073183,0.087051838094461,0.09736630405680231,0.09199073574791852,0.09199073574791852,0.0031466818046499644,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,9,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,520,,,,,,,,False,,,512,BF16,generic,none,CUDA_EVENT,True,[512],[0],"[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]",1,8,8,9,8,16384.0,0.1111111111111111,q512_8q1s16k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.014911999925971031,0.02489599958062172,0.017427200078964235,0.01595200039446354,0.003071906489592143,0.19733789497223914,0.2016295616652644,0.19860388994013833,0.19860388994013833,0.001512389485439305,0.44861409134062713,0.45837046456077934,0.45149211526120153,0.45149211526120153,0.0034381598874302305,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1040,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]",1,16,16,17,16,16384.0,0.058823529411764705,q1k_16q1s16k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.015039999969303608,0.03046399913728237,0.020643199887126686,0.016944000497460365,0.006320148675595729,0.2157924314537054,0.22215709640166995,0.21769400765743155,0.21769400765743155,0.0019679921844645526,0.4905115822753901,0.5049789194903589,0.4948340005651485,0.4948340005651485,0.0044733865493072344,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]",1,32,32,33,32,16384.0,0.030303030303030304,q2k_32q1s16k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.017152000218629837,0.02425600029528141,0.01865920014679432,0.018112000077962875,0.0019454553198986453,0.7430866512973927,0.7578834738720781,0.7455351005236347,0.7455351005236347,0.00416692487556653,0.7369773831645824,0.7516525540362471,0.7394057025273603,0.7394057025273603,0.004132666607961175,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,17,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,4112,,,,,,,,False,,,4096,BF16,generic,none,CUDA_EVENT,True,[4096],[0],"[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768]",1,16,16,17,16,32768.0,0.058823529411764705,q4k_16q1s32k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.015263999812304974,0.02470399998128414,0.018009600043296815,0.01657600048929453,0.0031544532774534346,0.25251173919752334,0.25566890792461106,0.2536729054481981,0.2536729054481981,0.0010279562245342983,1.0425282722084215,1.0555630628624373,1.0473223013846877,1.0473223013846877,0.004244053880724073,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,33,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,2080,,,,,,,,False,,,2048,BF16,generic,none,CUDA_EVENT,True,[2048],[0],"[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768]",1,32,32,33,32,32768.0,0.030303030303030304,q2k_32q1s32k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio +0.0,0.0,0.0,0.0,0.0,0.01500799972563982,0.027807999402284622,0.01912960009649396,0.01643200032413006,0.004688164695934997,0.1251379565220268,0.14341821167748953,0.12742497577885914,0.12742497577885914,0.005353812167343766,1.1355340167064276,1.3014137556763312,1.1562870179154463,1.1562870179154463,0.048581869194943325,0.0,0.0,0.0,0.0,0.0,2048,32,4,16,4,40960,65,0,0,True,FLASH_ATTN,False,true_mixed_fused_projected,,1088,,,,,,,,False,,,1024,BF16,generic,none,CUDA_EVENT,True,[1024],[0],"[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768]",1,64,64,65,64,32768.0,0.015384615384615385,q1k_64q1s32k,fused_total_conserving_projection_by_same_tp_pure_prefill_decode_reference_ratio diff --git a/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/attention_true_mixed_fused.csv b/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/attention_true_mixed_fused.csv new file mode 100644 index 0000000..1bcebdc --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/attention_true_mixed_fused.csv @@ -0,0 +1,49 @@ +num_tensor_parallel_workers,batch_spec,num_prefill_seqs,num_decode_seqs,total_prefill_tokens,total_decode_tokens,decode_avg_kv_cache_size,attention_core_mean_ms,attention_core_mean_as_median_ms,kv_cache_update_median_ms,pure_prefill_reference_mean_ms,pure_decode_reference_mean_ms,projected_prefill_mean_ms,projected_decode_mean_ms,projection_sum_error_ms,representation +1,q64_8q1s512,1,8,64,8,512.0,0.06604800000786781,0.06604800000786781,0.021359999664127827,0.05810240097343922,0.0600041144660541,0.032492258074592426,0.03355574193327539,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q128_8q1s1k,1,8,128,8,1024.0,0.07483199909329416,0.07483199909329416,0.024255999363958836,0.05587520003318787,0.06090559959411622,0.03580428402241854,0.03902771507087562,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q128_16q1s1k,1,16,128,16,1024.0,0.10597119852900506,0.10597119852900506,0.018112000077962875,0.05587520003318787,0.06891520097851753,0.04744885718421094,0.05852234134479412,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q256_16q1s2k,1,16,256,16,2048.0,0.13697599917650224,0.13697599917650224,0.01673599984496832,0.056396800279617305,0.11392960175871848,0.04535414343408448,0.09162185574241775,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q256_32q1s2k,1,32,256,32,2048.0,0.2163648009300232,0.2163648009300232,0.01515199989080429,0.056396800279617305,0.13807999789714814,0.06274415549817247,0.15362064543185072,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q512_32q1s4k,1,32,512,32,4096.0,0.3746495962142944,0.3746495962142944,0.016191999427974224,0.07432000041007995,0.2231455981731415,0.09360396051475776,0.2810456356995366,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q512_64q1s4k,1,64,512,64,4096.0,0.6718560099601746,0.6718560099601746,0.024848000146448612,0.07432000041007995,0.4058080047369003,0.10399797220840973,0.5678580377517648,-1.1102230246251565e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q1k_64q1s8k,1,64,1024,64,8192.0,1.3472000002861022,1.3472000002861022,0.019600000232458115,0.1307712011039257,0.7464000046253203,0.2008444429250931,1.1463555573610091,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q2k_32q1s4k,1,32,2048,32,4096.0,0.6351647913455962,0.6351647913455962,0.028256000019609928,0.33396480381488797,0.2231455981731415,0.38075520430942184,0.25440958703617433,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q4k_16q1s4k,1,16,4096,16,4096.0,1.2515872120857237,1.2515872120857237,0.04395199939608574,1.0999775886535645,0.1389280006289482,1.1112371236754128,0.14035008841031085,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q512_8q1s16k,1,8,512,8,16384.0,0.6654527902603149,0.6654527902603149,0.016848000697791576,0.07432000041007995,0.28281279802322384,0.13848196486571557,0.5269708253945994,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q1k_16q1s16k,1,16,1024,16,16384.0,0.7349536061286925,0.7349536061286925,0.01976000051945448,0.1307712011039257,0.39996159672737125,0.18109068484526028,0.5538629212834322,-1.1102230246251565e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q2k_32q1s16k,1,32,2048,32,16384.0,1.5246047973632815,1.5246047973632815,0.02759999968111515,0.33396480381488797,0.7430047929286957,0.4727750380198454,1.051829759343436,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q4k_16q1s32k,1,16,4096,16,32768.0,2.285759997367859,2.285759997367859,0.04617599956691265,1.0999775886535645,0.7502080142498017,1.3589365122072,0.9268234851606594,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q2k_32q1s32k,1,32,2048,32,32768.0,2.6958079814910887,2.6958079814910887,0.02817599941045046,0.33396480381488797,1.4325888037681578,0.5096392092471257,2.186168772243963,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +1,q1k_64q1s32k,1,64,1024,64,32768.0,4.842601585388183,4.842601585388183,0.018943999893963337,0.1307712011039257,2.8293471813201903,0.21393496609766816,4.628666619290515,-8.881784197001252e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q64_8q1s512,1,8,64,8,512.0,0.05652800053358078,0.05652800053358078,0.019600000232458115,0.05961279980838299,0.055968457407185014,0.029155180178492036,0.027372820355088746,6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q128_8q1s1k,1,8,128,8,1024.0,0.07362559959292413,0.07362559959292413,0.014431999996304512,0.05882879942655563,0.05778240002691746,0.03714313592014963,0.036482463672774496,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q128_16q1s1k,1,16,128,16,1024.0,0.0736224003136158,0.0736224003136158,0.014479999896138906,0.05882879942655563,0.06270079985260964,0.03563837490653034,0.037984025407085474,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q256_16q1s2k,1,16,256,16,2048.0,0.10047360062599181,0.10047360062599181,0.01425600005313754,0.05459520071744919,0.06628479920327664,0.045378692890289625,0.05509490773570219,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q256_32q1s2k,1,32,256,32,2048.0,0.14032640159130094,0.14032640159130094,0.01566399959847331,0.05459520071744919,0.09493440166115762,0.05123499252968345,0.0890914090616175,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q512_32q1s4k,1,32,512,32,4096.0,0.22105600088834762,0.22105600088834762,0.016032000072300434,0.06650560013949873,0.13783999979496003,0.07194410845270183,0.14911189243564577,-2.7755575615628914e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q512_64q1s4k,1,64,512,64,4096.0,0.36867519915103913,0.36867519915103913,0.015263999812304974,0.06650560013949873,0.22715839892625808,0.0834932625520734,0.2851819365989658,5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q1k_64q1s8k,1,64,1024,64,8192.0,0.6853824079036711,0.6853824079036711,0.015856000129133463,0.08607039973139763,0.40332479774951924,0.12053885718696096,0.5648435507167102,1.1102230246251565e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q2k_32q1s4k,1,32,2048,32,4096.0,0.3503839999437332,0.3503839999437332,0.018432000651955605,0.19546559900045393,0.13783999979496003,0.2054811520619412,0.14490284788179197,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q4k_16q1s4k,1,16,4096,16,4096.0,0.7185311973094941,0.7185311973094941,0.026575999334454536,0.5800191938877105,0.10921279862523078,0.6046757700946376,0.11385542721485654,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q512_8q1s16k,1,8,512,8,16384.0,0.6480192065238953,0.6480192065238953,0.016127999871969223,0.06650560013949873,0.18568639904260634,0.17088926838108623,0.47712993814280913,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q1k_16q1s16k,1,16,1024,16,16384.0,0.6820319950580597,0.6820319950580597,0.016959999687969685,0.08607039973139763,0.2815328001976013,0.15969057520605917,0.5223414198520004,-1.1102230246251565e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q2k_32q1s16k,1,32,2048,32,16384.0,0.7942592084407808,0.7942592084407808,0.0191040001809597,0.19546559900045393,0.40225600004196166,0.259736894547936,0.5345223138928448,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q4k_16q1s32k,1,16,4096,16,32768.0,1.759881591796875,1.759881591796875,0.02705600019544363,0.5800191938877105,0.511932796239853,0.93480767601568,0.8250739157811953,2.220446049250313e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q2k_32q1s32k,1,32,2048,32,32768.0,1.3758272171020507,1.3758272171020507,0.019024000503122807,0.19546559900045393,0.7483008027076722,0.2849506939696605,1.0908765231323903,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +2,q1k_64q1s32k,1,64,1024,64,32768.0,2.44380156993866,2.44380156993866,0.015728000551462173,0.08607039973139763,1.4435008168220522,0.13751499486424038,2.3062865750744197,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q64_8q1s512,1,8,64,8,512.0,0.0562208004295826,0.0562208004295826,0.015520000364631414,0.06054079942405224,0.0591108573866742,0.028446344104128624,0.027774456325453972,-6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q128_8q1s1k,1,8,128,8,1024.0,0.061199999228119854,0.061199999228119854,0.018240000121295452,0.05626560002565384,0.0588383998721838,0.029916029687899703,0.03128396954022015,-6.938893903907228e-18,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q128_16q1s1k,1,16,128,16,1024.0,0.06104319989681244,0.06104319989681244,0.016207999549806118,0.05626560002565384,0.05755840018391609,0.030174939058162802,0.03086826083864964,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q256_16q1s2k,1,16,256,16,2048.0,0.07356479987502099,0.07356479987502099,0.015584000386297703,0.0556256003677845,0.06238719932734965,0.0346749350032807,0.038889864871740294,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q256_32q1s2k,1,32,256,32,2048.0,0.09806400015950203,0.09806400015950203,0.015008000191301107,0.0556256003677845,0.06935679838061332,0.04364509673334097,0.05441890342616107,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q512_32q1s4k,1,32,512,32,4096.0,0.1509471982717514,0.1509471982717514,0.014864000026136637,0.0640383992344141,0.10621120035648347,0.05677791295527661,0.09416928531647478,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q512_64q1s4k,1,64,512,64,4096.0,0.21687040030956264,0.21687040030956264,0.015343999955803156,0.0640383992344141,0.14040640145540234,0.06793047918211377,0.1489399211274489,2.7755575615628914e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q1k_64q1s8k,1,64,1024,64,8192.0,0.3848575979471207,0.3848575979471207,0.015647999942302704,0.08209280073642732,0.22809920012950893,0.10185316839884552,0.2830044295482752,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q2k_32q1s4k,1,32,2048,32,4096.0,0.2785151988267898,0.2785151988267898,0.01726400014013052,0.12431039959192276,0.10621120035648347,0.1501912864839173,0.1283239123428725,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q4k_16q1s4k,1,16,4096,16,4096.0,0.456512001156807,0.456512001156807,0.017680000513792038,0.3261695951223373,0.06970879957079888,0.3761264483787333,0.08038555277807365,-5.551115123125783e-17,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q512_8q1s16k,1,8,512,8,16384.0,0.15639680176973342,0.15639680176973342,0.0266720000654459,0.0640383992344141,0.09146559983491898,0.0644060660218149,0.09199073574791852,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q1k_16q1s16k,1,16,1024,16,16384.0,0.6500960052013398,0.6500960052013398,0.01595200039446354,0.08209280073642732,0.1866239994764328,0.19860388994013833,0.45149211526120153,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q2k_32q1s16k,1,32,2048,32,16384.0,0.71252800822258,0.71252800822258,0.016944000497460365,0.12431039959192276,0.2825664013624191,0.21769400765743155,0.4948340005651485,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q4k_16q1s32k,1,16,4096,16,32768.0,1.484940803050995,1.484940803050995,0.018112000077962875,0.3261695951223373,0.32348800003528594,0.7455351005236347,0.7394057025273603,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q2k_32q1s32k,1,32,2048,32,32768.0,1.3009952068328858,1.3009952068328858,0.01657600048929453,0.12431039959192276,0.5132320046424866,0.2536729054481981,1.0473223013846877,0.0,one_fused_FA3_call_projected_for_Frontier_with_total_conservation +4,q1k_64q1s32k,1,64,1024,64,32768.0,1.2837119936943056,1.2837119936943056,0.01643200032413006,0.08209280073642732,0.7449311971664427,0.12742497577885914,1.1562870179154463,-2.220446049250313e-16,one_fused_FA3_call_projected_for_Frontier_with_total_conservation diff --git a/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/linear_op.csv b/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/linear_op.csv new file mode 100644 index 0000000..2803f02 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/linear_op.csv @@ -0,0 +1,37 @@ +time_stats.emb.min,time_stats.emb.max,time_stats.emb.mean,time_stats.emb.median,time_stats.emb.std,time_stats.input_layernorm.min,time_stats.input_layernorm.max,time_stats.input_layernorm.mean,time_stats.input_layernorm.median,time_stats.input_layernorm.std,time_stats.attn_pre_proj.min,time_stats.attn_pre_proj.max,time_stats.attn_pre_proj.mean,time_stats.attn_pre_proj.median,time_stats.attn_pre_proj.std,time_stats.attn_rope.min,time_stats.attn_rope.max,time_stats.attn_rope.mean,time_stats.attn_rope.median,time_stats.attn_rope.std,time_stats.attn_post_proj.min,time_stats.attn_post_proj.max,time_stats.attn_post_proj.mean,time_stats.attn_post_proj.median,time_stats.attn_post_proj.std,time_stats.post_attention_layernorm.min,time_stats.post_attention_layernorm.max,time_stats.post_attention_layernorm.mean,time_stats.post_attention_layernorm.median,time_stats.post_attention_layernorm.std,n_head,n_kv_head,n_embd,n_expanded_embd,vocab_size,use_gated_mlp,use_qk_norm,attn_output_gate,num_tokens,num_tensor_parallel_workers,padded_n_embd,padded_n_expanded_embd,model_arch,is_step2_mini,share_expert_dim,share_q_dim,measurement_type,profiling_precision,quant_signature +0.029184000566601753,0.06780800223350525,0.03157280012965202,0.030736000277101994,0.005874173435341216,0.033215999603271484,0.04825599864125252,0.03443359974771738,0.0337119996547699,0.0031825678429048183,1.438431978225708,1.505568027496338,1.446228802204132,1.4429279565811157,0.014128607642643025,0.538752019405365,0.5440319776535034,0.5416463971138,0.5420799851417542,0.0016099306164432847,1.0073280334472656,1.0163840055465698,1.0098415970802308,1.0081279873847961,0.0032980457197329728,0.04022400081157684,0.041728001087903976,0.04091359991580248,0.04081599973142147,0.0004728505416767937,32,4,2048,768,151936,True,True,False,8192,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.01360000018030405,0.06784000247716904,0.017755200061947106,0.0179840000346303,0.00837681421401443,0.01836800016462803,0.03651199862360954,0.019606399815529585,0.018719999119639397,0.0038821958848767424,0.7512000203132629,0.8208960294723511,0.7566704005002975,0.75382399559021,0.014793092586577971,0.28995200991630554,0.29337599873542786,0.29135999977588656,0.29150401055812836,0.000998381071258815,0.5149760246276855,0.5169600248336792,0.5160208016633987,0.5158880054950714,0.0005038652783291977,0.02051199972629547,0.021503999829292297,0.021067200042307378,0.021104000508785248,0.00023542855306902367,32,4,2048,768,151936,True,True,False,4096,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.0080960001796484,0.04281599819660187,0.011092799971811474,0.011039999779313803,0.005489135752949302,0.012223999947309494,0.026335999369621277,0.013187199970707298,0.01247999956831336,0.0030236510562153375,0.3928639888763428,0.45372799038887024,0.3976895987987518,0.39528000354766846,0.012905176716136006,0.1547199934720993,0.15884800255298615,0.15712319910526276,0.1573439985513687,0.001165121945135037,0.26633599400520325,0.268095999956131,0.26719200164079665,0.2671840041875839,0.0004242740199415328,0.013024000450968742,0.013887999579310417,0.013489600038155913,0.013520000036805868,0.0002382817554057738,32,4,2048,768,151936,True,True,False,2048,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.00825599953532219,0.04755200073122978,0.02398160002194345,0.01961600035429001,0.00874683840888523,0.018880000337958336,0.03868800029158592,0.021590400114655496,0.0208320003002882,0.004057015751746236,0.24316799640655518,0.2710399925708771,0.2521967992186546,0.25065599381923676,0.007998418528894075,0.09644799679517746,0.19120000302791595,0.10407840013504029,0.09963199868798256,0.020043722414992166,0.13600000739097595,0.18892799317836761,0.15760480016469955,0.15760000050067902,0.0112223677907762,0.009664000011980534,0.010015999898314476,0.009836799977347255,0.009824000298976898,9.016971952948177e-05,32,4,2048,768,151936,True,True,False,1024,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.017376000061631203,0.044704001396894455,0.026318399980664254,0.027312000282108784,0.007565344034680866,0.01836800016462803,0.03014400042593479,0.021276800055056812,0.020655999891459942,0.002632977855097951,0.1438719928264618,0.17132799327373505,0.152497598528862,0.15012799948453903,0.007947150319625347,0.1430719941854477,0.19305600225925446,0.16630879789590836,0.1685439944267273,0.014628024163894684,0.08899199962615967,0.10467199981212616,0.0943599995225668,0.09374399855732918,0.003472669494074113,0.007615999784320593,0.007935999892652035,0.007769599952735007,0.0077760000713169575,7.680004540222077e-05,32,4,2048,768,151936,True,True,False,512,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.016992000862956047,0.0544000007212162,0.02573199989274144,0.026016000658273697,0.00803323401720434,0.018432000651955605,0.02348800003528595,0.020648000109940768,0.02062400057911873,0.0013939985047930988,0.10220800340175629,0.1361600011587143,0.11850560046732425,0.11684799939393997,0.010637754898360304,0.16710400581359863,0.21110400557518005,0.19078560024499894,0.19409599900245667,0.013714352646558832,0.06265600025653839,0.0740479975938797,0.06842879951000214,0.0690080001950264,0.0031292240446560557,0.00979200005531311,0.033440001308918,0.01864320016466081,0.017280000261962414,0.006957998188876383,32,4,2048,768,151936,True,True,False,256,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.017152000218629837,0.04396799951791763,0.025907999789342284,0.02598400041460991,0.007714554737915267,0.018400000408291817,0.03667199984192848,0.024132800102233887,0.02112000063061714,0.006120348733354326,0.10678400099277496,0.1363839954137802,0.1193264003843069,0.11583999916911125,0.009838647443214228,0.17017599940299988,0.22748799622058868,0.18853759989142418,0.18433599919080734,0.015728078443174653,0.04569600149989128,0.06652799993753433,0.05192639995366335,0.05151999928057194,0.004516605694976449,0.021856000646948814,0.026335999369621277,0.02384479995816946,0.023599999956786633,0.0011301153013314744,32,4,2048,768,151936,True,True,False,128,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.017343999817967415,1.0683200359344482,0.05748240072280168,0.029504000209271908,0.16366824814254827,0.018688000738620758,0.3317759931087494,0.03685439983382821,0.021151999942958355,0.06767422466731164,0.10255999863147736,0.9434880018234253,0.16412640027701855,0.11956800147891045,0.17964606281728834,0.1714559942483902,2.1306240558624268,0.3011296011507511,0.1926399990916252,0.42310127734378766,0.03574400022625923,0.6859520077705383,0.08389280084520578,0.04279999993741512,0.14321647071615612,0.020479999482631683,0.1831360012292862,0.033024000097066165,0.023856000043451786,0.03470987082429836,32,4,2048,768,151936,True,True,False,64,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.016095999628305435,0.05142400041222572,0.026363200135529043,0.02676799986511469,0.008637725852473854,0.01849599927663803,0.03577600046992302,0.022193600237369538,0.020848000422120094,0.004615266181181815,0.10540799796581268,0.15014399588108063,0.12211520001292228,0.11896000057458878,0.012772013396624768,0.17315199971199036,0.21478399634361267,0.1881632000207901,0.1873439997434616,0.011761164657572015,0.03481600061058998,0.058079998940229416,0.042200000025331974,0.03969600051641464,0.006461281521769989,0.02143999934196472,0.038816001266241074,0.02466559996828437,0.023856000043451786,0.0035418033748569927,32,4,2048,768,151936,True,True,False,32,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.015904000028967857,0.03753599897027016,0.023127999808639287,0.02527999971061945,0.0054230027890305385,0.018400000408291817,0.03001599945127964,0.020488000102341176,0.020096000283956528,0.002585688394137697,0.10355199873447418,0.1438400000333786,0.11536479964852334,0.11124800145626068,0.011136028294919255,0.16502399742603302,0.2072959989309311,0.18646399974822997,0.19075199961662292,0.013296437761247597,0.03142400085926056,0.05215999856591225,0.03888959977775812,0.03750399872660637,0.0057399302067536314,0.020128000527620316,0.04064000025391579,0.023937600292265417,0.023648000322282314,0.004144197400898248,32,4,2048,768,151936,True,True,False,16,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.015200000256299973,0.037151999771595,0.02144480012357235,0.02195199951529503,0.005576364091933697,0.017952000722289085,0.0226879995316267,0.019934400077909233,0.019952000118792057,0.0011812499470458758,0.10063999891281128,0.13468800485134125,0.11595199964940547,0.1207519993185997,0.011974301621092394,0.16332800686359406,0.20748800039291382,0.18162400051951408,0.1767839938402176,0.01474120743720334,0.03222399950027466,0.043455999344587326,0.03829439990222454,0.03859200142323971,0.0031378131240041122,0.020959999412298203,0.03587200120091438,0.023795200139284135,0.023423999547958374,0.0031339489695198443,32,4,2048,768,151936,True,True,False,8,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +0.014944000169634819,0.04022400081157684,0.02162720002233982,0.022463999688625336,0.006017219317026815,0.018464000895619392,0.026528000831604004,0.021264000236988066,0.020896000787615776,0.001934095017586082,0.10156799852848053,0.1703999936580658,0.12565439902245998,0.1244799979031086,0.01641776722785409,0.1653759926557541,0.23865599930286407,0.1969360001385212,0.19223999977111816,0.02058903050274278,0.03254399821162224,0.06752000004053116,0.0443536002188921,0.041519999504089355,0.00984829071098989,0.020096000283956528,0.040031999349594116,0.026934400014579297,0.02478400059044361,0.00562071702648593,32,4,2048,768,151936,True,True,False,1,1,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.7531200051307678,0.8461440205574036,0.7618160009384155,0.7576479911804199,0.019464233617982506,0.2922559976577759,0.2985599935054779,0.2953856036067009,0.29576000571250916,0.001575901077689139,0.510047972202301,0.5140479803085327,0.5121696025133133,0.5123839974403381,0.0012263137313476844,,,,,,32,4,2048,768,151936,True,True,False,8192,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.39529600739479065,0.47279998660087585,0.40216960161924364,0.39825600385665894,0.0162749796240819,0.15625600516796112,0.16211199760437012,0.15959519892930984,0.15988799929618835,0.0011493418942396922,0.2635200023651123,0.2642880082130432,0.2639120012521744,0.26392000913619995,0.0001903593401384135,,,,,,32,4,2048,768,151936,True,True,False,4096,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.2531839907169342,0.32419198751449585,0.26924319565296173,0.26049599051475525,0.01657194262368116,0.10051199793815613,0.2375359982252121,0.12411200068891048,0.10311999917030334,0.039466165428540506,0.15881599485874176,0.19289599359035492,0.16896959990262986,0.1640480011701584,0.009162416086418127,,,,,,32,4,2048,768,151936,True,True,False,2048,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.15302400290966034,0.1780800074338913,0.16284480094909667,0.16113600134849548,0.00740355661356144,0.14521600306034088,0.20233599841594696,0.17807039842009545,0.1796799972653389,0.013908448560094403,0.0907519981265068,0.09750399738550186,0.09460479989647866,0.09478399902582169,0.0017965487667361475,,,,,,32,4,2048,768,151936,True,True,False,1024,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11740799993276596,0.17958399653434753,0.13262080028653145,0.12878400087356567,0.014146060532423056,0.17468799650669098,0.21161599457263947,0.1910431995987892,0.18966399878263474,0.01270903647700971,0.05926400050520897,0.07577600330114365,0.06530559975653887,0.06404799968004227,0.00479458462514625,,,,,,32,4,2048,768,151936,True,True,False,512,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11260800063610077,0.17209599912166595,0.13610880002379416,0.13809599727392197,0.017675922458993677,0.18729600310325623,0.26822400093078613,0.20815680101513861,0.2078079953789711,0.01700718593658771,0.04499199986457825,0.06537599861621857,0.0507551996037364,0.04787199944257736,0.005727123232692158,,,,,,32,4,2048,768,151936,True,True,False,256,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11257600039243698,0.23452800512313843,0.13866880126297473,0.13964799791574478,0.026590047697062115,0.1828799992799759,0.24751999974250793,0.20735519900918006,0.20670399814844131,0.01500438131586336,0.03654399886727333,0.05593600124120712,0.0408239996060729,0.03892800025641918,0.004864409450710354,,,,,,32,4,2048,768,151936,True,True,False,128,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11734399944543839,0.18726399540901184,0.13890240006148816,0.13308800011873245,0.019086167340006257,0.16991999745368958,0.2443840056657791,0.19185120090842248,0.19257599860429764,0.018017536159219673,0.03033600002527237,0.04956800118088722,0.038387199863791466,0.03750400058925152,0.00500897018702887,,,,,,32,4,2048,768,151936,True,True,False,64,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11420799791812897,0.18115200102329254,0.13938880078494548,0.1393439993262291,0.018141313962922536,0.1693439930677414,0.221343994140625,0.19187839925289155,0.19438399374485016,0.01430752121272052,0.03017600066959858,0.06019200012087822,0.03866560012102127,0.0364960003644228,0.0074199790031205266,,,,,,32,4,2048,768,151936,True,True,False,32,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11276800185441971,0.16223999857902527,0.1331360016018152,0.13305599987506866,0.01432493740801146,0.17187200486660004,0.23625600337982178,0.19287680014967917,0.1913280040025711,0.01754628831589704,0.029823999851942062,0.07574400305747986,0.03900959976017475,0.036927999928593636,0.009277465083962879,,,,,,32,4,2048,768,151936,True,True,False,16,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.1130559965968132,0.19814400374889374,0.13379519879817964,0.12531199678778648,0.020074427302248836,0.16841599345207214,0.21139200031757355,0.1893615983426571,0.19092799723148346,0.012418720676762711,0.029503999277949333,0.07558400183916092,0.040144000016152856,0.03728000074625015,0.010223239196498205,,,,,,32,4,2048,768,151936,True,True,False,8,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.115167997777462,0.2640640139579773,0.13805920109152794,0.13118399679660797,0.03132849683515479,0.17103999853134155,0.2977280020713806,0.19899839907884598,0.1976960003376007,0.028928046763067948,0.0297279991209507,0.05990400165319443,0.03834720011800528,0.0363520011305809,0.0070885330713495905,,,,,,32,4,2048,768,151936,True,True,False,1,2,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.3940800130367279,0.46483200788497925,0.39923040121793746,0.3957759886980057,0.015108903906233569,0.16022400557994843,0.1634880006313324,0.16203359961509706,0.16228799521923065,0.0010220720912414007,0.26073598861694336,0.2627840042114258,0.26138080209493636,0.2613760083913803,0.00041640363494172775,,,,,,32,4,2048,768,151936,True,True,False,8192,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.2536959946155548,0.27161601185798645,0.25960480123758317,0.2577280104160309,0.005142017404245015,0.09849599748849869,0.10281600058078766,0.10057279989123344,0.10063999891281128,0.0008836232237268523,0.13913600146770477,0.17606399953365326,0.15875840038061143,0.15988799929618835,0.010058952112389172,,,,,,32,4,2048,768,151936,True,True,False,4096,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.15142400562763214,0.1780479997396469,0.16139679849147798,0.1602879986166954,0.0070656055731385115,0.14601600170135498,0.23343999683856964,0.17913119941949845,0.179967999458313,0.023065274309176566,0.09388799965381622,0.12108799815177917,0.10317599996924401,0.10073599964380264,0.007364345618470178,,,,,,32,4,2048,768,151936,True,True,False,2048,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.12198399752378464,0.18406400084495544,0.13770400024950505,0.13232000172138214,0.01623164885687898,0.1773120015859604,0.20688000321388245,0.1870912007987499,0.18535999953746796,0.00825628058448234,0.05766399949789047,0.06739199906587601,0.06187200043350458,0.061216000467538834,0.003004312467037863,,,,,,32,4,2048,768,151936,True,True,False,1024,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11737599968910217,0.18614399433135986,0.13109439946711063,0.12771200388669968,0.01372168725934724,0.17587199807167053,0.2699519991874695,0.19926720038056372,0.19075199961662292,0.02327478982490538,0.041728001087903976,0.06224000081419945,0.05000480003654957,0.049375999718904495,0.006185850944273961,,,,,,32,4,2048,768,151936,True,True,False,512,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.1159679964184761,0.1610880047082901,0.12988320142030715,0.12494400516152382,0.011865375783184065,0.1767680048942566,0.27379199862480164,0.21035519987344742,0.21275199949741364,0.023206964017550125,0.037087999284267426,0.062144000083208084,0.04439679980278015,0.04283200018107891,0.006155226392511492,,,,,,32,4,2048,768,151936,True,True,False,256,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.12015999853610992,0.158720001578331,0.13573280088603495,0.1343199983239174,0.010666830836014414,0.1737920045852661,0.22127999365329742,0.20261440128087999,0.20321600139141083,0.011153965849067301,0.03587200120091438,0.04944000020623207,0.038265600241720675,0.037328001111745834,0.0030431580113575636,,,,,,32,4,2048,768,151936,True,True,False,128,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.1212799996137619,0.15139199793338776,0.13183839991688728,0.13014400005340576,0.00851207547022104,0.17257599532604218,0.2250880002975464,0.1872655987739563,0.18193599581718445,0.013638284975248818,0.02969600073993206,0.0525440014898777,0.03840640028938651,0.03444799967110157,0.007598511754254174,,,,,,32,4,2048,768,151936,True,True,False,64,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11270400136709213,0.1624639928340912,0.13508000001311302,0.13180799782276154,0.014243564451606775,0.17052799463272095,0.23715199530124664,0.1953311987221241,0.1966560035943985,0.01500670718978398,0.030239999294281006,0.05270399898290634,0.03807039987295866,0.03742399998009205,0.004906165602021684,,,,,,32,4,2048,768,151936,True,True,False,32,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.12319999933242798,0.17919999361038208,0.14059039913117885,0.14156799763441086,0.016217662982107223,0.1701119989156723,0.21987199783325195,0.18903039917349815,0.1870879977941513,0.014117854507841239,0.030368000268936157,0.06406400352716446,0.04045119984075427,0.03710399940609932,0.00878942205983628,,,,,,32,4,2048,768,151936,True,True,False,16,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11296000331640244,0.16761599481105804,0.13843199908733367,0.13814399391412735,0.014925286935582404,0.1711679995059967,0.21561600267887115,0.1906527981162071,0.1876479983329773,0.011803992622137974,0.0306560005992651,0.09216000139713287,0.041129599791020155,0.036847999319434166,0.013739721468154687,,,,,,32,4,2048,768,151936,True,True,False,8,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none +,,,,,,,,,,0.11929599940776825,0.2008959949016571,0.14535359852015972,0.13971199840307236,0.024769473061576282,0.16803200542926788,0.25123199820518494,0.19573760256171227,0.19366399943828583,0.023196011081705884,0.03049599938094616,0.08089599758386612,0.04739360017701984,0.0453919991850853,0.01286389846889463,,,,,,32,4,2048,768,151936,True,True,False,1,4,2048,768,generic,False,,,CUDA_EVENT,BF16,none diff --git a/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/manifest.json b/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/manifest.json new file mode 100644 index 0000000..a4f773a --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/manifest.json @@ -0,0 +1,129 @@ +{ + "attention_long_context_coverage": { + "by_tp": { + "1": { + "decode_kv_lengths": [ + 128, + 1024, + 2048, + 4096, + 8192, + 16384, + 32768, + 40960 + ], + "true_mixed_decode_avg_kv_lengths": [ + 512, + 1024, + 2048, + 4096, + 8192, + 16384, + 32768 + ] + }, + "2": { + "decode_kv_lengths": [ + 128, + 1024, + 2048, + 4096, + 8192, + 16384, + 32768, + 40960 + ], + "true_mixed_decode_avg_kv_lengths": [ + 512, + 1024, + 2048, + 4096, + 8192, + 16384, + 32768 + ] + }, + "4": { + "decode_kv_lengths": [ + 128, + 1024, + 2048, + 4096, + 8192, + 16384, + 32768, + 40960 + ], + "true_mixed_decode_avg_kv_lengths": [ + 512, + 1024, + 2048, + 4096, + 8192, + 16384, + 32768 + ] + } + }, + "included": true + }, + "attention_tp_coverage": [ + 1, + 2, + 4 + ], + "environment_contract": { + "dtype": "bfloat16", + "frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "hardware": "NVIDIA H20", + "model": "Qwen3-30B-A3B", + "tensor_parallel_sizes": [ + 1, + 2, + 4 + ], + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + }, + "inputs": { + "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-composition-tp1-20260717-v1-20260717T014550833983Z/artifacts/artifacts/attention-composition-tp1-v1/raw/flashattn-composition-tp1.json": "ce461be92a5059e2d168b2cbf21b55d9a55ee71c3b0586a3e63cd8288938168d", + "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-composition-tp2-20260717-v1-20260717T014551831545Z/artifacts/artifacts/attention-composition-tp2-v1/raw/flashattn-composition-tp2.json": "997d67fb3a28b488c3aa4945ae7c6d4ecf8a08726ee3cb4652d16c221ff395ad", + "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-composition-tp4-20260717-v1-20260717T014552897206Z/artifacts/artifacts/attention-composition-tp4-v1/raw/flashattn-composition-tp4.json": "046cffd4c11d8a36513167ec25d3e1ca016324715679ea688a58d4a71dfe62b2", + "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-long-context-tp1-20260717-v1-20260717T022700381447Z/artifacts/artifacts/attention-long-context-tp1-v1/raw/flashattn-long-context-tp1.json": "da3b5adaa0d06f6a0ab7394d3c85d8d0524f0bd571613785f1a6183f70eb457f", + "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-long-context-tp2-20260717-v1-20260717T022701357659Z/artifacts/artifacts/attention-long-context-tp2-v1/raw/flashattn-long-context-tp2.json": "d503acb959f91b1713ff02d3251c781c240c4b057208033f4fd4a637af76e2cb", + "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-long-context-tp4-20260717-v1-20260717T022702416725Z/artifacts/artifacts/attention-long-context-tp4-v1/raw/flashattn-long-context-tp4.json": "a24acd44ab4075a3ac0aad00b91e676b325b2427dbe14282260cae6769481cf7", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931" + }, + "outputs": { + "allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3", + "attention.csv": "4f0c42d2c47f52880d81b8d2b2d7d52bd62d2c34e0f3a7f94c4c6cbd64d751c8", + "attention_true_mixed_fused.csv": "c84c129d466aec97b42f1255f3b7363fe7f73948dc27ed65b6ce599d6fb9af6d", + "linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "moe.csv": "dd698113975e5ed77f51dfd8a69aebf764ed651c3f6f34b71914d25d6e001415" + }, + "profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-fused-mixed-total-conserving-pure-prefill-batch-composition-long-context-decode-mixed", + "projection_contract": { + "allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity", + "attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target; true mixed rows use a total-conserving compatibility projection", + "attention_long_context": "Direct FA3 measurements at decode KV lengths 16K/32K/40K, plus true-mixed prefill/decode measurements at 16K/32K, for TP1/2/4", + "attention_pure_prefill_batch_composition": "Direct FA3 measurements for 2/4 requests at query length 2048 and 2/4/8/16 requests at query length 512 for each TP; included only when batch-composition attention inputs are supplied", + "attention_true_mixed": "The directly measured fused total is preserved in diagnostics. Frontier's two targets are projected by the same-TP pure prefill/decode reference ratio, with projected prefill + decode exactly equal to the fused total; the split is a schema compatibility attribution, not an observation", + "linear": "Frontier profiler using vLLM 0.20 CUDA operators", + "moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero" + }, + "row_counts": { + "allreduce": 24, + "attention_frontier_compatible": 213, + "attention_true_mixed_fused_diagnostic": 48, + "linear": 36, + "moe": 72 + }, + "schema_version": "frontier_qwen30_vllm020_frozen_profile.v4" +} diff --git a/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/moe.csv b/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/moe.csv new file mode 100644 index 0000000..311097f --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final/moe.csv @@ -0,0 +1,73 @@ +time_stats.moe_gating_linear.min,time_stats.moe_gating_linear.max,time_stats.moe_gating_linear.mean,time_stats.moe_gating_linear.median,time_stats.moe_gating_linear.std,time_stats.moe_gating_routing_topk.min,time_stats.moe_gating_routing_topk.max,time_stats.moe_gating_routing_topk.mean,time_stats.moe_gating_routing_topk.median,time_stats.moe_gating_routing_topk.std,time_stats.moe_shuffling.min,time_stats.moe_shuffling.max,time_stats.moe_shuffling.mean,time_stats.moe_shuffling.median,time_stats.moe_shuffling.std,time_stats.moe_grouped_gemm.min,time_stats.moe_grouped_gemm.max,time_stats.moe_grouped_gemm.mean,time_stats.moe_grouped_gemm.median,time_stats.moe_grouped_gemm.std,num_tokens,num_experts,num_experts_per_device,expert_parallel_size,routing_runtime_path,routing_assignment_policy,routing_weight_policy,routing_uses_router_logits,gating_runtime_context,gating_runtime_context_impl,router_topk,hidden_dim,expert_hidden_dim,use_gated,num_tensor_parallel_workers,total_routed_tokens,model_expansion_ratio,tokens_per_expert_avg,tokens_to_experts_ratio,expert_utilization,min_load_ratio,load_imbalance_cv,max_load_ratio,load_entropy,load_gini_coefficient,load_distribution,seed,moe_grouped_gemm_backend,measurement_type,profiling_precision,model_arch,quant_signature,router_median_nonadditivity_ratio,projection_policy +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.33926400542259216,0.405023992061615,0.36780479848384856,0.36507199704647064,0.01690507644474779,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,1.1943039894104004,1.286784052848816,1.228384006023407,1.2273280024528503,0.02832547242381263,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.3984375,0.0,1.346291201783626,4.0,5.59375,0.661865234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,1.679744005203247,1.766144037246704,1.7095808148384095,1.7015680074691772,0.02438921262998535,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.625,0.0,1.015504800579495,5.0,6.15516433212955,0.529052734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,2.1576640605926514,2.2921600341796875,2.2097824096679686,2.188944101333618,0.045572321842012986,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.875,0.0,0.6343057228182637,2.5,6.64370748444639,0.35369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,2.422368049621582,2.5130879878997803,2.4516672134399413,2.434159994125366,0.03278900287381846,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.984375,0.0,0.4921254921257382,2.25,6.817190042344769,0.272369384765625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,2.2217600345611572,2.289599895477295,2.2571327924728393,2.263375997543335,0.021660416089449488,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,1.0,0.125,0.3486861500690843,1.875,6.908192310183997,0.197662353515625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,2.18668794631958,2.318079948425293,2.2218016147613526,2.211087942123413,0.035380897213135316,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,1.0,0.4375,0.2525504668006971,1.875,6.953347743053017,0.1410369873046875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,2.2291839122772217,2.3929600715637207,2.2908096313476562,2.2804640531539917,0.04479348924786221,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,1.0,0.65625,0.15765965680164504,1.5625,6.98229848728205,0.08779525756835938,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,2.0678720474243164,2.1297600269317627,2.0837119817733765,2.0779199600219727,0.017880044357986735,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,1.0,0.625,0.12169081635504074,1.3125,6.9892029662356325,0.06879425048828125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,2.916032075881958,3.0819520950317383,2.9805248022079467,2.9656319618225098,0.05482195799019572,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,1.0,0.796875,0.07935434147688751,1.1796875,6.9954297964750305,0.044734954833984375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,4.421599864959717,4.535359859466553,4.486294317245483,4.497056007385254,0.036990243787549344,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,1.0,0.8203125,0.060849326483103046,1.17578125,6.9973188375859685,0.033740997314453125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,7.302591800689697,7.402751922607422,7.354758310317993,7.3464319705963135,0.032142662400335566,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,1.0,0.890625,0.0412323087266341,1.08984375,6.998772433185578,0.02334284782409668,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.35280001163482666,0.39692801237106323,0.37662720382213594,0.37196800112724304,0.013401318050665304,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.4692479968070984,0.5523840188980103,0.5134752035140991,0.5100640058517456,0.02291433464135784,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.34652799367904663,0.4119040071964264,0.3789471983909607,0.38550400733947754,0.02073945105803335,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.31462401151657104,0.7456960082054138,0.38617280423641204,0.34545600414276123,0.12230201266253077,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.32521599531173706,0.419871985912323,0.36325119733810424,0.34968000650405884,0.03161798848223672,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.289792001247406,0.4663360118865967,0.4091839998960495,0.41655999422073364,0.0446001986506615,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.3761279881000519,0.4416320025920868,0.40686399936676027,0.40540799498558044,0.02257778769899645,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.7172480225563049,0.8663039803504944,0.7723807990550995,0.7591840028762817,0.04164772379451242,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.0195519924163818,1.2216639518737793,1.1253888130187988,1.1453600525856018,0.06548005322243594,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7490559816360474,1.9644800424575806,1.8529024004936219,1.814303994178772,0.08042565617327288,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,3.2479360103607178,3.385279893875122,3.296070408821106,3.2800960540771484,0.04525529026442046,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,6.344799995422363,6.517856121063232,6.464438438415527,6.478623867034912,0.05116674443145098,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,1,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.2648000121116638,0.325439989566803,0.28852800130844114,0.28390398621559143,0.01933778377635077,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.7347840070724487,0.862496018409729,0.7769344031810761,0.769216001033783,0.03290485328796285,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.421875,0.0,1.346291201783626,6.0,5.652114648336087,0.636962890625,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.9198399782180786,0.9646080136299133,0.9412063956260681,0.9411839842796326,0.014939365085478117,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.5703125,0.0,1.118033988749895,5.0,6.008641773518898,0.580810546875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,1.2796800136566162,1.3484159708023071,1.3006976008415223,1.2929120063781738,0.020807998177176254,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.8828125,0.0,0.6959705453537527,3.0,6.60872850615583,0.38055419921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,1.3630399703979492,1.4430400133132935,1.3909215927124023,1.3892319798469543,0.022335744492366926,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.984375,0.0,0.5201036555341637,3.0,6.798826509158851,0.28302001953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,1.27948796749115,1.3904000520706177,1.3176063895225525,1.309440016746521,0.038060887827312775,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,1.0,0.25,0.3511282039725661,1.875,6.91002266305238,0.1970977783203125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,1.264415979385376,1.3145920038223267,1.2791999936103822,1.2753440141677856,0.014130605249568332,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,1.0,0.375,0.24692938483248605,1.6875,6.955481130775285,0.13909912109375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,1.3081920146942139,1.347648024559021,1.3292255997657776,1.329967975616455,0.014558863679016933,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,1.0,0.625,0.17143053326165383,1.5625,6.9786675275754035,0.09520339965820312,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,1.242751955986023,1.3112000226974487,1.2747935891151427,1.266207993030548,0.021093073517695057,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,1.0,0.78125,0.11000099875256815,1.296875,6.991308871213679,0.062183380126953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.7388160228729248,1.8077759742736816,1.772764801979065,1.772704005241394,0.021056644077284283,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,1.0,0.78125,0.0864630150197678,1.1796875,6.994552526394139,0.048796653747558594,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,2.6563520431518555,2.7063679695129395,2.6785055875778196,2.6791679859161377,0.01639963463052944,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,1.0,0.8671875,0.06127686514721937,1.16015625,6.997291583027146,0.03497934341430664,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,4.386879920959473,4.452256202697754,4.4108480453491214,4.406303882598877,0.019768161791937636,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,1.0,0.884765625,0.041723768525324195,1.1171875,6.998746434318934,0.02298593521118164,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.24208000302314758,0.4028480052947998,0.3041536003351212,0.277103990316391,0.05660721484881584,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.2447039932012558,0.30502399802207947,0.26446720361709597,0.26265600323677063,0.016744548364435372,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.2337920069694519,0.2881599962711334,0.26074880361557007,0.264384001493454,0.016469850143940968,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.23369599878787994,0.28591999411582947,0.25465920120477675,0.25385600328445435,0.01593204652619194,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.2295999974012375,0.26556798815727234,0.24674240052700042,0.2497600018978119,0.010345732066199003,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.21721599996089935,0.29020801186561584,0.2394208014011383,0.2346400022506714,0.018747330357768585,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2717759907245636,0.305184006690979,0.28813759982585907,0.28809599578380585,0.01183422600545854,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.3917759954929352,0.43772798776626587,0.41130879521369934,0.4131519943475723,0.012992473640805227,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.6176639795303345,0.7009919881820679,0.642767995595932,0.6330719888210297,0.024074084919938756,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.0820800065994263,1.1674879789352417,1.1034304022789,1.0977439880371094,0.0233067292981566,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.9809919595718384,2.0415360927581787,2.003715181350708,1.992751955986023,0.022066076434645737,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,3.790112018585205,3.8651199340820312,3.829139161109924,3.8230879306793213,0.025177464160110564,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,2,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.212351992726326,0.24383999407291412,0.22760000079870224,0.22723200172185898,0.01050568575837594,1,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.47494399547576904,0.5184000134468079,0.4920704007148743,0.49169600009918213,0.011991064701471855,8,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.3984375,0.0,1.3919410907075054,6.0,5.570159765557392,0.667236328125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.6360960006713867,0.7004479765892029,0.6608384013175964,0.6572319865226746,0.020416877242438597,16,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.625,0.0,1.0307764064044151,4.0,6.138251855282827,0.5382080078125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.780896008014679,0.8301439881324768,0.80346559882164,0.8030399978160858,0.016801230312128875,32,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.859375,0.0,0.6903350635742038,3.0,6.5943747091218174,0.38067626953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.8607040047645569,0.9195200204849243,0.8783008038997651,0.8751039803028107,0.01719059253115595,64,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.9765625,0.0,0.49410588440130926,2.75,6.814452474347134,0.271270751953125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.833952009677887,0.894752025604248,0.8619967997074127,0.863215982913971,0.018716378851797198,128,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,1.0,0.25,0.33693529145074724,2.125,6.9186075263155535,0.1867218017578125,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.834879994392395,0.8871039748191833,0.8651552021503448,0.8638879954814911,0.015262430894400969,256,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,1.0,0.375,0.25567294018677456,1.8125,6.952441049154937,0.14349365234375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.8518080115318298,0.9097599983215332,0.8810272097587586,0.8751040101051331,0.017005819633271906,512,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,1.0,0.625,0.1747801353218523,1.53125,6.978069554482723,0.09820938110351562,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.8470079898834229,0.9010239839553833,0.8694015920162201,0.8716959953308105,0.016138881171190216,1024,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,1.0,0.65625,0.1158122428154187,1.3125,6.9901908183358845,0.06445503234863281,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,1.1698240041732788,1.2311359643936157,1.1888479948043824,1.1890720129013062,0.017978797794492758,2048,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,1.0,0.78125,0.08347181893108634,1.1796875,6.994921772573154,0.046871185302734375,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.7702720165252686,1.8097599744796753,1.7919103980064393,1.7956640124320984,0.012641295676021557,4096,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,1.0,0.78125,0.06866734477822484,1.20703125,6.996602562938728,0.03801727294921875,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.968672037124634,3.0278079509735107,2.9899007797241213,2.9824799299240112,0.01816414122631667,8192,128,128,1,standard_fused_topk,logit_topk,softmax_renorm,True,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,1.0,0.8984375,0.04399546833977376,1.126953125,6.998607314922362,0.024699926376342773,uniform_random_logits,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02502400055527687,0.06092799827456474,0.033839999698102474,0.028672000393271446,0.010315255343709818,0.019360000267624855,0.0352960005402565,0.023424000293016434,0.022064000368118286,0.0038898102289194572,0.0,0.0,0.0,0.0,0.0,0.19574399292469025,0.2512960135936737,0.21939200013875962,0.2199999988079071,0.017532156418212565,1,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8,0.375,0.0625,0.0625,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0233364439829928,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025280000641942024,0.05132799968123436,0.030641599837690593,0.027343999594449997,0.007562409232763304,0.020160000771284103,0.052960000932216644,0.024145600199699403,0.021424000151455402,0.007450198645599985,0.0,0.0,0.0,0.0,0.0,0.20483200252056122,0.24006399512290955,0.22215040028095245,0.22433599829673767,0.00969639786132892,8,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,64,0.375,0.5,0.5,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9806430689981738,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024831999093294144,0.046560000628232956,0.030371200107038022,0.02763199992477894,0.005878487205028602,0.020320000126957893,0.04560000076889992,0.02601920012384653,0.02270400058478117,0.00751129965906799,0.0,0.0,0.0,0.0,0.0,0.20559999346733093,0.24726399779319763,0.22126719802618028,0.22207999974489212,0.01328093478485499,16,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,128,0.375,1.0,1.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9103623678483975,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024288000538945198,0.049375999718904495,0.03086080001667142,0.0267359996214509,0.0070864531384997225,0.020479999482631683,0.030912000685930252,0.02274719988927245,0.021359999664127827,0.0029966813650672505,0.0,0.0,0.0,0.0,0.0,0.20003199577331543,0.2301120012998581,0.21453119963407516,0.21598400175571442,0.010402239855151332,32,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,256,0.375,2.0,2.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9610778571819444,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02393599972128868,0.04342399910092354,0.029380799923092126,0.026688000187277794,0.0057374782000526574,0.020031999796628952,0.036607999354600906,0.022487999964505435,0.020911999978125095,0.0038135203062103235,0.0,0.0,0.0,0.0,0.0,0.19551999866962433,0.22972799837589264,0.21238719969987868,0.21488000452518463,0.010706095835489097,64,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,512,0.375,4.0,4.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9952941013961014,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.025407999753952026,0.05510399863123894,0.031430399790406224,0.02798399981111288,0.007335050273982725,0.020479999482631683,0.03561599925160408,0.02275839988142252,0.021551999263465405,0.003545718365165811,0.0,0.0,0.0,0.0,0.0,0.19420799612998962,0.2903999984264374,0.2211231991648674,0.21476799994707108,0.025955008886196004,128,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,1024,0.375,8.0,8.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9273256282883522,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.0244159996509552,0.05395200103521347,0.03188959984108806,0.026031999848783016,0.00943995927387483,0.02051199972629547,0.036607999354600906,0.023247999791055917,0.02147199958562851,0.003910623837069648,0.0,0.0,0.0,0.0,0.0,0.2290560007095337,0.289247989654541,0.2543327987194061,0.24939200282096863,0.01663236753503115,256,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,2048,0.375,16.0,16.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,1.0380599882396606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.024512000381946564,0.04620800167322159,0.02884640023112297,0.026320000179111958,0.005516557583355925,0.02054399996995926,0.03846399858593941,0.023401600029319524,0.021263999864459038,0.0044742944198265374,0.0,0.0,0.0,0.0,0.0,0.30831998586654663,0.36953601241111755,0.3324000000953674,0.3288639932870865,0.018617999572156707,512,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,4096,0.375,32.0,32.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9569603278386984,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02412799932062626,0.06940799951553345,0.030817600432783365,0.026800000108778477,0.010047085187652939,0.020128000527620316,0.044096000492572784,0.024606400076299904,0.022304000332951546,0.00622857166823714,0.0,0.0,0.0,0.0,0.0,0.462911993265152,0.5497919917106628,0.4893856018781662,0.4816960096359253,0.023348152887178286,1024,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,8192,0.375,64.0,64.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.9524274993623046,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.02831999957561493,0.043487999588251114,0.031744000129401685,0.02991999965161085,0.003973415836428909,0.02070399932563305,0.029343999922275543,0.022886400017887353,0.021743999794125557,0.0026873065045088873,0.0,0.0,0.0,0.0,0.0,0.7662079930305481,0.8717759847640991,0.788454395532608,0.7744799852371216,0.03174746482604812,2048,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,16384,0.375,128.0,128.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8971818172100244,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.03830400109291077,0.06268800050020218,0.043337599746882914,0.040511999279260635,0.005823016247946116,0.023135999217629433,0.03747199848294258,0.025206399988383053,0.02393599972128868,0.003271381256231161,0.0,0.0,0.0,0.0,0.0,1.363935947418213,1.4143040180206299,1.3812703967094422,1.3798720240592957,0.014770450075530007,4096,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,32768,0.375,256.0,256.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.8113207890716184,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize +0.05660799890756607,0.07932800054550171,0.06249920018017292,0.06039999984204769,0.005636461691480845,0.02956799976527691,0.03747199848294258,0.031126399897038935,0.030287999659776688,0.002157571006631541,0.0,0.0,0.0,0.0,0.0,2.5507519245147705,2.680704116821289,2.579859209060669,2.566223978996277,0.03673283558365955,8192,128,128,1,standard_fused_topk,fixed_hotset8,softmax_renorm,False,standalone_legacy,vllm020_replicated_linear,8,2048,768,True,4,65536,0.375,512.0,512.0,0.0625,0.0,3.872983346207417,16.0,3.0,0.9375,hotset8,20260716,FlashInfer CUTLASS,CUDA_EVENT,BF16,generic,none,0.883909666885606,measured_gate+topk+modular_expert;shuffling_zero_because_expert_measurement_includes_prepare_finalize diff --git a/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py b/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py index ee2d8a4..ade95f0 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py +++ b/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py @@ -644,16 +644,51 @@ def main() -> None: args.output / "allreduce.json", ] batch_composition_augmented = len(args.attention) > 3 + long_context_augmented = any( + "long-context" in path.name for path in args.attention + ) + long_context_coverage: dict[str, Any] = {"included": long_context_augmented} + if long_context_augmented: + with (args.output / "attention.csv").open(newline="") as handle: + frozen_attention = list(csv.DictReader(handle)) + by_tp = {} + for tp in (1, 2, 4): + decode_kv = { + int(float(row["kv_cache_size"])) + for row in frozen_attention + if int(row["num_tensor_parallel_workers"]) == tp + and row["is_prefill"].lower() == "false" + } + mixed_kv = { + int(float(row["decode_avg_kv_cache_size"])) + for row in frozen_attention + if int(row["num_tensor_parallel_workers"]) == tp + and row.get("is_true_mixed_batch", "").lower() == "true" + } + if not {16384, 32768, 40960}.issubset(decode_kv): + raise ValueError(f"long-context decode coverage mismatch for TP{tp}") + if not {16384, 32768}.issubset(mixed_kv): + raise ValueError(f"long-context mixed coverage mismatch for TP{tp}") + by_tp[str(tp)] = { + "decode_kv_lengths": sorted(decode_kv), + "true_mixed_decode_avg_kv_lengths": sorted(mixed_kv), + } + long_context_coverage["by_tp"] = by_tp manifest = { "schema_version": ( - "frontier_qwen30_vllm020_frozen_profile.v3" - if batch_composition_augmented - else "frontier_qwen30_vllm020_frozen_profile.v2" + "frontier_qwen30_vllm020_frozen_profile.v4" + if long_context_augmented + else ( + "frontier_qwen30_vllm020_frozen_profile.v3" + if batch_composition_augmented + else "frontier_qwen30_vllm020_frozen_profile.v2" + ) ), "profile_id": ( "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-" "fused-mixed-total-conserving" + ("-pure-prefill-batch-composition" if batch_composition_augmented else "") + + ("-long-context-decode-mixed" if long_context_augmented else "") ), "environment_contract": { "hardware": "NVIDIA H20", @@ -672,6 +707,7 @@ def main() -> None: "allreduce": allreduce_rows, }, "attention_tp_coverage": attention_tps, + "attention_long_context_coverage": long_context_coverage, "projection_contract": { "linear": "Frontier profiler using vLLM 0.20 CUDA operators", "attention": ( @@ -692,6 +728,13 @@ def main() -> None: if batch_composition_augmented else "not included" ), + "attention_long_context": ( + "Direct FA3 measurements at decode KV lengths 16K/32K/40K, " + "plus true-mixed prefill/decode measurements at 16K/32K, " + "for TP1/2/4" + if long_context_augmented + else "not included" + ), "moe": ( "Replicated gate and fused top-k plus TP-local modular expert kernel; " "expert measurement already includes prepare/finalize so shuffling is zero" From ec775cdafd26e920b7eaeeb8acd17d78ec3b91ba Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 10:36:28 +0800 Subject: [PATCH 104/148] Correct F1 to a steady fixed-QPS workload --- runs/frontier-fidelity-envelope-v1/.gitignore | 2 + .../prepare_fixed_frontier_traces.py | 15 +- .../results/f1-pilot/README.md | 10 + .../frontier_trace_surface_frozen.json | 3912 +++++++++++++++++ .../test_fidelity_envelope.py | 10 + .../f1-isl512-osl1-steady10s/manifest.json | 57 + 6 files changed, 4004 insertions(+), 2 deletions(-) create mode 100644 runs/frontier-fidelity-envelope-v1/results/f1-pilot/README.md create mode 100644 runs/frontier-fidelity-envelope-v1/results/f1-pilot/frontier_trace_surface_frozen.json create mode 100644 runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1-steady10s/manifest.json diff --git a/runs/frontier-fidelity-envelope-v1/.gitignore b/runs/frontier-fidelity-envelope-v1/.gitignore index acd9559..4de000f 100644 --- a/runs/frontier-fidelity-envelope-v1/.gitignore +++ b/runs/frontier-fidelity-envelope-v1/.gitignore @@ -5,9 +5,11 @@ simulator-a1/ simulator-a2/ simulator-a3/ simulator-f1/ +simulator-f1-steady/ simulator-f2/ simulator-t1/ fleet-state/ fleet-artifacts/ profiles/profile-v3-batch/ trace-exact-v1-public/ +workloads/f1-isl512-osl1-steady10s/*.csv diff --git a/runs/frontier-fidelity-envelope-v1/prepare_fixed_frontier_traces.py b/runs/frontier-fidelity-envelope-v1/prepare_fixed_frontier_traces.py index 606e5dd..60d62cc 100644 --- a/runs/frontier-fidelity-envelope-v1/prepare_fixed_frontier_traces.py +++ b/runs/frontier-fidelity-envelope-v1/prepare_fixed_frontier_traces.py @@ -26,6 +26,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--input-tokens", type=int, required=True) parser.add_argument("--output-tokens", type=int, required=True) parser.add_argument("--requests", type=int, default=64) + parser.add_argument("--duration-seconds", type=float) parser.add_argument("--rate", type=float, action="append", required=True) parser.add_argument("--block-size", type=int, default=16) return parser.parse_args() @@ -47,6 +48,7 @@ def materialize( requests: int, rates: list[float], block_size: int, + duration_seconds: float | None = None, ) -> dict[str, object]: values = [input_tokens, output_tokens, requests, block_size] if any(value <= 0 for value in values): @@ -57,15 +59,21 @@ def materialize( raise ValueError("rates must be positive and finite") if len(set(rates)) != len(rates): raise ValueError("rates must be unique") + if duration_seconds is not None and duration_seconds <= 0: + raise ValueError("duration must be positive") root.mkdir(parents=True, exist_ok=True) anchors = [] blocks = math.ceil(input_tokens / block_size) for rate in rates: + request_count = max( + requests, + math.ceil(rate * duration_seconds) if duration_seconds is not None else 0, + ) path = root / f"{rate_key(rate)}.csv" with path.open("w", newline="") as output: writer = csv.DictWriter(output, fieldnames=FIELDS, lineterminator="\n") writer.writeheader() - for request_id in range(requests): + for request_id in range(request_count): first_block = request_id * blocks + 1 writer.writerow( { @@ -82,6 +90,7 @@ def materialize( { "label": rate_key(rate), "rate": rate, + "requests": request_count, "path": path.name, "sha256": sha256(path), } @@ -91,7 +100,8 @@ def materialize( "contract": { "input_tokens": input_tokens, "output_tokens": output_tokens, - "requests": requests, + "minimum_requests_per_anchor": requests, + "minimum_steady_duration_seconds": duration_seconds, "block_size": block_size, "arrival": "open_loop_uniform", "prefix_caching": False, @@ -114,6 +124,7 @@ def main() -> None: requests=args.requests, rates=args.rate, block_size=args.block_size, + duration_seconds=args.duration_seconds, ) print(json.dumps(manifest, sort_keys=True)) diff --git a/runs/frontier-fidelity-envelope-v1/results/f1-pilot/README.md b/runs/frontier-fidelity-envelope-v1/results/f1-pilot/README.md new file mode 100644 index 0000000..ef9e445 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/results/f1-pilot/README.md @@ -0,0 +1,10 @@ +# F1 64-request pilot（非 decision-bearing） + +该 surface 使用固定 `ISL=512/OSL=1`、uniform QPS、prefix off 和每个 anchor +64 requests。它完成了 runner correctness smoke，但不能用于 capacity/ranking claim：在 +512 req/s 时，64 个请求只覆盖 0.123 秒，TP4 的 64/64 SLO pass 只说明这个有限 burst +能在 TTFT SLO 内排空，并不说明系统能持续承载 512 req/s。 + +Decision-bearing F1 改为每个 anchor 至少 64 requests 且至少 10 秒 steady arrivals； +物化 contract 位于 `../../workloads/f1-isl512-osl1-steady10s/manifest.json`。原始 pilot +manifest 保留用于审计，不与 steady surface 合并。 diff --git a/runs/frontier-fidelity-envelope-v1/results/f1-pilot/frontier_trace_surface_frozen.json b/runs/frontier-fidelity-envelope-v1/results/f1-pilot/frontier_trace_surface_frozen.json new file mode 100644 index 0000000..bb21c11 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/results/f1-pilot/frontier_trace_surface_frozen.json @@ -0,0 +1,3912 @@ +{ + "collective": { + "allreduce_csv": "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv", + "allreduce_csv_sha256": "9d693fd406616b599e57bcde66c980c7fc2831b3acf37d3eb633cec80ea0070d", + "backend": "vidur" + }, + "config_results": [ + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "loads": [ + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 9.23979640007019, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "ad907fe8c9476f468f30802108ab33b012d857f0b2082a55159b4645e201980a", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 216.29366351634627, + "ttft_p95_ms": 283.4691372063174 + }, + "status": "completed", + "trace_label": "r16", + "trace_sha256": "dd00640d57184b1607fb466d2735bd4edf42e585ac6d6f2d6c6363c76b34aac8" + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 9.139918327331543, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "63476fae7272db8d2636c94e535f4f14bafa5e8b7b0e6d2cb9b1e840dd686115", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 259.0172330518654, + "ttft_p95_ms": 347.0536850751076 + }, + "status": "completed", + "trace_label": "r32", + "trace_sha256": "de448fa237c367c4910570586f8574a5020c3186a2adc1db5daa26acf0d2b9e1" + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 9.040833234786987, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "5b277eafe865f738e74074ea50ad124c7acc79e034f09c78ec4c588db7bcac95", + "score": { + "slos": { + "tpot_100ms": { + "feasible": false, + "pass_rate": 0.796875, + "passed": 51 + }, + "tpot_150ms": { + "feasible": false, + "pass_rate": 0.796875, + "passed": 51 + }, + "tpot_180ms": { + "feasible": false, + "pass_rate": 0.796875, + "passed": 51 + }, + "tpot_50ms": { + "feasible": false, + "pass_rate": 0.796875, + "passed": 51 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 749.1180253669007, + "ttft_p95_ms": 1235.420300361212 + }, + "status": "completed", + "trace_label": "r64", + "trace_sha256": "c796965d7afa1a9df4b8ba523412cef2e4dc7ddfc3cef6bce7e91265b20aaf2d" + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 8.89000153541565, + "offered_request_rate": 128.0, + "offered_request_rate_per_gpu": 128.0, + "request_count": 64, + "request_metrics_sha256": "07bd77c8416c5ff0e9b83861669f833c126ac2fb3b0a9a6ae2830fc8e2e9041b", + "score": { + "slos": { + "tpot_100ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_150ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_180ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_50ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 952.2430253669007, + "ttft_p95_ms": 1704.170300361212 + }, + "status": "completed", + "trace_label": "r128", + "trace_sha256": "30706d59e020ad0dcb670d367201de77ca7337b5fb41da0efd5af8c1915b2f04" + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 9.043700695037842, + "offered_request_rate": 256.0, + "offered_request_rate_per_gpu": 256.0, + "request_count": 64, + "request_metrics_sha256": "f939698c1a72ad9c6da784797fe3044c0eaeec87f311bcf2a9b35fc699aa9877", + "score": { + "slos": { + "tpot_100ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_150ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_180ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_50ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 1053.8055253669008, + "ttft_p95_ms": 1938.545300361212 + }, + "status": "completed", + "trace_label": "r256", + "trace_sha256": "f93c2390f084c4c60e7ca0d3096c7bdf024b7b286fd7ac72f4dc21d3d2af8cbf" + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "elapsed_seconds": 9.08841609954834, + "offered_request_rate": 512.0, + "offered_request_rate_per_gpu": 512.0, + "request_count": 64, + "request_metrics_sha256": "3734b160b72632c9d6eff25ab66e2db66a33685af385f22ff779454825604871", + "score": { + "slos": { + "tpot_100ms": { + "feasible": false, + "pass_rate": 0.390625, + "passed": 25 + }, + "tpot_150ms": { + "feasible": false, + "pass_rate": 0.390625, + "passed": 25 + }, + "tpot_180ms": { + "feasible": false, + "pass_rate": 0.390625, + "passed": 25 + }, + "tpot_50ms": { + "feasible": false, + "pass_rate": 0.390625, + "passed": 25 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 1104.5867753669008, + "ttft_p95_ms": 2055.732800361212 + }, + "status": "completed", + "trace_label": "r512", + "trace_sha256": "e2612362defdb10f876eb478a642ef78a9f6101833b7cc4ff1f0b02b8a453767" + } + ] + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "loads": [ + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 9.090706586837769, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "ad907fe8c9476f468f30802108ab33b012d857f0b2082a55159b4645e201980a", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 216.29366351634627, + "ttft_p95_ms": 283.4691372063174 + }, + "status": "completed", + "trace_label": "r16", + "trace_sha256": "dd00640d57184b1607fb466d2735bd4edf42e585ac6d6f2d6c6363c76b34aac8" + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 9.0899977684021, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "63476fae7272db8d2636c94e535f4f14bafa5e8b7b0e6d2cb9b1e840dd686115", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 259.0172330518654, + "ttft_p95_ms": 347.0536850751076 + }, + "status": "completed", + "trace_label": "r32", + "trace_sha256": "de448fa237c367c4910570586f8574a5020c3186a2adc1db5daa26acf0d2b9e1" + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 9.190520524978638, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "63a37dec7df065e7e8c8ef5ada849898c99795af0873bbe32c4120596580e435", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 0.984375, + "passed": 63 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 0.984375, + "passed": 63 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 0.984375, + "passed": 63 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 0.984375, + "passed": 63 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 733.829341199415, + "ttft_p95_ms": 1045.2671022385387 + }, + "status": "completed", + "trace_label": "r64", + "trace_sha256": "c796965d7afa1a9df4b8ba523412cef2e4dc7ddfc3cef6bce7e91265b20aaf2d" + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 9.088061571121216, + "offered_request_rate": 128.0, + "offered_request_rate_per_gpu": 128.0, + "request_count": 64, + "request_metrics_sha256": "043cb4e37ec062487ecf8bf2910d3660fe3d5846be13214ac2a8beb668e19192", + "score": { + "slos": { + "tpot_100ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_150ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_180ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_50ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 870.0662724508372, + "ttft_p95_ms": 1477.3167945290845 + }, + "status": "completed", + "trace_label": "r128", + "trace_sha256": "30706d59e020ad0dcb670d367201de77ca7337b5fb41da0efd5af8c1915b2f04" + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 9.192060708999634, + "offered_request_rate": 256.0, + "offered_request_rate_per_gpu": 256.0, + "request_count": 64, + "request_metrics_sha256": "e7f32c2152c42a95203201dba11f2c561eeaf3cfc9c6e9e68d89df2294475510", + "score": { + "slos": { + "tpot_100ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_150ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_180ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_50ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 940.3787724508372, + "ttft_p95_ms": 1680.4417945290845 + }, + "status": "completed", + "trace_label": "r256", + "trace_sha256": "f93c2390f084c4c60e7ca0d3096c7bdf024b7b286fd7ac72f4dc21d3d2af8cbf" + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "elapsed_seconds": 9.089672565460205, + "offered_request_rate": 512.0, + "offered_request_rate_per_gpu": 512.0, + "request_count": 64, + "request_metrics_sha256": "d879f833e5dc08d711f48fb6c6ccf48f7891fe7694470b107747ed0a1b95dac7", + "score": { + "slos": { + "tpot_100ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_150ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_180ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_50ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 975.5350224508372, + "ttft_p95_ms": 1782.0042945290845 + }, + "status": "completed", + "trace_label": "r512", + "trace_sha256": "e2612362defdb10f876eb478a642ef78a9f6101833b7cc4ff1f0b02b8a453767" + } + ] + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "loads": [ + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 9.191780090332031, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "ad907fe8c9476f468f30802108ab33b012d857f0b2082a55159b4645e201980a", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 216.29366351634627, + "ttft_p95_ms": 283.4691372063174 + }, + "status": "completed", + "trace_label": "r16", + "trace_sha256": "dd00640d57184b1607fb466d2735bd4edf42e585ac6d6f2d6c6363c76b34aac8" + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 9.189058303833008, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "63476fae7272db8d2636c94e535f4f14bafa5e8b7b0e6d2cb9b1e840dd686115", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 259.0172330518654, + "ttft_p95_ms": 347.0536850751076 + }, + "status": "completed", + "trace_label": "r32", + "trace_sha256": "de448fa237c367c4910570586f8574a5020c3186a2adc1db5daa26acf0d2b9e1" + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 9.190616607666016, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "63a37dec7df065e7e8c8ef5ada849898c99795af0873bbe32c4120596580e435", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 0.984375, + "passed": 63 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 0.984375, + "passed": 63 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 0.984375, + "passed": 63 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 0.984375, + "passed": 63 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 733.829341199415, + "ttft_p95_ms": 1045.2671022385387 + }, + "status": "completed", + "trace_label": "r64", + "trace_sha256": "c796965d7afa1a9df4b8ba523412cef2e4dc7ddfc3cef6bce7e91265b20aaf2d" + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 9.140897035598755, + "offered_request_rate": 128.0, + "offered_request_rate_per_gpu": 128.0, + "request_count": 64, + "request_metrics_sha256": "043cb4e37ec062487ecf8bf2910d3660fe3d5846be13214ac2a8beb668e19192", + "score": { + "slos": { + "tpot_100ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_150ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_180ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_50ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 870.0662724508372, + "ttft_p95_ms": 1477.3167945290845 + }, + "status": "completed", + "trace_label": "r128", + "trace_sha256": "30706d59e020ad0dcb670d367201de77ca7337b5fb41da0efd5af8c1915b2f04" + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 9.195079803466797, + "offered_request_rate": 256.0, + "offered_request_rate_per_gpu": 256.0, + "request_count": 64, + "request_metrics_sha256": "e7f32c2152c42a95203201dba11f2c561eeaf3cfc9c6e9e68d89df2294475510", + "score": { + "slos": { + "tpot_100ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_150ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_180ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_50ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 940.3787724508372, + "ttft_p95_ms": 1680.4417945290845 + }, + "status": "completed", + "trace_label": "r256", + "trace_sha256": "f93c2390f084c4c60e7ca0d3096c7bdf024b7b286fd7ac72f4dc21d3d2af8cbf" + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "elapsed_seconds": 9.291805744171143, + "offered_request_rate": 512.0, + "offered_request_rate_per_gpu": 512.0, + "request_count": 64, + "request_metrics_sha256": "d879f833e5dc08d711f48fb6c6ccf48f7891fe7694470b107747ed0a1b95dac7", + "score": { + "slos": { + "tpot_100ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_150ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_180ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_50ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 975.5350224508372, + "ttft_p95_ms": 1782.0042945290845 + }, + "status": "completed", + "trace_label": "r512", + "trace_sha256": "e2612362defdb10f876eb478a642ef78a9f6101833b7cc4ff1f0b02b8a453767" + } + ] + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "loads": [ + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 9.336015462875366, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "ad907fe8c9476f468f30802108ab33b012d857f0b2082a55159b4645e201980a", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 216.29366351634627, + "ttft_p95_ms": 283.4691372063174 + }, + "status": "completed", + "trace_label": "r16", + "trace_sha256": "dd00640d57184b1607fb466d2735bd4edf42e585ac6d6f2d6c6363c76b34aac8" + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 9.189040660858154, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "63476fae7272db8d2636c94e535f4f14bafa5e8b7b0e6d2cb9b1e840dd686115", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 259.0172330518654, + "ttft_p95_ms": 347.0536850751076 + }, + "status": "completed", + "trace_label": "r32", + "trace_sha256": "de448fa237c367c4910570586f8574a5020c3186a2adc1db5daa26acf0d2b9e1" + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 9.28804874420166, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "63a37dec7df065e7e8c8ef5ada849898c99795af0873bbe32c4120596580e435", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 0.984375, + "passed": 63 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 0.984375, + "passed": 63 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 0.984375, + "passed": 63 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 0.984375, + "passed": 63 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 733.829341199415, + "ttft_p95_ms": 1045.2671022385387 + }, + "status": "completed", + "trace_label": "r64", + "trace_sha256": "c796965d7afa1a9df4b8ba523412cef2e4dc7ddfc3cef6bce7e91265b20aaf2d" + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 9.286862850189209, + "offered_request_rate": 128.0, + "offered_request_rate_per_gpu": 128.0, + "request_count": 64, + "request_metrics_sha256": "043cb4e37ec062487ecf8bf2910d3660fe3d5846be13214ac2a8beb668e19192", + "score": { + "slos": { + "tpot_100ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_150ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_180ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_50ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 870.0662724508372, + "ttft_p95_ms": 1477.3167945290845 + }, + "status": "completed", + "trace_label": "r128", + "trace_sha256": "30706d59e020ad0dcb670d367201de77ca7337b5fb41da0efd5af8c1915b2f04" + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 9.135990619659424, + "offered_request_rate": 256.0, + "offered_request_rate_per_gpu": 256.0, + "request_count": 64, + "request_metrics_sha256": "e7f32c2152c42a95203201dba11f2c561eeaf3cfc9c6e9e68d89df2294475510", + "score": { + "slos": { + "tpot_100ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_150ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_180ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_50ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 940.3787724508372, + "ttft_p95_ms": 1680.4417945290845 + }, + "status": "completed", + "trace_label": "r256", + "trace_sha256": "f93c2390f084c4c60e7ca0d3096c7bdf024b7b286fd7ac72f4dc21d3d2af8cbf" + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "elapsed_seconds": 9.236075401306152, + "offered_request_rate": 512.0, + "offered_request_rate_per_gpu": 512.0, + "request_count": 64, + "request_metrics_sha256": "d879f833e5dc08d711f48fb6c6ccf48f7891fe7694470b107747ed0a1b95dac7", + "score": { + "slos": { + "tpot_100ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_150ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_180ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + }, + "tpot_50ms": { + "feasible": false, + "pass_rate": 0.515625, + "passed": 33 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 975.5350224508372, + "ttft_p95_ms": 1782.0042945290845 + }, + "status": "completed", + "trace_label": "r512", + "trace_sha256": "e2612362defdb10f876eb478a642ef78a9f6101833b7cc4ff1f0b02b8a453767" + } + ] + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "loads": [ + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 9.188935279846191, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "ef8e3157f73a0421abb1d6c8ef036db2c3858b52a81c47c52bf2b24803674ab6", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 145.3083076454922, + "ttft_p95_ms": 191.94509020206718 + }, + "status": "completed", + "trace_label": "r16", + "trace_sha256": "dd00640d57184b1607fb466d2735bd4edf42e585ac6d6f2d6c6363c76b34aac8" + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 9.238474130630493, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "b3e01bb71b1a148c2586b6f5a9cfe05bb9b31e656f4e7993ebd27a4d14d6c994", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 160.71212013197345, + "ttft_p95_ms": 215.40746023330092 + }, + "status": "completed", + "trace_label": "r32", + "trace_sha256": "de448fa237c367c4910570586f8574a5020c3186a2adc1db5daa26acf0d2b9e1" + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 9.044495582580566, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "be6729dc2729694448aa8a59685d2a1cfd6863cdc40baa8d8aff7edc86633c63", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 342.3859670790913, + "ttft_p95_ms": 516.3686608843987 + }, + "status": "completed", + "trace_label": "r64", + "trace_sha256": "c796965d7afa1a9df4b8ba523412cef2e4dc7ddfc3cef6bce7e91265b20aaf2d" + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 9.19176721572876, + "offered_request_rate": 128.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "4a23d18f2b8a29fd0e341ab68c7499b93864e91aebf164a39a819450d728e7fc", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 568.5763491205233, + "ttft_p95_ms": 976.9340429258309 + }, + "status": "completed", + "trace_label": "r128", + "trace_sha256": "30706d59e020ad0dcb670d367201de77ca7337b5fb41da0efd5af8c1915b2f04" + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 9.190044641494751, + "offered_request_rate": 256.0, + "offered_request_rate_per_gpu": 128.0, + "request_count": 64, + "request_metrics_sha256": "f1d8c6820f95946bcf3a4977585b99514ae35d870d2ff09bb9f7c7baef96464e", + "score": { + "slos": { + "tpot_100ms": { + "feasible": false, + "pass_rate": 0.796875, + "passed": 51 + }, + "tpot_150ms": { + "feasible": false, + "pass_rate": 0.796875, + "passed": 51 + }, + "tpot_180ms": { + "feasible": false, + "pass_rate": 0.796875, + "passed": 51 + }, + "tpot_50ms": { + "feasible": false, + "pass_rate": 0.796875, + "passed": 51 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 670.1388491205233, + "ttft_p95_ms": 1211.309042925831 + }, + "status": "completed", + "trace_label": "r256", + "trace_sha256": "f93c2390f084c4c60e7ca0d3096c7bdf024b7b286fd7ac72f4dc21d3d2af8cbf" + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "elapsed_seconds": 9.139874696731567, + "offered_request_rate": 512.0, + "offered_request_rate_per_gpu": 256.0, + "request_count": 64, + "request_metrics_sha256": "307d2d128d6032dd613bc67d5f78e0d3d9c9f05e7f1a72c5751fd1cec0b589b3", + "score": { + "slos": { + "tpot_100ms": { + "feasible": false, + "pass_rate": 0.765625, + "passed": 49 + }, + "tpot_150ms": { + "feasible": false, + "pass_rate": 0.765625, + "passed": 49 + }, + "tpot_180ms": { + "feasible": false, + "pass_rate": 0.765625, + "passed": 49 + }, + "tpot_50ms": { + "feasible": false, + "pass_rate": 0.765625, + "passed": 49 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 720.9200991205233, + "ttft_p95_ms": 1328.496542925831 + }, + "status": "completed", + "trace_label": "r512", + "trace_sha256": "e2612362defdb10f876eb478a642ef78a9f6101833b7cc4ff1f0b02b8a453767" + } + ] + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "loads": [ + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 9.338894128799438, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "ef8e3157f73a0421abb1d6c8ef036db2c3858b52a81c47c52bf2b24803674ab6", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 145.3083076454922, + "ttft_p95_ms": 191.94509020206718 + }, + "status": "completed", + "trace_label": "r16", + "trace_sha256": "dd00640d57184b1607fb466d2735bd4edf42e585ac6d6f2d6c6363c76b34aac8" + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 9.092449188232422, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "b3e01bb71b1a148c2586b6f5a9cfe05bb9b31e656f4e7993ebd27a4d14d6c994", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 160.71212013197345, + "ttft_p95_ms": 215.40746023330092 + }, + "status": "completed", + "trace_label": "r32", + "trace_sha256": "de448fa237c367c4910570586f8574a5020c3186a2adc1db5daa26acf0d2b9e1" + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 9.191263914108276, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "82720ff92616f5f6544e03f52ce3635328d72bd28cbfb26e18a0ad8f43e21abf", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 308.93499355358654, + "ttft_p95_ms": 484.23333144654856 + }, + "status": "completed", + "trace_label": "r64", + "trace_sha256": "c796965d7afa1a9df4b8ba523412cef2e4dc7ddfc3cef6bce7e91265b20aaf2d" + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 9.09642219543457, + "offered_request_rate": 128.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "e8dae4b0df880217f1c36c91435bf1afe920ccec1d776428611a7a49f3861d79", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 504.6855225446456, + "ttft_p95_ms": 731.2781755076838 + }, + "status": "completed", + "trace_label": "r128", + "trace_sha256": "30706d59e020ad0dcb670d367201de77ca7337b5fb41da0efd5af8c1915b2f04" + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 9.189780473709106, + "offered_request_rate": 256.0, + "offered_request_rate_per_gpu": 128.0, + "request_count": 64, + "request_metrics_sha256": "172a6cae007a956ca9640e313a3c82b2ffbb8b19c05bb1db808d1f390e9c8cc3", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 565.5028311011398, + "ttft_p95_ms": 970.7870068870639 + }, + "status": "completed", + "trace_label": "r256", + "trace_sha256": "f93c2390f084c4c60e7ca0d3096c7bdf024b7b286fd7ac72f4dc21d3d2af8cbf" + }, + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "elapsed_seconds": 9.236796617507935, + "offered_request_rate": 512.0, + "offered_request_rate_per_gpu": 256.0, + "request_count": 64, + "request_metrics_sha256": "18b637ba04030fc15b1b39a20e3c651c829f4bd06bc908ac441ebb640653ef93", + "score": { + "slos": { + "tpot_100ms": { + "feasible": false, + "pass_rate": 0.875, + "passed": 56 + }, + "tpot_150ms": { + "feasible": false, + "pass_rate": 0.875, + "passed": 56 + }, + "tpot_180ms": { + "feasible": false, + "pass_rate": 0.875, + "passed": 56 + }, + "tpot_50ms": { + "feasible": false, + "pass_rate": 0.875, + "passed": 56 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 600.6590811011398, + "ttft_p95_ms": 1072.3495068870639 + }, + "status": "completed", + "trace_label": "r512", + "trace_sha256": "e2612362defdb10f876eb478a642ef78a9f6101833b7cc4ff1f0b02b8a453767" + } + ] + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "loads": [ + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 9.48096251487732, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "ef8e3157f73a0421abb1d6c8ef036db2c3858b52a81c47c52bf2b24803674ab6", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 145.3083076454922, + "ttft_p95_ms": 191.94509020206718 + }, + "status": "completed", + "trace_label": "r16", + "trace_sha256": "dd00640d57184b1607fb466d2735bd4edf42e585ac6d6f2d6c6363c76b34aac8" + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 9.192594051361084, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "b3e01bb71b1a148c2586b6f5a9cfe05bb9b31e656f4e7993ebd27a4d14d6c994", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 160.71212013197345, + "ttft_p95_ms": 215.40746023330092 + }, + "status": "completed", + "trace_label": "r32", + "trace_sha256": "de448fa237c367c4910570586f8574a5020c3186a2adc1db5daa26acf0d2b9e1" + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 9.440584182739258, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "82720ff92616f5f6544e03f52ce3635328d72bd28cbfb26e18a0ad8f43e21abf", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 308.93499355358654, + "ttft_p95_ms": 484.23333144654856 + }, + "status": "completed", + "trace_label": "r64", + "trace_sha256": "c796965d7afa1a9df4b8ba523412cef2e4dc7ddfc3cef6bce7e91265b20aaf2d" + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 9.340449810028076, + "offered_request_rate": 128.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "e8dae4b0df880217f1c36c91435bf1afe920ccec1d776428611a7a49f3861d79", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 504.6855225446456, + "ttft_p95_ms": 731.2781755076838 + }, + "status": "completed", + "trace_label": "r128", + "trace_sha256": "30706d59e020ad0dcb670d367201de77ca7337b5fb41da0efd5af8c1915b2f04" + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 9.138880968093872, + "offered_request_rate": 256.0, + "offered_request_rate_per_gpu": 128.0, + "request_count": 64, + "request_metrics_sha256": "172a6cae007a956ca9640e313a3c82b2ffbb8b19c05bb1db808d1f390e9c8cc3", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 565.5028311011398, + "ttft_p95_ms": 970.7870068870639 + }, + "status": "completed", + "trace_label": "r256", + "trace_sha256": "f93c2390f084c4c60e7ca0d3096c7bdf024b7b286fd7ac72f4dc21d3d2af8cbf" + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "elapsed_seconds": 9.097595930099487, + "offered_request_rate": 512.0, + "offered_request_rate_per_gpu": 256.0, + "request_count": 64, + "request_metrics_sha256": "18b637ba04030fc15b1b39a20e3c651c829f4bd06bc908ac441ebb640653ef93", + "score": { + "slos": { + "tpot_100ms": { + "feasible": false, + "pass_rate": 0.875, + "passed": 56 + }, + "tpot_150ms": { + "feasible": false, + "pass_rate": 0.875, + "passed": 56 + }, + "tpot_180ms": { + "feasible": false, + "pass_rate": 0.875, + "passed": 56 + }, + "tpot_50ms": { + "feasible": false, + "pass_rate": 0.875, + "passed": 56 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 600.6590811011398, + "ttft_p95_ms": 1072.3495068870639 + }, + "status": "completed", + "trace_label": "r512", + "trace_sha256": "e2612362defdb10f876eb478a642ef78a9f6101833b7cc4ff1f0b02b8a453767" + } + ] + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "loads": [ + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 8.998417615890503, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "ef8e3157f73a0421abb1d6c8ef036db2c3858b52a81c47c52bf2b24803674ab6", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 145.3083076454922, + "ttft_p95_ms": 191.94509020206718 + }, + "status": "completed", + "trace_label": "r16", + "trace_sha256": "dd00640d57184b1607fb466d2735bd4edf42e585ac6d6f2d6c6363c76b34aac8" + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 9.042184114456177, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "b3e01bb71b1a148c2586b6f5a9cfe05bb9b31e656f4e7993ebd27a4d14d6c994", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 160.71212013197345, + "ttft_p95_ms": 215.40746023330092 + }, + "status": "completed", + "trace_label": "r32", + "trace_sha256": "de448fa237c367c4910570586f8574a5020c3186a2adc1db5daa26acf0d2b9e1" + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 9.095743656158447, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "82720ff92616f5f6544e03f52ce3635328d72bd28cbfb26e18a0ad8f43e21abf", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 308.93499355358654, + "ttft_p95_ms": 484.23333144654856 + }, + "status": "completed", + "trace_label": "r64", + "trace_sha256": "c796965d7afa1a9df4b8ba523412cef2e4dc7ddfc3cef6bce7e91265b20aaf2d" + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 9.151461124420166, + "offered_request_rate": 128.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "e8dae4b0df880217f1c36c91435bf1afe920ccec1d776428611a7a49f3861d79", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 504.6855225446456, + "ttft_p95_ms": 731.2781755076838 + }, + "status": "completed", + "trace_label": "r128", + "trace_sha256": "30706d59e020ad0dcb670d367201de77ca7337b5fb41da0efd5af8c1915b2f04" + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 9.000789642333984, + "offered_request_rate": 256.0, + "offered_request_rate_per_gpu": 128.0, + "request_count": 64, + "request_metrics_sha256": "172a6cae007a956ca9640e313a3c82b2ffbb8b19c05bb1db808d1f390e9c8cc3", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 565.5028311011398, + "ttft_p95_ms": 970.7870068870639 + }, + "status": "completed", + "trace_label": "r256", + "trace_sha256": "f93c2390f084c4c60e7ca0d3096c7bdf024b7b286fd7ac72f4dc21d3d2af8cbf" + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "elapsed_seconds": 8.99738073348999, + "offered_request_rate": 512.0, + "offered_request_rate_per_gpu": 256.0, + "request_count": 64, + "request_metrics_sha256": "18b637ba04030fc15b1b39a20e3c651c829f4bd06bc908ac441ebb640653ef93", + "score": { + "slos": { + "tpot_100ms": { + "feasible": false, + "pass_rate": 0.875, + "passed": 56 + }, + "tpot_150ms": { + "feasible": false, + "pass_rate": 0.875, + "passed": 56 + }, + "tpot_180ms": { + "feasible": false, + "pass_rate": 0.875, + "passed": 56 + }, + "tpot_50ms": { + "feasible": false, + "pass_rate": 0.875, + "passed": 56 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 600.6590811011398, + "ttft_p95_ms": 1072.3495068870639 + }, + "status": "completed", + "trace_label": "r512", + "trace_sha256": "e2612362defdb10f876eb478a642ef78a9f6101833b7cc4ff1f0b02b8a453767" + } + ] + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "loads": [ + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 9.145608186721802, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "cf7604290c74aa06e82791dccac8f57adbb43b861ec03601cf7353bcdb5cf85d", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 113.95995004232296, + "ttft_p95_ms": 148.6208545233651 + }, + "status": "completed", + "trace_label": "r16", + "trace_sha256": "dd00640d57184b1607fb466d2735bd4edf42e585ac6d6f2d6c6363c76b34aac8" + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 9.04130220413208, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "667039850bf100a1214dff99ba4a6f15ff47f9d3986736dfa2460979e2e644e6", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 123.08122913633257, + "ttft_p95_ms": 155.48122649863294 + }, + "status": "completed", + "trace_label": "r32", + "trace_sha256": "de448fa237c367c4910570586f8574a5020c3186a2adc1db5daa26acf0d2b9e1" + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 9.094721555709839, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "8fc12fe9c0c8ee1d2cae00e41bf57f36e90ecda35fed2e6c1b4fcaa52afb096b", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 132.5292253513205, + "ttft_p95_ms": 179.06372047949105 + }, + "status": "completed", + "trace_label": "r64", + "trace_sha256": "c796965d7afa1a9df4b8ba523412cef2e4dc7ddfc3cef6bce7e91265b20aaf2d" + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 9.047569274902344, + "offered_request_rate": 128.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "a657986eb6c2f0500d902176c3344861ffca678da2498bca87bbaf0989097229", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 369.5552485735299, + "ttft_p95_ms": 599.7693411300143 + }, + "status": "completed", + "trace_label": "r128", + "trace_sha256": "30706d59e020ad0dcb670d367201de77ca7337b5fb41da0efd5af8c1915b2f04" + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 9.150342464447021, + "offered_request_rate": 256.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "7f8328f0bba2873fb721543cb20ef17fd1c43610e8adff9d26f4033157f86d92", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 471.1177485735299, + "ttft_p95_ms": 834.1443411300143 + }, + "status": "completed", + "trace_label": "r256", + "trace_sha256": "f93c2390f084c4c60e7ca0d3096c7bdf024b7b286fd7ac72f4dc21d3d2af8cbf" + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "elapsed_seconds": 9.043598413467407, + "offered_request_rate": 512.0, + "offered_request_rate_per_gpu": 128.0, + "request_count": 64, + "request_metrics_sha256": "056ab2df7050c5080c7e03ba1bb9875d82fbcad2e8b1dc69b34a5c184fd89cb1", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 521.89899857353, + "ttft_p95_ms": 951.3318411300143 + }, + "status": "completed", + "trace_label": "r512", + "trace_sha256": "e2612362defdb10f876eb478a642ef78a9f6101833b7cc4ff1f0b02b8a453767" + } + ] + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "loads": [ + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 9.041609525680542, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "cf7604290c74aa06e82791dccac8f57adbb43b861ec03601cf7353bcdb5cf85d", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 113.95995004232296, + "ttft_p95_ms": 148.6208545233651 + }, + "status": "completed", + "trace_label": "r16", + "trace_sha256": "dd00640d57184b1607fb466d2735bd4edf42e585ac6d6f2d6c6363c76b34aac8" + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 9.040033102035522, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "667039850bf100a1214dff99ba4a6f15ff47f9d3986736dfa2460979e2e644e6", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 123.08122913633257, + "ttft_p95_ms": 155.48122649863294 + }, + "status": "completed", + "trace_label": "r32", + "trace_sha256": "de448fa237c367c4910570586f8574a5020c3186a2adc1db5daa26acf0d2b9e1" + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 9.14271068572998, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "8fc12fe9c0c8ee1d2cae00e41bf57f36e90ecda35fed2e6c1b4fcaa52afb096b", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 132.5292253513205, + "ttft_p95_ms": 179.06372047949105 + }, + "status": "completed", + "trace_label": "r64", + "trace_sha256": "c796965d7afa1a9df4b8ba523412cef2e4dc7ddfc3cef6bce7e91265b20aaf2d" + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 9.04430103302002, + "offered_request_rate": 128.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "a1ad1f58cce44b8bc7b72b2997413758c4d40e971ce51fc68f9530bb2279d281", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 302.6860632179096, + "ttft_p95_ms": 411.8879773090427 + }, + "status": "completed", + "trace_label": "r128", + "trace_sha256": "30706d59e020ad0dcb670d367201de77ca7337b5fb41da0efd5af8c1915b2f04" + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 8.998274326324463, + "offered_request_rate": 256.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "6271e503f9158379f1b1043acab957d96c452adb195e45c63b311667677dfd75", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 381.3555701081787, + "ttft_p95_ms": 623.3699841993117 + }, + "status": "completed", + "trace_label": "r256", + "trace_sha256": "f93c2390f084c4c60e7ca0d3096c7bdf024b7b286fd7ac72f4dc21d3d2af8cbf" + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "elapsed_seconds": 9.042869806289673, + "offered_request_rate": 512.0, + "offered_request_rate_per_gpu": 128.0, + "request_count": 64, + "request_metrics_sha256": "1bddc1f7a64764acb948087958ba9c20ed08b818545ccde67a3cec4cb48a9c35", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 416.5118201081787, + "ttft_p95_ms": 724.9324841993117 + }, + "status": "completed", + "trace_label": "r512", + "trace_sha256": "e2612362defdb10f876eb478a642ef78a9f6101833b7cc4ff1f0b02b8a453767" + } + ] + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "loads": [ + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.03968071937561, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "cf7604290c74aa06e82791dccac8f57adbb43b861ec03601cf7353bcdb5cf85d", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 113.95995004232296, + "ttft_p95_ms": 148.6208545233651 + }, + "status": "completed", + "trace_label": "r16", + "trace_sha256": "dd00640d57184b1607fb466d2735bd4edf42e585ac6d6f2d6c6363c76b34aac8" + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.040271520614624, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "667039850bf100a1214dff99ba4a6f15ff47f9d3986736dfa2460979e2e644e6", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 123.08122913633257, + "ttft_p95_ms": 155.48122649863294 + }, + "status": "completed", + "trace_label": "r32", + "trace_sha256": "de448fa237c367c4910570586f8574a5020c3186a2adc1db5daa26acf0d2b9e1" + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.089779615402222, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "8fc12fe9c0c8ee1d2cae00e41bf57f36e90ecda35fed2e6c1b4fcaa52afb096b", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 132.5292253513205, + "ttft_p95_ms": 179.06372047949105 + }, + "status": "completed", + "trace_label": "r64", + "trace_sha256": "c796965d7afa1a9df4b8ba523412cef2e4dc7ddfc3cef6bce7e91265b20aaf2d" + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.046946287155151, + "offered_request_rate": 128.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "a1ad1f58cce44b8bc7b72b2997413758c4d40e971ce51fc68f9530bb2279d281", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 302.6860632179096, + "ttft_p95_ms": 411.8879773090427 + }, + "status": "completed", + "trace_label": "r128", + "trace_sha256": "30706d59e020ad0dcb670d367201de77ca7337b5fb41da0efd5af8c1915b2f04" + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.149730682373047, + "offered_request_rate": 256.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "6271e503f9158379f1b1043acab957d96c452adb195e45c63b311667677dfd75", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 381.3555701081787, + "ttft_p95_ms": 623.3699841993117 + }, + "status": "completed", + "trace_label": "r256", + "trace_sha256": "f93c2390f084c4c60e7ca0d3096c7bdf024b7b286fd7ac72f4dc21d3d2af8cbf" + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "elapsed_seconds": 9.099130392074585, + "offered_request_rate": 512.0, + "offered_request_rate_per_gpu": 128.0, + "request_count": 64, + "request_metrics_sha256": "1bddc1f7a64764acb948087958ba9c20ed08b818545ccde67a3cec4cb48a9c35", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 416.5118201081787, + "ttft_p95_ms": 724.9324841993117 + }, + "status": "completed", + "trace_label": "r512", + "trace_sha256": "e2612362defdb10f876eb478a642ef78a9f6101833b7cc4ff1f0b02b8a453767" + } + ] + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "loads": [ + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.091226577758789, + "offered_request_rate": 16.0, + "offered_request_rate_per_gpu": 4.0, + "request_count": 64, + "request_metrics_sha256": "cf7604290c74aa06e82791dccac8f57adbb43b861ec03601cf7353bcdb5cf85d", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 113.95995004232296, + "ttft_p95_ms": 148.6208545233651 + }, + "status": "completed", + "trace_label": "r16", + "trace_sha256": "dd00640d57184b1607fb466d2735bd4edf42e585ac6d6f2d6c6363c76b34aac8" + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.049186944961548, + "offered_request_rate": 32.0, + "offered_request_rate_per_gpu": 8.0, + "request_count": 64, + "request_metrics_sha256": "667039850bf100a1214dff99ba4a6f15ff47f9d3986736dfa2460979e2e644e6", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 123.08122913633257, + "ttft_p95_ms": 155.48122649863294 + }, + "status": "completed", + "trace_label": "r32", + "trace_sha256": "de448fa237c367c4910570586f8574a5020c3186a2adc1db5daa26acf0d2b9e1" + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.14399766921997, + "offered_request_rate": 64.0, + "offered_request_rate_per_gpu": 16.0, + "request_count": 64, + "request_metrics_sha256": "8fc12fe9c0c8ee1d2cae00e41bf57f36e90ecda35fed2e6c1b4fcaa52afb096b", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 132.5292253513205, + "ttft_p95_ms": 179.06372047949105 + }, + "status": "completed", + "trace_label": "r64", + "trace_sha256": "c796965d7afa1a9df4b8ba523412cef2e4dc7ddfc3cef6bce7e91265b20aaf2d" + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.000115871429443, + "offered_request_rate": 128.0, + "offered_request_rate_per_gpu": 32.0, + "request_count": 64, + "request_metrics_sha256": "a1ad1f58cce44b8bc7b72b2997413758c4d40e971ce51fc68f9530bb2279d281", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 302.6860632179096, + "ttft_p95_ms": 411.8879773090427 + }, + "status": "completed", + "trace_label": "r128", + "trace_sha256": "30706d59e020ad0dcb670d367201de77ca7337b5fb41da0efd5af8c1915b2f04" + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.093112468719482, + "offered_request_rate": 256.0, + "offered_request_rate_per_gpu": 64.0, + "request_count": 64, + "request_metrics_sha256": "6271e503f9158379f1b1043acab957d96c452adb195e45c63b311667677dfd75", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 381.3555701081787, + "ttft_p95_ms": 623.3699841993117 + }, + "status": "completed", + "trace_label": "r256", + "trace_sha256": "f93c2390f084c4c60e7ca0d3096c7bdf024b7b286fd7ac72f4dc21d3d2af8cbf" + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "elapsed_seconds": 9.14099645614624, + "offered_request_rate": 512.0, + "offered_request_rate_per_gpu": 128.0, + "request_count": 64, + "request_metrics_sha256": "1bddc1f7a64764acb948087958ba9c20ed08b818545ccde67a3cec4cb48a9c35", + "score": { + "slos": { + "tpot_100ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_150ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_180ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + }, + "tpot_50ms": { + "feasible": true, + "pass_rate": 1.0, + "passed": 64 + } + }, + "tpot_p50_ms": null, + "tpot_p95_ms": null, + "ttft_p50_ms": 416.5118201081787, + "ttft_p95_ms": 724.9324841993117 + }, + "status": "completed", + "trace_label": "r512", + "trace_sha256": "e2612362defdb10f876eb478a642ef78a9f6101833b7cc4ff1f0b02b8a453767" + } + ] + } + ], + "contract": { + "arrival": "original_trace_timestamp_and_order", + "input_output": "exact_source_values", + "prefix_caching": false, + "primary_tpot_slo_ms": 150.0, + "rate_contract": "uniform-spacing", + "target_pass_rate": 0.95, + "tpot_slos_ms": [ + 50.0, + 100.0, + 150.0, + 180.0 + ], + "ttft_slo": "1000ms + 1000ms * input_tokens / 8000", + "window_seconds": null + }, + "frontier": { + "git_head": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "git_status_short": " M frontier/cc_backend/backends/vidur_cc_backend.py\n M frontier/config/config.py\n M frontier/entities/request.py\n M frontier/events/cluster_schedule_event.py\n M frontier/execution_time_predictor/sklearn_execution_time_predictor.py\n M frontier/metrics/constants.py\n M frontier/metrics/metrics_store.py\n M frontier/profiling/common/layers/rotary_embedding.py\n M frontier/profiling/moe/moe_impl.py\n M frontier/profiling/moe/moe_vllm_kernel.py\n M frontier/scheduler/cluster_scheduler/__init__.py\n M frontier/scheduler/cluster_scheduler/base_cluster_scheduler.py\n M frontier/scheduler/cluster_scheduler/cluster_scheduler_registry.py\n M frontier/scheduler/cluster_scheduler/sticky_lor_cluster_scheduler.py\n M frontier/scheduler/replica_scheduler/base_replica_scheduler.py\n M frontier/scheduler/replica_scheduler/vllm_v1_engine_replica_scheduler.py\n M frontier/scheduler/replica_stage_scheduler/replica_stage_schduler.py\n M frontier/simulator.py\n M frontier/types/cluster_scheduler_type.py\n?? data/profiling/compute/h20/\n?? frontier/scheduler/cluster_scheduler/prefix_lor_cluster_scheduler.py\n?? runs/\n?? tests/unit/test_attn_prefill_prediction_fallback.py\n?? tests/unit/test_vidur_cc_large_payload.py\n", + "source": "/tmp/replayserve-frontier-rs1b" + }, + "profiles": { + "coverage": { + "attention": { + "1": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + }, + "2": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + }, + "4": { + "exact_prefill_2048_rows": 1, + "profile_batch_size": 1 + } + }, + "manifest": { + "attention_tp_coverage": [ + 1, + 2, + 4 + ], + "environment_contract": { + "dtype": "bfloat16", + "frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "hardware": "NVIDIA H20", + "model": "Qwen3-30B-A3B", + "tensor_parallel_sizes": [ + 1, + 2, + 4 + ], + "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", + "vllm_version": "0.20.0" + }, + "inputs": { + "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-composition-tp1-20260717-v1-20260717T014550833983Z/artifacts/artifacts/attention-composition-tp1-v1/raw/flashattn-composition-tp1.json": "ce461be92a5059e2d168b2cbf21b55d9a55ee71c3b0586a3e63cd8288938168d", + "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-composition-tp2-20260717-v1-20260717T014551831545Z/artifacts/artifacts/attention-composition-tp2-v1/raw/flashattn-composition-tp2.json": "997d67fb3a28b488c3aa4945ae7c6d4ecf8a08726ee3cb4652d16c221ff395ad", + "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/fleet-artifacts/qwen30-attention-composition-tp4-20260717-v1-20260717T014552897206Z/artifacts/artifacts/attention-composition-tp4-v1/raw/flashattn-composition-tp4.json": "046cffd4c11d8a36513167ec25d3e1ca016324715679ea688a58d4a71dfe62b2", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp2-20260716-v1-dispatch-aware-20260716T140743025781Z/artifacts/artifacts/allreduce-full-tp2-v1/raw/allreduce-tp2.json": "97c3c76b5a04e95bd9192423c2b891667c668f39cc0dfecbd097d749939f2d0a", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-allreduce-full-tp4-20260716-v1-dispatch-aware-20260716T141106009788Z/artifacts/artifacts/allreduce-full-tp4-v1/raw/allreduce-tp4.json": "809df9baa6f468cf12bf0c99827475acc67894dd9f3f948976590b665fac0e76", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp1-20260716-v2-20260716T135132587012Z/artifacts/artifacts/flashattn-kv-full-v2-tp1/raw/flashattn-tp1.json": "dcb4c1bf7e76b9c765f78ddd2b8a734f2d7ba2adac13ce017689a8a77fe69a27", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp2-20260716-v2-20260716T135134194295Z/artifacts/artifacts/flashattn-kv-full-v2-tp2/raw/flashattn-tp2.json": "43ce042556ba887c8860614b43ccf0f564e5cebc1a0cffbce299d0acb9fa8d07", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-flashattn-kv-full-tp4-20260716-v2-20260716T135135197200Z/artifacts/artifacts/flashattn-kv-full-v2-tp4/raw/flashattn-tp4.json": "84eef31bcad0f556907a093318a420959d14fdc94474823d11f659704bdfec73", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-frontier-linear-full-20260716-v2-max-tokens-20260716T144444676943Z/artifacts/artifacts/frontier-linear-full-v2/profiles/compute/h20/qwen3-a3b-30b-moe/linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-moe-full-20260716-v1-local-shard-20260716T141334565164Z/artifacts/artifacts/moe-full-v1/raw/moe-full.json": "588f6ad0d69c9636d1b852e3df0a12d13cfe731f050ea7ec7aea457cceefbde8", + "/home/gahow/phd/aituner/runs/frontier-qwen30-vllm020-profile-v1/fleet-artifacts/qwen30-vllm020-router-full-20260716-v3-tp-context-20260716T145446098505Z/artifacts/artifacts/router-full-v3/raw/router.json": "1962972e983bff3e06a721ef4ae4ec65728ff669681497a4a7e7f769b88b4931" + }, + "outputs": { + "allreduce.json": "b38d14f990578d668523d25b107aceed433da5020d8ada3b6e44d3562261a3b3", + "attention.csv": "03180eec28087a301c9d8c185874c50bdb28b31e5e9af35fcce5a6a5a9e62147", + "attention_true_mixed_fused.csv": "cccf76e6cb6a9359f188b07d4dc2942442568a6e8cf3ff3d77ab3d35a052cecb", + "linear_op.csv": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "moe.csv": "dd698113975e5ed77f51dfd8a69aebf764ed651c3f6f34b71914d25d6e001415" + }, + "profile_id": "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-fused-mixed-total-conserving-pure-prefill-batch-composition", + "projection_contract": { + "allreduce": "Frozen exact runtime measurements; base profile-only comparison keeps the historical Frontier CC backend fixed to isolate compute profile fidelity", + "attention": "Pure prefill/extend/decode FA3 core plus separately measured KV update; input/output reshape assumed zero; exported mean is used as median target; true mixed rows use a total-conserving compatibility projection", + "attention_pure_prefill_batch_composition": "Direct FA3 measurements for 2/4 requests at query length 2048 and 2/4/8/16 requests at query length 512 for each TP; included only when batch-composition attention inputs are supplied", + "attention_true_mixed": "The directly measured fused total is preserved in diagnostics. Frontier's two targets are projected by the same-TP pure prefill/decode reference ratio, with projected prefill + decode exactly equal to the fused total; the split is a schema compatibility attribution, not an observation", + "linear": "Frontier profiler using vLLM 0.20 CUDA operators", + "moe": "Replicated gate and fused top-k plus TP-local modular expert kernel; expert measurement already includes prepare/finalize so shuffling is zero" + }, + "row_counts": { + "allreduce": 24, + "attention_frontier_compatible": 150, + "attention_true_mixed_fused_diagnostic": 30, + "linear": 36, + "moe": 72 + }, + "schema_version": "frontier_qwen30_vllm020_frozen_profile.v3" + } + }, + "root": "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/profiles/profile-v3-batch-final", + "sha256": { + "attention": "03180eec28087a301c9d8c185874c50bdb28b31e5e9af35fcce5a6a5a9e62147", + "linear": "67666cb0a4901b74599d468df2e31bcaa2a11a7842cc0cefba24ffce62508e0c", + "manifest": "70aecb6d70f9e509e9816b1a0929163b9505760c2ac30bbd4cd8e51badfb7635", + "moe": "dd698113975e5ed77f51dfd8a69aebf764ed651c3f6f34b71914d25d6e001415" + } + }, + "rankings": { + "tpot_100ms": [ + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 256.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": false + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 256.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": false + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 256.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": false + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 512.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": true + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 512.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": true + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 512.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": true + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 512.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": true + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 64.0, + "maximum_tested_feasible_request_rate_per_gpu": 64.0, + "upper_censored": false + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 64.0, + "maximum_tested_feasible_request_rate_per_gpu": 64.0, + "upper_censored": false + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 64.0, + "maximum_tested_feasible_request_rate_per_gpu": 64.0, + "upper_censored": false + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 128.0, + "maximum_tested_feasible_request_rate_per_gpu": 64.0, + "upper_censored": false + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 32.0, + "maximum_tested_feasible_request_rate_per_gpu": 32.0, + "upper_censored": false + } + ], + "tpot_150ms": [ + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 256.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": false + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 256.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": false + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 256.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": false + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 512.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": true + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 512.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": true + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 512.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": true + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 512.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": true + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 64.0, + "maximum_tested_feasible_request_rate_per_gpu": 64.0, + "upper_censored": false + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 64.0, + "maximum_tested_feasible_request_rate_per_gpu": 64.0, + "upper_censored": false + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 64.0, + "maximum_tested_feasible_request_rate_per_gpu": 64.0, + "upper_censored": false + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 128.0, + "maximum_tested_feasible_request_rate_per_gpu": 64.0, + "upper_censored": false + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 32.0, + "maximum_tested_feasible_request_rate_per_gpu": 32.0, + "upper_censored": false + } + ], + "tpot_180ms": [ + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 256.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": false + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 256.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": false + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 256.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": false + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 512.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": true + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 512.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": true + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 512.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": true + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 512.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": true + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 64.0, + "maximum_tested_feasible_request_rate_per_gpu": 64.0, + "upper_censored": false + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 64.0, + "maximum_tested_feasible_request_rate_per_gpu": 64.0, + "upper_censored": false + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 64.0, + "maximum_tested_feasible_request_rate_per_gpu": 64.0, + "upper_censored": false + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 128.0, + "maximum_tested_feasible_request_rate_per_gpu": 64.0, + "upper_censored": false + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 32.0, + "maximum_tested_feasible_request_rate_per_gpu": 32.0, + "upper_censored": false + } + ], + "tpot_50ms": [ + { + "config": { + "mns": 16, + "name": "tp2_mns16", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 256.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": false + }, + { + "config": { + "mns": 32, + "name": "tp2_mns32", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 256.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": false + }, + { + "config": { + "mns": 64, + "name": "tp2_mns64", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 256.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": false + }, + { + "config": { + "mns": 16, + "name": "tp4_mns16", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 512.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": true + }, + { + "config": { + "mns": 32, + "name": "tp4_mns32", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 512.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": true + }, + { + "config": { + "mns": 64, + "name": "tp4_mns64", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 512.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": true + }, + { + "config": { + "mns": 8, + "name": "tp4_mns8", + "tp": 4 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 512.0, + "maximum_tested_feasible_request_rate_per_gpu": 128.0, + "upper_censored": true + }, + { + "config": { + "mns": 16, + "name": "tp1_mns16", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 64.0, + "maximum_tested_feasible_request_rate_per_gpu": 64.0, + "upper_censored": false + }, + { + "config": { + "mns": 32, + "name": "tp1_mns32", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 64.0, + "maximum_tested_feasible_request_rate_per_gpu": 64.0, + "upper_censored": false + }, + { + "config": { + "mns": 64, + "name": "tp1_mns64", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 64.0, + "maximum_tested_feasible_request_rate_per_gpu": 64.0, + "upper_censored": false + }, + { + "config": { + "mns": 8, + "name": "tp2_mns8", + "tp": 2 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 128.0, + "maximum_tested_feasible_request_rate_per_gpu": 64.0, + "upper_censored": false + }, + { + "config": { + "mns": 8, + "name": "tp1_mns8", + "tp": 1 + }, + "lower_censored": false, + "maximum_tested_feasible_request_rate": 32.0, + "maximum_tested_feasible_request_rate_per_gpu": 32.0, + "upper_censored": false + } + ] + }, + "schema": "frontier-qwen30-exact-trace-surface-v1", + "status": "frozen_before_real", + "traces": [ + { + "first_arrival_s": 0.0, + "label": "r16", + "last_arrival_s": 3.9375, + "offered_request_rate": 16.0, + "path": "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r16.csv", + "requests": 64, + "sha256": "dd00640d57184b1607fb466d2735bd4edf42e585ac6d6f2d6c6363c76b34aac8" + }, + { + "first_arrival_s": 0.0, + "label": "r32", + "last_arrival_s": 1.96875, + "offered_request_rate": 32.0, + "path": "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r32.csv", + "requests": 64, + "sha256": "de448fa237c367c4910570586f8574a5020c3186a2adc1db5daa26acf0d2b9e1" + }, + { + "first_arrival_s": 0.0, + "label": "r64", + "last_arrival_s": 0.984375, + "offered_request_rate": 64.0, + "path": "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r64.csv", + "requests": 64, + "sha256": "c796965d7afa1a9df4b8ba523412cef2e4dc7ddfc3cef6bce7e91265b20aaf2d" + }, + { + "first_arrival_s": 0.0, + "label": "r128", + "last_arrival_s": 0.4921875, + "offered_request_rate": 128.0, + "path": "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r128.csv", + "requests": 64, + "sha256": "30706d59e020ad0dcb670d367201de77ca7337b5fb41da0efd5af8c1915b2f04" + }, + { + "first_arrival_s": 0.0, + "label": "r256", + "last_arrival_s": 0.24609375, + "offered_request_rate": 256.0, + "path": "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r256.csv", + "requests": 64, + "sha256": "f93c2390f084c4c60e7ca0d3096c7bdf024b7b286fd7ac72f4dc21d3d2af8cbf" + }, + { + "first_arrival_s": 0.0, + "label": "r512", + "last_arrival_s": 0.123046875, + "offered_request_rate": 512.0, + "path": "/home/gahow/phd/aituner/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1/r512.csv", + "requests": 64, + "sha256": "e2612362defdb10f876eb478a642ef78a9f6101833b7cc4ff1f0b02b8a453767" + } + ] +} diff --git a/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py b/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py index 20f4e32..4d8d224 100644 --- a/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py +++ b/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py @@ -203,6 +203,16 @@ class FidelityEnvelopeTest(unittest.TestCase): self.assertEqual(rows[1]["block_hash_ids"], "3|4") self.assertEqual(rows[1]["arrived_at"], "0.500000000000") self.assertEqual(manifest["contract"]["prefix_caching"], False) + steady = module.materialize( + root / "steady", + input_tokens=32, + output_tokens=4, + requests=2, + rates=[2.0], + block_size=16, + duration_seconds=10.0, + ) + self.assertEqual(steady["anchors"][0]["requests"], 20) if __name__ == "__main__": diff --git a/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1-steady10s/manifest.json b/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1-steady10s/manifest.json new file mode 100644 index 0000000..744a672 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/workloads/f1-isl512-osl1-steady10s/manifest.json @@ -0,0 +1,57 @@ +{ + "anchors": [ + { + "label": "r16", + "path": "r16.csv", + "rate": 16.0, + "requests": 160, + "sha256": "931604bdf085b1e0c2c70ffed9d10cbaeb6bfb3523976ce25c9bb3faf5fb20a9" + }, + { + "label": "r32", + "path": "r32.csv", + "rate": 32.0, + "requests": 320, + "sha256": "3f839823181c75149184df43326df4139c1f00cbc607dd305323d34e92467140" + }, + { + "label": "r64", + "path": "r64.csv", + "rate": 64.0, + "requests": 640, + "sha256": "b5bea5cf977f6a875a863a135eca70e643653b07ab92233f933f8b5e44e4a8c7" + }, + { + "label": "r128", + "path": "r128.csv", + "rate": 128.0, + "requests": 1280, + "sha256": "ea94212548621233172e430649e523f494064d035a7609d0b9ec3b1930222a3b" + }, + { + "label": "r256", + "path": "r256.csv", + "rate": 256.0, + "requests": 2560, + "sha256": "1e580f6ca30317f1f1ff81a86f318347341607cf1afcfef7ba53123335f02295" + }, + { + "label": "r512", + "path": "r512.csv", + "rate": 512.0, + "requests": 5120, + "sha256": "992ad52e6e6c08bcb0843d59ff7d529d637ad8ed812aca96d1318cb1ecfeb71b" + } + ], + "contract": { + "arrival": "open_loop_uniform", + "block_size": 16, + "input_tokens": 512, + "minimum_requests_per_anchor": 64, + "minimum_steady_duration_seconds": 10.0, + "output_tokens": 1, + "prefix_caching": false, + "prefix_relation": "all request block identities are disjoint" + }, + "schema": "qwen30-fixed-frontier-traces-v1" +} From e9ae04d852ea252aa4c44450816a51270173c39f Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 10:40:36 +0800 Subject: [PATCH 105/148] Add exact production-trace real replay client --- .../qwen30_exact_trace_client.py | 251 ++++++++++++++++++ .../run_qwen30_exact_trace_real_anchor.sh | 89 +++++++ .../test_fidelity_envelope.py | 18 ++ 3 files changed, 358 insertions(+) create mode 100644 runs/frontier-fidelity-envelope-v1/qwen30_exact_trace_client.py create mode 100755 runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh diff --git a/runs/frontier-fidelity-envelope-v1/qwen30_exact_trace_client.py b/runs/frontier-fidelity-envelope-v1/qwen30_exact_trace_client.py new file mode 100644 index 0000000..c23c66d --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/qwen30_exact_trace_client.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +"""Replay a private exact-trace anchor without emitting prompt text.""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import math +from pathlib import Path +from typing import Any + + +TARGET_PASS_RATE = 0.95 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, required=True) + parser.add_argument("--requests-file", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--tpot-slo-ms", type=float, default=150.0) + parser.add_argument("--timeout-seconds", type=float, default=1800.0) + return parser.parse_args() + + +def percentile(values: list[float], fraction: float) -> float | None: + if not values: + return None + ordered = sorted(values) + return ordered[math.ceil(fraction * len(ordered)) - 1] + + +def ttft_slo_ms(input_tokens: int) -> float: + return 1000.0 + 1000.0 * input_tokens / 8000.0 + + +def row_vector_sha256(rows: list[dict[str, Any]]) -> str: + digest = hashlib.sha256() + for row in rows: + digest.update( + json.dumps( + [ + row["source_index"], + row["arrived_at"], + row["input_length"], + row["output_length"], + row["session_id"], + row["runtime_block_ids"], + ], + separators=(",", ":"), + ).encode() + ) + digest.update(b"\n") + return digest.hexdigest() + + +async def request_one( + session: aiohttp.ClientSession, + row: dict[str, Any], + *, + scheduled_at: float, + benchmark_start: float, + tpot_slo_ms: float, +) -> dict[str, Any]: + loop = asyncio.get_running_loop() + delay = scheduled_at - loop.time() + if delay > 0: + await asyncio.sleep(delay) + admitted_at = loop.time() + record: dict[str, Any] = { + "source_index": int(row["source_index"]), + "session_id": int(row["session_id"]), + "scheduled_s": scheduled_at - benchmark_start, + "admitted_s": admitted_at - benchmark_start, + "admission_lag_ms": (admitted_at - scheduled_at) * 1000.0, + "input_tokens": int(row["input_length"]), + "requested_output_tokens": int(row["output_length"]), + "success": False, + } + body = dict(row["body"]) + body.update( + { + "temperature": 0, + "stream": True, + "stream_options": {"include_usage": True}, + "return_token_ids": True, + } + ) + try: + started = loop.time() + async with session.post("/v1/completions", json=body) as response: + if response.status != 200: + detail = (await response.text())[:1000] + raise RuntimeError(f"HTTP {response.status}: {detail}") + first_token_at = None + last_token_at = None + streamed_tokens = 0 + usage = None + while True: + raw = await response.content.readline() + if not raw: + break + line = raw.decode(errors="replace").strip() + if not line.startswith("data:"): + continue + data = line[5:].strip() + if data == "[DONE]": + break + payload = json.loads(data) + if payload.get("usage"): + usage = payload["usage"] + emitted = 0 + for choice in payload.get("choices") or []: + token_ids = choice.get("token_ids") or [] + emitted += len(token_ids) if token_ids else int(bool(choice.get("text"))) + if emitted: + now = loop.time() + first_token_at = first_token_at or now + last_token_at = now + streamed_tokens += emitted + finished = loop.time() + if first_token_at is None or last_token_at is None or usage is None: + raise RuntimeError("missing streaming token or usage") + actual_input = int(usage["prompt_tokens"]) + actual_output = int(usage["completion_tokens"]) + if actual_input != int(row["input_length"]) or actual_output != int( + row["output_length"] + ): + raise RuntimeError(f"usage mismatch: {actual_input}+{actual_output}") + ttft = (first_token_at - started) * 1000.0 + tpot = ( + (last_token_at - first_token_at) * 1000.0 / (actual_output - 1) + if actual_output > 1 + else None + ) + record.update( + { + "success": True, + "actual_input_tokens": actual_input, + "actual_output_tokens": actual_output, + "streamed_token_count": streamed_tokens, + "ttft_ms": ttft, + "tpot_ms": tpot, + "e2e_ms": (finished - started) * 1000.0, + "slo_pass": ttft <= ttft_slo_ms(actual_input) + and (tpot is None or tpot <= tpot_slo_ms), + } + ) + except Exception as error: + record.update( + { + "error": f"{type(error).__name__}: {error}", + "slo_pass": False, + } + ) + return record + + +async def replay(args: argparse.Namespace, rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + import aiohttp + + timeout = aiohttp.ClientTimeout(total=args.timeout_seconds) + connector = aiohttp.TCPConnector(limit=0, ttl_dns_cache=300) + benchmark_start = asyncio.get_running_loop().time() + 2.0 + async with aiohttp.ClientSession( + base_url=f"http://{args.host}:{args.port}", + timeout=timeout, + connector=connector, + ) as session: + tasks = [ + asyncio.create_task( + request_one( + session, + row, + scheduled_at=benchmark_start + float(row["arrived_at"]), + benchmark_start=benchmark_start, + tpot_slo_ms=args.tpot_slo_ms, + ) + ) + for row in rows + ] + return await asyncio.gather(*tasks) + + +def main() -> None: + args = parse_args() + if args.tpot_slo_ms <= 0 or args.timeout_seconds <= 0: + raise ValueError("SLO and timeout must be positive") + rows = [json.loads(line) for line in args.requests_file.open() if line.strip()] + if not rows: + raise ValueError("requests file is empty") + arrivals = [float(row["arrived_at"]) for row in rows] + if any(right < left for left, right in zip(arrivals, arrivals[1:])): + raise ValueError("request arrival order drift") + requests = asyncio.run(replay(args, rows)) + requests.sort(key=lambda row: int(row["source_index"])) + completed = [row for row in requests if row["success"]] + passed = sum(bool(row["slo_pass"]) for row in requests) + ttfts = [float(row["ttft_ms"]) for row in completed] + tpots = [ + float(row["tpot_ms"]) + for row in completed + if row["tpot_ms"] is not None + ] + pass_rate = passed / len(requests) + payload = { + "schema": "qwen30-exact-trace-anchor-v1", + "contract": { + "requests_file": str(args.requests_file.resolve()), + "requests_file_sha256": hashlib.sha256( + args.requests_file.read_bytes() + ).hexdigest(), + "row_vector_sha256": row_vector_sha256(rows), + "requests": len(rows), + "first_arrival_s": arrivals[0], + "last_arrival_s": arrivals[-1], + "arrival": "original_trace_timestamp_and_order", + "input_output_prompt": "exact_source_values", + "ttft_slo": "1000ms + 1000ms * input_tokens / 8000", + "tpot_slo_ms": args.tpot_slo_ms, + "target_pass_rate": TARGET_PASS_RATE, + "prompt_text_emitted": False, + }, + "summary": { + "completed": len(completed), + "failed": len(requests) - len(completed), + "passed": passed, + "pass_rate": pass_rate, + "feasible": pass_rate >= TARGET_PASS_RATE, + "ttft_p50_ms": percentile(ttfts, 0.50), + "ttft_p95_ms": percentile(ttfts, 0.95), + "tpot_p50_ms": percentile(tpots, 0.50), + "tpot_p95_ms": percentile(tpots, 0.95), + "admission_lag_p95_ms": percentile( + [float(row["admission_lag_ms"]) for row in requests], 0.95 + ), + }, + "requests": requests, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + print(json.dumps(payload["summary"], sort_keys=True), flush=True) + if len(completed) != len(requests): + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh new file mode 100755 index 0000000..8c03fe0 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash + +set -euo pipefail + +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}" +REQUESTS_FILE="${REQUESTS_FILE:?REQUESTS_FILE is required}" +TP="${TP:?TP is required}" +MNS="${MNS:?MNS is required}" +TRACE_LABEL="${TRACE_LABEL:?TRACE_LABEL is required}" +SERVER_PORT="${SERVER_PORT:?SERVER_PORT is required}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +SERVED_MODEL="qwen3-30b-exact-trace" +SERVER_PID="" + +mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/results" +exec > >(tee -a "${OUTPUT_ROOT}/logs/controller.log") 2>&1 + +cleanup() { + if [[ -n "${SERVER_PID}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then + kill -TERM -- "-${SERVER_PID}" 2>/dev/null || true + for _ in $(seq 1 30); do + kill -0 "${SERVER_PID}" 2>/dev/null || break + sleep 1 + done + kill -KILL -- "-${SERVER_PID}" 2>/dev/null || true + fi + SERVER_PID="" +} +trap cleanup EXIT INT TERM + +IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?fleet GPU allocation is required}" +if [[ "${#GPU_IDS[@]}" -ne "${TP}" ]]; then + echo "ERROR: expected ${TP} GPUs, got ${CUDA_VISIBLE_DEVICES}" >&2 + exit 1 +fi + +REQUEST_COUNT="$(wc -l < "${REQUESTS_FILE}")" +echo "QWEN30_EXACT_TRACE_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 trace=${TRACE_LABEL} requests=${REQUEST_COUNT} source=${REQUESTS_FILE} arrivals=original600s input_output_prompt=exact prefix=on block=16 tpot_slo=150ms output=${OUTPUT_ROOT} expected_wall=12-35m hard_wall=3600s" +date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" +sha256sum qwen30_exact_trace_client.py run_qwen30_exact_trace_real_anchor.sh \ + > "${OUTPUT_ROOT}/provenance/source.sha256" +sha256sum "${REQUESTS_FILE}" > "${OUTPUT_ROOT}/provenance/requests.sha256" +sha256sum "${MODEL_ROOT}/config.json" > "${OUTPUT_ROOT}/provenance/model.sha256" +nvidia-smi --query-gpu=index,name,uuid,driver_version --format=csv,noheader \ + > "${OUTPUT_ROOT}/provenance/gpus.csv" + +export TOKENIZERS_PARALLELISM=false +export VLLM_USE_V1=1 +export TORCH_CUDA_ARCH_LIST=9.0 +export HF_HUB_OFFLINE=1 +export TRANSFORMERS_OFFLINE=1 +ulimit -n 65536 + +setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \ + --host 127.0.0.1 --port "${SERVER_PORT}" --served-model-name "${SERVED_MODEL}" \ + --tensor-parallel-size "${TP}" --gpu-memory-utilization 0.92 \ + --max-model-len 40960 --max-num-batched-tokens 8192 --max-num-seqs "${MNS}" \ + --enable-prefix-caching --enable-chunked-prefill --no-enable-log-requests \ + > "${OUTPUT_ROOT}/logs/server.log" 2>&1 & +SERVER_PID=$! +READY=0 +for _ in $(seq 1 120); do + if curl -fsS --max-time 2 "http://127.0.0.1:${SERVER_PORT}/v1/models" \ + > "${OUTPUT_ROOT}/results/models.json" 2>/dev/null; then + READY=1 + break + fi + if ! kill -0 "${SERVER_PID}" 2>/dev/null; then + tail -200 "${OUTPUT_ROOT}/logs/server.log" + exit 1 + fi + sleep 3 +done +if [[ "${READY}" -ne 1 ]]; then + tail -200 "${OUTPUT_ROOT}/logs/server.log" + exit 1 +fi + +"${VENV_ROOT}/bin/python" qwen30_exact_trace_client.py \ + --port "${SERVER_PORT}" --requests-file "${REQUESTS_FILE}" \ + --output "${OUTPUT_ROOT}/results/result.json" --tpot-slo-ms 150 \ + --timeout-seconds 1800 + +cleanup +find "${OUTPUT_ROOT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 \ + | sort -z | xargs -0 sha256sum > "${OUTPUT_ROOT}/provenance/artifacts.sha256" +date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ" +echo "QWEN30_EXACT_TRACE_REAL_ANCHOR_COMPLETE" diff --git a/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py b/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py index 4d8d224..5ac9a6b 100644 --- a/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py +++ b/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py @@ -214,6 +214,24 @@ class FidelityEnvelopeTest(unittest.TestCase): ) self.assertEqual(steady["anchors"][0]["requests"], 20) + def test_exact_real_client_uses_the_prepared_row_vector_contract(self) -> None: + client = load("qwen30_exact_trace_client.py") + prepare = load("prepare_exact_trace.py") + rows = [ + { + "source_index": 7, + "arrived_at": 1.25, + "input_length": 32, + "output_length": 4, + "session_id": 3, + "runtime_block_ids": [11, 12], + } + ] + digest = client.hashlib.sha256() + prepare.update_digest(digest, [7, 1.25, 32, 4, 3, [11, 12]]) + self.assertEqual(client.row_vector_sha256(rows), digest.hexdigest()) + self.assertEqual(client.ttft_slo_ms(512), 1064.0) + if __name__ == "__main__": unittest.main() From 2aa713f5d6f72bba42ca824b67de32525c7968f8 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 11:55:12 +0800 Subject: [PATCH 106/148] Prepare sealed exact-trace real smoke --- runs/frontier-fidelity-envelope-v1/experiment-card.md | 8 ++++++++ .../run_qwen30_exact_trace_real_anchor.sh | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/runs/frontier-fidelity-envelope-v1/experiment-card.md b/runs/frontier-fidelity-envelope-v1/experiment-card.md index c1563db..0ef2d9b 100644 --- a/runs/frontier-fidelity-envelope-v1/experiment-card.md +++ b/runs/frontier-fidelity-envelope-v1/experiment-card.md @@ -48,6 +48,14 @@ F0 沿用已经冻结的 rate lattice与两个 fresh-server rounds。F1/F2 先 T1 保持原 600 秒 arrival window。先在 simulator 上冻结 `sampling_u` bracket；真机只运行 topological guard set `{TP1,TP2,TP4} × {MNS8,MNS32,MNS64}`，若 top set 或 bracket 仍可能被未测 MNS 反转再补 MNS16。每个入选 source-row vector在 real/sim 两侧必须有相同 digest；至少两个 session-hash folds，若本轮只完成一个 window则明确标为 single-window evidence。 +> **Pre-bracket smoke amendment（2026-07-17，任何 T1 真机结果产生前）：** +> 允许预先运行一个固定的 `TP4/MNS32 × u=0.005` real harness smoke。该点已属于上述 +> guard set，选择不依赖 simulator 输出；在完整 simulator surface 冻结前，性能结果保持 +> sealed，只检查进程退出状态、artifact 完整性、request usage 与 row-vector digest。该 smoke +> 不用于选择 config、移动负载 bracket 或形成 fidelity verdict；full real anchors 仍必须等待 +> simulator bracket 冻结。目的仅是让 600 秒真机 harness 验证与 CPU simulator surface 并行， +> 缩短 critical path。 + ## 诊断与停止规则 1. A1 必须报告每次 collective prediction 的 measured-model hit 与 analytical fallback 次数；不能只看最终 rank。 diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh index 8c03fe0..e194e0e 100755 --- a/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh @@ -39,6 +39,7 @@ REQUEST_COUNT="$(wc -l < "${REQUESTS_FILE}")" echo "QWEN30_EXACT_TRACE_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 trace=${TRACE_LABEL} requests=${REQUEST_COUNT} source=${REQUESTS_FILE} arrivals=original600s input_output_prompt=exact prefix=on block=16 tpot_slo=150ms output=${OUTPUT_ROOT} expected_wall=12-35m hard_wall=3600s" date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" sha256sum qwen30_exact_trace_client.py run_qwen30_exact_trace_real_anchor.sh \ + ../frontier-phase-factorial-v0/qwen30_prefill_client.py \ > "${OUTPUT_ROOT}/provenance/source.sha256" sha256sum "${REQUESTS_FILE}" > "${OUTPUT_ROOT}/provenance/requests.sha256" sha256sum "${MODEL_ROOT}/config.json" > "${OUTPUT_ROOT}/provenance/model.sha256" @@ -77,6 +78,14 @@ if [[ "${READY}" -ne 1 ]]; then exit 1 fi +# Warm execution kernels with four prefix-disjoint requests, then start the +# measured replay from an empty scheduler queue. The unrelated 2K-token KV +# footprint remains explicit and negligible relative to the configured cache. +"${VENV_ROOT}/bin/python" ../frontier-phase-factorial-v0/qwen30_prefill_client.py \ + --port "${SERVER_PORT}" --served-model "${SERVED_MODEL}" \ + --model-path "${MODEL_ROOT}" --rate 1 --requests 4 --input-tokens 512 \ + --output-tokens 1 --output "${OUTPUT_ROOT}/results/warmup.json" + "${VENV_ROOT}/bin/python" qwen30_exact_trace_client.py \ --port "${SERVER_PORT}" --requests-file "${REQUESTS_FILE}" \ --output "${OUTPUT_ROOT}/results/result.json" --tpot-slo-ms 150 \ From 03aa7944489e0e5221f89fa24839fb769309e53f Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 12:44:50 +0800 Subject: [PATCH 107/148] Isolate FlashInfer cache for trace replay --- .../run_qwen30_exact_trace_real_anchor.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh index e194e0e..1b582e0 100755 --- a/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh @@ -10,10 +10,12 @@ TRACE_LABEL="${TRACE_LABEL:?TRACE_LABEL is required}" SERVER_PORT="${SERVER_PORT:?SERVER_PORT is required}" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:-/tmp/wjh/flashinfer-workspace-vllm020-profiler-v1}" SERVED_MODEL="qwen3-30b-exact-trace" SERVER_PID="" -mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/results" +mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/results" \ + "${FLASHINFER_WORKSPACE_BASE}" exec > >(tee -a "${OUTPUT_ROOT}/logs/controller.log") 2>&1 cleanup() { @@ -36,7 +38,7 @@ if [[ "${#GPU_IDS[@]}" -ne "${TP}" ]]; then fi REQUEST_COUNT="$(wc -l < "${REQUESTS_FILE}")" -echo "QWEN30_EXACT_TRACE_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 trace=${TRACE_LABEL} requests=${REQUEST_COUNT} source=${REQUESTS_FILE} arrivals=original600s input_output_prompt=exact prefix=on block=16 tpot_slo=150ms output=${OUTPUT_ROOT} expected_wall=12-35m hard_wall=3600s" +echo "QWEN30_EXACT_TRACE_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 trace=${TRACE_LABEL} requests=${REQUEST_COUNT} source=${REQUESTS_FILE} arrivals=original600s input_output_prompt=exact prefix=on block=16 tpot_slo=150ms flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} output=${OUTPUT_ROOT} expected_wall=12-35m hard_wall=3600s" date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" sha256sum qwen30_exact_trace_client.py run_qwen30_exact_trace_real_anchor.sh \ ../frontier-phase-factorial-v0/qwen30_prefill_client.py \ @@ -51,6 +53,7 @@ export VLLM_USE_V1=1 export TORCH_CUDA_ARCH_LIST=9.0 export HF_HUB_OFFLINE=1 export TRANSFORMERS_OFFLINE=1 +export FLASHINFER_WORKSPACE_BASE ulimit -n 65536 setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \ From 903cbe682d7a8bc2accaba9fdbf1786f4e4a5d48 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 12:46:57 +0800 Subject: [PATCH 108/148] Record Frontier trace stalls without ranking them --- ...run_frontier_qwen30_exact_trace_surface.py | 86 +++++++++++++++++-- .../test_fidelity_envelope.py | 10 +++ 2 files changed, 90 insertions(+), 6 deletions(-) diff --git a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py index 02e30a9..fc102a7 100644 --- a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py +++ b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py @@ -68,6 +68,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--allreduce-csv", type=Path) parser.add_argument("--timeout-seconds", type=float, default=1800.0) parser.add_argument("--resume", action="store_true") + parser.add_argument("--continue-on-failure", action="store_true") return parser.parse_args() @@ -156,6 +157,12 @@ def trace_manifest_entry(trace: dict[str, Any]) -> dict[str, Any]: } +def classify_frontier_failure(stderr: str) -> str: + if "Sequential simulation ended with non-empty scheduler state" in stderr: + return "scheduler_stall" + return "frontier_error" + + def score(path: Path, expected_shapes: list[tuple[int, int]]) -> dict[str, Any]: with path.open(newline="") as source: rows = list(csv.DictReader(source)) @@ -312,9 +319,48 @@ def main() -> None: check=False, ) if completed.returncode != 0: - raise RuntimeError( - f"Frontier failed for {config.name}/{trace['label']}: {completed.returncode}" + stderr_path = run_dir / "stderr.log" + result = { + "status": "frontier_failed", + "failure_kind": classify_frontier_failure( + stderr_path.read_text(errors="replace") + ), + "returncode": completed.returncode, + "config": { + "tp": config.tp, + "mns": config.mns, + "name": config.name, + }, + "trace_label": trace["label"], + "offered_request_rate": trace["offered_request_rate"], + "offered_request_rate_per_gpu": ( + trace["offered_request_rate"] / config.tp + ), + "request_count": trace["requests"], + "elapsed_seconds": time.time() - started, + "trace_sha256": trace["sha256"], + "stderr_sha256": BASE.sha256(stderr_path), + } + BASE.write_json(result_path, result) + loads.append(result) + print( + json.dumps( + { + "config": config.name, + "trace": trace["label"], + "status": result["status"], + "failure_kind": result["failure_kind"], + }, + sort_keys=True, + ), + flush=True, ) + if not args.continue_on_failure: + raise RuntimeError( + f"Frontier failed for {config.name}/{trace['label']}: " + f"{completed.returncode}" + ) + continue metrics = find_request_metrics(run_dir) result = { "status": "completed", @@ -353,21 +399,38 @@ def main() -> None: for slo in (f"tpot_{int(value)}ms" for value in TPOT_SLOS_MS): records = [] for item in config_results: + completed_loads = [ + load for load in item["loads"] if load["status"] == "completed" + ] + invalid_loads = [ + load for load in item["loads"] if load["status"] != "completed" + ] feasible = [ load["offered_request_rate"] - for load in item["loads"] + for load in completed_loads if load["score"]["slos"][slo]["feasible"] ] capacity = max(feasible, default=None) records.append( { "config": item["config"], + "ranking_valid": not invalid_loads, + "invalid_loads": [ + { + "trace_label": load["trace_label"], + "failure_kind": load.get("failure_kind", "unknown"), + } + for load in invalid_loads + ], "maximum_tested_feasible_request_rate": capacity, "maximum_tested_feasible_request_rate_per_gpu": ( capacity / item["config"]["tp"] if capacity is not None else None ), - "lower_censored": capacity is None, - "upper_censored": capacity == traces[-1]["offered_request_rate"], + "lower_censored": capacity is None and not invalid_loads, + "upper_censored": ( + capacity == traces[-1]["offered_request_rate"] + and not invalid_loads + ), } ) records.sort( @@ -382,9 +445,20 @@ def main() -> None: ) rankings[slo] = records + has_invalid_cells = any( + load["status"] != "completed" + for item in config_results + for load in item["loads"] + ) + if selected != list(BASE.GRID): + manifest_status = "partial_not_decision_bearing" + elif has_invalid_cells: + manifest_status = "frozen_with_invalid_cells" + else: + manifest_status = "frozen_before_real" manifest = { "schema": "frontier-qwen30-exact-trace-surface-v1", - "status": "frozen_before_real" if selected == list(BASE.GRID) else "partial_not_decision_bearing", + "status": manifest_status, "contract": { "window_seconds": ( WINDOW_SECONDS if args.rate_contract == "trace-window" else None diff --git a/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py b/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py index 5ac9a6b..ac0f3e0 100644 --- a/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py +++ b/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py @@ -106,6 +106,16 @@ class FidelityEnvelopeTest(unittest.TestCase): def test_exact_trace_parser_and_joint_slo_score(self) -> None: module = load("run_frontier_qwen30_exact_trace_surface.py") + self.assertEqual( + module.classify_frontier_failure( + "Sequential simulation ended with non-empty scheduler state" + ), + "scheduler_stall", + ) + self.assertEqual( + module.classify_frontier_failure("unexpected failure"), + "frontier_error", + ) with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) trace = root / "trace.csv" From 8b054ffcb118220439128c65e12e603279d7c4a2 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 13:02:08 +0800 Subject: [PATCH 109/148] Bound Frontier predictor training parallelism --- .../run_frontier_qwen30_exact_trace_surface.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py index fc102a7..a504d8d 100644 --- a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py +++ b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py @@ -67,6 +67,7 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument("--allreduce-csv", type=Path) parser.add_argument("--timeout-seconds", type=float, default=1800.0) + parser.add_argument("--predictor-training-job-threads", type=int, default=1) parser.add_argument("--resume", action="store_true") parser.add_argument("--continue-on-failure", action="store_true") return parser.parse_args() @@ -222,6 +223,8 @@ def score(path: Path, expected_shapes: list[tuple[int, int]]) -> dict[str, Any]: def main() -> None: args = parse_args() + if args.predictor_training_job_threads <= 0: + raise ValueError("predictor training job threads must be positive") for name in ( "frontier_source", "replayserve_root", @@ -298,6 +301,12 @@ def main() -> None: run_id=f"qwen30_trace_{config.name}_{trace['label']}", knobs=config_knobs, ) + command.extend( + [ + "--random_forrest_execution_time_predictor_config_num_training_job_threads", + str(args.predictor_training_job_threads), + ] + ) command = BASE.configure_cc_command( command, backend=args.cc_backend, @@ -471,6 +480,7 @@ def main() -> None: "tpot_slos_ms": TPOT_SLOS_MS, "primary_tpot_slo_ms": 150.0, "target_pass_rate": TARGET_PASS_RATE, + "predictor_training_job_threads": args.predictor_training_job_threads, }, "frontier": { "source": str(args.frontier_source), From e7b482658e8aa9ec34880075abaede96a3d96a2a Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 13:29:18 +0800 Subject: [PATCH 110/148] Prepare cache-consistent T1 real smoke --- .../jobs_t1_real_smoke.toml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 runs/frontier-fidelity-envelope-v1/jobs_t1_real_smoke.toml diff --git a/runs/frontier-fidelity-envelope-v1/jobs_t1_real_smoke.toml b/runs/frontier-fidelity-envelope-v1/jobs_t1_real_smoke.toml new file mode 100644 index 0000000..d366eb2 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/jobs_t1_real_smoke.toml @@ -0,0 +1,22 @@ +version = 1 + +[[jobs]] +name = "qwen30-t1-exact-real-smoke-tp4-mns32-u0p005-20260717-v2-cache-consistent" +gpus = 4 +gpu_model = "H20" +hosts = ["dash0"] +command = "cd /home/admin/cpfs/wjh/aituner/aituner-qwen30-vllm020-profile-v1/runs/frontier-fidelity-envelope-v1 && timeout --signal=TERM --kill-after=30s 3600 bash run_qwen30_exact_trace_real_anchor.sh" + +[jobs.env] +HOME = "/tmp/wjh" +XDG_CACHE_HOME = "/tmp/wjh/.cache" +VLLM_CACHE_ROOT = "/tmp/wjh/.cache/vllm" +FLASHINFER_WORKSPACE_BASE = "/tmp/wjh" +TP = "4" +MNS = "32" +TRACE_LABEL = "u0p005" +SERVER_PORT = "8124" +OUTPUT_ROOT = "/home/admin/cpfs/wjh/aituner/fidelity-envelope-private/trace-exact-v1/real-sealed-prebracket/tp4_mns32/u0p005/round1-retry2-cache-consistent" +REQUESTS_FILE = "/home/admin/cpfs/wjh/aituner/fidelity-envelope-private/trace-exact-v1/private/u0p005/real_requests.jsonl" +VENV_ROOT = "/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1" +MODEL_ROOT = "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B" From d3be91dd585e03096d53aa9b3269a7953043ec6f Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 22:42:57 +0800 Subject: [PATCH 111/148] Audit Frontier prefix-cache trace contract --- .../qwen30_exact_trace_client.py | 10 + ...run_frontier_qwen30_exact_trace_surface.py | 39 ++- .../experiment-card.md | 228 +++++++++++++++++ .../materialize_qwen30_tp_normalized_trace.py | 240 ++++++++++++++++++ .../qwen30-tracepd-u0p01-card.md | 191 ++++++++++++++ 5 files changed, 705 insertions(+), 3 deletions(-) create mode 100644 runs/simulator-tuning-latency-matrix-v0/experiment-card.md create mode 100644 runs/simulator-tuning-latency-matrix-v0/materialize_qwen30_tp_normalized_trace.py create mode 100644 runs/simulator-tuning-latency-matrix-v0/qwen30-tracepd-u0p01-card.md diff --git a/runs/frontier-fidelity-envelope-v1/qwen30_exact_trace_client.py b/runs/frontier-fidelity-envelope-v1/qwen30_exact_trace_client.py index c23c66d..b130b73 100644 --- a/runs/frontier-fidelity-envelope-v1/qwen30_exact_trace_client.py +++ b/runs/frontier-fidelity-envelope-v1/qwen30_exact_trace_client.py @@ -21,6 +21,11 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--port", type=int, required=True) parser.add_argument("--requests-file", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--served-model", + default=None, + help="Optional server-side model alias used only for HTTP routing.", + ) parser.add_argument("--tpot-slo-ms", type=float, default=150.0) parser.add_argument("--timeout-seconds", type=float, default=1800.0) return parser.parse_args() @@ -64,6 +69,7 @@ async def request_one( scheduled_at: float, benchmark_start: float, tpot_slo_ms: float, + served_model: str | None, ) -> dict[str, Any]: loop = asyncio.get_running_loop() delay = scheduled_at - loop.time() @@ -81,6 +87,8 @@ async def request_one( "success": False, } body = dict(row["body"]) + if served_model is not None: + body["model"] = served_model body.update( { "temperature": 0, @@ -178,6 +186,7 @@ async def replay(args: argparse.Namespace, rows: list[dict[str, Any]]) -> list[d scheduled_at=benchmark_start + float(row["arrived_at"]), benchmark_start=benchmark_start, tpot_slo_ms=args.tpot_slo_ms, + served_model=args.served_model, ) ) for row in rows @@ -219,6 +228,7 @@ def main() -> None: "last_arrival_s": arrivals[-1], "arrival": "original_trace_timestamp_and_order", "input_output_prompt": "exact_source_values", + "served_model_alias": args.served_model, "ttft_slo": "1000ms + 1000ms * input_tokens / 8000", "tpot_slo_ms": args.tpot_slo_ms, "target_pass_rate": TARGET_PASS_RATE, diff --git a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py index a504d8d..8afd928 100644 --- a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py +++ b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py @@ -84,8 +84,21 @@ def percentile(values: list[float], fraction: float) -> float | None: return ordered[math.ceil(fraction * len(ordered)) - 1] +def manifest_offered_rate(path: Path) -> tuple[float | None, str | None]: + manifest_path = path.with_name("manifest.json") + if not manifest_path.is_file(): + return None, None + manifest = json.loads(manifest_path.read_text()) + if manifest.get("public_csv") != str(path): + return None, None + rate = manifest.get("global_offered_request_rate") + if not isinstance(rate, (int, float)) or not math.isfinite(rate) or rate <= 0: + raise ValueError(f"invalid global_offered_request_rate in {manifest_path}") + return float(rate), str(manifest_path) + + def parse_trace( - specification: str, *, rate_contract: str = "trace-window" + specification: str, *, rate_contract: str = "trace-window", prefix_caching: bool ) -> dict[str, Any]: if "=" not in specification: raise ValueError(f"trace must be LABEL=PATH: {specification}") @@ -114,7 +127,12 @@ def parse_trace( if any(right < left for left, right in zip(arrivals, arrivals[1:])): raise ValueError(f"arrival order drift in {path}") if rate_contract == "trace-window": - offered_request_rate = len(rows) / WINDOW_SECONDS + offered_request_rate, rate_manifest = manifest_offered_rate(path) + if offered_request_rate is None: + offered_request_rate = len(rows) / WINDOW_SECONDS + rate_source = "legacy_fixed_600_second_window" + else: + rate_source = f"manifest:{rate_manifest}" elif rate_contract == "uniform-spacing": if len(rows) < 2 or arrivals[-1] <= arrivals[0]: raise ValueError("uniform-spacing traces require at least two arrivals") @@ -123,6 +141,7 @@ def parse_trace( if any(abs(value - expected_interval) > 1e-9 for value in intervals): raise ValueError(f"non-uniform fixed trace: {path}") offered_request_rate = 1.0 / expected_interval + rate_source = "uniform_spacing" else: raise ValueError(f"unknown rate contract: {rate_contract}") shapes = [ @@ -131,12 +150,22 @@ def parse_trace( ] if any(prefill <= 0 or decode <= 0 or prefill + decode > 40960 for prefill, decode in shapes): raise ValueError(f"out-of-contract shape in {path}") + if prefix_caching: + for index, (row, (prefill, _)) in enumerate(zip(rows, shapes, strict=True)): + block_ids = [value for value in row["block_hash_ids"].split("|") if value] + if len(block_ids) != prefill // 16: + raise ValueError( + "Frontier prefix-cache trace must expose only complete " + f"16-token blocks: row={index}, ISL={prefill}, " + f"ids={len(block_ids)}, expected={prefill // 16}" + ) return { "label": label, "path": path, "sha256": BASE.sha256(path), "requests": len(rows), "offered_request_rate": offered_request_rate, + "offered_request_rate_source": rate_source, "first_arrival_s": arrivals[0], "last_arrival_s": arrivals[-1], "shapes": shapes, @@ -236,7 +265,11 @@ def main() -> None: if args.allreduce_csv is not None: args.allreduce_csv = args.allreduce_csv.resolve() traces = [ - parse_trace(specification, rate_contract=args.rate_contract) + parse_trace( + specification, + rate_contract=args.rate_contract, + prefix_caching=args.prefix_caching, + ) for specification in args.trace ] if len({trace["label"] for trace in traces}) != len(traces): diff --git a/runs/simulator-tuning-latency-matrix-v0/experiment-card.md b/runs/simulator-tuning-latency-matrix-v0/experiment-card.md new file mode 100644 index 0000000..72edd6e --- /dev/null +++ b/runs/simulator-tuning-latency-matrix-v0/experiment-card.md @@ -0,0 +1,228 @@ +# EXP-SIM-TUNING-LATENCY-MATRIX-V0：现有 simulator 能否选择真机低延迟 serving config？ + +> **状态：** protocol 已冻结；Qwen3.6 release-runtime topology smoke 已通过（2026-07-17）。 +> 没有 Frontier Qwen3.6 contract/profile、trace replay 或 latency-selection 结果；本 card +> 不授权 600-s matrix launch。 + +## Claim 与决策 + +- **研究问题：** 对冻结的 `(H20 hardware, model, serving engine, request trace, config + candidate surface)`，现有 simulator 能否为一个明确的 latency objective 选出真机最优、或 + 统计上等价的 serving-engine config？ +- **Parent claim：** “simulator 已经解决 config tuning”只有在完整 candidate surface 上产生 + 正确的真机 latency selection 时才成立；绝对 latency error 或单个吞吐/SLO 结果均不充分。 +- **Hypothesis：** simulator 在 fixed-shape / prefill-only 条件下可能足够，但在 decode、真实 + arrival 与 prefix/KV state 同时存在时会因 state-conditioned residual 或 coverage failure 而 + 失去 selection fidelity。 +- **Competing hypothesis：** 经冻结、非 evaluation 的 operator/collective calibration 后， + simulator 能跨 non-MoE hybrid/MoE 与四种 workload condition 稳定选出真机低延迟 config。 +- **判定：** 每个 `model × workload × metric` 是独立 tuning task，且 config 越低 latency 越好。 + 只有所有预注册且 simulator 声称支持的 task 均通过下面的 coverage 与 selection gates，才能 + 写“该 simulator 已解决其声明范围内的 tuning”；任一失败 cell 是明确 boundary，不被平均值 + 掩盖。 + +## 固定系统与模型覆盖 + +| 角色 | 状态 | 运行时契约 | 当前可复用 artifact / 含义 | +|---|---|---|---| +| MoE | **可开始准备** | Qwen3-30B-A3B，BF16，community vLLM 0.20.0，dash0 H20；Frontier/Vidur-class fork | `frontier-phase-factorial-v0` 有 `ISL=2048, OSL=1, r=4` 的 request-level real/sim pilot；`frontier-fidelity-envelope-v1` 有 fixed-PD 与 exact Trace-PD materializers。历史 pilot 不是本 card 的最终 600-s evaluation。 | +| Non-MoE hybrid (not all-attention dense) | **TP1/2/4 smoke passed; profile pending** | `Qwen/Qwen3.6-27B@cea40373b9214dd387123e68841890af30dcd469`，community release `vLLM 0.20.2` / Torch `2.11.0` / Transformers `5.14.1`，dash0 H20；Frontier profile 必须在此 exact stack 采集 | Exact release serves text-only 2048→1 at TP1/2/4. vLLM resolves a 784-token attention page for its GDN/Mamba state, so requested block=16 is not the physical engine state. Qwen3.6 trace, Frontier hybrid model contract, and profile remain missing. | + +所有公开结果固定在单机 `dash0`（8×H20）；每个 model 使用相同 engine build 的 real +server 与 simulator contract。Qwen3.6 bring-up 已确认真实 runtime/page legality；其余最低成本 +gate 是冻结 trace/tokenizer 与完成 Frontier hybrid profile-schema audit。若不能做到，论文结论 +必须限于 MoE，不得写跨模型 generalization。 + +### Qwen3.6 prerequisite audit (runtime + smoke, 2026-07-17) + +| Prerequisite | Evidence | Verdict | Decision | +|---|---|---|---| +| Exact model snapshot | Immutable ModelScope snapshot `Qwen/Qwen3.6-27B@cea40373b9214dd387123e68841890af30dcd469` is staged at `/home/admin/cpfs/wjh/models/Qwen/Qwen3.6-27B`; its sorted-file inventory digest is `031cea103726cbf27732b6971e898e210aee77f0c32e4bc703c947e0372349c4`. | **PASS** | Bind every Qwen3.6 trace, real server, and profile manifest to this revision and inventory. | +| Community vLLM runtime | Clean environment `/home/admin/cpfs/wjh/venvs/qwen36-vllm-0.20.2` contains official wheel `vLLM 0.20.2`, Torch `2.11.0`, and Transformers `5.14.1`; `pip check` passes and CPU-only `get_config(..., trust_remote_code=false)` resolves `Qwen3_5ForConditionalGeneration`. | **PASS (CPU)** | Use exactly this release environment for real serving and profile collection. | +| Topology compatibility | Fresh TP1/2/4 servers each completed a text-only 2048-prompt/1-completion request with exact usage; all GPUs were released after cleanup. Every topology resolved the attention page from requested 16 to 784 tokens for Mamba/GDN state. | **PASS (GPU smoke)** | This is an engine-configuration legality result, not a latency rank or simulator result. Profile/simulator input must expose the 784-token state/page contract. | +| Qwen3.6 Frontier model/profile | No local or remote Qwen3.6-named Frontier profile/model directory was found | **MISSING** | New dense profile root is required; Qwen30/MoE profile CSVs are forbidden inputs. | +| Old dense artifacts | Existing Qwen3.5-27B trace is internal-vLLM/hybrid/speculative | **N/A as baseline** | Keep only as historical context; never feed it to this matrix or profile calibration. | + +This audit is not a failed simulator result. The topology gate is closed, but the model contract, +profile root, and frozen trace are still reproducibility blockers: the Qwen3.6 half is `NOT RUN`, +not `FAIL`. For a paper claim about a conventional all-attention dense transformer, Qwen3.6 is not +admissible evidence; it is only a non-MoE hybrid case. + +## Workload matrix + +每个 model 都运行四个固定 trace。`P` 是 PD disaggregation 的 P-node workload，不是缺失 +decode 的坏数据；它的 `OSL=1` 是设计变量。固定 cases 与 trace cases都使用 open-loop replay， +600 秒 observation window；`Fixed` 的 request count 为 `ceil(QPS × 600)`。 + +| ID | trace contract | output contract | prefix/KV contract | 可评 latency objectives | +|---|---|---|---|---| +| Fixed-P | 固定 `ISL=2048`、uniform fixed QPS=`4 req/s`，token vector deterministic | `OSL=1` | no shared logical prefix；prefix cache off | mean/p90 TTFT；mean/p90 E2E；TPOT=`N/A` | +| Fixed-PD | 固定 `ISL=2048`、uniform fixed QPS=`4 req/s` | `OSL=128`，`min=max=128`，`ignore_eos=true` | no shared logical prefix；prefix cache off | mean/p90 TTFT、TPOT、E2E | +| Trace-P | held-out production trace 的逐 request prompt-token vector、arrival time、request order、session root 与 prefix block relation 原样保留 | **只将 replay output 强制为 `OSL=1`** | source prefix equivalence、warm/cold state、cache policy 原样保留 | mean/p90 TTFT；mean/p90 E2E；TPOT=`N/A` | +| Trace-PD | held-out production trace 的逐 request input/output lengths、arrival time、order、session root 与 prefix block relation 原样保留 | 原始 `min=max=source OSL`，`ignore_eos=true` | source prefix equivalence、warm/cold state、cache policy 原样保留 | mean/p90 TTFT、TPOT、E2E | + +`Trace-P` 应明确称为 **trace-derived prefill-only**：它忠实保留输入、到达、会话与 cache +state，但不忠实保留原始 output-length distribution；不能被描述成完整 trace-faithful replay。 +`Trace-PD` 才是完整 input/output/arrival/prefix-KV joint replay。 + +### Trace、correctness 与 cache 的不可变契约 + +- 每个 trace manifest 列出：request ID、arrival timestamp、input/output token counts、session + ID、runtime block identities、trace vector SHA-256、source trace SHA-256、requested and + resolved physical page/block size、cache policy、initial cache state、warmup trace digest 与 + model tokenizer/version。 +- 当 simulator 要模拟 vLLM prefix caching 时，`block_hash_ids` 只能描述可复用的**完整** + runtime blocks：adapter 必须导出 `floor(ISL / block_size)` 个 prompt identities，不能将 + `ceil` 得到的 partial final block 当作 cache hit。该 simulator-facing projection 有独立 + digest；private real replay 仍保留完整 prompt 与原始 token vector。 +- `Trace-P` 产生单独 manifest：source OSL 和 replay OSL 都写入 digest；只允许 source + OSL → 1 这一个变换。`Trace-PD`、real client 与 simulator CSV 的 row-vector digest 必须完全 + 相同。 +- 真实端逐 request 核验 prompt/completion usage 与 manifest 一致；simulator 核验 request ID、 + input/output shapes、完成数与 trace digest。一项不符即该 cell 的 metric 无效。 +- 每个 `(config, workload, trial)` fresh server；server warmup 用独立 manifest，warmup request + 不进入指标，也不得改变 evaluation cache 初态。prefix-on trace 以同一明示 initial cache + state重放，不能用随机前史替代。 + +## Candidate-surface 与 baseline 规则 + +1. 对每个 model 冻结 `C = TP∈{1,2,4} × MNS∈{8,16,32,64}`（12 个 configs），并固定 + `DP=PP=EP=1`、MBT=8192、chunked prefill on。每个 candidate 还必须冻结 **真实 engine + resolved** page/block and state-cache contract；requested CLI `--block-size 16` 不能替代它。 + Qwen3.6 smoke 在所有 TP 都将其改写为 784 tokens。任何不被某 model/engine 静态支持的 + config 要在运行前从该 model 的 `C` 整体删除，并记录理由；不能看结果后删除。 +2. `C_real-valid` 是 server launch 成功、所有三个 fresh trials 全部请求完成且 usage/hash 合法的 + configs。真机 OOM、timeout 或 request error 保留为 real infeasible/safety outcome，而非给它 + 赋无限 latency。 +3. 对每个 simulator，selection coverage 必须涵盖全部 `C_real-valid`。crash、scheduler stall、 + trace mismatch、缺 request metrics 都是 simulator **coverage failure**，不可当成高 latency 或 + infeasible 来产生排序。只有完整 coverage 才能形成 tuning verdict。 +4. Frontier、官方 Vidur 与 APEX 分别独立报告版本、patch、profile 与 capability surface。APEX + 若只支持 DP/PP/TP parallel plan，则它只评 `parallel-plan tuning` 交集；不得用该子问题结果 + 宣称解决 MNS/MBT/cache 等完整 engine tuning。 + +## Metrics、统计与 selection score + +每个完整 request 使用同一 client-boundary 定义： + +- `TTFT`：客户端发出请求至收到第一个 generated token；包含 serving queue。 +- `E2E`：客户端发出请求至完成 `[DONE]`。 +- `TPOT`：`(E2E - TTFT) / (completion_tokens - 1)`；只对 `completion_tokens > 1` 的请求定义。 +- 每 trial 的 `mean` 是 request-level arithmetic mean；`p90` 是 nearest-rank + `ceil(0.90 × N)`。P-only 的 TPOT 是 JSON `null` / table `N/A`，绝不写 0。 + +同一 config 的 real 运行三次 fresh-server trial，随机化 config order；每个 trial 都对整个 +trace 计算六项或四项指标。real ranking point estimate 是三个 trial metric 的 median，且报告 +pooled request statistic、每-trial statistic 和 95% hierarchical bootstrap interval（先重采样 +trial，再在 trial 内重采样 request）。simulator 的 seed、determinism 和 repeat policy同样记录。 + +对每一个 objective，输出：absolute real/sim surface、real bootstrap top set、simulator exact +top set、top-set hit、sim-chosen config 的 best/worst tie-break regret、Kendall τ-b、non-tied +pair direction accuracy、coverage 与 error/infeasible table。Latency regret 定义为 +`(real_metric(sim_choice) - real_metric(real_best)) / real_metric(real_best)`。 + +## 预注册 decision gates + +| Gate | Pass | Failure handling | +|---|---|---| +| Trace/metric validity | 100% planned requests have matching usage/shape and finite applicable metrics | cell invalid；诊断 harness，不作 performance conclusion | +| Simulator coverage | 100% of `C_real-valid` output legal request metrics | `FAIL: coverage`；不把 missing cell 排成最差 | +| Selection | simulator top set 与 bootstrap real top set 有交集 | `FAIL: selection` | +| Risk | simulator top-set 的 **worst** real tie-break regret `≤5%` | `FAIL: regret`，即使 best tie-break 碰巧为 0 | +| Global rank diagnostic | τ-b `≥0.8`；若没有非并列 pair 则写 `N/A`，不当通过 | `NEEDS EVIDENCE` / `FAIL` 与 top-set gate 分开报告 | + +这里不使用 SLO、capacity 或 throughput 作 selection objective；它们最多作为安全/诊断附录指标。 +P-only 共 4 tasks，PD 共 6 tasks，所以完整 `non-MoE-hybrid/MoE × 4 workloads` matrix 共有 40 个 +独立 latency-selection tasks。 + +## Calibration 与 information boundary + +- 允许 simulator 使用同硬件/同 model/同 runtime 的 offline operator、collective 或 memory + profiles，但 profile command、shape grid、cost 和 hash 必须在 evaluation 前冻结。 +- calibration 不得读取 evaluation 的 real request latency、top config 或 E2E scalar；禁止 per-TP + 或 per-config serving E2E scale。若有 trace-driven calibration，必须只使用按 session root 切分 + 的 calibration fold；Trace-P/PD 使用完全不相交的 held-out root sessions。 +- Fixed traces 与 trace evaluation 都不能同时作为 calibration workload。任何 required simulator + patch 在查看该 cell real result 前冻结；patched baseline 与 original release 分开作图。 + +## Minimal Qwen3.6-27B bring-up and profile plan + +This is the only permitted path to add the dense case. It intentionally separates cheap provenance +work from GPU work, and it does not authorize either stage by itself. + +1. **Preflight and topology smoke (complete):** the snapshot/runtime are locked; fresh TP1/2/4 + servers completed exact-usage text requests. The actual vLLM state/page setting is 784 tokens, + not the requested 16. This establishes the profile input contract only. +2. **Hybrid Frontier contract (not run):** verify Frontier can represent Gated DeltaNet/linear-state, + gated attention, FFN, true mixed execution, TP collectives, the 784-token page rule and capacity, + without substituting Qwen3.5/Qwen30 fields. +3. **Minimal profile set (GPU, separately approved):** on the exact Qwen3.6/community-vLLM/dash0 + stack, collect per-TP Gated-DeltaNet, gated-attention prefill/decode, state/KV and TP2/TP4 all-reduce + rows. The shape grid must cover the pre-registered Fixed-P/PD and held-out Trace-P/PD ranges, but + must not contain evaluation E2E latency or a fitted time scale. Freeze CSV/schema/command hashes + before running any evaluation trace. +4. **Profile acceptance (CPU):** verify model/runtime identity in every profile manifest; require + requested shape coverage, finite values, and no fallback to a MoE or Qwen3.5 profile. Only then may + a parent-approved real/simulator latency surface be scheduled. + +## 预期产物、复现与成本 + +- **Figure prototype：** [latency-selection-matrix-schematic.svg](latency-selection-matrix-schematic.svg) + （明确标注 `SCHEMATIC — NOT MEASURED DATA`）。最终图替换为每个 task 的 top-set hit / regret / + coverage heatmap，并附 per-config latency surfaces。 +- **Raw layout：** `artifacts/////trial-*/`；每个目录含 + command、environment/machine fingerprint、server flags、trace/profile hash、raw request records、 + stdout/stderr、duration 与 SHA-256 inventory。 +- **Existing reuse audit：** MoE F0 可检验 P-only raw metric reader；现有 F2 可检验 fixed-PD + trace materialization；T1 可检验 Trace-PD digest and simulator coverage reporting。它们并不构成 +这个 600-s no-SLO matrix 的完整 real surface。Qwen3.6 requires a new same-stack trace, + model contract and profile root; old Qwen3.5 internal-vLLM artifacts are excluded. +- **Cost estimate, not authorization：** 单 config×trial 假定 `10 min replay + 5 min fresh-server + startup/warmup = 15 min`。完整 12-config sweep 的 GPU allocation 为 + `4 × (TP1 + TP2 + TP4) = 28 GPU`; 因而每个 `model×workload` 的三 trial 约 + `28 × 0.25 × 3 = 21 H20-GPUh`。8 cells 的 real serving core is **168 H20-GPUh**. The Qwen3.6 + bring-up/profile set adds at most **6 H20-GPUh** (three topology smokes plus dense/attention/ + collective rows), giving a **174 H20-GPUh** core estimate and a **190–215 H20-GPUh** planning + envelope with retries. Every launch must recompute this from the staged trace duration and actual + server preflight; this paragraph is not authorization. + +### Required remote launch echo (not executed) + +```text +NOT EXECUTED — awaiting explicit approval after Qwen3.6 model/runtime/profile gates +host=dash0; remote_root=/home/admin/cpfs/wjh/aituner/aituner +models=non-moe-hybrid:Qwen/Qwen3.6-27B@,moe:Qwen3-30B-A3B-BF16 +engine=community-open-source-vLLM@; simulator=Frontier/Vidur-class, +workloads=Fixed-P(ISL2048,OSL1,QPS4,600s),Fixed-PD(ISL2048,OSL128,QPS4,600s), + Trace-P(heldout exact input/arrival/prefix,OSL1),Trace-PD(heldout exact joint trace) +surface=TP{1,2,4}×MNS{8,16,32,64}; trials=3; fresh_server=true +real_paths=runs/simulator-tuning-latency-matrix-v0/artifacts///real/ +expected_cost=174 H20-GPUh core; 190–215 H20-GPUh planning envelope; expected_wall=multiple queued hours +gate=Qwen3.6 snapshot + community-vLLM fingerprint + resolved-784 hybrid Frontier profile completeness + trace/profile hashes + user/parent approval +``` + +## Expected outcomes and stop/pivot rules + +- If a simulator passes every supported task, stop diagnosing that simulator and look for a new problem + outside its evaluated scope (e.g., unsupported engine knobs or alignment cost). +- If it fails coverage, repair/liveness is a separate intervention; original capability result remains + frozen. First rerun simulator only; do not spend GPU-hours until it can produce a complete surface. +- If it covers but selects incorrectly, retain the counterexample and run one minimal discriminative + mechanism experiment (scheduler state, batch composition, graph/kernel family, or MoE/collective), + changing one factor at a time. +- If Qwen3.6 cannot be profiled on its exact community-vLLM stack, stop the non-MoE-hybrid half rather + than treating any internal-vLLM or Qwen3.5 result as a substitute. If the headline requires a pure + all-attention dense-transformer case, add that model explicitly rather than relabelling Qwen3.6. + +## Benchmark-design audit + +| Risk | Pre-registered control | +|---|---| +| SLO/capacity result misrepresented as tuning | selection uses only per-request mean/p90 TTFT, TPOT, E2E on the same trace | +| P-only judged by nonexistent TPOT | OSL=1 and TPOT=`N/A` are explicit workload/metric contracts | +| Trace simplification hidden | Trace-P is labelled output-normalized; only Trace-PD claims full joint replay | +| calibration equals evaluation | microprofile/trace calibration folds are disjoint; no E2E correction on evaluation | +| simulator crash treated as a bad config | coverage gate separates missing metrics from latency ranking | +| cross-stack or wrong-architecture reuse | only Qwen3.6-27B hybrid profiles collected on its exact community-vLLM stack are admissible; old Qwen3.5/internal-vLLM or Qwen30/MoE artifacts are excluded | +| selected winning cells only | all 8 cells and all legal configs are frozen before real results | +| tail/noise omitted | p90, three fresh trials, per-trial values and bootstrap top sets are mandatory | diff --git a/runs/simulator-tuning-latency-matrix-v0/materialize_qwen30_tp_normalized_trace.py b/runs/simulator-tuning-latency-matrix-v0/materialize_qwen30_tp_normalized_trace.py new file mode 100644 index 0000000..cc7fe98 --- /dev/null +++ b/runs/simulator-tuning-latency-matrix-v0/materialize_qwen30_tp_normalized_trace.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +"""Derive a per-GPU-normalized real/Frontier Qwen30 trace pair. + +Arrival times change as ``t' = t / TP``. The private JSONL preserves the +complete prompt identity vector for real replay; the Frontier CSV exports only +the completed 16-token prompt blocks that vLLM can legally reuse from prefix +cache. Prompt bodies remain exclusively in the private output. +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +from pathlib import Path +from typing import Any + + +CSV_FIELDS = ( + "arrived_at", + "num_prefill_tokens", + "num_decode_tokens", + "session_id", + "block_hash_ids", +) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def update_digest(digest: Any, values: list[Any]) -> None: + digest.update(json.dumps(values, separators=(",", ":")).encode()) + digest.update(b"\n") + + +def full_prompt_block_ids( + runtime_block_ids: list[int], input_tokens: int, block_size: int +) -> list[int]: + """Project exact prompt identities to vLLM-cacheable full blocks only.""" + if input_tokens <= 0 or block_size <= 0: + raise ValueError("input_tokens and block_size must be positive") + expected_runtime_blocks = math.ceil(input_tokens / block_size) + if len(runtime_block_ids) != expected_runtime_blocks: + raise ValueError( + "runtime block identity count does not match the exact prompt: " + f"got {len(runtime_block_ids)}, expected {expected_runtime_blocks}" + ) + return runtime_block_ids[: input_tokens // block_size] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--base-public-csv", type=Path, required=True) + parser.add_argument("--base-private-jsonl", type=Path, required=True) + parser.add_argument("--tp", type=int, required=True) + parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument("--base-window-seconds", type=float, default=600.0) + parser.add_argument("--runtime-block-size", type=int, default=16) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + if ( + args.tp <= 0 + or args.runtime_block_size <= 0 + or not math.isfinite(args.base_window_seconds) + or args.base_window_seconds <= 0 + ): + raise ValueError("TP, runtime block size, and base window must be positive") + + public_rows = list(csv.DictReader(args.base_public_csv.open(newline=""))) + private_rows = [ + json.loads(line) for line in args.base_private_jsonl.open() if line.strip() + ] + if not public_rows or len(public_rows) != len(private_rows): + raise ValueError("public/private request counts must match and be non-zero") + if set(public_rows[0]) != set(CSV_FIELDS): + raise ValueError("unexpected public trace schema") + + normalized_digest = hashlib.sha256() + semantic_digest = hashlib.sha256() + normalized_private: list[dict[str, Any]] = [] + normalized_public: list[dict[str, str]] = [] + original_arrivals: list[float] = [] + frontier_prefix_digest = hashlib.sha256() + partial_prompt_count = 0 + raw_runtime_block_count = 0 + frontier_full_block_count = 0 + + for index, (public, private) in enumerate( + zip(public_rows, private_rows, strict=True) + ): + original_arrival = float(public["arrived_at"]) + if not math.isfinite(original_arrival) or original_arrival < 0: + raise ValueError(f"invalid arrival at row {index}") + if not math.isclose(original_arrival, float(private["arrived_at"]), abs_tol=1e-9): + raise ValueError(f"public/private arrival mismatch at row {index}") + runtime_block_ids = [int(value) for value in private["runtime_block_ids"]] + expected = ( + int(public["num_prefill_tokens"]), + int(public["num_decode_tokens"]), + int(public["session_id"]), + public["block_hash_ids"], + ) + actual = ( + int(private["input_length"]), + int(private["output_length"]), + int(private["session_id"]), + "|".join(str(value) for value in runtime_block_ids), + ) + if expected != actual: + raise ValueError(f"public/private payload mismatch at row {index}") + + input_tokens = int(private["input_length"]) + frontier_block_ids = full_prompt_block_ids( + runtime_block_ids, input_tokens, args.runtime_block_size + ) + if not frontier_block_ids: + raise ValueError( + "Frontier prefix-cache replay requires at least one complete " + f"runtime block; request {index} has ISL={input_tokens}" + ) + partial_prompt_count += int(input_tokens % args.runtime_block_size != 0) + raw_runtime_block_count += len(runtime_block_ids) + frontier_full_block_count += len(frontier_block_ids) + + normalized_arrival = original_arrival / args.tp + updated = dict(private) + updated["arrived_at"] = normalized_arrival + normalized_private.append(updated) + normalized_public.append( + { + "arrived_at": f"{normalized_arrival:.12f}", + "num_prefill_tokens": public["num_prefill_tokens"], + "num_decode_tokens": public["num_decode_tokens"], + "session_id": public["session_id"], + "block_hash_ids": "|".join(str(value) for value in frontier_block_ids), + } + ) + update_digest( + semantic_digest, + [ + int(private["source_index"]), + int(private["input_length"]), + int(private["output_length"]), + int(private["session_id"]), + runtime_block_ids, + ], + ) + update_digest( + normalized_digest, + [ + int(private["source_index"]), + normalized_arrival, + int(private["input_length"]), + int(private["output_length"]), + int(private["session_id"]), + runtime_block_ids, + ], + ) + update_digest( + frontier_prefix_digest, + [ + int(private["source_index"]), + normalized_arrival, + input_tokens, + frontier_block_ids, + ], + ) + original_arrivals.append(original_arrival) + + if any( + right < left for left, right in zip(original_arrivals, original_arrivals[1:]) + ): + raise ValueError("base arrival order is not monotonic") + + public_dir = args.output_root / "public" + private_dir = args.output_root / "private" + public_dir.mkdir(parents=True, exist_ok=True) + private_dir.mkdir(parents=True, exist_ok=True) + public_path = public_dir / "frontier.csv" + private_path = private_dir / "real_requests.jsonl" + with public_path.open("w", newline="") as output: + writer = csv.DictWriter(output, fieldnames=CSV_FIELDS, lineterminator="\n") + writer.writeheader() + writer.writerows(normalized_public) + with private_path.open("w") as output: + for row in normalized_private: + output.write(json.dumps(row, separators=(",", ":")) + "\n") + + manifest = { + "schema": "qwen30-tp-normalized-trace-v2", + "privacy": "prompt text exists only under private/ and must not be harvested", + "transform": "arrival_seconds_prime = arrival_seconds / tensor_parallel_size", + "tensor_parallel_size": args.tp, + "requests": len(normalized_private), + "base_window_seconds": args.base_window_seconds, + "global_offered_request_rate": len(normalized_private) + / args.base_window_seconds + * args.tp, + "per_gpu_offered_request_rate": len(normalized_private) + / args.base_window_seconds, + "frontier_prefix_block_projection": { + "runtime_block_size": args.runtime_block_size, + "rule": "full_prompt_blocks_only=floor(input_tokens/runtime_block_size)", + "partial_prompt_count": partial_prompt_count, + "raw_runtime_block_count": raw_runtime_block_count, + "frontier_full_block_count": frontier_full_block_count, + }, + "original_first_arrival_s": original_arrivals[0], + "original_last_arrival_s": original_arrivals[-1], + "normalized_first_arrival_s": original_arrivals[0] / args.tp, + "normalized_last_arrival_s": original_arrivals[-1] / args.tp, + "base_public_csv": str(args.base_public_csv.resolve()), + "base_public_csv_sha256": sha256(args.base_public_csv), + "base_private_jsonl": str(args.base_private_jsonl.resolve()), + "base_private_jsonl_sha256": sha256(args.base_private_jsonl), + "public_csv": str(public_path.resolve()), + "public_csv_sha256": sha256(public_path), + "private_jsonl": str(private_path.resolve()), + "private_jsonl_sha256": sha256(private_path), + "semantic_vector_sha256": semantic_digest.hexdigest(), + "normalized_row_vector_sha256": normalized_digest.hexdigest(), + "frontier_prefix_vector_sha256": frontier_prefix_digest.hexdigest(), + } + manifest_path = public_dir / "manifest.json" + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + print(json.dumps(manifest, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/runs/simulator-tuning-latency-matrix-v0/qwen30-tracepd-u0p01-card.md b/runs/simulator-tuning-latency-matrix-v0/qwen30-tracepd-u0p01-card.md new file mode 100644 index 0000000..74b2fed --- /dev/null +++ b/runs/simulator-tuning-latency-matrix-v0/qwen30-tracepd-u0p01-card.md @@ -0,0 +1,191 @@ +# Qwen3-30B-A3B Trace-PD `u0p01` latency-selection comparison + +> **Status:** the 36-run real surface is valid. The initially reported Frontier +> surface is **invalidated for simulator fidelity/selection** by a post-hoc +> prefix-cache trace-contract audit (2026-07-17): its simulator-facing block +> metadata includes non-cacheable partial final blocks. The raw CPU artifacts +> are retained for diagnosis, but are not Frontier evidence. This remains the +> first MoE case of `EXP-SIM-TUNING-LATENCY-MATRIX-V0`, not a claim about the +> remaining three workload classes. + +## Question and decision + +Can the frozen Frontier surface select the same low-latency Qwen3-30B-A3B +configuration as a fresh community-vLLM deployment when both replay exactly +the same trace-derived prefill+decode request vector? The objective contains +no SLO or capacity threshold: lower mean/p90 TTFT, TPOT, and E2E is better. + +The result is a bounded Trace-PD result only. It cannot establish a claim +about Fixed-P, Fixed-PD, Trace-P, other trace rates, another model, or a +modified Frontier. + +## Frozen contract + +| Item | Value | +|---|---| +| Hardware | `dash0`, H20, one replica; fresh vLLM server per config/trial | +| Model / engine | `/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B`; community `vLLM 0.20.0+cu129`, BF16 | +| Config surface | `TP ∈ {1,2,4} × MNS ∈ {8,16,32,64}`; `DP=PP=EP=1`, `MBT=8192`, GPU-memory utilization `0.92`, chunked prefill and prefix caching on | +| Trace | `trace-exact-v1`, anchor `u0p01`; **129 requests**, original input/output length, order and session are preserved. The first Frontier materialization also copied every 16-token runtime identity, including a final partial block; that is not a legal vLLM prefix-cache-hit representation and invalidates this simulator attempt. For a config of tensor parallelism `TP`, every arrival is transformed as `t′ = t / TP`; the TP1/2/4 traces therefore end at 597.037/298.519/149.259 s. | +| Offered load | Base rate is `129 / 600 = 0.215 req/s`. TP1/2/4 replay at 0.215/0.430/0.860 req/s globally, respectively, so all three topologies receive 0.215 req/s/GPU. This is a **per-GPU-normalized trace**, not an unchanged-wall-clock trace. | +| Simulator baseline | The existing same-global-rate T1 surface is excluded: it evaluates TP2/TP4 at lower per-GPU load. A new Frontier surface must use the identical TP-normalized arrival vector for each topology before coverage or latency can be reported. | +| Trials | Three fresh-server trials per config; independent warmup excluded from the measured trace | + +## Validity gates + +1. Each real trial must complete all 129 requests, preserve exact prompt and + completion-token usage, and have the TP-normalized row-vector digest. +2. A simulator metric is valid only if its cell has all 129 finite request + metrics with matching request ID, prompt/output lengths and transformed + arrival vector. A simulator `scheduler_stall` is a coverage result, never + an infeasible-latency label. +3. TTFT, TPOT and E2E are kept in milliseconds. `TPOT` is recorded only for + outputs longer than one token (all requests in this Trace-PD anchor + qualify). +4. A failed real run remains an infeasible/safety observation; it is not + silently converted to infinite latency or omitted from ranking. + +## Analysis and decision rule + +For every `config × trial`, retain raw per-request records and compute mean +and nearest-rank p90 for TTFT, TPOT and E2E. Report pooled 387-request values +alongside the three per-trial values. A selection-stability bootstrap is +deferred; it is not needed to evaluate Frontier's coverage gate. + +The primary gate is Frontier decision-valid coverage on all 12 cells. If any +real-valid candidate lacks a legal simulator metric, Frontier receives no +full-surface selection, top-set overlap, regret, or Kendall-tau score. We +will still show all valid per-cell latency comparisons and the complete real +surface, so any coverage failure and its ground truth remain auditable. + +The simulator's pre-existing SLO fields do not influence request generation, +scheduling, or this no-SLO analysis; they are excluded from scoring. + +## Post-hoc Frontier trace-contract audit (2026-07-17) + +The CPU attempt initially passed only a syntactic trace check (row count, +IDs, lengths, and SHA). It did **not** validate the semantic invariant needed +by vLLM prefix caching: only completed 16-token blocks may be cache hits. + +- The trace exporter emitted `ceil(ISL / 16)` identities. Thus 122/129 prompts + contained one partial final block; the public trace has 36,443 identities, + while a full-block Frontier/vLLM cache adapter must expose 36,321. +- Exact community-vLLM 0.20 source (`vllm.v1.core.kv_cache_manager`) documents + that computed prefix blocks "must be full" and bounds hits by + `prompt_length - 1`. Frontier's fixture contract only requires a non-empty + `block_hash_ids` field and does not validate this full-block invariant. +- Frontier's prefix manager converts every hit to `len(blocks) * 16`. For the + final TP2/MNS16 state, requests 89/111/117 had `(ISL, IDs)` of + `(722,46)`, `(678,43)`, and `(706,45)`. If all supplied identities hit, it + obtains 736/688/720 computed tokens and hence `num_new_tokens` of + -14/-10/-14. The scheduler preserves zero-or-negative-token requests in its + waiting queue; no event remains, so the simulator reports + `Sequential simulation ended with non-empty scheduler state`. + +This proves a trace-adapter/Frontier prefix-cache semantic mismatch, not a +GPU capacity failure: a targeted CPU hook found no failed KV allocation and +no PP-admission deferral, only the three non-admitted requests with a full +8192-token batch budget. The correct next input is a **separate Frontier +adapter** that exports only `floor(ISL / 16)` complete prompt-block identities; +it must not modify the private real trace or its input/output/arrival vector. + +## Invalid raw Frontier attempt (2026-07-17; retained for diagnosis only) + +The TP-normalized surface completed in 156 seconds of CPU-only wall time on +`dash0`, with Frontier commit `deadc4a321f0baaa534c6ebd17f974123733cdc2`. +All twelve input traces passed the 129-request, ID/shape, and TP-specific +trace-SHA checks. Only two cells reached valid request metrics; the other ten +ended in `scheduler_stall`. + +| Config | Status | mean / p90 TTFT (ms) | mean / p90 TPOT (ms) | mean / p90 E2E (ms) | +|---|---|---:|---:|---:| +| TP1/MNS32 | complete | 440032 / 972628 | 145.8 / 151.5 | 936301 / 1645895 | +| TP1/MNS64 | complete | 174335 / 510962 | 163.6 / 178.9 | 724328 / 1285374 | +| remaining 10 configs | `scheduler_stall` | N/A | N/A | N/A | + +The apparent **2/12 (16.7%)** coverage is not a Frontier coverage result: +both the ten stalls and the two completed cells consumed the invalid partial +block metadata. In particular, the two completed cells may receive false +prefix hits. No latency, ranking, coverage, top-set overlap, regret, or +Kendall-tau statement about Frontier may be derived from this attempt. + +The legacy surface runner writes an offered-rate summary assuming a fixed +600-second window. That field is not used here because TP2/TP4 have compressed +arrival clocks. The materialized trace manifests are authoritative: they +record global 0.215/0.430/0.860 req/s and the matched 0.215 req/s/GPU rate. + +## Real result (2026-07-17) + +All **36/36** `config × trial` replays completed with exit code zero. Each +trial passed all validity gates: 129/129 successful requests, exact requested +and observed input/output token usage, the TP-specific private-trace SHA and +normalized arrival-vector SHA, and HTTP routing to the explicit alias +`qwen3-30b-exact-trace`. The cache-populating smoke and earlier failed launch +attempts are excluded from this table. Values pool 387 requests/config (three +trials); p90 is nearest-rank. + +| Config | mean / p90 TTFT (ms) | mean / p90 TPOT (ms) | mean / p90 E2E (ms) | +|---|---:|---:|---:| +| TP1/MNS8 | 88479.9 / 168185.5 | 14.3 / 16.3 | 137996.4 / 222136.0 | +| TP1/MNS16 | 17378.2 / 42739.8 | 20.6 / 25.3 | 87355.2 / 156177.2 | +| TP1/MNS32 | 620.5 / 1931.5 | 23.8 / 30.0 | 81051.2 / 162908.6 | +| TP1/MNS64 | 619.8 / 1956.7 | 24.1 / 30.4 | 82253.7 / 165355.3 | +| TP2/MNS8 | 95920.1 / 183878.8 | 9.2 / 10.2 | 128296.9 / 215848.3 | +| TP2/MNS16 | 34443.0 / 77922.4 | 14.1 / 16.1 | 82936.7 / 137726.4 | +| TP2/MNS32 | 1050.6 / 2521.8 | 18.0 / 21.3 | 62267.5 / 117239.3 | +| TP2/MNS64 | 375.8 / 1148.6 | 18.2 / 21.6 | 62353.6 / 118392.3 | +| TP4/MNS8 | 102605.9 / 193359.7 | **6.8 / 7.2** | 126405.4 / 209092.4 | +| TP4/MNS16 | 35042.6 / 74281.3 | 8.8 / 9.6 | 65779.1 / 110374.3 | +| TP4/MNS32 | 6296.2 / 19159.9 | 12.2 / 13.8 | 47944.5 / 83825.7 | +| TP4/MNS64 | **246.0 / 685.5** | 13.2 / 15.4 | **44985.1 / 83763.6** | + +Thus real-vLLM chooses TP4/MNS64 for mean/p90 TTFT and E2E, and TP4/MNS8 +for mean/p90 TPOT. These are different single-metric objectives, not a +claim that one configuration simultaneously optimizes all three. + +## Correct conclusion and next gate + +This run establishes a **harness finding**, not a Frontier capability finding: +the initial simulator-facing Trace-PD prefix contract was invalid. The valid +real surface remains a frozen ground truth, but all raw Frontier numbers above +are excluded from the research claim. We therefore retract the prior bounded +counterexample and do not yet know whether Frontier selects the correct config +on this case. + +Before a CPU-only Frontier rerun, the comparison must additionally record +these remaining alignment gaps: + +1. real vLLM uses `FULL_AND_PIECEWISE` CUDA graphs, whereas the raw Frontier + command explicitly used `--decode_cuda_graph_mode none`; Frontier exposes + `full_decode_only` and `piecewise`, not the identical combined mode; +2. Frontier used one explicit KV-block count per TP, while real vLLM's graph + capture changes the count with MNS (the mismatch is small but measurable); +3. Frontier skipped CPU-overhead modeling, and its MoE CSV has only + `standalone_legacy` gating rows, so the code warned and fell back rather + than training the requested `prefill_hot` pseudo-model; +4. the runner's rate *metadata* incorrectly fixed the TP2/TP4 numerator to a + 600-second window. Arrival timestamps supplied to the simulator were + correct, so this did not alter the raw execution, but the metadata must be + fixed before reporting a rerun. + +The next admissible Frontier result uses complete-block prefix metadata, +per-`TP×MNS` real KV capacity, correct TP-normalized rate metadata, and an +explicit CUDA-graph compatibility decision. Only a complete, semantically +valid surface may receive the selection metrics. Any remaining mismatch after +that CPU rerun is then evidence about Frontier's scheduler/profile fidelity, +not this adapter. + +## Cost and output + +The measured replay horizons are 597/299/149 s at TP1/2/4. Including five +minutes of per-launch server start/warmup, 36 launches consume about **13 +H20-GPUh nominally**; applying the launcher's historical 12--35 minute +fresh-server envelope yields an upper bound of **41 H20-GPUh**. A separate +CPU-only simulator rerun is mandatory before this GPU stage; no profile +collection is included. + +The remote, experiment-specific output root is +`/home/admin/cpfs/wjh/aituner/simulator-tuning-latency-q30-tp-normalized-u0p01-20260717`. +It contains the prompt-free real audit JSON/Markdown, commands, environment, +GPU/trace/model hashes, raw prompt-free request records, and cache inventory. +Private prompt text remains on `dash0` and is never copied into the repository. From 47355a9411da3e6683c4fc45d8b69cb0ce741455 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 22:50:00 +0800 Subject: [PATCH 112/148] Record corrected Frontier liveness probe --- .../qwen30-tracepd-u0p01-card.md | 44 ++++++++++++++++--- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/runs/simulator-tuning-latency-matrix-v0/qwen30-tracepd-u0p01-card.md b/runs/simulator-tuning-latency-matrix-v0/qwen30-tracepd-u0p01-card.md index 74b2fed..9d768b8 100644 --- a/runs/simulator-tuning-latency-matrix-v0/qwen30-tracepd-u0p01-card.md +++ b/runs/simulator-tuning-latency-matrix-v0/qwen30-tracepd-u0p01-card.md @@ -114,6 +114,32 @@ The legacy surface runner writes an offered-rate summary assuming a fixed arrival clocks. The materialized trace manifests are authoritative: they record global 0.215/0.430/0.860 req/s and the matched 0.215 req/s/GPU rate. +## Corrected-adapter CPU liveness probe (2026-07-17) + +One deliberately narrow CPU-only rerun tested the causal fix before launching +another surface: TP2/MNS16, the same 129-request Trace-PD input, MBT=8192, +prefix caching and chunked prefill enabled, and the original Frontier commit. +It changed only the simulator-facing prefix projection from +`ceil(ISL / 16)` to `floor(ISL / 16)`; the private real replay remained bytewise +unchanged. The materialized trace records 36,321 complete blocks (36,443 raw +runtime identities; 122 prompts with a discarded partial final block), global +arrival rate 0.430 req/s, and the required 0.215 req/s/GPU. + +The run completed **129/129** requests in 105.8 CPU wall-clock seconds with no +stall. This is a direct liveness validation of the diagnosis above: the +partial-block adapter, rather than KV capacity, caused the old deadlock. + +It is **not** a valid latency or selection result. Even after the repair, +Frontier reports p50 TTFT=946,973 ms, p50 TPOT=96.37 ms, p50 E2E=1,305,128 ms, +and 112.2 generated decode tokens/s. The trace supplies 460,490 output tokens +over a 298.5-s TP2 arrival horizon (about 1,543 output tokens/s), so the +simulated service rate is below the offered decode work and its queue must +grow. The corresponding real TP2/MNS16 replay has mean/p90 TPOT 14.1/16.1 ms +and mean/p90 TTFT 34,443/77,922 ms. Thus the post-fix discrepancy is a +decode-service-model / runtime-semantics question, not residual evidence from +the invalid cache metadata. The probe artifact is +`/home/admin/cpfs/wjh/aituner/simulator-tuning-latency-q30-tp-normalized-u0p01-20260717/frontier-contract-probe-v1`. + ## Real result (2026-07-17) All **36/36** `config × trial` replays completed with exit code zero. Each @@ -145,19 +171,23 @@ claim that one configuration simultaneously optimizes all three. ## Correct conclusion and next gate -This run establishes a **harness finding**, not a Frontier capability finding: -the initial simulator-facing Trace-PD prefix contract was invalid. The valid -real surface remains a frozen ground truth, but all raw Frontier numbers above -are excluded from the research claim. We therefore retract the prior bounded -counterexample and do not yet know whether Frontier selects the correct config -on this case. +This run establishes two bounded findings, neither of which is a full Frontier +selection verdict: (1) the initial simulator-facing Trace-PD prefix contract +was invalid, and (2) its corrected TP2/MNS16 liveness probe estimates a much +smaller decode service rate than the real system. The valid real surface remains +a frozen ground truth, but all raw Frontier numbers above are excluded from the +research claim. We therefore retract the prior bounded counterexample and do +not yet know whether Frontier selects the correct config on this case. Before a CPU-only Frontier rerun, the comparison must additionally record these remaining alignment gaps: 1. real vLLM uses `FULL_AND_PIECEWISE` CUDA graphs, whereas the raw Frontier command explicitly used `--decode_cuda_graph_mode none`; Frontier exposes - `full_decode_only` and `piecewise`, not the identical combined mode; + `full_decode_only` and `piecewise`, not the identical combined mode. More + importantly, Frontier chooses kernel-only predictor families whenever this + mode is non-`none`, so a graph-compatible rerun also requires corresponding + kernel-only profile coverage rather than merely changing the CLI flag; 2. Frontier used one explicit KV-block count per TP, while real vLLM's graph capture changes the count with MNS (the mismatch is small but measurable); 3. Frontier skipped CPU-overhead modeling, and its MoE CSV has only From bdc357dc6c18146fc23e78476880db9072879fdc Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 23:22:42 +0800 Subject: [PATCH 113/148] Align Frontier piecewise graph profiles --- .../graph-piecewise-experiment-card.md | 40 ++++ .../graph-piecewise-profile-prototype.svg | 28 +++ ...run_frontier_qwen30_exact_trace_surface.py | 176 ++++++++++++++++++ .../run_graph_kernel_only_attention.sh | 71 +++++++ .../run_graph_kernel_only_linear.sh | 43 +++++ .../run_graph_kernel_only_moe.sh | 48 +++++ .../freeze_frontier_profiles.py | 132 +++++++++---- .../profile_vllm020_flashattn.py | 168 ++++++++++++++++- .../profile_vllm020_moe.py | 97 +++++++--- .../profile_vllm020_router.py | 66 ++++++- 10 files changed, 804 insertions(+), 65 deletions(-) create mode 100644 runs/frontier-fidelity-envelope-v1/graph-piecewise-experiment-card.md create mode 100644 runs/frontier-fidelity-envelope-v1/graph-piecewise-profile-prototype.svg create mode 100644 runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_attention.sh create mode 100644 runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_linear.sh create mode 100644 runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_moe.sh diff --git a/runs/frontier-fidelity-envelope-v1/graph-piecewise-experiment-card.md b/runs/frontier-fidelity-envelope-v1/graph-piecewise-experiment-card.md new file mode 100644 index 0000000..272fe9c --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/graph-piecewise-experiment-card.md @@ -0,0 +1,40 @@ +# EXP-SIMFID-Q30-GRAPH-PIECEWISE：graph-compatible kernel-only profile 是否修正 Frontier trace replay？ + +> **状态：** approved and running（2026-07-17）。本卡是已纠正 prefix-trace contract 后的最小判别实验；不复用此前 `decode_cuda_graph_mode=none` 的数值作 fidelity verdict。 + +## Purpose and hypotheses + +- **Parent claim：** Frontier 是否已经足以为 Qwen3-30B-A3B 的真实 trace serving surface 选择 config。 +- **Question：** 旧 Frontier replay 低估 decode service rate，是否主要是 simulator 使用 `none` 而真机使用 `FULL_AND_PIECEWISE`、并且没有向 Frontier 提供独立 `KERNEL_ONLY` profile family？ +- **G1 (graph-family omission)：** 用同一 vLLM 0.20/FA3/FlashInfer-CUTLASS stack 的 `RecordFunctionTracer` kernel-only measurements，加真实 capture buckets 和 Frontier `piecewise`，会显著缩小 TP2/MNS16 的 TPOT/service-rate gap，并至少改变一个 config 的 latency ranking。 +- **G2 (remaining composition error)：** 即使 graph family 对齐，TPOT、TTFT 或 E2E ranking 仍与真机不一致；则 graph omission 只是必要修正，不是 simulator 已解决 tuning 的证据。 + +## Controlled setup + +| Item | Frozen choice | +|---|---| +| model/runtime/hardware | Qwen3-30B-A3B BF16; community vLLM 0.20.0 (`88d34c…`); dash0 NVIDIA H20 | +| simulator | Frontier `deadc4a321f0baaa534c6ebd17f974123733cdc2`; no local source patch | +| workload | exact 129-request Trace-PD public projection; exact ISL/OSL/arrival order; TP-normalized arrival time and complete 16-token prefix blocks | +| surface | TP in {1,2,4}; MNS in {8,16,32,64}; MBT=8192; prefix/chunked prefill on | +| real graph contract | observed vLLM capture sizes: MNS8=[1,2,4,8,16], MNS16=[1,2,4,8,16,24,32], MNS32=[1,2,4,8,16,24,32,40,48,56,64], MNS64=[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128] | +| profile intervention | CUDA-event profile stays frozen for prefill/mixed batches. New `KERNEL_ONLY` linear, FA3 decode + KV-update, MoE, and router rows use Frontier's actual `RecordFunctionTracer` semantics; no relabeling of CUDA-event numbers. | +| exact capacity | per-cell real observed KV block count and capture list; Frontier CPU-overhead model remains disabled on both old/new simulator runs because the intervention is GPU-kernel family only. | + +Frontier source inspection fixes the semantic boundary: `piecewise` emits `PIECEWISE` whenever a capture hits, but the MONOLITHIC predictor selects `KERNEL_ONLY` only when `num_prefill_tokens == 0`. Hence new profile coverage is pure decode only; captured mixed/prefill work continues to consume the existing CUDA-event family. + +## Measurement and decision rule + +- **Primary outputs：** per-config mean/p90 TTFT, TPOT, E2E; ranking for each metric; TP2/MNS16 per-request TPOT gap against the already frozen three-trial real audit. +- **Validity gates：** every kernel CSV hash matches its manifest; every row says `KERNEL_ONLY`; every TP/capture-bucket/KV-context required by the runner is present; command records `piecewise`, per-cell blocks and capture sizes; each simulator cell completes all 129 requests. +- **Decision:** G1 is supported only if the graph-aligned TP2/MNS16 TPOT median moves toward real **and** full-surface rank/error evidence improves. A single-cell timing improvement does not establish tuning sufficiency. If G2 holds, update the research claim to “Frontier has not solved tuning under trace-faithful MoE serving after graph-family alignment,” then profile stage/state composition rather than add arbitrary kernel rows. + +## Expected figure + +`graph-piecewise-profile-prototype.svg` is deliberately schematic. The final figure uses the same axes and adds real data only after the profile and replay validity gates pass. + +## Cost and provenance + +- **GPU cost:** three 1-GPU FA3 decode profile shards, plus one 1-GPU linear shard and one 1-GPU MoE/router shard; expected 1.5--3.0 H20-GPU-hours, hard cap 4.0 GPU-hours. +- **CPU cost:** 12 exact-trace simulations, expected 20--45 CPU minutes; a one-cell TP2/MNS16 smoke precedes the full surface. +- **Calibration separation:** kernel microprofiles are independent measurements, never fitted to trace E2E latency. The frozen real trace audit is evaluation only. diff --git a/runs/frontier-fidelity-envelope-v1/graph-piecewise-profile-prototype.svg b/runs/frontier-fidelity-envelope-v1/graph-piecewise-profile-prototype.svg new file mode 100644 index 0000000..3713255 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/graph-piecewise-profile-prototype.svg @@ -0,0 +1,28 @@ + + + SCHEMATIC — no measured data + Does graph-compatible KERNEL_ONLY profiling make Frontier select the real trace-serving configuration? + + A. TP2/MNS16 TPOT prediction + + + latencymeasurement family + + + + nonepiecewisereal + old simG1: moves closer + + + B. Full 12-cell ranking agreement + + + rank errorsimulator variant + + + + nonepiecewisereal rank + G2: stays wrongG1: error falls + + Final figure reports mean/p90 TTFT, TPOT, E2E for the identical 129-request trace, not an SLO-derived proxy. + diff --git a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py index 8afd928..454d0b2 100644 --- a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py +++ b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py @@ -19,6 +19,27 @@ from typing import Any TARGET_PASS_RATE = 0.95 TPOT_SLOS_MS = (50.0, 100.0, 150.0, 180.0) WINDOW_SECONDS = 600.0 +GRAPH_CAPTURE_SIZES_BY_MNS = { + 8: (1, 2, 4, 8, 16), + 16: (1, 2, 4, 8, 16, 24, 32), + 32: (1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64), + 64: (1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128), +} +REAL_NUM_BLOCKS_BY_CONFIG = { + (1, 8): 20137, + (1, 16): 20128, + (1, 32): 20108, + (1, 64): 20069, + (2, 8): 76639, + (2, 16): 76620, + (2, 32): 76583, + (2, 64): 76505, + (4, 8): 191930, + (4, 16): 191882, + (4, 32): 191786, + (4, 64): 191589, +} +KERNEL_DECODE_KV_CONTEXTS = (128, 1024, 2048, 4096, 8192, 16384, 32768, 40960) BASE_RUNNER = ( Path(__file__).resolve().parents[1] / "frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py" @@ -43,6 +64,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--frontier-source", type=Path, required=True) parser.add_argument("--replayserve-root", type=Path, required=True) parser.add_argument("--profile-root", type=Path, required=True) + parser.add_argument("--kernel-profile-root", type=Path) parser.add_argument("--python-deps", type=Path, required=True) parser.add_argument("--output-root", type=Path, required=True) parser.add_argument( @@ -68,6 +90,21 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--allreduce-csv", type=Path) parser.add_argument("--timeout-seconds", type=float, default=1800.0) parser.add_argument("--predictor-training-job-threads", type=int, default=1) + parser.add_argument( + "--decode-cuda-graph-mode", + choices=("none", "full_decode_only", "piecewise"), + default="none", + ) + parser.add_argument( + "--align-real-graph-runtime", + action="store_true", + help="Use real observed capture lists and per-(TP,MNS) KV blocks.", + ) + parser.add_argument( + "--fresh-predictor-cache", + action="store_true", + help="Disable Frontier predictor cache reuse for this profile family.", + ) parser.add_argument("--resume", action="store_true") parser.add_argument("--continue-on-failure", action="store_true") return parser.parse_args() @@ -241,15 +278,99 @@ def score(path: Path, expected_shapes: list[tuple[int, int]]) -> dict[str, Any]: } ttfts = [float(row["ttft_ms"]) for row in request_metrics] tpots = [float(row["tpot_ms"]) for row in request_metrics if row["tpot_ms"] is not None] + e2es = [float(row["e2e_ms"]) for row in request_metrics] return { + "ttft_mean_ms": sum(ttfts) / len(ttfts), "ttft_p50_ms": percentile(ttfts, 0.50), + "ttft_p90_ms": percentile(ttfts, 0.90), "ttft_p95_ms": percentile(ttfts, 0.95), + "tpot_mean_ms": sum(tpots) / len(tpots), "tpot_p50_ms": percentile(tpots, 0.50), + "tpot_p90_ms": percentile(tpots, 0.90), "tpot_p95_ms": percentile(tpots, 0.95), + "e2e_mean_ms": sum(e2es) / len(e2es), + "e2e_p50_ms": percentile(e2es, 0.50), + "e2e_p90_ms": percentile(e2es, 0.90), + "e2e_p95_ms": percentile(e2es, 0.95), "slos": slos, } +def kernel_profile_paths(root: Path) -> dict[str, Path]: + paths = { + "linear": root / "linear_op.csv", + "attention": root / "attention.csv", + "moe": root / "moe.csv", + "manifest": root / "manifest.json", + } + missing = [str(path) for path in paths.values() if not path.is_file()] + if missing: + raise FileNotFoundError(missing) + return paths + + +def validate_kernel_profile(paths: dict[str, Path]) -> dict[str, Any]: + manifest = json.loads(paths["manifest"].read_text()) + outputs = manifest.get("outputs", {}) + for filename, name in ( + ("linear_op.csv", "linear"), + ("attention.csv", "attention"), + ("moe.csv", "moe"), + ): + if outputs.get(filename) != BASE.sha256(paths[name]): + raise ValueError(f"kernel-only profile hash mismatch for {filename}") + + with paths["linear"].open(newline="") as source: + linear_rows = list(csv.DictReader(source)) + with paths["attention"].open(newline="") as source: + attention_rows = list(csv.DictReader(source)) + with paths["moe"].open(newline="") as source: + moe_rows = list(csv.DictReader(source)) + for label, rows in (("linear", linear_rows), ("attention", attention_rows), ("moe", moe_rows)): + if not rows or {row.get("measurement_type") for row in rows} != {"KERNEL_ONLY"}: + raise ValueError(f"{label} lacks an exclusive KERNEL_ONLY measurement family") + + required_buckets = set(GRAPH_CAPTURE_SIZES_BY_MNS[64]) + coverage: dict[str, Any] = {} + for tp in (1, 2, 4): + linear_tokens = { + int(float(row["num_tokens"])) + for row in linear_rows + if int(float(row["num_tensor_parallel_workers"])) == tp + } + moe_tokens = { + int(float(row["num_tokens"])) + for row in moe_rows + if int(float(row["num_tensor_parallel_workers"])) == tp + } + attention_pairs = { + (int(float(row["batch_size"])), int(float(row["kv_cache_size"]))) + for row in attention_rows + if int(float(row["num_tensor_parallel_workers"])) == tp + and row["is_prefill"].lower() == "false" + and row.get("is_true_mixed_batch", "").lower() != "true" + } + missing_linear = required_buckets - linear_tokens + missing_moe = required_buckets - moe_tokens + missing_attention = { + (bucket, kv) + for bucket in required_buckets + for kv in KERNEL_DECODE_KV_CONTEXTS + if (bucket, kv) not in attention_pairs + } + if missing_linear or missing_moe or missing_attention: + raise ValueError( + f"kernel-only profile coverage TP{tp}: linear={sorted(missing_linear)}, " + f"moe={sorted(missing_moe)}, attention={sorted(missing_attention)}" + ) + coverage[str(tp)] = { + "linear_tokens": sorted(linear_tokens), + "moe_tokens": sorted(moe_tokens), + "attention_decode_pairs": len(attention_pairs), + } + return {"manifest": manifest, "coverage": coverage} + + def main() -> None: args = parse_args() if args.predictor_training_job_threads <= 0: @@ -264,6 +385,10 @@ def main() -> None: setattr(args, name, getattr(args, name).resolve()) if args.allreduce_csv is not None: args.allreduce_csv = args.allreduce_csv.resolve() + if args.kernel_profile_root is not None: + args.kernel_profile_root = args.kernel_profile_root.resolve() + if args.decode_cuda_graph_mode == "none": + raise ValueError("--kernel-profile-root requires a non-none graph mode") traces = [ parse_trace( specification, @@ -288,6 +413,11 @@ def main() -> None: raise ValueError(f"unknown configs: {wanted - {config.name for config in selected}}") paths = BASE.profile_paths(args.profile_root) coverage = BASE.validate_profile(paths) + kernel_paths = None + kernel_coverage = None + if args.kernel_profile_root is not None: + kernel_paths = kernel_profile_paths(args.kernel_profile_root) + kernel_coverage = validate_kernel_profile(kernel_paths) builder = BASE.load_module( "qwen30_exact_trace_frontier_builder", args.replayserve_root / "tools/run_frontier_sweep.py", @@ -320,6 +450,18 @@ def main() -> None: config_knobs = BASE.knobs(config, paths, args.output_root / "cache") config_knobs["enable_prefix_caching"] = args.prefix_caching config_knobs["prediction_max_tokens_per_request"] = 40960 + config_knobs["decode_cuda_graph_mode"] = args.decode_cuda_graph_mode + config_knobs["no_cache"] = args.fresh_predictor_cache + if args.align_real_graph_runtime: + config_knobs["num_blocks"] = REAL_NUM_BLOCKS_BY_CONFIG[(config.tp, config.mns)] + if kernel_paths is not None: + config_knobs.update( + { + "linear_op_kernel_only_input_file": str(kernel_paths["linear"]), + "atten_kernel_only_input_file": str(kernel_paths["attention"]), + "moe_kernel_only_input_file": str(kernel_paths["moe"]), + } + ) for trace in traces: run_dir = args.output_root / "runs" / config.name / trace["label"] result_path = run_dir / "result.json" @@ -340,6 +482,13 @@ def main() -> None: str(args.predictor_training_job_threads), ] ) + if args.align_real_graph_runtime: + command.extend( + [ + "--cudagraph_capture_sizes", + *(str(size) for size in GRAPH_CAPTURE_SIZES_BY_MNS[config.mns]), + ] + ) command = BASE.configure_cc_command( command, backend=args.cc_backend, @@ -514,6 +663,9 @@ def main() -> None: "primary_tpot_slo_ms": 150.0, "target_pass_rate": TARGET_PASS_RATE, "predictor_training_job_threads": args.predictor_training_job_threads, + "decode_cuda_graph_mode": args.decode_cuda_graph_mode, + "align_real_graph_runtime": args.align_real_graph_runtime, + "fresh_predictor_cache": args.fresh_predictor_cache, }, "frontier": { "source": str(args.frontier_source), @@ -530,6 +682,30 @@ def main() -> None: "coverage": coverage, "sha256": {name: BASE.sha256(path) for name, path in paths.items()}, }, + "kernel_only_profiles": ( + None + if kernel_paths is None + else { + "root": str(args.kernel_profile_root), + "coverage": kernel_coverage, + "sha256": { + name: BASE.sha256(path) for name, path in kernel_paths.items() + }, + } + ), + "runtime_alignment": { + "capture_sizes_by_mns": ( + GRAPH_CAPTURE_SIZES_BY_MNS if args.align_real_graph_runtime else None + ), + "num_blocks_by_config": ( + { + f"tp{tp}_mns{mns}": blocks + for (tp, mns), blocks in REAL_NUM_BLOCKS_BY_CONFIG.items() + } + if args.align_real_graph_runtime + else None + ), + }, "collective": { "backend": args.cc_backend, "allreduce_csv": str(args.allreduce_csv) if args.allreduce_csv else None, diff --git a/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_attention.sh b/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_attention.sh new file mode 100644 index 0000000..2d1ff9e --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_attention.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +set -euo pipefail + +TP="${TP:?TP must be 1, 2, or 4}" +case "${TP}" in + 1|2|4) ;; + *) echo "invalid TP=${TP}" >&2; exit 1 ;; +esac + +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" +RUN_DIR="$(pwd -P)" +PROFILE_DIR="${PROFILE_DIR:-${RUN_DIR%/runs/frontier-fidelity-envelope-v1}/runs/frontier-qwen30-vllm020-profile-v1}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-t1-dash0-deadc4a}" +MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +CAPTURE_BUCKETS="${CAPTURE_BUCKETS:-1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128}" +KV_CONTEXTS="${KV_CONTEXTS:-128 1024 2048 4096 8192 16384 32768 40960}" +WARMUP_ITERS="${WARMUP_ITERS:-3}" +REPEATS="${REPEATS:-5}" + +mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/raw" +exec > >(tee -a "${OUTPUT_ROOT}/logs/attention-tp${TP}.log") 2>&1 + +IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?a fleet-allocated GPU is required}" +if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then + echo "expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2 + exit 1 +fi + +BATCH_SPECS=() +for bucket in ${CAPTURE_BUCKETS}; do + prefix="" + if [[ "${bucket}" -ne 1 ]]; then + prefix="${bucket}" + fi + for context in ${KV_CONTEXTS}; do + BATCH_SPECS+=("${prefix}q1s${context}") + done +done + +echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} role=FA3-decode-kernel-only tp=${TP} buckets='${CAPTURE_BUCKETS}' kv_contexts='${KV_CONTEXTS}' method=Frontier-RecordFunctionTracer output=${OUTPUT_ROOT} expected_wall=10-35m expected_gpu_cap=1.0_H20h" +date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" +nvidia-smi --query-gpu=index,name,driver_version,memory.used,utilization.gpu --format=csv,noheader +test "$(git -C "${FRONTIER_SOURCE}" rev-parse HEAD)" = "deadc4a321f0baaa534c6ebd17f974123733cdc2" +test "$(git -C "${VLLM_SOURCE}" rev-parse HEAD)" = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1" +test -f "${MODEL_ROOT}/config.json" +git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit" +git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit" +git -C "${VLLM_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/vllm.commit" +printf '%s\n' "${BATCH_SPECS[@]}" > "${OUTPUT_ROOT}/provenance/batch-specs.txt" +sha256sum "${PROFILE_DIR}/profile_vllm020_flashattn.py" "${RUN_DIR}/run_graph_kernel_only_attention.sh" > "${OUTPUT_ROOT}/provenance/source.sha256" + +timeout --signal=TERM --kill-after=30s 2400 \ + "${VENV_ROOT}/bin/python" "${PROFILE_DIR}/profile_vllm020_flashattn.py" \ + --vllm-source "${VLLM_SOURCE}" \ + --frontier-source "${FRONTIER_SOURCE}" \ + --model "${MODEL_ROOT}" \ + --output "${OUTPUT_ROOT}/raw/attention-tp${TP}.json" \ + --tp "${TP}" \ + --batch-specs "${BATCH_SPECS[@]}" \ + --warmup-iters "${WARMUP_ITERS}" \ + --repeats "${REPEATS}" \ + --profile-kv-update \ + --profile-method record_function + +test -s "${OUTPUT_ROOT}/raw/attention-tp${TP}.json" +sha256sum "${OUTPUT_ROOT}/raw/attention-tp${TP}.json" "${OUTPUT_ROOT}/provenance"/* > "${OUTPUT_ROOT}/artifacts.sha256" +date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ" +echo "GRAPH_KERNEL_ONLY_ATTENTION_COMPLETE tp=${TP} rows=${#BATCH_SPECS[@]}" diff --git a/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_linear.sh b/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_linear.sh new file mode 100644 index 0000000..7af7161 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_linear.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -euo pipefail + +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" +RUN_DIR="$(pwd -P)" +PROFILE_DIR="${PROFILE_DIR:-${RUN_DIR%/runs/frontier-fidelity-envelope-v1}/runs/frontier-qwen30-vllm020-profile-v1}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-t1-dash0-deadc4a}" +MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +TOKENS=(1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128) +export MODEL_ROOT + +mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/profiles" +exec > >(tee -a "${OUTPUT_ROOT}/logs/linear.log") 2>&1 +IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?a fleet-allocated GPU is required}" +if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then + echo "expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2 + exit 1 +fi + +echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} role=linear-kernel-only tp=1,2,4 tokens='${TOKENS[*]}' method=Frontier-RecordFunctionTracer output=${OUTPUT_ROOT} expected_wall=15-35m expected_gpu_cap=1.0_H20h" +date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" +test "$(git -C "${FRONTIER_SOURCE}" rev-parse HEAD)" = "deadc4a321f0baaa534c6ebd17f974123733cdc2" +test "$(git -C "${VLLM_SOURCE}" rev-parse HEAD)" = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1" +git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit" +git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit" +sha256sum "${PROFILE_DIR}/frontier_vllm020_compat.py" "${RUN_DIR}/run_graph_kernel_only_linear.sh" > "${OUTPUT_ROOT}/provenance/source.sha256" + +cd "${FRONTIER_SOURCE}" +timeout --signal=TERM --kill-after=30s 2400 \ + "${VENV_ROOT}/bin/python" "${PROFILE_DIR}/frontier_vllm020_compat.py" \ + --disable_ray --num_gpus 1 --output_dir "${OUTPUT_ROOT}/profiles" \ + --device h20 --models qwen3-a3b-30b-moe \ + --num_tensor_parallel_workers 1 2 4 --max_tokens 128 \ + --num_tokens_list "${TOKENS[@]}" --profile_method record_function \ + --precision BF16 --is_moe --yes + +find "${OUTPUT_ROOT}/profiles" -name linear_op_kernel_only.csv -type f -size +0c -print -quit > "${OUTPUT_ROOT}/provenance/linear-path.txt" +test -s "${OUTPUT_ROOT}/provenance/linear-path.txt" +date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ" +echo "GRAPH_KERNEL_ONLY_LINEAR_COMPLETE" diff --git a/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_moe.sh b/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_moe.sh new file mode 100644 index 0000000..5a36966 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_moe.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +set -euo pipefail + +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" +RUN_DIR="$(pwd -P)" +PROFILE_DIR="${PROFILE_DIR:-${RUN_DIR%/runs/frontier-fidelity-envelope-v1}/runs/frontier-qwen30-vllm020-profile-v1}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" +FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-t1-dash0-deadc4a}" +MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +TOKENS="${TOKENS:-1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128}" + +mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/raw" +exec > >(tee -a "${OUTPUT_ROOT}/logs/moe-router.log") 2>&1 +IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?a fleet-allocated GPU is required}" +if [[ "${#GPU_IDS[@]}" -ne 1 ]]; then + echo "expected exactly one GPU, got ${CUDA_VISIBLE_DEVICES}" >&2 + exit 1 +fi + +echo "PROFILE_LAUNCH_ECHO host=$(hostname) gpu=${CUDA_VISIBLE_DEVICES} role=MoE+router-kernel-only tp=1,2,4 tokens='${TOKENS}' method=Frontier-RecordFunctionTracer output=${OUTPUT_ROOT} expected_wall=10-30m expected_gpu_cap=1.0_H20h" +date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" +test "$(git -C "${FRONTIER_SOURCE}" rev-parse HEAD)" = "deadc4a321f0baaa534c6ebd17f974123733cdc2" +test "$(git -C "${VLLM_SOURCE}" rev-parse HEAD)" = "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1" +git rev-parse HEAD > "${OUTPUT_ROOT}/provenance/aituner.commit" +git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit" +printf '%s\n' ${TOKENS} > "${OUTPUT_ROOT}/provenance/tokens.txt" +sha256sum "${PROFILE_DIR}/profile_vllm020_moe.py" "${PROFILE_DIR}/profile_vllm020_router.py" "${RUN_DIR}/run_graph_kernel_only_moe.sh" > "${OUTPUT_ROOT}/provenance/source.sha256" + +timeout --signal=TERM --kill-after=30s 2400 \ + "${VENV_ROOT}/bin/python" "${PROFILE_DIR}/profile_vllm020_moe.py" \ + --vllm-source "${VLLM_SOURCE}" --frontier-source "${FRONTIER_SOURCE}" \ + --model "${MODEL_ROOT}" --output "${OUTPUT_ROOT}/raw/moe.json" \ + --tp 1 2 4 --num-tokens ${TOKENS} --routing-modes uniform_random_logits \ + --warmup-iters 3 --repeats 5 --profile-method record_function --check-reference + +timeout --signal=TERM --kill-after=30s 1800 \ + "${VENV_ROOT}/bin/python" "${PROFILE_DIR}/profile_vllm020_router.py" \ + --vllm-source "${VLLM_SOURCE}" --frontier-source "${FRONTIER_SOURCE}" \ + --model "${MODEL_ROOT}" --output "${OUTPUT_ROOT}/raw/router.json" \ + --num-tokens ${TOKENS} --warmup-iters 3 --repeats 5 --profile-method record_function + +test -s "${OUTPUT_ROOT}/raw/moe.json" +test -s "${OUTPUT_ROOT}/raw/router.json" +sha256sum "${OUTPUT_ROOT}/raw"/*.json "${OUTPUT_ROOT}/provenance"/* > "${OUTPUT_ROOT}/artifacts.sha256" +date -u +"END_UTC=%Y-%m-%dT%H:%M:%SZ" +echo "GRAPH_KERNEL_ONLY_MOE_COMPLETE" diff --git a/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py b/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py index ade95f0..c6f8a2a 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py +++ b/runs/frontier-qwen30-vllm020-profile-v1/freeze_frontier_profiles.py @@ -117,8 +117,18 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--attention", type=Path, nargs="+", required=True) parser.add_argument("--moe", type=Path, required=True) parser.add_argument("--router", type=Path, required=True) - parser.add_argument("--allreduce", type=Path, nargs=2, required=True) + parser.add_argument("--allreduce", type=Path, nargs=2) + parser.add_argument("--allreduce-frozen", type=Path) parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--measurement-type", + choices=("CUDA_EVENT", "KERNEL_ONLY"), + default="CUDA_EVENT", + ) + parser.add_argument( + "--frontier-commit", + default="d9cfeb6d8791fbf2f295dd9744c56a666171776e", + ) return parser.parse_args() @@ -196,7 +206,7 @@ def write_csv(path: Path, fieldnames: list[str], rows: list[dict[str, Any]]) -> def freeze_attention( - inputs: list[Path], output: Path + inputs: list[Path], output: Path, *, measurement_type: str ) -> tuple[int, int, list[str]]: rows: list[dict[str, Any]] = [] mixed_rows: list[dict[str, Any]] = [] @@ -208,6 +218,13 @@ def freeze_attention( raise ValueError(f"unexpected attention schema in {path}") if payload["environment"].get("vllm_version") != "0.20.0": raise ValueError(f"unexpected vLLM version in {path}") + expected_method = ( + "record_function" if measurement_type == "KERNEL_ONLY" else "cuda_event" + ) + if payload["environment"].get("profile_method", "cuda_event") != expected_method: + raise ValueError( + f"attention profile method mismatch in {path}: expected {expected_method}" + ) for raw in payload["rows"]: if raw.get("error") is not None: raise ValueError(f"failed attention row in {path}: {raw['error']}") @@ -321,7 +338,7 @@ def freeze_attention( "profiling_precision": "BF16", "model_arch": "generic", "quant_signature": "none", - "measurement_type": "CUDA_EVENT", + "measurement_type": measurement_type, "is_true_mixed_batch": True, "prefill_seq_lens": json.dumps(prefill_queries), "prefill_kv_cache_sizes": json.dumps(prefill_contexts), @@ -418,7 +435,7 @@ def freeze_attention( "profiling_precision": "BF16", "model_arch": "generic", "quant_signature": "none", - "measurement_type": "CUDA_EVENT", + "measurement_type": measurement_type, "is_true_mixed_batch": False, "prefill_seq_lens": "", "prefill_kv_cache_sizes": "", @@ -498,13 +515,23 @@ def load_features(counts: list[int]) -> dict[str, float]: } -def freeze_moe(moe_path: Path, router_path: Path, output: Path) -> int: +def freeze_moe( + moe_path: Path, router_path: Path, output: Path, *, measurement_type: str +) -> int: moe = load_json(moe_path) router = load_json(router_path) if moe.get("schema_version") != "qwen30_vllm020_moe_raw.v1": raise ValueError(f"unexpected MoE schema in {moe_path}") if router.get("schema_version") != "qwen30_vllm020_router_raw.v1": raise ValueError(f"unexpected router schema in {router_path}") + expected_method = ( + "record_function" if measurement_type == "KERNEL_ONLY" else "cuda_event" + ) + for payload, label in ((moe, "moe"), (router, "router")): + if payload["environment"].get("profile_method", "cuda_event") != expected_method: + raise ValueError( + f"{label} profile method mismatch in {payload}: expected {expected_method}" + ) router_by_tokens = {int(row["num_tokens"]): row for row in router["rows"]} rows: list[dict[str, Any]] = [] seen_pairs: set[tuple[int, int, str]] = set() @@ -550,7 +577,7 @@ def freeze_moe(moe_path: Path, router_path: Path, output: Path) -> int: "load_distribution": routing_mode, "seed": 20260716, "moe_grouped_gemm_backend": raw["backend"], - "measurement_type": "CUDA_EVENT", + "measurement_type": measurement_type, "profiling_precision": "BF16", "model_arch": "generic", "quant_signature": "none", @@ -565,9 +592,17 @@ def freeze_moe(moe_path: Path, router_path: Path, output: Path) -> int: ) rows.append(row) - expected = 3 * 12 * 2 - if len(rows) != expected: - raise ValueError(f"expected {expected} MoE rows, got {len(rows)}") + tokens = {int(row["num_tokens"]) for row in router["rows"]} + modes = {str(row["routing_mode"]) for row in moe["rows"]} + expected = {(tp, tokens_value, mode) for tp in (1, 2, 4) for tokens_value in tokens for mode in modes} + actual = { + (int(row["num_tensor_parallel_workers"]), int(row["num_tokens"]), str(row["load_distribution"])) + for row in rows + } + if actual != expected: + raise ValueError( + f"MoE TP/token/routing coverage mismatch: missing={expected - actual}, extra={actual - expected}" + ) moe_fields = [ f"time_stats.{op}.{stat}" for op in MOE_OPS for stat in STAT_NAMES ] + list(MOE_METADATA) @@ -617,7 +652,13 @@ def freeze_allreduce(inputs: list[Path], output: Path) -> int: def main() -> None: args = parse_args() - all_inputs = [args.linear, *args.attention, args.moe, args.router, *args.allreduce] + if args.allreduce is not None and args.allreduce_frozen is not None: + raise SystemExit("provide either --allreduce or --allreduce-frozen, not both") + all_inputs = [args.linear, *args.attention, args.moe, args.router] + if args.allreduce is not None: + all_inputs.extend(args.allreduce) + if args.allreduce_frozen is not None: + all_inputs.append(args.allreduce_frozen) for path in all_inputs: if not path.is_file(): raise SystemExit(f"missing input: {path}") @@ -627,24 +668,39 @@ def main() -> None: shutil.copyfile(args.linear, linear_output) with linear_output.open(newline="") as handle: linear_rows = list(csv.DictReader(handle)) - if len(linear_rows) != 36: - raise ValueError(f"expected 36 linear rows, got {len(linear_rows)}") + if not linear_rows: + raise ValueError("linear profile has no rows") + if {row.get("measurement_type") for row in linear_rows} != {args.measurement_type}: + raise ValueError( + f"linear measurement family mismatch: expected {args.measurement_type}" + ) attention_rows, mixed_rows, attention_tps = freeze_attention( - list(args.attention), args.output + list(args.attention), args.output, measurement_type=args.measurement_type ) - moe_rows = freeze_moe(args.moe, args.router, args.output) - allreduce_rows = freeze_allreduce(list(args.allreduce), args.output) + moe_rows = freeze_moe( + args.moe, args.router, args.output, measurement_type=args.measurement_type + ) + allreduce_rows = 0 + allreduce_source = "not_included" + if args.allreduce is not None: + allreduce_rows = freeze_allreduce(list(args.allreduce), args.output) + allreduce_source = "raw_vllm020_measurements" + elif args.allreduce_frozen is not None: + shutil.copyfile(args.allreduce_frozen, args.output / "allreduce.json") + allreduce_rows = len(load_json(args.allreduce_frozen).get("rows", [])) + allreduce_source = "carried_forward_frozen_measurements" output_files = [ linear_output, args.output / "attention.csv", args.output / "attention_true_mixed_fused.csv", args.output / "moe.csv", - args.output / "allreduce.json", ] + if (args.output / "allreduce.json").is_file(): + output_files.append(args.output / "allreduce.json") batch_composition_augmented = len(args.attention) > 3 - long_context_augmented = any( + long_context_augmented = args.measurement_type == "KERNEL_ONLY" or any( "long-context" in path.name for path in args.attention ) long_context_coverage: dict[str, Any] = {"included": long_context_augmented} @@ -659,34 +715,45 @@ def main() -> None: if int(row["num_tensor_parallel_workers"]) == tp and row["is_prefill"].lower() == "false" } - mixed_kv = { - int(float(row["decode_avg_kv_cache_size"])) - for row in frozen_attention - if int(row["num_tensor_parallel_workers"]) == tp - and row.get("is_true_mixed_batch", "").lower() == "true" - } - if not {16384, 32768, 40960}.issubset(decode_kv): - raise ValueError(f"long-context decode coverage mismatch for TP{tp}") - if not {16384, 32768}.issubset(mixed_kv): - raise ValueError(f"long-context mixed coverage mismatch for TP{tp}") + required_decode = ( + {128, 1024, 2048, 4096, 8192, 16384, 32768, 40960} + if args.measurement_type == "KERNEL_ONLY" + else {16384, 32768, 40960} + ) + if not required_decode.issubset(decode_kv): + raise ValueError(f"decode KV coverage mismatch for TP{tp}") by_tp[str(tp)] = { "decode_kv_lengths": sorted(decode_kv), - "true_mixed_decode_avg_kv_lengths": sorted(mixed_kv), } + if args.measurement_type != "KERNEL_ONLY": + mixed_kv = { + int(float(row["decode_avg_kv_cache_size"])) + for row in frozen_attention + if int(row["num_tensor_parallel_workers"]) == tp + and row.get("is_true_mixed_batch", "").lower() == "true" + } + if not {16384, 32768}.issubset(mixed_kv): + raise ValueError(f"long-context mixed coverage mismatch for TP{tp}") + by_tp[str(tp)]["true_mixed_decode_avg_kv_lengths"] = sorted(mixed_kv) long_context_coverage["by_tp"] = by_tp manifest = { "schema_version": ( - "frontier_qwen30_vllm020_frozen_profile.v4" - if long_context_augmented + "frontier_qwen30_vllm020_kernel_only_profile.v1" + if args.measurement_type == "KERNEL_ONLY" + else ( + "frontier_qwen30_vllm020_frozen_profile.v4" + if long_context_augmented else ( "frontier_qwen30_vllm020_frozen_profile.v3" if batch_composition_augmented else "frontier_qwen30_vllm020_frozen_profile.v2" + ) ) ), "profile_id": ( "qwen3-30b-a3b-bf16-vllm020-h20-tp1-2-4-" "fused-mixed-total-conserving" + + ("-kernel-only-record-function" if args.measurement_type == "KERNEL_ONLY" else "") + ("-pure-prefill-batch-composition" if batch_composition_augmented else "") + ("-long-context-decode-mixed" if long_context_augmented else "") ), @@ -696,7 +763,7 @@ def main() -> None: "dtype": "bfloat16", "vllm_version": "0.20.0", "vllm_source_commit": "88d34c6409e9fb3c7b8ca0c04756f061d2099eb1", - "frontier_commit": "d9cfeb6d8791fbf2f295dd9744c56a666171776e", + "frontier_commit": args.frontier_commit, "tensor_parallel_sizes": [1, 2, 4], }, "row_counts": { @@ -740,8 +807,7 @@ def main() -> None: "expert measurement already includes prepare/finalize so shuffling is zero" ), "allreduce": ( - "Frozen exact runtime measurements; base profile-only comparison keeps the " - "historical Frontier CC backend fixed to isolate compute profile fidelity" + "Frozen exact runtime measurements; source=" + allreduce_source ), }, "inputs": {str(path.resolve()): sha256(path) for path in all_inputs}, diff --git a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_flashattn.py b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_flashattn.py index 0e25d35..200762e 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_flashattn.py +++ b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_flashattn.py @@ -39,6 +39,12 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--repeats", type=int, default=5) parser.add_argument("--device", default="cuda:0") parser.add_argument("--profile-kv-update", action="store_true") + parser.add_argument( + "--profile-method", + choices=("cuda_event", "record_function"), + default="cuda_event", + ) + parser.add_argument("--frontier-source", type=Path) return parser.parse_args() @@ -66,12 +72,17 @@ def main() -> None: raise SystemExit(f"expected vLLM source {VLLM_COMMIT}, got {source_head}") if not args.model.joinpath("config.json").is_file(): raise SystemExit(f"missing model config: {args.model / 'config.json'}") + if args.profile_method == "record_function" and args.frontier_source is None: + raise SystemExit("--frontier-source is required for --profile-method record_function") bench_dir = args.vllm_source / "benchmarks" / "attention_benchmarks" sys.path.insert(0, str(bench_dir)) import runner # type: ignore[import-not-found] # noqa: PLC0415 from batch_spec import parse_batch_spec # type: ignore[import-not-found] # noqa: PLC0415 - from common import BenchmarkConfig # type: ignore[import-not-found] # noqa: PLC0415 + from common import ( # type: ignore[import-not-found] # noqa: PLC0415 + BenchmarkConfig, + BenchmarkResult, + ) from vllm.config import ( # noqa: PLC0415 CacheConfig, CompilationConfig, @@ -90,6 +101,13 @@ def main() -> None: from vllm.v1.kv_cache_interface import FullAttentionSpec # noqa: PLC0415 from vllm.v1.worker.workspace import init_workspace_manager # noqa: PLC0415 + record_function_tracer = None + if args.profile_method == "record_function": + sys.path.insert(0, str(args.frontier_source.resolve())) + from frontier.profiling.utils.record_function_tracer import RecordFunctionTracer + + record_function_tracer = RecordFunctionTracer + def create_vllm_config(config: BenchmarkConfig, max_num_blocks: int) -> VllmConfig: model_config = ModelConfig( model=str(args.model), @@ -146,6 +164,9 @@ def main() -> None: runner._create_vllm_config = create_vllm_config init_workspace_manager(args.device) + args.output.parent.mkdir(parents=True, exist_ok=True) + if args.profile_method == "record_function": + (args.output.parent / "profiler_traces").mkdir(exist_ok=True) def profile_kv_cache_update(config: BenchmarkConfig) -> dict[str, float]: device = torch.device(config.device) @@ -218,6 +239,137 @@ def main() -> None: "std_ms": statistics.pstdev(samples), } + def profile_kernel_only( + config: BenchmarkConfig, + ) -> tuple[BenchmarkResult, dict[str, float] | None]: + """Trace exactly one vLLM FA3 forward/KV-update per annotation. + + `RecordFunctionTracer` is Frontier's actual KERNEL_ONLY collector: it + follows CUDA launch correlations and sums kernels under the annotation. + The profiling loop therefore contains no CUDA-event value relabeling. + """ + device = torch.device(config.device) + torch.accelerator.set_device_index(device) + backend_config = runner._get_backend_config(config.backend) + requests = parse_batch_spec(config.batch_spec) + q_lens = [request.q_len for request in requests] + kv_lens = [request.kv_len for request in requests] + total_q = sum(q_lens) + max_kv = max(kv_lens) + max_blocks_per_request = (max_kv + config.block_size - 1) // config.block_size + max_num_blocks = len(requests) * max_blocks_per_request + + with runner.log_warnings_and_errors_only(): + vllm_config = create_vllm_config(config, max_num_blocks) + dtype = vllm_config.model_config.dtype + with set_current_vllm_config(vllm_config): + backend_class, impl, layer = runner._create_backend_impl( + backend_config, config, device, dtype + ) + required_layout = backend_class.get_required_kv_cache_layout() + if required_layout is not None: + set_kv_cache_layout(required_layout) + get_kv_cache_layout.cache_clear() + common_metadata = runner._build_common_attn_metadata( + q_lens, kv_lens, config.block_size, device + ) + kv_cache_spec = FullAttentionSpec( + block_size=config.block_size, + num_kv_heads=config.num_kv_heads, + head_size=config.head_dim, + dtype=dtype, + ) + builder = runner._create_metadata_builder( + backend_class, kv_cache_spec, vllm_config, device, config.backend + ) + attn_metadata = builder.build( + common_prefix_len=0, common_attn_metadata=common_metadata + ) + quantize_query = config.kv_cache_dtype.startswith("fp8") and getattr( + impl, "supports_quant_query_input", False + ) + q_list, k_list, v_list = runner._create_input_tensors( + config, total_q, device, dtype, quantize_query=quantize_query + ) + cache_list = runner._create_kv_cache( + config, max_num_blocks, backend_class, device, dtype + ) + output = torch.empty( + total_q, + config.num_q_heads, + config.head_dim, + device=device, + dtype=dtype, + ) + + def run_core() -> None: + for layer_index in range(config.num_layers): + impl.forward( + layer, + q_list[layer_index], + k_list[layer_index], + v_list[layer_index], + cache_list[layer_index], + attn_metadata, + output=output, + ) + + for _ in range(config.warmup_iters): + run_core() + torch.accelerator.synchronize() + core_tracer = record_function_tracer(str(args.output.parent)) + with core_tracer: + for _ in range(config.repeats): + with torch.profiler.record_function("vidur_attention_core"): + run_core() + core_stats = core_tracer.get_operation_time_stats() + if "attention_core" not in core_stats: + raise RuntimeError("missing KERNEL_ONLY FlashAttention core stats") + core = { + name: float(value) / config.num_layers + for name, value in core_stats["attention_core"].items() + } + + kv_stats = None + if args.profile_kv_update: + def run_kv_cache_update() -> None: + for layer_index in range(config.num_layers): + impl.do_kv_cache_update( + layer, + k_list[layer_index], + v_list[layer_index], + cache_list[layer_index], + common_metadata.slot_mapping, + ) + + for _ in range(config.warmup_iters): + run_kv_cache_update() + torch.accelerator.synchronize() + kv_tracer = record_function_tracer(str(args.output.parent)) + with kv_tracer: + for _ in range(config.repeats): + with torch.profiler.record_function("vidur_attn_kv_cache_save"): + run_kv_cache_update() + kv_time_stats = kv_tracer.get_operation_time_stats() + if "attn_kv_cache_save" not in kv_time_stats: + raise RuntimeError("missing KERNEL_ONLY KV-update stats") + kv_stats = { + f"{name}_ms": float(value) / config.num_layers + for name, value in kv_time_stats["attn_kv_cache_save"].items() + } + + result = BenchmarkResult( + config=config, + mean_time=core["mean"] / 1000.0, + std_time=core["std"] / 1000.0, + min_time=core["min"] / 1000.0, + max_time=core["max"] / 1000.0, + throughput_tokens_per_sec=( + total_q / (core["mean"] / 1000.0) if core["mean"] > 0 else 0.0 + ), + ) + return result, kv_stats + rows: list[dict[str, object]] = [] for tp in args.tp: for batch_spec in args.batch_specs: @@ -237,12 +389,18 @@ def main() -> None: kv_cache_dtype="auto", use_cuda_graphs=False, ) - result = runner.run_attention_benchmark(config) + if args.profile_method == "record_function": + result, kv_stats = profile_kernel_only(config) + else: + result = runner.run_attention_benchmark(config) + kv_stats = ( + profile_kv_cache_update(config) if args.profile_kv_update else None + ) row = result.to_dict() row["tensor_parallel_size"] = tp row["attention_core_excludes_kv_cache_update"] = True - if args.profile_kv_update: - row["kv_cache_update_time"] = profile_kv_cache_update(config) + if kv_stats is not None: + row["kv_cache_update_time"] = kv_stats rows.append(row) print( json.dumps( @@ -273,10 +431,10 @@ def main() -> None: "attention_backend": "FLASH_ATTN", "block_size": 16, "profile_kv_update": args.profile_kv_update, + "profile_method": args.profile_method, }, "rows": rows, } - args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text( json.dumps(payload, indent=2, sort_keys=True, default=json_default) + "\n" ) diff --git a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_moe.py b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_moe.py index 74da2d1..2eecef2 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_moe.py +++ b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_moe.py @@ -9,7 +9,7 @@ import math import statistics import subprocess from pathlib import Path -from typing import Any +from typing import Any, Callable import torch import vllm @@ -40,6 +40,12 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--repeats", type=int, default=5) parser.add_argument("--device", default="cuda:0") parser.add_argument("--check-reference", action="store_true") + parser.add_argument( + "--profile-method", + choices=("cuda_event", "record_function"), + default="cuda_event", + ) + parser.add_argument("--frontier-source", type=Path) return parser.parse_args() @@ -59,6 +65,34 @@ def stats_ms(samples: list[float]) -> dict[str, float]: } +def measure_kernel_only_ms( + fn: Callable[[], torch.Tensor], + *, + warmup_iters: int, + repeats: int, + trace_root: Path, + operation_name: str, + record_function_tracer: type, +) -> tuple[torch.Tensor, dict[str, float]]: + """Use Frontier's KERNEL_ONLY contract, not a CUDA-event relabel.""" + result = None + for _ in range(warmup_iters): + result = fn() + torch.accelerator.synchronize() + + tracer = record_function_tracer(str(trace_root)) + with tracer: + for _ in range(repeats): + with torch.profiler.record_function(f"vidur_{operation_name}"): + result = fn() + stats = tracer.get_operation_time_stats() + if operation_name not in stats: + raise RuntimeError(f"missing RecordFunctionTracer stats for {operation_name}") + if result is None: + raise RuntimeError("kernel-only profiler executed no MoE step") + return result, {name: float(value) for name, value in stats[operation_name].items()} + + def routing_inputs( mode: str, num_tokens: int, device: torch.device ) -> tuple[torch.Tensor, torch.Tensor, dict[str, Any]]: @@ -145,6 +179,8 @@ def main() -> None: raise SystemExit( f"model contract mismatch: expected {expected_model}, got {observed_model}" ) + if args.profile_method == "record_function" and args.frontier_source is None: + raise SystemExit("--frontier-source is required for --profile-method record_function") from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config from vllm.model_executor.layers.fused_moe.activation import MoEActivation @@ -163,10 +199,22 @@ def main() -> None: from vllm.utils.math_utils import next_power_of_2 from vllm.v1.worker.workspace import init_workspace_manager + record_function_tracer = None + if args.profile_method == "record_function": + import sys + + sys.path.insert(0, str(args.frontier_source.resolve())) + from frontier.profiling.utils.record_function_tracer import RecordFunctionTracer + + record_function_tracer = RecordFunctionTracer + device = torch.device(args.device) torch.accelerator.set_device_index(device) torch.manual_seed(20260716) init_workspace_manager(args.device) + args.output.parent.mkdir(parents=True, exist_ok=True) + if args.profile_method == "record_function": + (args.output.parent / "profiler_traces").mkdir(exist_ok=True) max_num_tokens = next_power_of_2(max(args.num_tokens)) rows: list[dict[str, Any]] = [] @@ -257,8 +305,8 @@ def main() -> None: routing_mode, num_tokens, device ) - for _ in range(args.warmup_iters): - output = kernel.apply( + def run_kernel() -> torch.Tensor: + return kernel.apply( hidden_states=hidden, w1=w13_kernel, w2=w2_kernel, @@ -269,27 +317,30 @@ def main() -> None: expert_map=None, apply_router_weight_on_input=False, ) - torch.accelerator.synchronize() - samples: list[float] = [] - for _ in range(args.repeats): - start = torch.cuda.Event(enable_timing=True) - end = torch.cuda.Event(enable_timing=True) - start.record() - output = kernel.apply( - hidden_states=hidden, - w1=w13_kernel, - w2=w2_kernel, - topk_weights=topk_weights, - topk_ids=topk_ids, - activation=MoEActivation.SILU, - global_num_experts=NUM_EXPERTS, - expert_map=None, - apply_router_weight_on_input=False, + if args.profile_method == "record_function": + output, time_ms = measure_kernel_only_ms( + run_kernel, + warmup_iters=args.warmup_iters, + repeats=args.repeats, + trace_root=args.output.parent, + operation_name="moe_grouped_gemm", + record_function_tracer=record_function_tracer, ) - end.record() + else: + for _ in range(args.warmup_iters): + output = run_kernel() torch.accelerator.synchronize() - samples.append(float(start.elapsed_time(end))) + samples: list[float] = [] + for _ in range(args.repeats): + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + output = run_kernel() + end.record() + torch.accelerator.synchronize() + samples.append(float(start.elapsed_time(end))) + time_ms = stats_ms(samples) if output.shape != hidden.shape or not torch.isfinite(output).all(): raise SystemExit( @@ -316,7 +367,7 @@ def main() -> None: "backend": backend.value, "intermediate_size_per_partition": INTERMEDIATE_DIM // tp, "output_is_reduced": kernel.output_is_reduced(), - "time_ms": stats_ms(samples), + "time_ms": time_ms, "routing_load": load, } rows.append(row) @@ -350,6 +401,7 @@ def main() -> None: "weight_quantization": "none", "top_k": TOP_K, "norm_topk_prob": True, + "profile_method": args.profile_method, }, "measurement_scope": ( "one TP-local weight shard: vLLM modular MoE prepare+FlashInfer " @@ -357,7 +409,6 @@ def main() -> None: ), "rows": rows, } - args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") diff --git a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_router.py b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_router.py index 8da2070..eac341f 100644 --- a/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_router.py +++ b/runs/frontier-qwen30-vllm020-profile-v1/profile_vllm020_router.py @@ -31,6 +31,12 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--warmup-iters", type=int, default=5) parser.add_argument("--repeats", type=int, default=20) parser.add_argument("--device", default="cuda:0") + parser.add_argument( + "--profile-method", + choices=("cuda_event", "record_function"), + default="cuda_event", + ) + parser.add_argument("--frontier-source", type=Path) return parser.parse_args() @@ -51,12 +57,34 @@ def stats_ms(samples: list[float]) -> dict[str, float]: def measure_ms( - fn: Callable[[], Any], warmup_iters: int, repeats: int + fn: Callable[[], Any], + warmup_iters: int, + repeats: int, + *, + profile_method: str, + trace_root: Path | None = None, + operation_name: str | None = None, + record_function_tracer: type | None = None, ) -> tuple[Any, dict[str, float]]: result = None for _ in range(warmup_iters): result = fn() torch.accelerator.synchronize() + if profile_method == "record_function": + if trace_root is None or operation_name is None or record_function_tracer is None: + raise ValueError("record_function profiling requires tracer metadata") + tracer = record_function_tracer(str(trace_root)) + with tracer: + for _ in range(repeats): + with torch.profiler.record_function(f"vidur_{operation_name}"): + result = fn() + stats = tracer.get_operation_time_stats() + if operation_name not in stats: + raise RuntimeError(f"missing RecordFunctionTracer stats for {operation_name}") + return result, { + name: float(value) for name, value in stats[operation_name].items() + } + samples: list[float] = [] for _ in range(repeats): start = torch.cuda.Event(enable_timing=True) @@ -76,6 +104,8 @@ def main() -> None: source_head = git_head(args.vllm_source) if source_head != VLLM_COMMIT: raise SystemExit(f"expected vLLM source {VLLM_COMMIT}, got {source_head}") + if args.profile_method == "record_function" and args.frontier_source is None: + raise SystemExit("--frontier-source is required for --profile-method record_function") raw_model_config = json.loads(args.model.joinpath("config.json").read_text()) observed = { @@ -103,6 +133,15 @@ def main() -> None: from vllm.model_executor.layers.fused_moe import fused_topk from vllm.model_executor.layers.linear import ReplicatedLinear + record_function_tracer = None + if args.profile_method == "record_function": + import sys + + sys.path.insert(0, str(args.frontier_source.resolve())) + from frontier.profiling.utils.record_function_tracer import RecordFunctionTracer + + record_function_tracer = RecordFunctionTracer + device = torch.device(args.device) torch.accelerator.set_device_index(device) torch.manual_seed(20260716) @@ -113,6 +152,9 @@ def main() -> None: skip_tokenizer_init=True, generation_config="vllm", ) + args.output.parent.mkdir(parents=True, exist_ok=True) + if args.profile_method == "record_function": + (args.output.parent / "profiler_traces").mkdir(exist_ok=True) rows: list[dict[str, Any]] = [] with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: @@ -141,12 +183,22 @@ def main() -> None: (num_tokens, HIDDEN_DIM), device=device, dtype=torch.bfloat16 ).uniform_(-0.1, 0.1) logits, gate_time = measure_ms( - lambda: gate(hidden)[0], args.warmup_iters, args.repeats + lambda: gate(hidden)[0], + args.warmup_iters, + args.repeats, + profile_method=args.profile_method, + trace_root=args.output.parent, + operation_name="moe_gating_linear", + record_function_tracer=record_function_tracer, ) topk_result, topk_time = measure_ms( lambda: fused_topk(hidden, logits, TOP_K, renormalize=True), args.warmup_iters, args.repeats, + profile_method=args.profile_method, + trace_root=args.output.parent, + operation_name="moe_gating_routing_topk", + record_function_tracer=record_function_tracer, ) def gate_and_topk() -> tuple[ @@ -158,7 +210,13 @@ def main() -> None: ) combined_result, combined_time = measure_ms( - gate_and_topk, args.warmup_iters, args.repeats + gate_and_topk, + args.warmup_iters, + args.repeats, + profile_method=args.profile_method, + trace_root=args.output.parent, + operation_name="moe_gating_linear_and_routing_topk", + record_function_tracer=record_function_tracer, ) topk_weights, topk_ids, _ = topk_result combined_weights, combined_ids, _ = combined_result @@ -207,6 +265,7 @@ def main() -> None: "gate_replication": "replicated_across_tp", "top_k": TOP_K, "norm_topk_prob": True, + "profile_method": args.profile_method, }, "measurement_scope": ( "vLLM ReplicatedLinear gate and fused_topk; measured separately and " @@ -214,7 +273,6 @@ def main() -> None: ), "rows": rows, } - args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") From 2deb53cb7243390db7d94a009caf30157b423fdc Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 23:25:15 +0800 Subject: [PATCH 114/148] Allow graph linear profile smoke --- .../run_graph_kernel_only_linear.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_linear.sh b/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_linear.sh index 7af7161..c75bfe7 100644 --- a/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_linear.sh +++ b/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_linear.sh @@ -9,7 +9,8 @@ VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-t1-dash0-deadc4a}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" -TOKENS=(1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128) +TOKENS_TEXT="${TOKENS:-1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128}" +read -r -a TOKENS <<< "${TOKENS_TEXT}" export MODEL_ROOT mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/profiles" From 27a2a3468a9db3ab5ec09e9c1d57b20c9de63494 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 23:27:07 +0800 Subject: [PATCH 115/148] Use verified FlashInfer profile cache --- .../run_graph_kernel_only_attention.sh | 4 ++++ .../run_graph_kernel_only_linear.sh | 4 ++++ .../run_graph_kernel_only_moe.sh | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_attention.sh b/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_attention.sh index 2d1ff9e..4d6ca07 100644 --- a/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_attention.sh +++ b/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_attention.sh @@ -12,6 +12,7 @@ OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" RUN_DIR="$(pwd -P)" PROFILE_DIR="${PROFILE_DIR:-${RUN_DIR%/runs/frontier-fidelity-envelope-v1}/runs/frontier-qwen30-vllm020-profile-v1}" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +PROFILE_HOME="${PROFILE_HOME:-/tmp/wjh}" VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-t1-dash0-deadc4a}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" @@ -19,6 +20,9 @@ CAPTURE_BUCKETS="${CAPTURE_BUCKETS:-1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 KV_CONTEXTS="${KV_CONTEXTS:-128 1024 2048 4096 8192 16384 32768 40960}" WARMUP_ITERS="${WARMUP_ITERS:-3}" REPEATS="${REPEATS:-5}" +export HOME="${PROFILE_HOME}" +export XDG_CACHE_HOME="${PROFILE_HOME}/.cache" +export VLLM_CACHE_ROOT="${XDG_CACHE_HOME}/vllm" mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/raw" exec > >(tee -a "${OUTPUT_ROOT}/logs/attention-tp${TP}.log") 2>&1 diff --git a/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_linear.sh b/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_linear.sh index c75bfe7..5764f97 100644 --- a/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_linear.sh +++ b/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_linear.sh @@ -6,12 +6,16 @@ OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" RUN_DIR="$(pwd -P)" PROFILE_DIR="${PROFILE_DIR:-${RUN_DIR%/runs/frontier-fidelity-envelope-v1}/runs/frontier-qwen30-vllm020-profile-v1}" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +PROFILE_HOME="${PROFILE_HOME:-/tmp/wjh}" VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-t1-dash0-deadc4a}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" TOKENS_TEXT="${TOKENS:-1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128}" read -r -a TOKENS <<< "${TOKENS_TEXT}" export MODEL_ROOT +export HOME="${PROFILE_HOME}" +export XDG_CACHE_HOME="${PROFILE_HOME}/.cache" +export VLLM_CACHE_ROOT="${XDG_CACHE_HOME}/vllm" mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/profiles" exec > >(tee -a "${OUTPUT_ROOT}/logs/linear.log") 2>&1 diff --git a/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_moe.sh b/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_moe.sh index 5a36966..27bb472 100644 --- a/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_moe.sh +++ b/runs/frontier-fidelity-envelope-v1/run_graph_kernel_only_moe.sh @@ -6,10 +6,14 @@ OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT must be set}" RUN_DIR="$(pwd -P)" PROFILE_DIR="${PROFILE_DIR:-${RUN_DIR%/runs/frontier-fidelity-envelope-v1}/runs/frontier-qwen30-vllm020-profile-v1}" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +PROFILE_HOME="${PROFILE_HOME:-/tmp/wjh}" VLLM_SOURCE="${VLLM_SOURCE:-/home/admin/cpfs/wjh/agentic-kv/third_party/vllm_v20_build}" FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-t1-dash0-deadc4a}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" TOKENS="${TOKENS:-1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128}" +export HOME="${PROFILE_HOME}" +export XDG_CACHE_HOME="${PROFILE_HOME}/.cache" +export VLLM_CACHE_ROOT="${XDG_CACHE_HOME}/vllm" mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/raw" exec > >(tee -a "${OUTPUT_ROOT}/logs/moe-router.log") 2>&1 From e2cd43808d5b4638191520b32d69aff87e53cf50 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Fri, 17 Jul 2026 23:41:01 +0800 Subject: [PATCH 116/148] Bound graph predictor grid to runtime --- .../run_frontier_qwen30_exact_trace_surface.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py index 454d0b2..cac0e74 100644 --- a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py +++ b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py @@ -454,6 +454,13 @@ def main() -> None: config_knobs["no_cache"] = args.fresh_predictor_cache if args.align_real_graph_runtime: config_knobs["num_blocks"] = REAL_NUM_BLOCKS_BY_CONFIG[(config.tp, config.mns)] + # These are predictor-grid limits, not scheduler knobs. Keep the + # grid inside the observed vLLM graph/runtime envelope: no batch + # can use a capture bucket above this cell's list and MBT is 8192. + config_knobs["prediction_max_batch_size"] = max( + GRAPH_CAPTURE_SIZES_BY_MNS[config.mns] + ) + config_knobs["prediction_max_prefill_chunk_size"] = 8192 if kernel_paths is not None: config_knobs.update( { From e6f3e4a69086a38bfa6be2d1ad5929676e28a65f Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sat, 18 Jul 2026 00:09:06 +0800 Subject: [PATCH 117/148] Audit graph-aligned Frontier surface --- .../analyze_graph_piecewise_surface.py | 272 ++++++++++++++++++ 1 file changed, 272 insertions(+) create mode 100644 runs/frontier-fidelity-envelope-v1/analyze_graph_piecewise_surface.py diff --git a/runs/frontier-fidelity-envelope-v1/analyze_graph_piecewise_surface.py b/runs/frontier-fidelity-envelope-v1/analyze_graph_piecewise_surface.py new file mode 100644 index 0000000..968e8bb --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/analyze_graph_piecewise_surface.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +"""Compare a graph-aligned Frontier Qwen30 surface with audited real vLLM data. + +The input surface may have been produced by independent TP jobs. Therefore +this script deliberately reads the per-cell ``result.json`` files rather than +the runner's root manifest, which is only a convenience artifact and can be +overwritten by concurrent dispatch. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +from itertools import combinations +from pathlib import Path +from typing import Any + + +CONFIGS = tuple(f"tp{tp}_mns{mns}" for tp in (1, 2, 4) for mns in (8, 16, 32, 64)) +METRICS = ("ttft_ms", "tpot_ms", "e2e_ms") +STATISTICS = ("mean", "p90") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--sim-root", type=Path, required=True) + parser.add_argument("--real-audit", type=Path, required=True) + parser.add_argument("--json-output", type=Path, required=True) + parser.add_argument("--markdown-output", type=Path, required=True) + return parser.parse_args() + + +def read_json(path: Path) -> dict[str, Any]: + with path.open(encoding="utf-8") as source: + payload = json.load(source) + if not isinstance(payload, dict): + raise ValueError(f"expected JSON object: {path}") + return payload + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def numeric(value: Any, label: str) -> float: + if not isinstance(value, (int, float)) or isinstance(value, bool): + raise ValueError(f"{label} is not numeric: {value!r}") + result = float(value) + if not math.isfinite(result) or result < 0: + raise ValueError(f"{label} is invalid: {value!r}") + return result + + +def parse_config(name: str) -> tuple[int, int]: + try: + tp_part, mns_part = name.split("_", 1) + return int(tp_part.removeprefix("tp")), int(mns_part.removeprefix("mns")) + except ValueError as error: + raise ValueError(f"invalid config name: {name}") from error + + +def load_sim_cell(root: Path, name: str) -> dict[str, Any]: + tp, mns = parse_config(name) + path = root / "runs" / name / f"tp{tp}" / "result.json" + result = read_json(path) + if result.get("status") != "completed": + raise ValueError(f"{path}: status is not completed") + config = result.get("config") + if config != {"tp": tp, "mns": mns, "name": name}: + raise ValueError(f"{path}: config drift: {config!r}") + if result.get("trace_label") != f"tp{tp}": + raise ValueError(f"{path}: trace label does not match TP") + if result.get("request_count") != 129: + raise ValueError(f"{path}: expected 129 requests") + score = result.get("score") + if not isinstance(score, dict): + raise ValueError(f"{path}: missing score") + metrics: dict[str, dict[str, float]] = {} + for metric in METRICS: + prefix = metric.removesuffix("_ms") + metrics[metric] = { + statistic: numeric(score.get(f"{prefix}_{statistic}_ms"), f"{path}:{metric}:{statistic}") + for statistic in STATISTICS + } + return { + "config": name, + "tp": tp, + "mns": mns, + "trace_label": result["trace_label"], + "trace_sha256": result.get("trace_sha256"), + "offered_request_rate": numeric(result.get("offered_request_rate"), f"{path}:rate"), + "offered_request_rate_per_gpu": numeric( + result.get("offered_request_rate_per_gpu"), f"{path}:rate_per_gpu" + ), + "request_count": result["request_count"], + "elapsed_seconds": numeric(result.get("elapsed_seconds"), f"{path}:elapsed"), + "result_path": str(path.resolve()), + "result_sha256": sha256_file(path), + "metrics": metrics, + } + + +def load_real_cell(audit: dict[str, Any], name: str) -> dict[str, dict[str, float]]: + configs = audit.get("configs") + if not isinstance(configs, dict) or name not in configs: + raise ValueError(f"real audit lacks {name}") + result = configs[name] + metrics = result.get("metrics") + if not isinstance(metrics, dict): + raise ValueError(f"real audit {name} lacks metrics") + parsed: dict[str, dict[str, float]] = {} + for metric in METRICS: + values = metrics.get(metric) + if not isinstance(values, dict): + raise ValueError(f"real audit {name} lacks {metric}") + parsed[metric] = { + "mean": numeric(values.get("pooled_mean_ms"), f"real:{name}:{metric}:mean"), + "p90": numeric(values.get("pooled_p90_ms"), f"real:{name}:{metric}:p90"), + } + return parsed + + +def ranking(values: dict[str, float]) -> list[str]: + return [name for name, _ in sorted(values.items(), key=lambda item: (item[1], item[0]))] + + +def pairwise_agreement(sim: dict[str, float], real: dict[str, float]) -> dict[str, int]: + concordant = discordant = ties = 0 + for left, right in combinations(sorted(sim), 2): + sim_delta = sim[left] - sim[right] + real_delta = real[left] - real[right] + if math.isclose(sim_delta, 0.0, abs_tol=1e-9) or math.isclose( + real_delta, 0.0, abs_tol=1e-9 + ): + ties += 1 + elif (sim_delta > 0) == (real_delta > 0): + concordant += 1 + else: + discordant += 1 + return { + "concordant_pairs": concordant, + "discordant_pairs": discordant, + "tied_pairs": ties, + "informative_pairs": concordant + discordant, + } + + +def comparison_summary(cells: dict[str, dict[str, Any]], real_audit: dict[str, Any]) -> dict[str, Any]: + summaries: dict[str, Any] = {} + for metric in METRICS: + for statistic in STATISTICS: + sim_values = { + name: cells[name]["metrics"][metric][f"sim_{statistic}_ms"] + for name in CONFIGS + } + real_values = { + name: load_real_cell(real_audit, name)[metric][statistic] for name in CONFIGS + } + sim_ranking = ranking(sim_values) + real_ranking = ranking(real_values) + pairwise = pairwise_agreement(sim_values, real_values) + summaries[f"{metric}:{statistic}"] = { + "sim_winner": sim_ranking[0], + "real_winner": real_ranking[0], + "winner_match": sim_ranking[0] == real_ranking[0], + "sim_ranking": sim_ranking, + "real_ranking": real_ranking, + **pairwise, + "pairwise_agreement_fraction": ( + pairwise["concordant_pairs"] / pairwise["informative_pairs"] + if pairwise["informative_pairs"] + else None + ), + } + return summaries + + +def render_markdown(payload: dict[str, Any]) -> str: + lines = [ + "# Frontier piecewise graph-profile vs. real vLLM", + "", + "Each cell uses its TP-normalized, 129-request trace. Frontier values are one deterministic simulation; " + "real values pool three fresh-server trials (387 requests/cell).", + "", + "| Config | TTFT sim / real mean (ms) | TPOT sim / real mean (ms) | E2E sim / real mean (ms) |", + "|---|---:|---:|---:|", + ] + for name in CONFIGS: + cell = payload["cells"][name] + entries = [] + for metric in METRICS: + values = cell["metrics"][metric] + entries.append(f"{values['sim_mean_ms']:.1f} / {values['real_mean_ms']:.1f}") + lines.append(f"| {name} | " + " | ".join(entries) + " |") + + lines.extend(["", "## Selection agreement", "", "| Target | Frontier winner | Real winner | Match | Pairwise agreement |", "|---|---|---|---:|---:|"]) + for target, summary in payload["selection"].items(): + agreement = summary["pairwise_agreement_fraction"] + agreement_text = "n/a" if agreement is None else f"{agreement:.1%}" + lines.append( + f"| {target} | {summary['sim_winner']} | {summary['real_winner']} | " + f"{'yes' if summary['winner_match'] else 'no'} | {agreement_text} " + f"({summary['concordant_pairs']}/{summary['informative_pairs']}) |" + ) + lines.append("") + return "\n".join(lines) + + +def main() -> None: + args = parse_args() + sim_root = args.sim_root.resolve() + real_path = args.real_audit.resolve() + real_audit = read_json(real_path) + if set(real_audit.get("configs", {})) != set(CONFIGS): + raise ValueError("real audit config set does not match the 12-cell surface") + + cells: dict[str, dict[str, Any]] = {} + for name in CONFIGS: + sim = load_sim_cell(sim_root, name) + real = load_real_cell(real_audit, name) + metrics = {} + for metric in METRICS: + metrics[metric] = { + f"sim_{statistic}_ms": sim["metrics"][metric][statistic] + for statistic in STATISTICS + } + metrics[metric].update( + { + f"real_{statistic}_ms": real[metric][statistic] + for statistic in STATISTICS + } + ) + metrics[metric].update( + { + f"{statistic}_ratio_sim_over_real": sim["metrics"][metric][statistic] + / real[metric][statistic] + for statistic in STATISTICS + } + ) + cells[name] = {key: value for key, value in sim.items() if key != "metrics"} + cells[name]["metrics"] = metrics + + payload = { + "schema": "frontier-qwen30-piecewise-graph-comparison-v1", + "contract": { + "configs": list(CONFIGS), + "request_count_per_cell": 129, + "real_trials_per_cell": 3, + "trace_policy": "TP-normalized arrivals with full 16-token prefix blocks only", + "graph_semantics": "Frontier piecewise; CUDA_EVENT for prefill/mixed and KERNEL_ONLY for captured decode", + }, + "sim_root": str(sim_root), + "real_audit": str(real_path), + "real_audit_sha256": sha256_file(real_path), + "cells": cells, + "selection": comparison_summary(cells, real_audit), + } + args.json_output.parent.mkdir(parents=True, exist_ok=True) + args.markdown_output.parent.mkdir(parents=True, exist_ok=True) + args.json_output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + args.markdown_output.write_text(render_markdown(payload), encoding="utf-8") + print(json.dumps(payload["selection"], indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() From 65fee8450ab9173479a7e7a7bcf3e4d85ad9cab9 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sat, 18 Jul 2026 00:30:59 +0800 Subject: [PATCH 118/148] Record graph-aligned Frontier result --- .../graph-piecewise-experiment-card.md | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/runs/frontier-fidelity-envelope-v1/graph-piecewise-experiment-card.md b/runs/frontier-fidelity-envelope-v1/graph-piecewise-experiment-card.md index 272fe9c..6a1e0e3 100644 --- a/runs/frontier-fidelity-envelope-v1/graph-piecewise-experiment-card.md +++ b/runs/frontier-fidelity-envelope-v1/graph-piecewise-experiment-card.md @@ -1,6 +1,6 @@ # EXP-SIMFID-Q30-GRAPH-PIECEWISE：graph-compatible kernel-only profile 是否修正 Frontier trace replay？ -> **状态：** approved and running（2026-07-17）。本卡是已纠正 prefix-trace contract 后的最小判别实验；不复用此前 `decode_cuda_graph_mode=none` 的数值作 fidelity verdict。 +> **状态：** completed（2026-07-17）。本卡是已纠正 prefix-trace contract 后的最小判别实验；不复用此前 `decode_cuda_graph_mode=none` 的数值作 fidelity verdict。 ## Purpose and hypotheses @@ -29,9 +29,39 @@ Frontier source inspection fixes the semantic boundary: `piecewise` emits `PIECE - **Validity gates：** every kernel CSV hash matches its manifest; every row says `KERNEL_ONLY`; every TP/capture-bucket/KV-context required by the runner is present; command records `piecewise`, per-cell blocks and capture sizes; each simulator cell completes all 129 requests. - **Decision:** G1 is supported only if the graph-aligned TP2/MNS16 TPOT median moves toward real **and** full-surface rank/error evidence improves. A single-cell timing improvement does not establish tuning sufficiency. If G2 holds, update the research claim to “Frontier has not solved tuning under trace-faithful MoE serving after graph-family alignment,” then profile stage/state composition rather than add arbitrary kernel rows. -## Expected figure +## Result (completed 2026-07-17) -`graph-piecewise-profile-prototype.svg` is deliberately schematic. The final figure uses the same axes and adds real data only after the profile and replay validity gates pass. +### Validity gates + +- The fresh profile family is exclusive `KERNEL_ONLY`: linear=57, attention=456, MoE=57 rows. Attention covers every TP in {1,2,4}, all MNS64 capture buckets, and the eight decode KV contexts; the final manifest records Frontier `deadc4a…`. +- All three TP traces contain the same 129 ISL/OSL requests and prompt semantic vector. Their offered rates are 0.215/0.430/0.860 req/s for TP=1/2/4, while the per-GPU rate is fixed. Prefix IDs use `floor(ISL/16)`: 36,321 legal complete blocks, not the invalid 36,443 `ceil` projection. +- All 12 Frontier cells completed 129/129 requests. No result has a Frontier error or scheduler stall. Commands record `piecewise`, the observed capture list, the real KV-block count, frozen CUDA-event prefill/mixed CSVs, and the new kernel-only CSVs. + +### Frontier versus real vLLM selection + +| Target | Frontier winner | Real winner | Pairwise order agreement | +|---|---|---|---:| +| TTFT mean / p90 | TP4/MNS64 / TP4/MNS64 | TP4/MNS64 / TP4/MNS64 | 56/66 (84.8%) / 56/66 (84.8%) | +| TPOT mean / p90 | TP4/MNS8 / TP4/MNS8 | TP4/MNS8 / TP4/MNS8 | 64/66 (97.0%) / 64/66 (97.0%) | +| E2E mean / p90 | TP4/MNS64 / TP4/MNS64 | TP4/MNS64 / TP4/MNS64 | 61/66 (92.4%) / 55/66 (83.3%) | + +Thus, for this one Qwen3-30B-A3B Trace-PD surface, graph-aligned Frontier selects the same argmin config for all six reported latency objectives. This is evidence for *selection fidelity on this surface*, not a general claim about dense models, fixed-shape traffic, prefill-only traffic, another serving engine, or SLO capacity. + +### Absolute latency is still unusable + +The same simulator is not calibrated for absolute latency. Across all 12 cells, sim/real ratios are: TTFT mean 8.13--510.88x (median 38.29x), TPOT mean 4.06--5.72x (median 4.86x), and E2E mean 4.66--13.99x (median 8.24x). At the TP2/MNS16 anchor, Frontier reports mean/p90 TTFT=606,885/1,194,098 ms, TPOT=64.42/66.54 ms, E2E=828,082/1,404,520 ms; real vLLM reports 34,443/77,922 ms, 14.10/16.15 ms, and 82,937/137,726 ms, respectively. + +The TPOT error is service-time error, not merely queueing: Frontier's `tpot_computation` equals its TPOT at the anchor. The large TTFT/E2E error then amplifies through the simulated queue. The profile rows themselves are measured kernel rows, so the next diagnosis must inspect Frontier's layer/operation composition and batching-state model rather than relabeling or adding arbitrary graph kernels. + +### Decision + +G1 is only partially supported: `piecewise` plus a true kernel-only decode family improves the prior `none` smoke's 96.37 ms TPOT median to 65.60 ms, but leaves a large absolute gap. G2 is rejected for the *argmin selection* question on this surface because all six winners match. The defensible claim is therefore: + +> Frontier has solved config selection for this evaluated MoE trace surface, despite failing absolute latency prediction; it has **not** yet been shown to solve the broader tuning problem. + +The next discriminating tests are the remaining planned dense/MoE × fixed-shape/trace-faithful × prefill+decode/prefill-only surfaces. In parallel, diagnose the decode service-time composition error; do not use Frontier's current absolute values for SLO, cost, or capacity claims. + +Artifacts: `/home/admin/cpfs/wjh/aituner/graph-piecewise-qwen30-20260717/full/frozen-kernel-only/manifest.json` and `/home/admin/cpfs/wjh/aituner/graph-piecewise-qwen30-20260717/simulator-piecewise-surface-v2/analysis/{comparison.json,comparison.md}`. ## Cost and provenance From 44104bd96e37f02d92f3e061af44c3945b5f27a2 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sat, 18 Jul 2026 01:00:41 +0800 Subject: [PATCH 119/148] Prepare remaining Qwen30 latency cases --- .../analyze_qwen30_latency_case.py | 136 ++++++++++ .../audit_qwen30_latency_case.py | 186 +++++++++++++ .../prepare_qwen30_latency_case.py | 245 ++++++++++++++++++ .../qwen30-latency-expansion-card.md | 49 ++++ ...run_frontier_qwen30_exact_trace_surface.py | 28 +- .../run_qwen30_exact_trace_real_anchor.sh | 21 +- 6 files changed, 650 insertions(+), 15 deletions(-) create mode 100644 runs/frontier-fidelity-envelope-v1/analyze_qwen30_latency_case.py create mode 100644 runs/frontier-fidelity-envelope-v1/audit_qwen30_latency_case.py create mode 100644 runs/frontier-fidelity-envelope-v1/prepare_qwen30_latency_case.py create mode 100644 runs/frontier-fidelity-envelope-v1/qwen30-latency-expansion-card.md diff --git a/runs/frontier-fidelity-envelope-v1/analyze_qwen30_latency_case.py b/runs/frontier-fidelity-envelope-v1/analyze_qwen30_latency_case.py new file mode 100644 index 0000000..2b7e734 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/analyze_qwen30_latency_case.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Compare one complete Qwen30 Frontier latency surface with real vLLM.""" + +from __future__ import annotations + +import argparse +import json +import math +from itertools import combinations +from pathlib import Path +from typing import Any + + +CONFIGS = tuple(f"tp{tp}_mns{mns}" for tp in (1, 2, 4) for mns in (8, 16, 32, 64)) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--sim-root", type=Path, required=True) + parser.add_argument("--real-audit", type=Path, required=True) + parser.add_argument("--json-output", type=Path, required=True) + parser.add_argument("--markdown-output", type=Path, required=True) + return parser.parse_args() + + +def number(value: Any, field: str) -> float: + if not isinstance(value, (int, float)) or isinstance(value, bool): + raise ValueError(f"{field} is not numeric: {value!r}") + result = float(value) + if not math.isfinite(result) or result < 0: + raise ValueError(f"{field} is invalid: {value!r}") + return result + + +def parse_config(name: str) -> tuple[int, int]: + tp, mns = name.split("_", 1) + return int(tp.removeprefix("tp")), int(mns.removeprefix("mns")) + + +def rank(values: dict[str, float]) -> list[str]: + return [name for name, _ in sorted(values.items(), key=lambda item: (item[1], item[0]))] + + +def pairwise(sim: dict[str, float], real: dict[str, float]) -> dict[str, int | float | None]: + concordant = discordant = ties = 0 + for left, right in combinations(CONFIGS, 2): + sim_delta = sim[left] - sim[right] + real_delta = real[left] - real[right] + if math.isclose(sim_delta, 0.0, abs_tol=1e-9) or math.isclose(real_delta, 0.0, abs_tol=1e-9): + ties += 1 + elif (sim_delta > 0) == (real_delta > 0): + concordant += 1 + else: + discordant += 1 + informative = concordant + discordant + return { + "concordant_pairs": concordant, + "discordant_pairs": discordant, + "tied_pairs": ties, + "informative_pairs": informative, + "agreement": concordant / informative if informative else None, + } + + +def main() -> None: + args = parse_args() + root = args.sim_root.resolve() + audit = json.loads(args.real_audit.read_text()) + metrics = tuple(audit.get("applicable_metrics") or []) + if not metrics or set(audit.get("configs") or {}) != set(CONFIGS): + raise ValueError("real audit is incomplete") + expected_requests = int(audit["trace_manifests"]["tp1"]["requests"]) + cells: dict[str, Any] = {} + for name in CONFIGS: + tp, mns = parse_config(name) + path = root / "runs" / name / f"tp{tp}" / "result.json" + result = json.loads(path.read_text()) + if result.get("status") != "completed" or result.get("request_count") != expected_requests: + raise ValueError(f"incomplete simulator result: {path}") + if result.get("config") != {"tp": tp, "mns": mns, "name": name}: + raise ValueError(f"simulator config drift: {path}") + score = result.get("score") + if not isinstance(score, dict): + raise ValueError(f"simulator score missing: {path}") + values: dict[str, Any] = {} + for metric in metrics: + prefix = metric.removesuffix("_ms") + values[metric] = { + statistic: number(score.get(f"{prefix}_{statistic}_ms"), f"{path}:{metric}:{statistic}") + for statistic in ("mean", "p90") + } + cells[name] = {"path": str(path), "metrics": values} + selection: dict[str, Any] = {} + for metric in metrics: + for statistic in ("mean", "p90"): + sim_values = {name: cells[name]["metrics"][metric][statistic] for name in CONFIGS} + real_values = { + name: number( + audit["configs"][name]["metrics"][metric][f"pooled_{statistic}_ms"], + f"real:{name}:{metric}:{statistic}", + ) + for name in CONFIGS + } + sim_ranking, real_ranking = rank(sim_values), rank(real_values) + sim_choice, real_best = sim_ranking[0], real_ranking[0] + selection[f"{metric}:{statistic}"] = { + "sim_winner": sim_choice, + "real_winner": real_best, + "winner_match": sim_choice == real_best, + "selected_config_real_regret": (real_values[sim_choice] - real_values[real_best]) / real_values[real_best], + "sim_ranking": sim_ranking, + "real_ranking": real_ranking, + **pairwise(sim_values, real_values), + } + payload = { + "schema": "qwen30-latency-case-frontier-real-comparison-v1", + "sim_root": str(root), + "real_audit": str(args.real_audit.resolve()), + "prefill_only": bool(audit["prefill_only"]), + "applicable_metrics": metrics, + "selection": selection, + "cells": cells, + } + lines = ["# Qwen30 Frontier vs real latency selection", "", "| Objective | Frontier | Real | Match | Regret | Pairwise |", "|---|---|---|---:|---:|---:|"] + for objective, row in selection.items(): + agreement = "N/A" if row["agreement"] is None else f"{row['agreement']:.1%}" + lines.append(f"| {objective} | {row['sim_winner']} | {row['real_winner']} | {'yes' if row['winner_match'] else 'no'} | {row['selected_config_real_regret']:.1%} | {agreement} |") + args.json_output.parent.mkdir(parents=True, exist_ok=True) + args.markdown_output.parent.mkdir(parents=True, exist_ok=True) + args.json_output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + args.markdown_output.write_text("\n".join(lines) + "\n") + print(json.dumps(selection, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-fidelity-envelope-v1/audit_qwen30_latency_case.py b/runs/frontier-fidelity-envelope-v1/audit_qwen30_latency_case.py new file mode 100644 index 0000000..ed06274 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/audit_qwen30_latency_case.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""Audit one completed Qwen30 latency-selection real surface. + +This reader intentionally treats a prefill-only case as a four-objective +surface: TTFT/E2E mean and p90. TPOT is omitted rather than coerced to zero. +""" + +from __future__ import annotations + +import argparse +import json +import math +import statistics +from pathlib import Path +from typing import Any + + +CONFIGS = tuple(f"tp{tp}_mns{mns}" for tp in (1, 2, 4) for mns in (8, 16, 32, 64)) +TRIALS = (1, 2, 3) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--case-root", type=Path, required=True) + parser.add_argument("--json-output", type=Path, required=True) + parser.add_argument("--markdown-output", type=Path, required=True) + return parser.parse_args() + + +def nearest_rank(values: list[float], fraction: float) -> float: + if not values: + raise ValueError("cannot calculate a percentile of an empty metric") + return sorted(values)[math.ceil(len(values) * fraction) - 1] + + +def finite(value: Any, field: str) -> float: + if not isinstance(value, (int, float)) or isinstance(value, bool): + raise ValueError(f"{field} is not numeric: {value!r}") + value = float(value) + if not math.isfinite(value) or value < 0: + raise ValueError(f"{field} is invalid: {value!r}") + return value + + +def read_manifest(root: Path, tp: int) -> dict[str, Any]: + path = root / "traces" / f"tp{tp}" / "public" / "manifest.json" + manifest = json.loads(path.read_text()) + if manifest.get("schema") != "qwen30-latency-case-v1": + raise ValueError(f"unexpected trace schema in {path}") + if manifest.get("tensor_parallel_size") != tp or int(manifest.get("requests", 0)) <= 0: + raise ValueError(f"invalid trace contract in {path}") + outputs = manifest.get("output_tokens") + if not isinstance(outputs, list) or len(outputs) != 1 or int(outputs[0]) <= 0: + raise ValueError(f"non-uniform output contract in {path}") + return manifest + + +def metric_stats(values: list[float]) -> dict[str, float]: + return {"samples": len(values), "mean_ms": statistics.fmean(values), "p90_ms": nearest_rank(values, 0.90)} + + +def validate_trial(path: Path, manifest: dict[str, Any]) -> tuple[dict[str, list[float]], dict[str, Any]]: + payload = json.loads(path.read_text()) + if payload.get("schema") != "qwen30-exact-trace-anchor-v1": + raise ValueError(f"unexpected real result schema in {path}") + contract = payload.get("contract") + summary = payload.get("summary") + records = payload.get("requests") + if not isinstance(contract, dict) or not isinstance(summary, dict) or not isinstance(records, list): + raise ValueError(f"malformed result payload: {path}") + expected = int(manifest["requests"]) + expected_contract = { + "requests": expected, + "requests_file_sha256": manifest["private_jsonl_sha256"], + "row_vector_sha256": manifest["row_vector_sha256"], + "first_arrival_s": manifest["first_arrival_s"], + "last_arrival_s": manifest["last_arrival_s"], + } + for name, wanted in expected_contract.items(): + actual = contract.get(name) + if isinstance(wanted, float): + if not isinstance(actual, (int, float)) or not math.isclose(float(actual), wanted, abs_tol=1e-9): + raise ValueError(f"{path}: contract drift for {name}") + elif actual != wanted: + raise ValueError(f"{path}: contract drift for {name}") + if len(records) != expected or summary.get("completed") != expected or summary.get("failed") != 0: + raise ValueError(f"{path}: incomplete real replay") + + metrics: dict[str, list[float]] = {"ttft_ms": [], "tpot_ms": [], "e2e_ms": []} + indices: set[int] = set() + expected_output = int(manifest["output_tokens"][0]) + for record in records: + if record.get("success") is not True: + raise ValueError(f"{path}: failed request record") + index = record.get("source_index") + if not isinstance(index, int) or index in indices: + raise ValueError(f"{path}: duplicate/non-integer source index") + indices.add(index) + if record.get("actual_input_tokens") != record.get("input_tokens"): + raise ValueError(f"{path}: input usage drift") + if record.get("requested_output_tokens") != expected_output or record.get("actual_output_tokens") != expected_output: + raise ValueError(f"{path}: output usage drift") + for metric in ("ttft_ms", "e2e_ms"): + metrics[metric].append(finite(record.get(metric), metric)) + tpot = record.get("tpot_ms") + if expected_output == 1: + if tpot is not None: + raise ValueError(f"{path}: OSL=1 must report TPOT=null") + else: + metrics["tpot_ms"].append(finite(tpot, "tpot_ms")) + if len(indices) != expected: + raise ValueError(f"{path}: missing source rows") + return metrics, {"result_path": str(path), "requests": expected} + + +def main() -> None: + args = parse_args() + root = args.case_root.resolve() + manifests = {tp: read_manifest(root, tp) for tp in (1, 2, 4)} + prefill_only = int(manifests[1]["output_tokens"][0]) == 1 + if any((int(manifest["output_tokens"][0]) == 1) != prefill_only for manifest in manifests.values()): + raise ValueError("TP-specific output contracts differ") + applicable = ("ttft_ms", "e2e_ms") if prefill_only else ("ttft_ms", "tpot_ms", "e2e_ms") + configs: dict[str, Any] = {} + for name in CONFIGS: + tp = int(name.split("_", 1)[0].removeprefix("tp")) + trial_rows = [] + pooled = {metric: [] for metric in applicable} + for trial in TRIALS: + path = root / "real" / name / f"trial{trial}" / "results" / "result.json" + values, provenance = validate_trial(path, manifests[tp]) + trial_rows.append({ + **provenance, + "trial": trial, + "metrics": {metric: metric_stats(values[metric]) for metric in applicable}, + }) + for metric in applicable: + pooled[metric].extend(values[metric]) + configs[name] = { + "trials": trial_rows, + "metrics": { + metric: { + "pooled_samples": len(pooled[metric]), + "pooled_mean_ms": statistics.fmean(pooled[metric]), + "pooled_p90_ms": nearest_rank(pooled[metric], 0.90), + "trial_mean_of_means_ms": statistics.fmean( + row["metrics"][metric]["mean_ms"] for row in trial_rows + ), + "trial_stddev_of_means_ms": statistics.stdev( + row["metrics"][metric]["mean_ms"] for row in trial_rows + ), + } + for metric in applicable + }, + } + winners: dict[str, Any] = {} + for metric in applicable: + for statistic in ("pooled_mean_ms", "pooled_p90_ms"): + ranked = sorted((row["metrics"][metric][statistic], name) for name, row in configs.items()) + winners[f"{metric}:{statistic}"] = { + "winner": ranked[0][1], + "winner_value_ms": ranked[0][0], + "ranking": [name for _, name in ranked], + } + payload = { + "schema": "qwen30-latency-case-real-audit-v1", + "case_root": str(root), + "prefill_only": prefill_only, + "applicable_metrics": list(applicable), + "trace_manifests": {f"tp{tp}": manifests[tp] for tp in manifests}, + "configs": configs, + "winners": winners, + } + lines = ["# Qwen30 real latency case audit", "", f"Prefill-only: `{prefill_only}`.", ""] + lines += ["| Objective | Real winner | Value (ms) |", "|---|---|---:|"] + for objective, winner in winners.items(): + lines.append(f"| {objective} | {winner['winner']} | {winner['winner_value_ms']:.2f} |") + args.json_output.parent.mkdir(parents=True, exist_ok=True) + args.markdown_output.parent.mkdir(parents=True, exist_ok=True) + args.json_output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + args.markdown_output.write_text("\n".join(lines) + "\n") + print(json.dumps(winners, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-fidelity-envelope-v1/prepare_qwen30_latency_case.py b/runs/frontier-fidelity-envelope-v1/prepare_qwen30_latency_case.py new file mode 100644 index 0000000..e4bfa6a --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/prepare_qwen30_latency_case.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +"""Materialize Qwen30 Fixed/Trace latency cases without changing their contract. + +The private request file deliberately contains either an exact source prompt +(trace cases) or deterministic token IDs (fixed cases). The public Frontier +fixture contains only arrivals, shapes, sessions, and legal complete prefix +block identities. +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +from pathlib import Path +from typing import Any + + +FIELDS = ( + "arrived_at", + "num_prefill_tokens", + "num_decode_tokens", + "session_id", + "block_hash_ids", +) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def row_digest(rows: list[dict[str, Any]]) -> str: + digest = hashlib.sha256() + for row in rows: + digest.update( + json.dumps( + [ + row["source_index"], + row["arrived_at"], + row["input_length"], + row["output_length"], + row["session_id"], + row["runtime_block_ids"], + ], + separators=(",", ":"), + ).encode() + ) + digest.update(b"\n") + return digest.hexdigest() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="kind", required=True) + + trace = sub.add_parser("trace") + trace.add_argument("--base-public", type=Path, required=True) + trace.add_argument("--base-private", type=Path, required=True) + trace.add_argument("--output-tokens", type=int, required=True) + trace.add_argument("--tp", type=int, required=True) + trace.add_argument("--output-root", type=Path, required=True) + + fixed = sub.add_parser("fixed") + fixed.add_argument("--model", type=Path, required=True) + fixed.add_argument("--input-tokens", type=int, required=True) + fixed.add_argument("--output-tokens", type=int, required=True) + fixed.add_argument("--requests", type=int, required=True) + fixed.add_argument("--per-gpu-rate", type=float, required=True) + fixed.add_argument("--tp", type=int, required=True) + fixed.add_argument("--output-root", type=Path, required=True) + return parser.parse_args() + + +def materialize_trace(args: argparse.Namespace) -> tuple[list[dict[str, Any]], bool, str, float]: + with args.base_public.open(newline="") as source: + public = list(csv.DictReader(source)) + private = [json.loads(line) for line in args.base_private.open() if line.strip()] + if not public or len(public) != len(private): + raise ValueError("trace public/private request count mismatch") + rows: list[dict[str, Any]] = [] + for public_row, private_row in zip(public, private, strict=True): + if int(public_row["num_prefill_tokens"]) != int(private_row["input_length"]): + raise ValueError("trace input-length drift") + if float(public_row["arrived_at"]) != float(private_row["arrived_at"]): + raise ValueError("trace arrival drift") + # The real vLLM request artifact retains its final partial prompt block + # for provenance, while the Frontier CSV deliberately projects only + # legal complete cache blocks. Use the latter as the shared + # simulator-facing identity vector; vLLM itself derives cache hashes + # from the exact private prompt text. + runtime_ids = [int(value) for value in public_row["block_hash_ids"].split("|") if value] + if len(runtime_ids) != int(private_row["input_length"]) // 16: + raise ValueError("public trace exposes an incomplete runtime prefix block") + if int(public_row["session_id"]) != int(private_row["session_id"]): + raise ValueError("trace session-id drift") + body = dict(private_row["body"]) + body.update( + { + "min_tokens": args.output_tokens, + "max_tokens": args.output_tokens, + "ignore_eos": True, + } + ) + rows.append( + { + "source_index": int(private_row["source_index"]), + "arrived_at": float(private_row["arrived_at"]), + "input_length": int(private_row["input_length"]), + "output_length": args.output_tokens, + "session_id": int(private_row["session_id"]), + "runtime_block_ids": runtime_ids, + "body": body, + } + ) + return ( + rows, + True, + "trace-derived: exact input/arrival/session/prefix; output override only", + len(rows) / 600.0 * args.tp, + ) + + +def materialize_fixed(args: argparse.Namespace) -> tuple[list[dict[str, Any]], bool, str, float]: + if min(args.input_tokens, args.output_tokens, args.requests, args.tp) <= 0: + raise ValueError("fixed dimensions must be positive") + if args.input_tokens + args.output_tokens > 40960: + raise ValueError("fixed shape exceeds max model length") + if not math.isfinite(args.per_gpu_rate) or args.per_gpu_rate <= 0: + raise ValueError("per-GPU rate must be positive") + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True) + candidates = [ + token for token in range(tokenizer.vocab_size) if token not in set(tokenizer.all_special_ids) + ] + if len(candidates) < args.requests + 1: + raise ValueError("tokenizer does not provide enough non-special tokens") + rate = args.per_gpu_rate * args.tp + base = candidates[0] + rows = [] + for index in range(args.requests): + rows.append( + { + "source_index": index, + "arrived_at": index / rate, + "input_length": args.input_tokens, + "output_length": args.output_tokens, + "session_id": index, + "runtime_block_ids": [], + "body": { + "prompt": [candidates[index + 1], *([base] * (args.input_tokens - 1))], + "min_tokens": args.output_tokens, + "max_tokens": args.output_tokens, + "ignore_eos": True, + }, + } + ) + return ( + rows, + False, + "fixed-shape: deterministic token IDs, uniform TP-normalized QPS, no prefix reuse", + rate, + ) + + +def write_case( + rows: list[dict[str, Any]], + *, + prefix_caching: bool, + description: str, + global_rate: float, + tp: int, + root: Path, +) -> None: + if not rows: + raise ValueError("empty case") + root = root.resolve() + public_root = root / "public" + private_root = root / "private" + public_root.mkdir(parents=True, exist_ok=True) + private_root.mkdir(parents=True, exist_ok=True) + public_path = public_root / "frontier.csv" + private_path = private_root / "real_requests.jsonl" + with public_path.open("w", newline="") as output: + writer = csv.DictWriter(output, fieldnames=FIELDS, lineterminator="\n") + writer.writeheader() + for row in rows: + writer.writerow( + { + "arrived_at": f"{row['arrived_at']:.12f}", + "num_prefill_tokens": row["input_length"], + "num_decode_tokens": row["output_length"], + "session_id": row["session_id"], + "block_hash_ids": "|".join(str(value) for value in row["runtime_block_ids"]), + } + ) + with private_path.open("w") as output: + for row in rows: + output.write(json.dumps(row, separators=(",", ":")) + "\n") + arrivals = [float(row["arrived_at"]) for row in rows] + manifest = { + "schema": "qwen30-latency-case-v1", + "description": description, + "tensor_parallel_size": tp, + "requests": len(rows), + "output_tokens": sorted({int(row["output_length"]) for row in rows}), + "prefix_caching": prefix_caching, + "public_csv": str(public_path), + "public_csv_sha256": sha256(public_path), + "private_jsonl": str(private_path), + "private_jsonl_sha256": sha256(private_path), + "row_vector_sha256": row_digest(rows), + "first_arrival_s": arrivals[0], + "last_arrival_s": arrivals[-1], + "global_offered_request_rate": global_rate, + "per_gpu_offered_request_rate": global_rate / tp, + } + (public_root / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + print(json.dumps(manifest, sort_keys=True)) + + +def main() -> None: + args = parse_args() + if args.kind == "trace": + rows, prefix, description, global_rate = materialize_trace(args) + else: + rows, prefix, description, global_rate = materialize_fixed(args) + write_case( + rows, + prefix_caching=prefix, + description=description, + global_rate=global_rate, + tp=args.tp, + root=args.output_root, + ) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-fidelity-envelope-v1/qwen30-latency-expansion-card.md b/runs/frontier-fidelity-envelope-v1/qwen30-latency-expansion-card.md new file mode 100644 index 0000000..8ed2b8c --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/qwen30-latency-expansion-card.md @@ -0,0 +1,49 @@ +# EXP-SIMFID-Q30-LATENCY-EXPANSION: remaining Fixed/Trace × PD/P cases + +> Status: prepared for runtime-alignment preflight (2026-07-18). The already +> completed Qwen3-30B-A3B Trace-PD surface is excluded from this card. + +## Question and fixed boundary + +Does the same graph-aligned Frontier configuration that selected the correct +winner on Trace-PD also select the real vLLM winner when either the arrival/ +prefix state or the decode phase is removed? This is a selection test, not an +SLO/capacity evaluation. + +| Case | Input / output | arrival and cache | Applicable objectives | +|---|---|---|---| +| Fixed-PD | 2048 / 128 | 129 uniform requests; `t'=t/TP`; prefix cache off | mean/p90 TTFT, TPOT, E2E | +| Trace-P | exact held-out input, arrival, session and complete block-16 prefix relation / 1 | same TP-normalized 129-row trace; prefix cache on | mean/p90 TTFT, E2E; TPOT=N/A | +| Fixed-P | 2048 / 1 | 129 uniform requests; `t'=t/TP`; prefix cache off | mean/p90 TTFT, E2E; TPOT=N/A | + +All cases use Qwen3-30B-A3B BF16, community vLLM 0.20.0, H20, `TP∈{1,2,4}` +and `MNS∈{8,16,32,64}`, MBT=8192, chunked prefill, three fresh-server trials +per cell. The fixed QPS is the Trace-PD base offered rate (`129/600` req/s per +GPU), hence the global arrival rate is multiplied by TP. It makes throughput +per GPU comparable without claiming that different TP values see identical +cluster-level load. + +## Simulator and real contracts + +- Frontier uses commit `deadc4a…`, `piecewise`, the frozen CUDA-event profile + for prefill/mixed batches and KERNEL_ONLY profile for captured pure decode. +- Trace-P reuses the already verified graph buckets/KV capacities because the + server CLI is unchanged from Trace-PD. The only workload change is `OSL→1`. +- Fixed-P/PD disables vLLM prefix caching on both sides. Before freezing their + simulator surfaces, a no-request server-start preflight records the actual + graph captures and KV-block capacities for every `(TP,MNS)`; no request + latency or winner is used as calibration. +- Each real result verifies every request's input/output usage and row-vector + digest. `OSL=1` produces JSON `null` TPOT samples and is never converted to + zero. + +## Decision rule and cost + +For every applicable objective, compare the complete 12-cell simulator and +pooled three-trial real surface: winner match, selected-config real regret, +and non-tied pair direction agreement. A simulator crash or missing request +metric is a coverage failure, not a high-latency cell. + +The no-request Fixed runtime preflight is capped at 2 H20-GPUh. Each 36-run +real surface is estimated at 13 nominal / 41 worst-case H20-GPUh, plus CPU-only +Frontier replay. Launch logs record the resolved inputs, paths, and duration. diff --git a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py index cac0e74..a2c39a0 100644 --- a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py +++ b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen30_exact_trace_surface.py @@ -279,19 +279,23 @@ def score(path: Path, expected_shapes: list[tuple[int, int]]) -> dict[str, Any]: ttfts = [float(row["ttft_ms"]) for row in request_metrics] tpots = [float(row["tpot_ms"]) for row in request_metrics if row["tpot_ms"] is not None] e2es = [float(row["e2e_ms"]) for row in request_metrics] + def summary(values: list[float]) -> dict[str, float | None]: + if not values: + return {"mean": None, "p50": None, "p90": None, "p95": None} + return { + "mean": sum(values) / len(values), + "p50": percentile(values, 0.50), + "p90": percentile(values, 0.90), + "p95": percentile(values, 0.95), + } + + ttft_summary = summary(ttfts) + tpot_summary = summary(tpots) + e2e_summary = summary(e2es) return { - "ttft_mean_ms": sum(ttfts) / len(ttfts), - "ttft_p50_ms": percentile(ttfts, 0.50), - "ttft_p90_ms": percentile(ttfts, 0.90), - "ttft_p95_ms": percentile(ttfts, 0.95), - "tpot_mean_ms": sum(tpots) / len(tpots), - "tpot_p50_ms": percentile(tpots, 0.50), - "tpot_p90_ms": percentile(tpots, 0.90), - "tpot_p95_ms": percentile(tpots, 0.95), - "e2e_mean_ms": sum(e2es) / len(e2es), - "e2e_p50_ms": percentile(e2es, 0.50), - "e2e_p90_ms": percentile(e2es, 0.90), - "e2e_p95_ms": percentile(e2es, 0.95), + **{f"ttft_{name}_ms": value for name, value in ttft_summary.items()}, + **{f"tpot_{name}_ms": value for name, value in tpot_summary.items()}, + **{f"e2e_{name}_ms": value for name, value in e2e_summary.items()}, "slos": slos, } diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh index 1b582e0..1037a54 100755 --- a/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh @@ -8,10 +8,12 @@ TP="${TP:?TP is required}" MNS="${MNS:?MNS is required}" TRACE_LABEL="${TRACE_LABEL:?TRACE_LABEL is required}" SERVER_PORT="${SERVER_PORT:?SERVER_PORT is required}" +PREFIX_CACHING="${PREFIX_CACHING:-true}" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:-/tmp/wjh/flashinfer-workspace-vllm020-profiler-v1}" SERVED_MODEL="qwen3-30b-exact-trace" +EXACT_TRACE_CLIENT="${EXACT_TRACE_CLIENT:-qwen30_exact_trace_client.py}" SERVER_PID="" mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" "${OUTPUT_ROOT}/results" \ @@ -37,8 +39,21 @@ if [[ "${#GPU_IDS[@]}" -ne "${TP}" ]]; then exit 1 fi +case "${PREFIX_CACHING}" in + true) + PREFIX_CACHING_FLAG="--enable-prefix-caching" + ;; + false) + PREFIX_CACHING_FLAG="--no-enable-prefix-caching" + ;; + *) + echo "ERROR: PREFIX_CACHING must be true or false, got ${PREFIX_CACHING}" >&2 + exit 1 + ;; +esac + REQUEST_COUNT="$(wc -l < "${REQUESTS_FILE}")" -echo "QWEN30_EXACT_TRACE_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 trace=${TRACE_LABEL} requests=${REQUEST_COUNT} source=${REQUESTS_FILE} arrivals=original600s input_output_prompt=exact prefix=on block=16 tpot_slo=150ms flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} output=${OUTPUT_ROOT} expected_wall=12-35m hard_wall=3600s" +echo "QWEN30_EXACT_TRACE_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 trace=${TRACE_LABEL} requests=${REQUEST_COUNT} source=${REQUESTS_FILE} arrivals=manifest prefix=${PREFIX_CACHING} block=16 metrics=TTFT,TPOT-if-OSL-gt-1,E2E flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} output=${OUTPUT_ROOT} expected_wall=12-35m hard_wall=3600s" date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" sha256sum qwen30_exact_trace_client.py run_qwen30_exact_trace_real_anchor.sh \ ../frontier-phase-factorial-v0/qwen30_prefill_client.py \ @@ -60,7 +75,7 @@ setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \ --host 127.0.0.1 --port "${SERVER_PORT}" --served-model-name "${SERVED_MODEL}" \ --tensor-parallel-size "${TP}" --gpu-memory-utilization 0.92 \ --max-model-len 40960 --max-num-batched-tokens 8192 --max-num-seqs "${MNS}" \ - --enable-prefix-caching --enable-chunked-prefill --no-enable-log-requests \ + "${PREFIX_CACHING_FLAG}" --enable-chunked-prefill --no-enable-log-requests \ > "${OUTPUT_ROOT}/logs/server.log" 2>&1 & SERVER_PID=$! READY=0 @@ -89,7 +104,7 @@ fi --model-path "${MODEL_ROOT}" --rate 1 --requests 4 --input-tokens 512 \ --output-tokens 1 --output "${OUTPUT_ROOT}/results/warmup.json" -"${VENV_ROOT}/bin/python" qwen30_exact_trace_client.py \ +"${VENV_ROOT}/bin/python" "${EXACT_TRACE_CLIENT}" \ --port "${SERVER_PORT}" --requests-file "${REQUESTS_FILE}" \ --output "${OUTPUT_ROOT}/results/result.json" --tpot-slo-ms 150 \ --timeout-seconds 1800 From a7e3c0fdf33463cbccb0d5cb22997d032973c18f Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sat, 18 Jul 2026 01:05:46 +0800 Subject: [PATCH 120/148] Record fixed-case vLLM runtime state --- .../run_qwen30_fixed_runtime_preflight.sh | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_runtime_preflight.sh diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_runtime_preflight.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_runtime_preflight.sh new file mode 100644 index 0000000..a64da12 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_runtime_preflight.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash + +# Capture vLLM's resolved runtime state for Fixed-P/PD before their Frontier +# surfaces are frozen. No benchmark request is issued in this preflight. + +set -euo pipefail + +OUT="${1:?output root is required}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:-${OUT}/flashinfer-shared-workspace}" +PORT=8600 + +mkdir -p "${OUT}/runs" "${OUT}/provenance" "${FLASHINFER_WORKSPACE_BASE}" +exec > >(tee -a "${OUT}/controller.log") 2>&1 + +echo "FIXED_RUNTIME_PREFLIGHT_LAUNCH_ECHO host=$(hostname) model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 configs=TP{1,2,4}xMNS{8,16,32,64} MBT=8192 prefix=false chunked_prefill=true requests=0 resolved_state=graph_capture_and_kv_blocks output=${OUT} expected_wall=8-20m expected_gpu_cap=2_H20h" +date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ +sha256sum "${BASH_SOURCE[0]}" "${MODEL_ROOT}/config.json" > "${OUT}/provenance/input.sha256" +"${VENV_ROOT}/bin/vllm" --version > "${OUT}/provenance/vllm.version" + +if ! nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '$1 != 0 {exit 1}'; then + echo 'ERROR: a GPU is not idle before runtime preflight' >&2 + exit 1 +fi + +declare -a PIDS=() + +cleanup_pid() { + local pid="$1" + if kill -0 "${pid}" 2>/dev/null; then + kill -TERM -- "-${pid}" 2>/dev/null || true + for _ in $(seq 1 30); do + kill -0 "${pid}" 2>/dev/null || break + sleep 1 + done + kill -KILL -- "-${pid}" 2>/dev/null || true + fi +} + +cleanup_all() { + local pid + for pid in "${PIDS[@]:-}"; do + cleanup_pid "${pid}" + done +} +trap cleanup_all EXIT INT TERM + +launch_one() { + local tp="$1" mns="$2" gpus="$3" port="$4" + local name="tp${tp}_mns${mns}" + local root="${OUT}/runs/${name}" + mkdir -p "${root}" + ( + export HOME=/tmp/wjh + export XDG_CACHE_HOME=/tmp/wjh/.cache + export VLLM_CACHE_ROOT=/tmp/wjh/.cache/vllm + export TOKENIZERS_PARALLELISM=false + export VLLM_USE_V1=1 + export TORCH_CUDA_ARCH_LIST=9.0 + export HF_HUB_OFFLINE=1 + export TRANSFORMERS_OFFLINE=1 + export CUDA_VISIBLE_DEVICES="${gpus}" + setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \ + --host 127.0.0.1 --port "${port}" --served-model-name qwen3-30b-fixed-preflight \ + --tensor-parallel-size "${tp}" --gpu-memory-utilization 0.92 \ + --max-model-len 40960 --max-num-batched-tokens 8192 --max-num-seqs "${mns}" \ + --no-enable-prefix-caching --enable-chunked-prefill --no-enable-log-requests \ + > "${root}/server.log" 2>&1 & + local server_pid=$! + local ready=0 + for _ in $(seq 1 150); do + if curl -fsS --max-time 2 "http://127.0.0.1:${port}/v1/models" > "${root}/models.json" 2>/dev/null; then + ready=1 + break + fi + if ! kill -0 "${server_pid}" 2>/dev/null; then + break + fi + sleep 2 + done + grep -E -i 'cuda graph|cudagraph|capture|gpu blocks|num_gpu_blocks|kv cache' "${root}/server.log" > "${root}/resolved-state.log" || true + if [[ "${ready}" -ne 1 ]]; then + echo "status=failed" > "${root}/status" + cleanup_pid "${server_pid}" + exit 1 + fi + echo "status=ready" > "${root}/status" + cleanup_pid "${server_pid}" + ) & + PIDS+=("$!") +} + +wait_wave() { + local failed=0 pid + for pid in "${PIDS[@]}"; do + wait "${pid}" || failed=1 + done + PIDS=() + if [[ "${failed}" -ne 0 ]]; then + echo 'ERROR: one or more runtime preflight servers failed' >&2 + exit 1 + fi + if ! nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '$1 != 0 {exit 1}'; then + echo 'ERROR: a GPU remains allocated after runtime preflight wave' >&2 + exit 1 + fi +} + +for tp in 4 2 1; do + echo "PREFLIGHT_WAVE_START tp=${tp}" + case "${tp}" in + 4) + launch_one 4 8 '0,1,2,3' "${PORT}"; PORT=$((PORT + 1)) + launch_one 4 16 '4,5,6,7' "${PORT}"; PORT=$((PORT + 1)) + wait_wave + launch_one 4 32 '0,1,2,3' "${PORT}"; PORT=$((PORT + 1)) + launch_one 4 64 '4,5,6,7' "${PORT}"; PORT=$((PORT + 1)) + ;; + 2) + launch_one 2 8 '0,1' "${PORT}"; PORT=$((PORT + 1)) + launch_one 2 16 '2,3' "${PORT}"; PORT=$((PORT + 1)) + launch_one 2 32 '4,5' "${PORT}"; PORT=$((PORT + 1)) + launch_one 2 64 '6,7' "${PORT}"; PORT=$((PORT + 1)) + ;; + 1) + launch_one 1 8 '0' "${PORT}"; PORT=$((PORT + 1)) + launch_one 1 16 '1' "${PORT}"; PORT=$((PORT + 1)) + launch_one 1 32 '2' "${PORT}"; PORT=$((PORT + 1)) + launch_one 1 64 '3' "${PORT}"; PORT=$((PORT + 1)) + ;; + esac + wait_wave + echo "PREFLIGHT_WAVE_COMPLETE tp=${tp}" +done + +find "${OUT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 | sort -z | xargs -0 sha256sum > "${OUT}/provenance/artifacts.sha256" +date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ +echo 'FIXED_RUNTIME_PREFLIGHT_COMPLETE' From ddacc5f7a61947b9b7148c3670f99928014449b3 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sat, 18 Jul 2026 01:08:08 +0800 Subject: [PATCH 121/148] Add Qwen235 vLLM 0.20 compatibility gate --- .../run_qwen235_v020_smoke.sh | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh new file mode 100644 index 0000000..29a98f7 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +# Compatibility gate only: this is not a simulator-fidelity measurement. + +set -euo pipefail + +OUT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}" +TP="${TP:?TP is required}" +MNS="${MNS:?MNS is required}" +MBT="${MBT:?MBT is required}" +SERVER_PORT="${SERVER_PORT:?SERVER_PORT is required}" +ENABLE_EXPERT_PARALLEL="${ENABLE_EXPERT_PARALLEL:?ENABLE_EXPERT_PARALLEL is required}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}" +CLIENT="${CLIENT:?CLIENT is required}" +SERVED_MODEL="qwen3-235b-v020-smoke" +SERVER_PID="" + +mkdir -p "${OUT}/logs" "${OUT}/provenance" "${OUT}/results" +exec > >(tee -a "${OUT}/logs/controller.log") 2>&1 + +cleanup() { + if [[ -n "${SERVER_PID}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then + kill -TERM -- "-${SERVER_PID}" 2>/dev/null || true + for _ in $(seq 1 30); do + kill -0 "${SERVER_PID}" 2>/dev/null || break + sleep 1 + done + kill -KILL -- "-${SERVER_PID}" 2>/dev/null || true + fi + SERVER_PID="" +} +trap cleanup EXIT INT TERM + +IFS=',' read -r -a GPU_IDS <<< "${CUDA_VISIBLE_DEVICES:?GPU allocation is required}" +if [[ "${#GPU_IDS[@]}" -ne "${TP}" ]]; then + echo "ERROR: expected TP=${TP} GPUs, got ${CUDA_VISIBLE_DEVICES}" >&2 + exit 1 +fi +case "${ENABLE_EXPERT_PARALLEL}" in + true) EP_FLAG=(--enable-expert-parallel) ;; + false) EP_FLAG=() ;; + *) echo "ERROR: ENABLE_EXPERT_PARALLEL must be true or false" >&2; exit 1 ;; +esac + +echo "QWEN235_V020_SMOKE_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 checkpoint=FP8_compute_kv=auto config=TP${TP}_EP${ENABLE_EXPERT_PARALLEL}_MNS${MNS}_MBT${MBT} prefix=false requests=1 shape=2048x1 role=compatibility_gate_not_latency_measurement output=${OUT} expected_wall=8-20m expected_gpu_cap=$((${TP} * 1))_H20h" +date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ +sha256sum "${BASH_SOURCE[0]}" "${CLIENT}" "${MODEL_ROOT}/config.json" > "${OUT}/provenance/input.sha256" +"${VENV_ROOT}/bin/vllm" --version > "${OUT}/provenance/vllm.version" +"${VENV_ROOT}/bin/python" - <<'PY' > "${OUT}/provenance/runtime.json" +import importlib.metadata as metadata +import json +import vllm +print(json.dumps({"vllm": metadata.version("vllm"), "vllm_file": vllm.__file__}, sort_keys=True)) +PY + +export TOKENIZERS_PARALLELISM=false +export VLLM_USE_V1=1 +export TORCH_CUDA_ARCH_LIST=9.0 +export HF_HUB_OFFLINE=1 +export TRANSFORMERS_OFFLINE=1 +export HOME=/tmp/wjh +export XDG_CACHE_HOME=/tmp/wjh/.cache +export VLLM_CACHE_ROOT=/tmp/wjh/.cache/vllm +setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \ + --host 127.0.0.1 --port "${SERVER_PORT}" --served-model-name "${SERVED_MODEL}" \ + --tensor-parallel-size "${TP}" --disable-custom-all-reduce --quantization fp8 \ + --gpu-memory-utilization 0.80 --kv-cache-dtype auto --max-model-len 40960 \ + --max-num-batched-tokens "${MBT}" --max-num-seqs "${MNS}" \ + --no-enable-prefix-caching --enable-chunked-prefill --no-enable-log-requests \ + "${EP_FLAG[@]}" > "${OUT}/logs/server.log" 2>&1 & +SERVER_PID=$! +READY=0 +for _ in $(seq 1 180); do + if curl -fsS --max-time 2 "http://127.0.0.1:${SERVER_PORT}/v1/models" > "${OUT}/results/models.json" 2>/dev/null; then + READY=1 + break + fi + if ! kill -0 "${SERVER_PID}" 2>/dev/null; then + tail -200 "${OUT}/logs/server.log" + exit 1 + fi + sleep 3 +done +if [[ "${READY}" -ne 1 ]]; then + tail -200 "${OUT}/logs/server.log" + exit 1 +fi + +"${VENV_ROOT}/bin/python" "${CLIENT}" --port "${SERVER_PORT}" --served-model "${SERVED_MODEL}" \ + --model-path "${MODEL_ROOT}" --rate 1 --requests 1 --input-tokens 2048 --output-tokens 1 \ + --output "${OUT}/results/one_request.json" +grep -E -i 'quant|fp8|gpu blocks|num_gpu_blocks|kv cache|expert.parallel|expert parallel' "${OUT}/logs/server.log" > "${OUT}/provenance/resolved-runtime.log" || true +cleanup +find "${OUT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 | sort -z | xargs -0 sha256sum > "${OUT}/provenance/artifacts.sha256" +date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ +echo 'QWEN235_V020_SMOKE_COMPLETE' From 12705411f97347b8150ba1b1bb0f69203f0fbdb0 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sat, 18 Jul 2026 01:08:54 +0800 Subject: [PATCH 122/148] Record Qwen235 portability experiment gate --- .../qwen235-v020-portability-card.md | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 runs/frontier-fidelity-envelope-v1/qwen235-v020-portability-card.md diff --git a/runs/frontier-fidelity-envelope-v1/qwen235-v020-portability-card.md b/runs/frontier-fidelity-envelope-v1/qwen235-v020-portability-card.md new file mode 100644 index 0000000..159e7de --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/qwen235-v020-portability-card.md @@ -0,0 +1,41 @@ +# EXP-SIMFID-Q235-V020-PORTABILITY: large FP8 MoE boundary case + +> Status: compatibility gate prepared. No Qwen235 vLLM 0.20/deadc4a profile or +> real/simulator latency result exists yet; historical vLLM 0.10.2 SLO data is +> explicitly excluded. + +## Scope + +This is the requested Qwen3-235B-A22B-FP8 case, kept separate from the Qwen30 +surface. Its purpose is first to establish whether the current Frontier stack +can even represent the same large-FP8-MoE runtime; it is not evidence that a +Qwen30 profile generalizes. + +| Item | Frozen proposal | +|---|---| +| model | Qwen3-235B-A22B-FP8: FP8 e4m3/dynamic-block checkpoint; BF16 compute/KV contract recorded at runtime | +| engine | community vLLM 0.20.0 (`88d34c640…`) | +| simulator | Frontier `deadc4a…`, with `frontier.__file__` recorded by each profiler | +| candidate plans | TP4/EP1 and TP8/EP8, each MNS 64/128, MBT=8192 | +| first workload | Fixed-P: ISL=2048, OSL=1, prefix off, provisional global QPS=1.6 | +| objective | mean/p90 TTFT and E2E; TPOT=N/A | + +TP1/TP2 do not fit the 239-GB checkpoint. TP8/EP1 is not assumed legal: only +the two plans above have existing historical memory/topology evidence. + +## Gates + +1. A fresh vLLM 0.20 TP4 and TP8 server must load the checkpoint and return an + exact 2048→1 response without FP8 fallback. This is a compatibility smoke, + not a latency sample. +2. New model-specific vLLM 0.20/deadc4a profiles must cover linear, router, + FP8 MoE, prefill attention, and TP collectives for TP4/TP8. Old vLLM 0.10.2 + CUDA-event CSVs are forbidden inputs. +3. The profile manifest binds model config/quantization, vLLM, exact Frontier + import path/commit, hardware, and finite shape coverage. CPU replay must + emit legal request metrics before a real surface is launched. +4. Only then run the four-plan real/simulator Fixed-P surface with three fresh + trials; a missing simulator cell is coverage failure, not a poor config. + +This ordering prevents a profile-runtime mismatch from being misreported as a +simulator tuning failure. From 7a437b4d91040383d2b9298c08a02417799fc124 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sat, 18 Jul 2026 01:11:08 +0800 Subject: [PATCH 123/148] Allow fixed runtime preflight resume --- .../run_qwen30_fixed_runtime_preflight.sh | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_runtime_preflight.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_runtime_preflight.sh index a64da12..1844faa 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_runtime_preflight.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_runtime_preflight.sh @@ -9,17 +9,28 @@ OUT="${1:?output root is required}" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:-${OUT}/flashinfer-shared-workspace}" +TPS="${TPS:-4 2 1}" PORT=8600 mkdir -p "${OUT}/runs" "${OUT}/provenance" "${FLASHINFER_WORKSPACE_BASE}" exec > >(tee -a "${OUT}/controller.log") 2>&1 -echo "FIXED_RUNTIME_PREFLIGHT_LAUNCH_ECHO host=$(hostname) model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 configs=TP{1,2,4}xMNS{8,16,32,64} MBT=8192 prefix=false chunked_prefill=true requests=0 resolved_state=graph_capture_and_kv_blocks output=${OUT} expected_wall=8-20m expected_gpu_cap=2_H20h" +echo "FIXED_RUNTIME_PREFLIGHT_LAUNCH_ECHO host=$(hostname) model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 tps=${TPS}xMNS{8,16,32,64} MBT=8192 prefix=false chunked_prefill=true requests=0 resolved_state=graph_capture_and_kv_blocks output=${OUT} expected_wall=8-20m expected_gpu_cap=2_H20h" date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ sha256sum "${BASH_SOURCE[0]}" "${MODEL_ROOT}/config.json" > "${OUT}/provenance/input.sha256" "${VENV_ROOT}/bin/vllm" --version > "${OUT}/provenance/vllm.version" -if ! nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '$1 != 0 {exit 1}'; then +wait_for_idle() { + for _ in $(seq 1 30); do + if nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '$1 != 0 {exit 1}'; then + return 0 + fi + sleep 2 + done + return 1 +} + +if ! wait_for_idle; then echo 'ERROR: a GPU is not idle before runtime preflight' >&2 exit 1 fi @@ -101,13 +112,13 @@ wait_wave() { echo 'ERROR: one or more runtime preflight servers failed' >&2 exit 1 fi - if ! nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '$1 != 0 {exit 1}'; then + if ! wait_for_idle; then echo 'ERROR: a GPU remains allocated after runtime preflight wave' >&2 exit 1 fi } -for tp in 4 2 1; do +for tp in ${TPS}; do echo "PREFLIGHT_WAVE_START tp=${tp}" case "${tp}" in 4) From 5fc0b48fa1dc073a3177e50ba04372c7c1f38408 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sat, 18 Jul 2026 10:32:21 +0800 Subject: [PATCH 124/148] Extend Qwen235 smoke readiness deadline --- .../run_qwen235_v020_smoke.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh index 29a98f7..235cb91 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh @@ -13,6 +13,7 @@ ENABLE_EXPERT_PARALLEL="${ENABLE_EXPERT_PARALLEL:?ENABLE_EXPERT_PARALLEL is requ VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}" CLIENT="${CLIENT:?CLIENT is required}" +READY_TIMEOUT_SECONDS="${READY_TIMEOUT_SECONDS:-1800}" SERVED_MODEL="qwen3-235b-v020-smoke" SERVER_PID="" @@ -42,8 +43,12 @@ case "${ENABLE_EXPERT_PARALLEL}" in false) EP_FLAG=() ;; *) echo "ERROR: ENABLE_EXPERT_PARALLEL must be true or false" >&2; exit 1 ;; esac +if ! [[ "${READY_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]]; then + echo "ERROR: READY_TIMEOUT_SECONDS must be a positive integer" >&2 + exit 1 +fi -echo "QWEN235_V020_SMOKE_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 checkpoint=FP8_compute_kv=auto config=TP${TP}_EP${ENABLE_EXPERT_PARALLEL}_MNS${MNS}_MBT${MBT} prefix=false requests=1 shape=2048x1 role=compatibility_gate_not_latency_measurement output=${OUT} expected_wall=8-20m expected_gpu_cap=$((${TP} * 1))_H20h" +echo "QWEN235_V020_SMOKE_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 checkpoint=FP8_compute_kv=auto config=TP${TP}_EP${ENABLE_EXPERT_PARALLEL}_MNS${MNS}_MBT${MBT} prefix=false requests=1 shape=2048x1 ready_timeout=${READY_TIMEOUT_SECONDS}s role=compatibility_gate_not_latency_measurement output=${OUT} expected_wall=20-35m expected_gpu_cap=$((${TP} * 35 / 60))_H20h" date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ sha256sum "${BASH_SOURCE[0]}" "${CLIENT}" "${MODEL_ROOT}/config.json" > "${OUT}/provenance/input.sha256" "${VENV_ROOT}/bin/vllm" --version > "${OUT}/provenance/vllm.version" @@ -71,7 +76,8 @@ setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \ "${EP_FLAG[@]}" > "${OUT}/logs/server.log" 2>&1 & SERVER_PID=$! READY=0 -for _ in $(seq 1 180); do +READY_DEADLINE=$((SECONDS + READY_TIMEOUT_SECONDS)) +while (( SECONDS < READY_DEADLINE )); do if curl -fsS --max-time 2 "http://127.0.0.1:${SERVER_PORT}/v1/models" > "${OUT}/results/models.json" 2>/dev/null; then READY=1 break From a01c1206abedfd70e2c94096d9f7ecb6c38bac65 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sat, 18 Jul 2026 10:48:45 +0800 Subject: [PATCH 125/148] Launch frozen Qwen30 latency case surfaces --- .../audit_qwen30_latency_case.py | 13 +- .../run_qwen30_latency_case_real_surface.sh | 180 ++++++++++++++++++ 2 files changed, 190 insertions(+), 3 deletions(-) create mode 100644 runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh diff --git a/runs/frontier-fidelity-envelope-v1/audit_qwen30_latency_case.py b/runs/frontier-fidelity-envelope-v1/audit_qwen30_latency_case.py index ed06274..617b443 100644 --- a/runs/frontier-fidelity-envelope-v1/audit_qwen30_latency_case.py +++ b/runs/frontier-fidelity-envelope-v1/audit_qwen30_latency_case.py @@ -22,6 +22,11 @@ TRIALS = (1, 2, 3) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--case-root", type=Path, required=True) + parser.add_argument( + "--traces-root", + type=Path, + help="Frozen TP-normalized traces; defaults to /traces.", + ) parser.add_argument("--json-output", type=Path, required=True) parser.add_argument("--markdown-output", type=Path, required=True) return parser.parse_args() @@ -42,8 +47,8 @@ def finite(value: Any, field: str) -> float: return value -def read_manifest(root: Path, tp: int) -> dict[str, Any]: - path = root / "traces" / f"tp{tp}" / "public" / "manifest.json" +def read_manifest(traces_root: Path, tp: int) -> dict[str, Any]: + path = traces_root / f"tp{tp}" / "public" / "manifest.json" manifest = json.loads(path.read_text()) if manifest.get("schema") != "qwen30-latency-case-v1": raise ValueError(f"unexpected trace schema in {path}") @@ -116,7 +121,8 @@ def validate_trial(path: Path, manifest: dict[str, Any]) -> tuple[dict[str, list def main() -> None: args = parse_args() root = args.case_root.resolve() - manifests = {tp: read_manifest(root, tp) for tp in (1, 2, 4)} + traces_root = (args.traces_root or root / "traces").resolve() + manifests = {tp: read_manifest(traces_root, tp) for tp in (1, 2, 4)} prefill_only = int(manifests[1]["output_tokens"][0]) == 1 if any((int(manifest["output_tokens"][0]) == 1) != prefill_only for manifest in manifests.values()): raise ValueError("TP-specific output contracts differ") @@ -165,6 +171,7 @@ def main() -> None: payload = { "schema": "qwen30-latency-case-real-audit-v1", "case_root": str(root), + "traces_root": str(traces_root), "prefill_only": prefill_only, "applicable_metrics": list(applicable), "trace_manifests": {f"tp{tp}": manifests[tp] for tp in manifests}, diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh new file mode 100644 index 0000000..553a93d --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash + +# One three-trial Qwen30 latency-selection surface. The caller supplies a +# frozen trace root and runs different cases sequentially to keep GPU use +# exclusive. + +set -euo pipefail + +OUT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}" +TRACE_ROOT="${TRACE_ROOT:?TRACE_ROOT is required}" +CASE_NAME="${CASE_NAME:?CASE_NAME is required}" +PREFIX_CACHING="${PREFIX_CACHING:?PREFIX_CACHING is required}" +RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +RUNNER="${RUNNER_DIR}/run_qwen30_exact_trace_real_anchor.sh" +CLIENT="${RUNNER_DIR}/qwen30_exact_trace_client.py" +PREFILL_CLIENT="${RUNNER_DIR}/../frontier-phase-factorial-v0/qwen30_prefill_client.py" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +FLASHINFER_SHARED_WORKSPACE="${FLASHINFER_SHARED_WORKSPACE:-${OUT}/flashinfer-shared-workspace}" +PORT="${BASE_PORT:-8300}" + +case "${PREFIX_CACHING}" in + true|false) ;; + *) echo "ERROR: PREFIX_CACHING must be true or false" >&2; exit 1 ;; +esac + +case "${CASE_NAME}" in + trace-po|fixed-pd|fixed-po) ;; + *) echo "ERROR: unsupported CASE_NAME=${CASE_NAME}" >&2; exit 1 ;; +esac + +for path in "${RUNNER}" "${CLIENT}" "${PREFILL_CLIENT}" "${MODEL_ROOT}/config.json"; do + [[ -f "${path}" ]] || { echo "ERROR: required path is missing: ${path}" >&2; exit 1; } +done + +declare -a WAVE_PIDS=() + +wait_for_wave() { + local failed=0 pid + for pid in "${WAVE_PIDS[@]}"; do + if ! wait "${pid}"; then + failed=1 + fi + done + WAVE_PIDS=() + if [[ "${failed}" -ne 0 ]]; then + printf '%s\n' 'ERROR one or more real-serving runs failed; aborting surface.' \ + | tee -a "${OUT}/controller.log" >&2 + return 1 + fi +} + +preflight_gpus() { + nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader \ + | tee -a "${OUT}/controller.log" + if nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits \ + | awk '$1 != 0 {exit 1}'; then + return 0 + fi + printf '%s\n' 'ERROR non-idle GPU before a real-serving wave; refusing to start.' \ + | tee -a "${OUT}/controller.log" >&2 + return 1 +} + +assert_no_our_server() { + if pgrep -fa 'qwen3-30b-exact-trace' > "${OUT}/provenance/unexpected_server_processes.txt"; then + printf '%s\n' 'ERROR a prior Qwen30 server remains; refusing the next wave.' \ + | tee -a "${OUT}/controller.log" >&2 + return 1 + fi +} + +launch_config() { + local trial="$1" tp="$2" mns="$3" gpus="$4" + local config="tp${tp}_mns${mns}" + local run_out="${OUT}/real/${config}/trial${trial}" + local requests="${TRACE_ROOT}/tp${tp}/private/real_requests.jsonl" + local port="${PORT}" + PORT=$((PORT + 1)) + mkdir -p "${run_out}" + ( + cd "${RUNNER_DIR}" + set +e + env \ + HOME=/tmp/wjh \ + XDG_CACHE_HOME=/tmp/wjh/.cache \ + VLLM_CACHE_ROOT=/tmp/wjh/.cache/vllm \ + CUDA_VISIBLE_DEVICES="${gpus}" \ + TP="${tp}" \ + MNS="${mns}" \ + TRACE_LABEL="${CASE_NAME}/tp${tp}-normalized" \ + PREFIX_CACHING="${PREFIX_CACHING}" \ + SERVER_PORT="${port}" \ + OUTPUT_ROOT="${run_out}" \ + REQUESTS_FILE="${requests}" \ + VENV_ROOT="${VENV_ROOT}" \ + MODEL_ROOT="${MODEL_ROOT}" \ + FLASHINFER_WORKSPACE_BASE="${FLASHINFER_SHARED_WORKSPACE}" \ + EXACT_TRACE_CLIENT="${CLIENT}" \ + timeout --signal=TERM --kill-after=60s 4200 bash "${RUNNER}" \ + > "${run_out}/launcher.stdout.log" 2> "${run_out}/launcher.stderr.log" + local_status="$?" + printf '%s\n' "${local_status}" > "${run_out}/launcher.exit_code" + exit "${local_status}" + ) & + WAVE_PIDS+=("$!") +} + +run_tp_wave() { + local trial="$1" tp="$2" order="$3" + IFS=',' read -r -a mnss <<< "${order}" + WAVE_PIDS=() + preflight_gpus + assert_no_our_server + printf 'WAVE_START case=%s trial=%s tp=%s mns=%s\n' "${CASE_NAME}" "${trial}" "${tp}" "${order}" \ + | tee -a "${OUT}/controller.log" + case "${tp}" in + 4) + launch_config "${trial}" 4 "${mnss[0]}" '0,1,2,3' + launch_config "${trial}" 4 "${mnss[1]}" '4,5,6,7' + wait_for_wave + preflight_gpus + assert_no_our_server + launch_config "${trial}" 4 "${mnss[2]}" '0,1,2,3' + launch_config "${trial}" 4 "${mnss[3]}" '4,5,6,7' + ;; + 2) + launch_config "${trial}" 2 "${mnss[0]}" '0,1' + launch_config "${trial}" 2 "${mnss[1]}" '2,3' + launch_config "${trial}" 2 "${mnss[2]}" '4,5' + launch_config "${trial}" 2 "${mnss[3]}" '6,7' + ;; + 1) + launch_config "${trial}" 1 "${mnss[0]}" '0' + launch_config "${trial}" 1 "${mnss[1]}" '1' + launch_config "${trial}" 1 "${mnss[2]}" '2' + launch_config "${trial}" 1 "${mnss[3]}" '3' + ;; + esac + wait_for_wave + preflight_gpus + assert_no_our_server + printf 'WAVE_COMPLETE case=%s trial=%s tp=%s\n' "${CASE_NAME}" "${trial}" "${tp}" \ + | tee -a "${OUT}/controller.log" +} + +{ + printf '%s\n' "REAL_LAUNCH_ECHO host=dash0 model=Qwen3-30B-A3B engine=vLLM-0.20.0+cu129 dtype=BF16 case=${CASE_NAME} traces=${TRACE_ROOT}/tp{1,2,4}/private/real_requests.jsonl prefix=${PREFIX_CACHING} requests=129 transform=t_prime=t/TP surface=TP{1,2,4}xMNS{8,16,32,64} trials=3 fresh_server=true metrics=mean,p90(TTFT,TPOT-if-OSL-gt-1,E2E) expected_cost=13_H20-GPUh_nominal__41_H20-GPUh_max output=${OUT}/real" + date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ + mkdir -p "${OUT}/provenance" "${FLASHINFER_SHARED_WORKSPACE}" + sha256sum "${BASH_SOURCE[0]}" "${RUNNER}" "${CLIENT}" "${PREFILL_CLIENT}" \ + "${MODEL_ROOT}/config.json" > "${OUT}/provenance/real-input.sha256" + for tp in 1 2 4; do + sha256sum "${TRACE_ROOT}/tp${tp}/public/manifest.json" \ + "${TRACE_ROOT}/tp${tp}/private/real_requests.jsonl" \ + >> "${OUT}/provenance/real-input.sha256" + done + "${VENV_ROOT}/bin/vllm" --version > "${OUT}/provenance/vllm.version" + "${VENV_ROOT}/bin/python" -c 'import torch, transformers, vllm; print(f"torch={torch.__version__}"); print(f"transformers={transformers.__version__}"); print(f"vllm={vllm.__version__}")' \ + > "${OUT}/provenance/runtime.versions" + nvidia-smi --query-gpu=index,name,uuid,driver_version,memory.total --format=csv,noheader \ + > "${OUT}/provenance/gpus.before.csv" + + run_tp_wave 1 4 '8,16,32,64' + run_tp_wave 1 2 '8,16,32,64' + run_tp_wave 1 1 '8,16,32,64' + run_tp_wave 2 1 '64,32,16,8' + run_tp_wave 2 2 '64,32,16,8' + run_tp_wave 2 4 '64,32,16,8' + run_tp_wave 3 2 '16,32,64,8' + run_tp_wave 3 4 '16,32,64,8' + run_tp_wave 3 1 '16,32,64,8' + + nvidia-smi --query-gpu=index,name,uuid,driver_version,memory.total --format=csv,noheader \ + > "${OUT}/provenance/gpus.after.csv" + find "${OUT}/real" -type f ! -path '*/provenance/artifacts.sha256' -print0 \ + | sort -z | xargs -0 sha256sum > "${OUT}/provenance/real-artifacts.sha256" + date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ + printf '%s\n' 'REAL_SURFACE_COMPLETE' +} >> "${OUT}/controller.log" 2>&1 From 28ffb22beea09679435e6e4c38137809521c4b12 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sat, 18 Jul 2026 10:50:11 +0800 Subject: [PATCH 126/148] Create Qwen30 case output before launch logging --- .../run_qwen30_latency_case_real_surface.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh index 553a93d..f8136b3 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh @@ -32,6 +32,7 @@ esac for path in "${RUNNER}" "${CLIENT}" "${PREFILL_CLIENT}" "${MODEL_ROOT}/config.json"; do [[ -f "${path}" ]] || { echo "ERROR: required path is missing: ${path}" >&2; exit 1; } done +mkdir -p "${OUT}" declare -a WAVE_PIDS=() From 686b0505176ea0ce90021d9a47906eadc3e01d68 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sat, 18 Jul 2026 10:57:48 +0800 Subject: [PATCH 127/148] Isolate Qwen235 DeepGEMM JIT cache --- .../run_qwen235_v020_smoke.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh index 235cb91..4b33f35 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh @@ -14,6 +14,8 @@ VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}" CLIENT="${CLIENT:?CLIENT is required}" READY_TIMEOUT_SECONDS="${READY_TIMEOUT_SECONDS:-1800}" +TRTLLM_DG_CACHE_DIR="${TRTLLM_DG_CACHE_DIR:-${OUT}/deep_gemm}" +DEEP_GEMM_JIT_DEBUG="${DEEP_GEMM_JIT_DEBUG:-false}" SERVED_MODEL="qwen3-235b-v020-smoke" SERVER_PID="" @@ -47,8 +49,12 @@ if ! [[ "${READY_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]]; then echo "ERROR: READY_TIMEOUT_SECONDS must be a positive integer" >&2 exit 1 fi +case "${DEEP_GEMM_JIT_DEBUG}" in + true|false) ;; + *) echo "ERROR: DEEP_GEMM_JIT_DEBUG must be true or false" >&2; exit 1 ;; +esac -echo "QWEN235_V020_SMOKE_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 checkpoint=FP8_compute_kv=auto config=TP${TP}_EP${ENABLE_EXPERT_PARALLEL}_MNS${MNS}_MBT${MBT} prefix=false requests=1 shape=2048x1 ready_timeout=${READY_TIMEOUT_SECONDS}s role=compatibility_gate_not_latency_measurement output=${OUT} expected_wall=20-35m expected_gpu_cap=$((${TP} * 35 / 60))_H20h" +echo "QWEN235_V020_SMOKE_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 checkpoint=FP8_compute_kv=auto config=TP${TP}_EP${ENABLE_EXPERT_PARALLEL}_MNS${MNS}_MBT${MBT} prefix=false requests=1 shape=2048x1 ready_timeout=${READY_TIMEOUT_SECONDS}s deep_gemm_cache=${TRTLLM_DG_CACHE_DIR} deep_gemm_jit_debug=${DEEP_GEMM_JIT_DEBUG} role=compatibility_gate_not_latency_measurement output=${OUT} expected_wall=20-35m expected_gpu_cap=$((${TP} * 35 / 60))_H20h" date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ sha256sum "${BASH_SOURCE[0]}" "${CLIENT}" "${MODEL_ROOT}/config.json" > "${OUT}/provenance/input.sha256" "${VENV_ROOT}/bin/vllm" --version > "${OUT}/provenance/vllm.version" @@ -67,6 +73,11 @@ export TRANSFORMERS_OFFLINE=1 export HOME=/tmp/wjh export XDG_CACHE_HOME=/tmp/wjh/.cache export VLLM_CACHE_ROOT=/tmp/wjh/.cache/vllm +export TRTLLM_DG_CACHE_DIR +if [[ "${DEEP_GEMM_JIT_DEBUG}" == true ]]; then + export TRTLLM_DG_JIT_DEBUG=1 +fi +mkdir -p "${TRTLLM_DG_CACHE_DIR}" setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \ --host 127.0.0.1 --port "${SERVER_PORT}" --served-model-name "${SERVED_MODEL}" \ --tensor-parallel-size "${TP}" --disable-custom-all-reduce --quantization fp8 \ From ecb45cf762b0ee5aa7933fb7273b9331db49f360 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sat, 18 Jul 2026 11:09:22 +0800 Subject: [PATCH 128/148] Extend Qwen30 JIT readiness for latency surface --- .../run_qwen30_exact_trace_real_anchor.sh | 6 ++++-- .../run_qwen30_latency_case_real_surface.sh | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh index 1037a54..0d28d0a 100755 --- a/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh @@ -12,6 +12,7 @@ PREFIX_CACHING="${PREFIX_CACHING:-true}" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:-/tmp/wjh/flashinfer-workspace-vllm020-profiler-v1}" +SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-120}" SERVED_MODEL="qwen3-30b-exact-trace" EXACT_TRACE_CLIENT="${EXACT_TRACE_CLIENT:-qwen30_exact_trace_client.py}" SERVER_PID="" @@ -53,7 +54,7 @@ case "${PREFIX_CACHING}" in esac REQUEST_COUNT="$(wc -l < "${REQUESTS_FILE}")" -echo "QWEN30_EXACT_TRACE_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 trace=${TRACE_LABEL} requests=${REQUEST_COUNT} source=${REQUESTS_FILE} arrivals=manifest prefix=${PREFIX_CACHING} block=16 metrics=TTFT,TPOT-if-OSL-gt-1,E2E flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} output=${OUTPUT_ROOT} expected_wall=12-35m hard_wall=3600s" +echo "QWEN30_EXACT_TRACE_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 trace=${TRACE_LABEL} requests=${REQUEST_COUNT} source=${REQUESTS_FILE} arrivals=manifest prefix=${PREFIX_CACHING} block=16 metrics=TTFT,TPOT-if-OSL-gt-1,E2E flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} output=${OUTPUT_ROOT} ready_budget_s=$((SERVER_READY_ATTEMPTS * 3)) hard_wall=3600s" date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" sha256sum qwen30_exact_trace_client.py run_qwen30_exact_trace_real_anchor.sh \ ../frontier-phase-factorial-v0/qwen30_prefill_client.py \ @@ -79,7 +80,7 @@ setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \ > "${OUTPUT_ROOT}/logs/server.log" 2>&1 & SERVER_PID=$! READY=0 -for _ in $(seq 1 120); do +for _ in $(seq 1 "${SERVER_READY_ATTEMPTS}"); do if curl -fsS --max-time 2 "http://127.0.0.1:${SERVER_PORT}/v1/models" \ > "${OUTPUT_ROOT}/results/models.json" 2>/dev/null; then READY=1 @@ -92,6 +93,7 @@ for _ in $(seq 1 120); do sleep 3 done if [[ "${READY}" -ne 1 ]]; then + echo "ERROR server did not become ready within $((SERVER_READY_ATTEMPTS * 3)) seconds" >&2 tail -200 "${OUTPUT_ROOT}/logs/server.log" exit 1 fi diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh index f8136b3..435cf76 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh @@ -16,7 +16,8 @@ CLIENT="${RUNNER_DIR}/qwen30_exact_trace_client.py" PREFILL_CLIENT="${RUNNER_DIR}/../frontier-phase-factorial-v0/qwen30_prefill_client.py" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" -FLASHINFER_SHARED_WORKSPACE="${FLASHINFER_SHARED_WORKSPACE:-${OUT}/flashinfer-shared-workspace}" +FLASHINFER_SHARED_WORKSPACE="${FLASHINFER_SHARED_WORKSPACE:-/tmp/wjh/flashinfer-workspace-vllm020-profiler-v1}" +SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-300}" PORT="${BASE_PORT:-8300}" case "${PREFIX_CACHING}" in @@ -97,6 +98,7 @@ launch_config() { VENV_ROOT="${VENV_ROOT}" \ MODEL_ROOT="${MODEL_ROOT}" \ FLASHINFER_WORKSPACE_BASE="${FLASHINFER_SHARED_WORKSPACE}" \ + SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS}" \ EXACT_TRACE_CLIENT="${CLIENT}" \ timeout --signal=TERM --kill-after=60s 4200 bash "${RUNNER}" \ > "${run_out}/launcher.stdout.log" 2> "${run_out}/launcher.stderr.log" From b7f9cef9c5f422426b24b20ccc926810a1fbec80 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sat, 18 Jul 2026 11:11:10 +0800 Subject: [PATCH 129/148] Keep Qwen30 JIT cache scoped to experiment --- .../run_qwen30_latency_case_real_surface.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh index 435cf76..d9b1d65 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh @@ -16,7 +16,7 @@ CLIENT="${RUNNER_DIR}/qwen30_exact_trace_client.py" PREFILL_CLIENT="${RUNNER_DIR}/../frontier-phase-factorial-v0/qwen30_prefill_client.py" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" -FLASHINFER_SHARED_WORKSPACE="${FLASHINFER_SHARED_WORKSPACE:-/tmp/wjh/flashinfer-workspace-vllm020-profiler-v1}" +FLASHINFER_SHARED_WORKSPACE="${FLASHINFER_SHARED_WORKSPACE:-${OUT}/flashinfer-shared-workspace}" SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-300}" PORT="${BASE_PORT:-8300}" From b9523cef5c9b6791420acbec5ece566a8d06e63f Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sat, 18 Jul 2026 11:15:22 +0800 Subject: [PATCH 130/148] Pin DeepGEMM nvcc for Qwen235 smoke --- .../run_qwen235_v020_smoke.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh index 4b33f35..f2affde 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_smoke.sh @@ -15,6 +15,7 @@ MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}" CLIENT="${CLIENT:?CLIENT is required}" READY_TIMEOUT_SECONDS="${READY_TIMEOUT_SECONDS:-1800}" TRTLLM_DG_CACHE_DIR="${TRTLLM_DG_CACHE_DIR:-${OUT}/deep_gemm}" +TRTLLM_DG_NVCC_COMPILER="${TRTLLM_DG_NVCC_COMPILER:-/usr/local/cuda/bin/nvcc}" DEEP_GEMM_JIT_DEBUG="${DEEP_GEMM_JIT_DEBUG:-false}" SERVED_MODEL="qwen3-235b-v020-smoke" SERVER_PID="" @@ -49,12 +50,16 @@ if ! [[ "${READY_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]]; then echo "ERROR: READY_TIMEOUT_SECONDS must be a positive integer" >&2 exit 1 fi +if [[ ! -x "${TRTLLM_DG_NVCC_COMPILER}" ]]; then + echo "ERROR: TRTLLM_DG_NVCC_COMPILER is not executable: ${TRTLLM_DG_NVCC_COMPILER}" >&2 + exit 1 +fi case "${DEEP_GEMM_JIT_DEBUG}" in true|false) ;; *) echo "ERROR: DEEP_GEMM_JIT_DEBUG must be true or false" >&2; exit 1 ;; esac -echo "QWEN235_V020_SMOKE_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 checkpoint=FP8_compute_kv=auto config=TP${TP}_EP${ENABLE_EXPERT_PARALLEL}_MNS${MNS}_MBT${MBT} prefix=false requests=1 shape=2048x1 ready_timeout=${READY_TIMEOUT_SECONDS}s deep_gemm_cache=${TRTLLM_DG_CACHE_DIR} deep_gemm_jit_debug=${DEEP_GEMM_JIT_DEBUG} role=compatibility_gate_not_latency_measurement output=${OUT} expected_wall=20-35m expected_gpu_cap=$((${TP} * 35 / 60))_H20h" +echo "QWEN235_V020_SMOKE_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 checkpoint=FP8_compute_kv=auto config=TP${TP}_EP${ENABLE_EXPERT_PARALLEL}_MNS${MNS}_MBT${MBT} prefix=false requests=1 shape=2048x1 ready_timeout=${READY_TIMEOUT_SECONDS}s deep_gemm_cache=${TRTLLM_DG_CACHE_DIR} deep_gemm_nvcc=${TRTLLM_DG_NVCC_COMPILER} deep_gemm_jit_debug=${DEEP_GEMM_JIT_DEBUG} role=compatibility_gate_not_latency_measurement output=${OUT} expected_wall=20-35m expected_gpu_cap=$((${TP} * 35 / 60))_H20h" date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ sha256sum "${BASH_SOURCE[0]}" "${CLIENT}" "${MODEL_ROOT}/config.json" > "${OUT}/provenance/input.sha256" "${VENV_ROOT}/bin/vllm" --version > "${OUT}/provenance/vllm.version" @@ -74,6 +79,7 @@ export HOME=/tmp/wjh export XDG_CACHE_HOME=/tmp/wjh/.cache export VLLM_CACHE_ROOT=/tmp/wjh/.cache/vllm export TRTLLM_DG_CACHE_DIR +export TRTLLM_DG_NVCC_COMPILER if [[ "${DEEP_GEMM_JIT_DEBUG}" == true ]]; then export TRTLLM_DG_JIT_DEBUG=1 fi From f11daa67768bfb075b24d268866144a1e5b69659 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sat, 18 Jul 2026 11:56:43 +0800 Subject: [PATCH 131/148] Fix Qwen30 exact trace served-model routing --- .../run_qwen30_exact_trace_real_anchor.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh index 0d28d0a..8d64119 100755 --- a/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh @@ -108,6 +108,7 @@ fi "${VENV_ROOT}/bin/python" "${EXACT_TRACE_CLIENT}" \ --port "${SERVER_PORT}" --requests-file "${REQUESTS_FILE}" \ + --served-model "${SERVED_MODEL}" \ --output "${OUTPUT_ROOT}/results/result.json" --tpot-slo-ms 150 \ --timeout-seconds 1800 From e98608911e6dd913d8acf496e5c9fb9ea76081c2 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sat, 18 Jul 2026 17:27:12 +0800 Subject: [PATCH 132/148] Resume valid Qwen30 latency cells safely --- .../run_qwen30_latency_case_real_surface.sh | 44 +++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh index d9b1d65..7de86fc 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh @@ -18,6 +18,8 @@ VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" FLASHINFER_SHARED_WORKSPACE="${FLASHINFER_SHARED_WORKSPACE:-${OUT}/flashinfer-shared-workspace}" SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-300}" +IDLE_GPU_MEMORY_TOLERANCE_MIB="${IDLE_GPU_MEMORY_TOLERANCE_MIB:-16}" +RESUME_VALID_CELLS="${RESUME_VALID_CELLS:-false}" PORT="${BASE_PORT:-8300}" case "${PREFIX_CACHING}" in @@ -30,6 +32,16 @@ case "${CASE_NAME}" in *) echo "ERROR: unsupported CASE_NAME=${CASE_NAME}" >&2; exit 1 ;; esac +case "${RESUME_VALID_CELLS}" in + true|false) ;; + *) echo "ERROR: RESUME_VALID_CELLS must be true or false" >&2; exit 1 ;; +esac + +[[ "${IDLE_GPU_MEMORY_TOLERANCE_MIB}" =~ ^[0-9]+$ ]] || { + echo "ERROR: IDLE_GPU_MEMORY_TOLERANCE_MIB must be a non-negative integer" >&2 + exit 1 +} + for path in "${RUNNER}" "${CLIENT}" "${PREFILL_CLIENT}" "${MODEL_ROOT}/config.json"; do [[ -f "${path}" ]] || { echo "ERROR: required path is missing: ${path}" >&2; exit 1; } done @@ -37,6 +49,23 @@ mkdir -p "${OUT}" declare -a WAVE_PIDS=() +has_valid_result() { + local result="$1" expected_requests="$2" + [[ -s "${result}" ]] || return 1 + "${VENV_ROOT}/bin/python" - "${result}" "${expected_requests}" <<'PY' +import json +import sys + +with open(sys.argv[1]) as source: + summary = json.load(source).get("summary", {}) +raise SystemExit( + 0 + if summary.get("completed") == int(sys.argv[2]) and summary.get("failed") == 0 + else 1 +) +PY +} + wait_for_wave() { local failed=0 pid for pid in "${WAVE_PIDS[@]}"; do @@ -56,10 +85,11 @@ preflight_gpus() { nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader \ | tee -a "${OUT}/controller.log" if nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits \ - | awk '$1 != 0 {exit 1}'; then + | awk -v tolerance="${IDLE_GPU_MEMORY_TOLERANCE_MIB}" '$1 > tolerance {exit 1}'; then return 0 fi - printf '%s\n' 'ERROR non-idle GPU before a real-serving wave; refusing to start.' \ + printf 'ERROR GPU memory exceeds idle tolerance (%s MiB) before a real-serving wave; refusing to start.\n' \ + "${IDLE_GPU_MEMORY_TOLERANCE_MIB}" \ | tee -a "${OUT}/controller.log" >&2 return 1 } @@ -77,8 +107,16 @@ launch_config() { local config="tp${tp}_mns${mns}" local run_out="${OUT}/real/${config}/trial${trial}" local requests="${TRACE_ROOT}/tp${tp}/private/real_requests.jsonl" + local request_count local port="${PORT}" PORT=$((PORT + 1)) + request_count="$(wc -l < "${requests}")" + if [[ "${RESUME_VALID_CELLS}" == true ]] \ + && has_valid_result "${run_out}/results/result.json" "${request_count}"; then + printf 'CONFIG_REUSED_VALID case=%s trial=%s tp=%s mns=%s\n' \ + "${CASE_NAME}" "${trial}" "${tp}" "${mns}" + return 0 + fi mkdir -p "${run_out}" ( cd "${RUNNER_DIR}" @@ -148,7 +186,7 @@ run_tp_wave() { } { - printf '%s\n' "REAL_LAUNCH_ECHO host=dash0 model=Qwen3-30B-A3B engine=vLLM-0.20.0+cu129 dtype=BF16 case=${CASE_NAME} traces=${TRACE_ROOT}/tp{1,2,4}/private/real_requests.jsonl prefix=${PREFIX_CACHING} requests=129 transform=t_prime=t/TP surface=TP{1,2,4}xMNS{8,16,32,64} trials=3 fresh_server=true metrics=mean,p90(TTFT,TPOT-if-OSL-gt-1,E2E) expected_cost=13_H20-GPUh_nominal__41_H20-GPUh_max output=${OUT}/real" + printf '%s\n' "REAL_LAUNCH_ECHO host=dash0 model=Qwen3-30B-A3B engine=vLLM-0.20.0+cu129 dtype=BF16 case=${CASE_NAME} traces=${TRACE_ROOT}/tp{1,2,4}/private/real_requests.jsonl prefix=${PREFIX_CACHING} requests=129 transform=t_prime=t/TP surface=TP{1,2,4}xMNS{8,16,32,64} trials=3 fresh_server=true resume_valid_cells=${RESUME_VALID_CELLS} idle_gpu_memory_tolerance_mib=${IDLE_GPU_MEMORY_TOLERANCE_MIB} metrics=mean,p90(TTFT,TPOT-if-OSL-gt-1,E2E) expected_cost=13_H20-GPUh_nominal__41_H20-GPUh_max output=${OUT}/real" date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ mkdir -p "${OUT}/provenance" "${FLASHINFER_SHARED_WORKSPACE}" sha256sum "${BASH_SOURCE[0]}" "${RUNNER}" "${CLIENT}" "${PREFILL_CLIENT}" \ From 672631879288ee0c85393229b4ebe84705b5da5c Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sun, 19 Jul 2026 00:54:48 +0800 Subject: [PATCH 133/148] Profile fixed PD workload pressure against trace anchor --- .../fixed-pd-pressure-matching-card.md | 51 ++++ .../run_qwen30_fixed_pd_pressure_probe.sh | 232 ++++++++++++++++++ 2 files changed, 283 insertions(+) create mode 100644 runs/frontier-fidelity-envelope-v1/fixed-pd-pressure-matching-card.md create mode 100644 runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pd_pressure_probe.sh diff --git a/runs/frontier-fidelity-envelope-v1/fixed-pd-pressure-matching-card.md b/runs/frontier-fidelity-envelope-v1/fixed-pd-pressure-matching-card.md new file mode 100644 index 0000000..220cb9a --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/fixed-pd-pressure-matching-card.md @@ -0,0 +1,51 @@ +# EXP-SIMFID-Q30-FIXED-PD-PRESSURE: calibrate a nontrivial fixed-shape workload + +> Status: planned. This is a real-only workload calibration; its measurements +> are not part of a Frontier-versus-vLLM winner comparison. + +## Question + +The former Fixed-PD workload (`2048 -> 128`, 0.215 request/s/GPU) was nearly +single-request at TP4: its real mean E2E was 604 ms at a TP4 cluster rate of +0.86 request/s, or about 0.52 in-flight requests by Little's law. It therefore +does not test the decode batching regime of Trace-PD. + +Can a no-prefix, uniform `4096 -> 256` workload be assigned a fixed offered +rate that reaches the healthy Trace-PD operating point without using a +simulator result for calibration? + +## Controlled calibration + +The reference is the real Trace-PD `TP4/MNS64` cell, chosen before this probe +because it is the real TTFT/E2E winner on that surface: + +| Reference metric | Target | +|---|---:| +| mean TTFT | 245.95 ms | +| mean TPOT | 13.18 ms | +| mean E2E | 44.99 s | +| offered rate | 0.215 request/s/GPU; 0.86 request/s at TP4 | +| in-flight proxy | 38.69 requests | + +The probe fixes Qwen3-30B-A3B BF16, community vLLM 0.20.0, H20, TP4, MNS64, +MBT=8192, chunked prefill, and prefix caching off. It compares per-GPU rates +`{1, 2, 3, 4}` (global TP4 rates `{4, 8, 12, 16}` request/s). Every rate has +257 exact `4096 -> 256` requests in each of three fresh-server trials; rate +orders are rotated across trials. + +## Decision rule + +Pool the three trials for each rate. Among rates with every request completed, +choose the rate minimizing the Euclidean distance of the two *relative* errors +for mean TTFT and mean TPOT from the above targets. Report p90 TTFT/TPOT/E2E, +trial variation, and `global_rate * mean_E2E` as an in-flight proxy, but do not +turn any of them into an SLO. The selected rate becomes a frozen workload +contract. The subsequent 12-cell real/simulator surface must be fresh and is +not allowed to reuse this calibration data. + +## Interpretation boundary + +This selects a workload regime, not a simulator parameter and not a serving +capacity point. It may make Fixed-PD more comparable to Trace-PD in batching +pressure, but it deliberately continues to exclude trace-shaped arrivals and +prefix reuse; those are separately evaluated workload dimensions. diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pd_pressure_probe.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pd_pressure_probe.sh new file mode 100644 index 0000000..d22067e --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pd_pressure_probe.sh @@ -0,0 +1,232 @@ +#!/usr/bin/env bash + +# Real-only, pressure-matching probe for the next Fixed-PD workload. This +# intentionally profiles one anchor, then freezes the workload before any +# Frontier-vs-real selection comparison. + +set -euo pipefail + +OUT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}" +RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +CLIENT="${CLIENT:-${RUNNER_DIR}/../frontier-phase-factorial-v0/qwen30_prefill_client.py}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:-${OUT}/flashinfer-workspace}" +GPU_IDS="${GPU_IDS:-0,1,2,3}" +TP="${TP:-4}" +MNS="${MNS:-64}" +REQUESTS="${REQUESTS:-257}" +GLOBAL_RATES="${GLOBAL_RATES:-4 8 12 16}" +SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-180}" +PORT="${PORT:-8930}" +SERVED_MODEL="qwen3-30b-fixed-pd-pressure" +SERVER_PID="" + +[[ "${TP}" == "4" ]] || { echo 'ERROR: this calibrated probe is TP4-only' >&2; exit 1; } +[[ "${MNS}" == "64" ]] || { echo 'ERROR: this calibrated probe is MNS64-only' >&2; exit 1; } +[[ "${REQUESTS}" =~ ^[1-9][0-9]*$ ]] || { echo 'ERROR: REQUESTS must be positive' >&2; exit 1; } +[[ -f "${CLIENT}" ]] || { echo "ERROR: client missing: ${CLIENT}" >&2; exit 1; } +[[ -f "${MODEL_ROOT}/config.json" ]] || { echo "ERROR: model missing: ${MODEL_ROOT}" >&2; exit 1; } + +mkdir -p "${OUT}/provenance" "${OUT}/trials" "${FLASHINFER_WORKSPACE_BASE}" +exec > >(tee -a "${OUT}/controller.log") 2>&1 + +cleanup_server() { + if [[ -n "${SERVER_PID}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then + kill -TERM -- "-${SERVER_PID}" 2>/dev/null || true + for _ in $(seq 1 30); do + kill -0 "${SERVER_PID}" 2>/dev/null || break + sleep 1 + done + kill -KILL -- "-${SERVER_PID}" 2>/dev/null || true + fi + SERVER_PID="" +} +trap cleanup_server EXIT INT TERM + +assert_idle() { + nvidia-smi --query-gpu=index,memory.used,utilization.gpu --format=csv,noheader + nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits \ + | awk '$1 > 16 {exit 1}' +} + +wait_ready() { + local target="$1" + for _ in $(seq 1 "${SERVER_READY_ATTEMPTS}"); do + if curl -fsS --max-time 2 "http://127.0.0.1:${PORT}/v1/models" > "${target}/models.json" 2>/dev/null; then + return 0 + fi + if ! kill -0 "${SERVER_PID}" 2>/dev/null; then + tail -200 "${target}/server.log" >&2 || true + return 1 + fi + sleep 3 + done + echo "ERROR: vLLM did not become ready in $((SERVER_READY_ATTEMPTS * 3)) seconds" >&2 + return 1 +} + +start_server() { + local target="$1" + export TOKENIZERS_PARALLELISM=false + export VLLM_USE_V1=1 + export TORCH_CUDA_ARCH_LIST=9.0 + export HF_HUB_OFFLINE=1 + export TRANSFORMERS_OFFLINE=1 + export FLASHINFER_WORKSPACE_BASE + export HOME=/tmp/wjh + export XDG_CACHE_HOME=/tmp/wjh/.cache + export VLLM_CACHE_ROOT=/tmp/wjh/.cache/vllm + export CUDA_VISIBLE_DEVICES="${GPU_IDS}" + setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \ + --host 127.0.0.1 --port "${PORT}" --served-model-name "${SERVED_MODEL}" \ + --tensor-parallel-size "${TP}" --gpu-memory-utilization 0.92 \ + --max-model-len 40960 --max-num-batched-tokens 8192 --max-num-seqs "${MNS}" \ + --no-enable-prefix-caching --enable-chunked-prefill --no-enable-log-requests \ + > "${target}/server.log" 2>&1 & + SERVER_PID=$! + wait_ready "${target}" +} + +run_client() { + local target="$1" rate="$2" + timeout --signal=TERM --kill-after=60s 1800 \ + "${VENV_ROOT}/bin/python" "${CLIENT}" \ + --port "${PORT}" --served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" \ + --rate "${rate}" --requests "${REQUESTS}" --input-tokens 4096 --output-tokens 256 \ + --timeout-seconds 1200 --output "${target}/result.json" +} + +warmup_server() { + local target="$1" + timeout --signal=TERM --kill-after=60s 600 \ + "${VENV_ROOT}/bin/python" "${CLIENT}" \ + --port "${PORT}" --served-model "${SERVED_MODEL}" --model-path "${MODEL_ROOT}" \ + --rate 1 --requests 4 --input-tokens 512 --output-tokens 1 \ + --timeout-seconds 300 --output "${target}/result.json" +} + +analyze() { + "${VENV_ROOT}/bin/python" - "${OUT}" "${GLOBAL_RATES}" <<'PY' +import json +import math +import statistics +import sys +from pathlib import Path + +root = Path(sys.argv[1]) +rates = [float(value) for value in sys.argv[2].split()] +target = {"ttft_ms": 245.9527667526406, "tpot_ms": 13.178025610291787} + +def p90(values): + return sorted(values)[math.ceil(0.9 * len(values)) - 1] + +rows = [] +for rate in rates: + label = f"r{rate:g}" + trial_means = {"ttft_ms": [], "tpot_ms": [], "e2e_ms": []} + pooled = {key: [] for key in trial_means} + for trial in range(1, 4): + path = root / "trials" / f"trial{trial}" / label / "result.json" + payload = json.loads(path.read_text()) + workload = payload["workload"] + if (float(workload["offered_request_rate"]) != rate or workload["request_count"] != 257 + or workload["input_tokens"] != 4096 or workload["output_tokens"] != 256 + or workload["prefix_caching"] is not False): + raise ValueError(f"workload drift: {path}") + requests = payload["requests"] + if len(requests) != 257 or any(not request["success"] for request in requests): + raise ValueError(f"incomplete client result: {path}") + for key in pooled: + values = [float(request[key]) for request in requests] + pooled[key].extend(values) + trial_means[key].append(statistics.mean(values)) + row = { + "global_rate": rate, + "per_gpu_rate": rate / 4.0, + "requests_per_trial": 257, + "trials": 3, + "metrics": { + key: { + "pooled_mean_ms": statistics.mean(values), + "pooled_p90_ms": p90(values), + "trial_mean_stdev_ms": statistics.stdev(trial_means[key]), + } + for key, values in pooled.items() + }, + } + row["inflight_proxy"] = rate * row["metrics"]["e2e_ms"]["pooled_mean_ms"] / 1000.0 + row["relative_distance"] = math.sqrt(sum( + ((row["metrics"][key]["pooled_mean_ms"] - target[key]) / target[key]) ** 2 + for key in target + )) + rows.append(row) + +winner = min(rows, key=lambda row: (row["relative_distance"], row["global_rate"])) +payload = { + "schema": "qwen30-fixed-pd-pressure-probe-v1", + "target_trace_pd_tp4_mns64": target, + "decision_rule": "minimum Euclidean distance of relative mean TTFT and TPOT errors", + "rates": rows, + "recommended_global_rate": winner["global_rate"], + "recommended_per_gpu_rate": winner["per_gpu_rate"], +} +(root / "pressure-analysis.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") +lines = [ + "# Fixed-PD pressure probe", + "", + "| Global / per-GPU rps | TTFT mean / p90 (ms) | TPOT mean / p90 (ms) | E2E mean / p90 (ms) | In-flight proxy | Relative distance |", + "|---|---:|---:|---:|---:|---:|", +] +for row in rows: + metric = row["metrics"] + lines.append( + f"| {row['global_rate']:g} / {row['per_gpu_rate']:g} | " + f"{metric['ttft_ms']['pooled_mean_ms']:.2f} / {metric['ttft_ms']['pooled_p90_ms']:.2f} | " + f"{metric['tpot_ms']['pooled_mean_ms']:.2f} / {metric['tpot_ms']['pooled_p90_ms']:.2f} | " + f"{metric['e2e_ms']['pooled_mean_ms']:.2f} / {metric['e2e_ms']['pooled_p90_ms']:.2f} | " + f"{row['inflight_proxy']:.2f} | {row['relative_distance']:.3f} |" + ) +lines.extend([ + "", + f"Recommended frozen rate: **{winner['global_rate']:g} global rps / {winner['per_gpu_rate']:g} rps per GPU**.", + "Selection uses only pooled mean TTFT and TPOT; p90 and in-flight proxy are audit outputs.", +]) +(root / "pressure-analysis.md").write_text("\n".join(lines) + "\n") +print(json.dumps(payload, sort_keys=True)) +PY +} + +{ + echo "FIXED_PD_PRESSURE_PROBE_LAUNCH_ECHO host=$(hostname) model=${MODEL_ROOT} engine=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 gpus=${GPU_IDS} prefix=false shape=4096_to_256 requests_per_rate=${REQUESTS} global_rates={${GLOBAL_RATES}} rate_contract=global_rate_divided_by_TP trials=3 fresh_server=true metric_target=TracePD_TP4_MNS64_meanTTFT245.95ms_meanTPOT13.18ms expected_wall=12-20m expected_cost=0.8-1.4_H20-GPUh output=${OUT}" + date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ + assert_idle + sha256sum "${BASH_SOURCE[0]}" "${CLIENT}" "${MODEL_ROOT}/config.json" > "${OUT}/provenance/input.sha256" + "${VENV_ROOT}/bin/vllm" --version > "${OUT}/provenance/vllm.version" + "${VENV_ROOT}/bin/python" -c 'import torch, transformers, vllm; print(f"torch={torch.__version__}"); print(f"transformers={transformers.__version__}"); print(f"vllm={vllm.__version__}")' > "${OUT}/provenance/runtime.versions" + nvidia-smi --query-gpu=index,name,uuid,driver_version,memory.total --format=csv,noheader > "${OUT}/provenance/gpus.before.csv" + + declare -a ORDERS=("4 8 12 16" "16 12 8 4" "8 16 4 12") + for trial in 1 2 3; do + trial_root="${OUT}/trials/trial${trial}" + mkdir -p "${trial_root}" + echo "TRIAL_START trial=${trial} order=${ORDERS[$((trial - 1))]}" + start_server "${trial_root}" + warmup_server "${trial_root}/warmup" + for rate in ${ORDERS[$((trial - 1))]}; do + rate_root="${trial_root}/r${rate}" + mkdir -p "${rate_root}" + echo "RATE_START trial=${trial} global_rate=${rate} per_gpu_rate=$(awk -v value="${rate}" 'BEGIN {printf "%.3f", value / 4}')" + run_client "${rate_root}" "${rate}" + echo "RATE_COMPLETE trial=${trial} global_rate=${rate}" + done + cleanup_server + assert_idle + echo "TRIAL_COMPLETE trial=${trial}" + done + analyze + find "${OUT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 | sort -z | xargs -0 sha256sum > "${OUT}/provenance/artifacts.sha256" + nvidia-smi --query-gpu=index,name,uuid,driver_version,memory.total --format=csv,noheader > "${OUT}/provenance/gpus.after.csv" + date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ + echo 'FIXED_PD_PRESSURE_PROBE_COMPLETE' +} >> "${OUT}/controller.log" 2>&1 From 9837fa5133119ecd80b140fe0b0e715ff2a3de12 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sun, 19 Jul 2026 01:07:57 +0800 Subject: [PATCH 134/148] Match Qwen30 probe file descriptor limit --- .../run_qwen30_fixed_pd_pressure_probe.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pd_pressure_probe.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pd_pressure_probe.sh index d22067e..85aed68 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pd_pressure_probe.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pd_pressure_probe.sh @@ -31,6 +31,12 @@ SERVER_PID="" mkdir -p "${OUT}/provenance" "${OUT}/trials" "${FLASHINFER_WORKSPACE_BASE}" exec > >(tee -a "${OUT}/controller.log") 2>&1 +# Match the validated Qwen30 real-surface runner's file-descriptor budget for +# vLLM's multiprocessing shared-memory transport. The failed r1 probe had the +# default remote limit (1024) and stalled before KV-cache creation; r2 tests +# whether this runner-contract difference is causal. +ulimit -n 65536 + cleanup_server() { if [[ -n "${SERVER_PID}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then kill -TERM -- "-${SERVER_PID}" 2>/dev/null || true @@ -204,6 +210,7 @@ PY sha256sum "${BASH_SOURCE[0]}" "${CLIENT}" "${MODEL_ROOT}/config.json" > "${OUT}/provenance/input.sha256" "${VENV_ROOT}/bin/vllm" --version > "${OUT}/provenance/vllm.version" "${VENV_ROOT}/bin/python" -c 'import torch, transformers, vllm; print(f"torch={torch.__version__}"); print(f"transformers={transformers.__version__}"); print(f"vllm={vllm.__version__}")' > "${OUT}/provenance/runtime.versions" + ulimit -n > "${OUT}/provenance/open-file-limit" nvidia-smi --query-gpu=index,name,uuid,driver_version,memory.total --format=csv,noheader > "${OUT}/provenance/gpus.before.csv" declare -a ORDERS=("4 8 12 16" "16 12 8 4" "8 16 4 12") From 909a80f0a66ccf068a947b5649a755b15dfeabaf Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sun, 19 Jul 2026 01:13:50 +0800 Subject: [PATCH 135/148] Reuse validated FlashInfer cache for pressure probe --- .../fixed-pd-pressure-matching-card.md | 4 +++- .../run_qwen30_fixed_pd_pressure_probe.sh | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/runs/frontier-fidelity-envelope-v1/fixed-pd-pressure-matching-card.md b/runs/frontier-fidelity-envelope-v1/fixed-pd-pressure-matching-card.md index 220cb9a..6b2ec07 100644 --- a/runs/frontier-fidelity-envelope-v1/fixed-pd-pressure-matching-card.md +++ b/runs/frontier-fidelity-envelope-v1/fixed-pd-pressure-matching-card.md @@ -31,7 +31,9 @@ The probe fixes Qwen3-30B-A3B BF16, community vLLM 0.20.0, H20, TP4, MNS64, MBT=8192, chunked prefill, and prefix caching off. It compares per-GPU rates `{1, 2, 3, 4}` (global TP4 rates `{4, 8, 12, 16}` request/s). Every rate has 257 exact `4096 -> 256` requests in each of three fresh-server trials; rate -orders are rotated across trials. +orders are rotated across trials. The launcher reuses the validated vLLM 0.20 +FlashInfer kernel cache; this avoids including one-off custom-kernel JIT in +server startup and does not alter request latency measurement. ## Decision rule diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pd_pressure_probe.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pd_pressure_probe.sh index 85aed68..c1b4623 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pd_pressure_probe.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pd_pressure_probe.sh @@ -11,7 +11,10 @@ RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" CLIENT="${CLIENT:-${RUNNER_DIR}/../frontier-phase-factorial-v0/qwen30_prefill_client.py}" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" -FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:-${OUT}/flashinfer-workspace}" +# Reuse the validated vLLM 0.20 H20 kernel cache. A per-output workspace +# starts costly FlashInfer MoE JIT compilation and changes startup behavior +# without changing the serving configuration being profiled. +FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:-/tmp/wjh/flashinfer-workspace-vllm020-profiler-v1}" GPU_IDS="${GPU_IDS:-0,1,2,3}" TP="${TP:-4}" MNS="${MNS:-64}" @@ -211,6 +214,7 @@ PY "${VENV_ROOT}/bin/vllm" --version > "${OUT}/provenance/vllm.version" "${VENV_ROOT}/bin/python" -c 'import torch, transformers, vllm; print(f"torch={torch.__version__}"); print(f"transformers={transformers.__version__}"); print(f"vllm={vllm.__version__}")' > "${OUT}/provenance/runtime.versions" ulimit -n > "${OUT}/provenance/open-file-limit" + readlink -f "${FLASHINFER_WORKSPACE_BASE}" > "${OUT}/provenance/flashinfer-workspace" nvidia-smi --query-gpu=index,name,uuid,driver_version,memory.total --format=csv,noheader > "${OUT}/provenance/gpus.before.csv" declare -a ORDERS=("4 8 12 16" "16 12 8 4" "8 16 4 12") From 33b73afe9b9b988b95a44dc3c616c932f54c9550 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sun, 19 Jul 2026 01:26:09 +0800 Subject: [PATCH 136/148] Narrow fixed PD pressure sweep around load knee --- .../fixed-pd-pressure-matching-card.md | 15 ++++++++----- .../run_qwen30_fixed_pd_pressure_probe.sh | 22 ++++++++++++++----- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/runs/frontier-fidelity-envelope-v1/fixed-pd-pressure-matching-card.md b/runs/frontier-fidelity-envelope-v1/fixed-pd-pressure-matching-card.md index 6b2ec07..97b74ca 100644 --- a/runs/frontier-fidelity-envelope-v1/fixed-pd-pressure-matching-card.md +++ b/runs/frontier-fidelity-envelope-v1/fixed-pd-pressure-matching-card.md @@ -28,12 +28,15 @@ because it is the real TTFT/E2E winner on that surface: | in-flight proxy | 38.69 requests | The probe fixes Qwen3-30B-A3B BF16, community vLLM 0.20.0, H20, TP4, MNS64, -MBT=8192, chunked prefill, and prefix caching off. It compares per-GPU rates -`{1, 2, 3, 4}` (global TP4 rates `{4, 8, 12, 16}` request/s). Every rate has -257 exact `4096 -> 256` requests in each of three fresh-server trials; rate -orders are rotated across trials. The launcher reuses the validated vLLM 0.20 -FlashInfer kernel cache; this avoids including one-off custom-kernel JIT in -server startup and does not alter request latency measurement. +MBT=8192, chunked prefill, and prefix caching off. An incomplete one-trial +range-finding pilot showed global 4 rps stable and global 8 rps already badly +overloaded; it is not used in the decision. The formal probe therefore compares +per-GPU rates `{1, 1.125, 1.25, 1.5}` (global TP4 rates `{4, 4.5, 5, 6}` +request/s). Every rate has 257 exact `4096 -> 256` requests in each of three +fresh-server trials; rate orders are rotated across trials. The launcher +requires a validated vLLM 0.20 FlashInfer kernel cache; this avoids including +one-off custom-kernel JIT in server startup and does not alter request latency +measurement. ## Decision rule diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pd_pressure_probe.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pd_pressure_probe.sh index c1b4623..b8ada40 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pd_pressure_probe.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pd_pressure_probe.sh @@ -14,12 +14,12 @@ MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" # Reuse the validated vLLM 0.20 H20 kernel cache. A per-output workspace # starts costly FlashInfer MoE JIT compilation and changes startup behavior # without changing the serving configuration being profiled. -FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:-/tmp/wjh/flashinfer-workspace-vllm020-profiler-v1}" +FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:?FLASHINFER_WORKSPACE_BASE is required}" GPU_IDS="${GPU_IDS:-0,1,2,3}" TP="${TP:-4}" MNS="${MNS:-64}" REQUESTS="${REQUESTS:-257}" -GLOBAL_RATES="${GLOBAL_RATES:-4 8 12 16}" +GLOBAL_RATES="${GLOBAL_RATES:-4 4.5 5 6}" SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-180}" PORT="${PORT:-8930}" SERVED_MODEL="qwen3-30b-fixed-pd-pressure" @@ -30,6 +30,12 @@ SERVER_PID="" [[ "${REQUESTS}" =~ ^[1-9][0-9]*$ ]] || { echo 'ERROR: REQUESTS must be positive' >&2; exit 1; } [[ -f "${CLIENT}" ]] || { echo "ERROR: client missing: ${CLIENT}" >&2; exit 1; } [[ -f "${MODEL_ROOT}/config.json" ]] || { echo "ERROR: model missing: ${MODEL_ROOT}" >&2; exit 1; } +[[ -d "${FLASHINFER_WORKSPACE_BASE}" ]] || { echo "ERROR: FlashInfer workspace missing: ${FLASHINFER_WORKSPACE_BASE}" >&2; exit 1; } +read -r -a RATE_VALUES <<< "${GLOBAL_RATES}" +[[ "${#RATE_VALUES[@]}" -eq 4 ]] || { echo 'ERROR: GLOBAL_RATES must contain exactly four rates' >&2; exit 1; } +for rate in "${RATE_VALUES[@]}"; do + awk -v value="${rate}" 'BEGIN {exit !(value > 0)}' || { echo "ERROR: invalid rate: ${rate}" >&2; exit 1; } +done mkdir -p "${OUT}/provenance" "${OUT}/trials" "${FLASHINFER_WORKSPACE_BASE}" exec > >(tee -a "${OUT}/controller.log") 2>&1 @@ -51,7 +57,9 @@ cleanup_server() { fi SERVER_PID="" } -trap cleanup_server EXIT INT TERM +trap cleanup_server EXIT +trap 'cleanup_server; exit 130' INT +trap 'cleanup_server; exit 143' TERM assert_idle() { nvidia-smi --query-gpu=index,memory.used,utilization.gpu --format=csv,noheader @@ -207,7 +215,7 @@ PY } { - echo "FIXED_PD_PRESSURE_PROBE_LAUNCH_ECHO host=$(hostname) model=${MODEL_ROOT} engine=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 gpus=${GPU_IDS} prefix=false shape=4096_to_256 requests_per_rate=${REQUESTS} global_rates={${GLOBAL_RATES}} rate_contract=global_rate_divided_by_TP trials=3 fresh_server=true metric_target=TracePD_TP4_MNS64_meanTTFT245.95ms_meanTPOT13.18ms expected_wall=12-20m expected_cost=0.8-1.4_H20-GPUh output=${OUT}" + echo "FIXED_PD_PRESSURE_PROBE_LAUNCH_ECHO host=$(hostname) model=${MODEL_ROOT} engine=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 gpus=${GPU_IDS} prefix=false shape=4096_to_256 requests_per_rate=${REQUESTS} global_rates={${GLOBAL_RATES}} rate_contract=global_rate_divided_by_TP flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} trials=3 fresh_server=true metric_target=TracePD_TP4_MNS64_meanTTFT245.95ms_meanTPOT13.18ms expected_wall=12-20m expected_cost=0.8-1.4_H20-GPUh output=${OUT}" date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ assert_idle sha256sum "${BASH_SOURCE[0]}" "${CLIENT}" "${MODEL_ROOT}/config.json" > "${OUT}/provenance/input.sha256" @@ -217,7 +225,11 @@ PY readlink -f "${FLASHINFER_WORKSPACE_BASE}" > "${OUT}/provenance/flashinfer-workspace" nvidia-smi --query-gpu=index,name,uuid,driver_version,memory.total --format=csv,noheader > "${OUT}/provenance/gpus.before.csv" - declare -a ORDERS=("4 8 12 16" "16 12 8 4" "8 16 4 12") + declare -a ORDERS=( + "${RATE_VALUES[*]}" + "${RATE_VALUES[3]} ${RATE_VALUES[2]} ${RATE_VALUES[1]} ${RATE_VALUES[0]}" + "${RATE_VALUES[1]} ${RATE_VALUES[3]} ${RATE_VALUES[0]} ${RATE_VALUES[2]}" + ) for trial in 1 2 3; do trial_root="${OUT}/trials/trial${trial}" mkdir -p "${trial_root}" From 7ea96358788c54f3d8a5f9bd5178c34ba3fe7973 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sun, 19 Jul 2026 01:45:17 +0800 Subject: [PATCH 137/148] Record fixed PD pressure calibration result --- .../fixed-pd-pressure-matching-card.md | 37 ++++++++++++++++++- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/runs/frontier-fidelity-envelope-v1/fixed-pd-pressure-matching-card.md b/runs/frontier-fidelity-envelope-v1/fixed-pd-pressure-matching-card.md index 97b74ca..2efedad 100644 --- a/runs/frontier-fidelity-envelope-v1/fixed-pd-pressure-matching-card.md +++ b/runs/frontier-fidelity-envelope-v1/fixed-pd-pressure-matching-card.md @@ -1,7 +1,7 @@ # EXP-SIMFID-Q30-FIXED-PD-PRESSURE: calibrate a nontrivial fixed-shape workload -> Status: planned. This is a real-only workload calibration; its measurements -> are not part of a Frontier-versus-vLLM winner comparison. +> Status: complete (2026-07-19). This is a real-only workload calibration; its +> measurements are not part of a Frontier-versus-vLLM winner comparison. ## Question @@ -54,3 +54,36 @@ This selects a workload regime, not a simulator parameter and not a serving capacity point. It may make Fixed-PD more comparable to Trace-PD in batching pressure, but it deliberately continues to exclude trace-shaped arrivals and prefix reuse; those are separately evaluated workload dimensions. + +## Result + +All 12 planned cells completed (`257` requests × `3` fresh-server trials per +rate, zero client failures). The raw artifact is +`dash0:/home/admin/cpfs/wjh/aituner/qwen30-fixed-pd-pressure-20260719-r5`; +it records the vLLM version, model/config checksums, H20 inventory, FlashInfer +workspace, and an artifact checksum manifest. Pooled results are: + +| Global / per-GPU rps | TTFT mean / p90 (ms) | TPOT mean / p90 (ms) | E2E mean / p90 (ms) | In-flight proxy | +|---|---:|---:|---:|---:| +| 4 / 1 | 121.53 / 125.12 | 10.75 / 11.83 | 2862.42 / 3138.92 | 11.45 | +| 4.5 / 1.125 | 122.76 / 126.29 | 11.76 / 12.55 | 3122.32 / 3322.26 | 14.05 | +| 5 / 1.25 | 125.09 / 129.66 | 15.16 / 16.58 | 3989.73 / 4354.22 | 19.95 | +| 6 / 1.5 | 129.27 / 134.69 | 21.33 / 23.03 | 5569.14 / 6005.12 | 33.41 | + +The predeclared two-metric rule picks `4.5` global rps (`1.125` rps/GPU), but +that is only the *least mismatched* tested point: TTFT is still 50.1% below +the Trace-PD target (122.76 vs 245.95 ms), while TPOT is 10.7% below it (11.76 +vs 13.18 ms). Raising load to 5 global rps moves TPOT past target (+15.0%) +while TTFT barely changes (125.09 ms); at 6 global rps TPOT is +61.9% while +TTFT remains 47.4% below target. Trial-level mean standard deviations are at +most 0.78 ms for TTFT and 0.34 ms for TPOT, so this is not trial-order noise. + +**Conclusion:** `4096 -> 256` with uniform no-prefix arrivals cannot be called +a pressure-matched Fixed-PD control simply by setting RPS. It has much shorter +request residence time (3.12 s versus 44.99 s) and lower concurrency (14.05 +versus 38.69 by the same Little-law proxy) at its closest point. For the +paper's fixed-versus-trace comparison, retain this case only as a deliberately +different fixed-shape workload; do not interpret any simulator gap as being +caused solely by trace fidelity. A future pressure-matched fixed control must +also increase output length (or otherwise preserve residence time), then repeat +this real-only calibration before reopening the Frontier-versus-vLLM surface. From c8383c9c4c16d69f6e0fc1e1944c2b8dedb88bcd Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sun, 19 Jul 2026 02:25:15 +0800 Subject: [PATCH 138/148] Launch Qwen30 fixed pressure comparison --- .../qwen30-fixed-pressure-surface-card.md | 49 +++++++ .../run_qwen30_fixed_pressure_campaign.sh | 138 ++++++++++++++++++ .../run_qwen30_latency_case_real_surface.sh | 10 +- 3 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 runs/frontier-fidelity-envelope-v1/qwen30-fixed-pressure-surface-card.md create mode 100644 runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pressure_campaign.sh diff --git a/runs/frontier-fidelity-envelope-v1/qwen30-fixed-pressure-surface-card.md b/runs/frontier-fidelity-envelope-v1/qwen30-fixed-pressure-surface-card.md new file mode 100644 index 0000000..365943c --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/qwen30-fixed-pressure-surface-card.md @@ -0,0 +1,49 @@ +# EXP-SIMFID-Q30-FIXED-PRESSURE-SURFACE: high-pressure Fixed-PD / Fixed-PO + +> Status: approved and launching (2026-07-19). + +## Question + +When uniform Fixed-PD is moved from the near-singleton workload to +`4096 -> 256` at `1.125` request/s/GPU, does Frontier still choose a different +latency-optimal configuration from real community vLLM 0.20? Does removing +decode (`4096 -> 1`) change that selection boundary? + +## Setup and decision rule + +- Qwen3-30B-A3B BF16 on dash0 H20; community vLLM 0.20.0. +- Fixed-PD `4096 -> 256` and Fixed-PO `4096 -> 1`; 257 uniform, + prefix-disjoint requests; prefix caching off. +- Rate is frozen at 1.125 request/s/GPU, so global rates are 1.125, 2.25, and + 4.5 request/s for TP1, TP2, and TP4. +- Surface: `TP in {1,2,4}` x `MNS in {8,16,32,64}`, MBT=8192. Real + measurements use three fresh-server trials with rotated order. +- Frontier is `deadc4a`, `piecewise`, the frozen CUDA-event profile plus the + graph-compatible KERNEL_ONLY profile and measured collectives used by the + prior Trace-PD comparison. No latency calibration is fitted to this case. +- Compare exact winners, selected-config real regret, and pairwise order for + mean/p90 TTFT, E2E, and TPOT where decode exists. SLO is not scored. + +The pressure-probe measurements selected the workload but are excluded from +the real evaluation pool. A failed or incomplete Frontier cell is a coverage +failure, not a high-latency observation. + +## Expected output and decision + +The final table has one row per case/objective with Frontier winner, real +winner, regret, and pairwise agreement. If the old Fixed-PD reversal persists, +the failure is not an artifact of singleton load. If it disappears, simulator +fidelity has a load-regime boundary that must be localized. Fixed-PO isolates +whether decode-state composition is necessary for either outcome. + +The previously reviewed matrix schematic remains the figure prototype: +`../simulator-tuning-latency-matrix-v0/latency-selection-matrix-schematic.svg`. + +## Provenance + +Remote output root: +`dash0:/home/admin/cpfs/wjh/aituner/qwen30-fixed-pressure-surface-20260719-r1`. +The campaign records source/profile/model hashes, runtime versions, GPU +inventory, per-cell commands, raw request records, and an artifact checksum +manifest. + diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pressure_campaign.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pressure_campaign.sh new file mode 100644 index 0000000..2f0ade6 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_fixed_pressure_campaign.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash + +# Frozen Qwen30 Fixed-PD/Fixed-PO Frontier-versus-real campaign. The pressure +# calibration artifacts are deliberately not reused as evaluation results. + +set -euo pipefail + +CAMPAIGN_ROOT="${CAMPAIGN_ROOT:?CAMPAIGN_ROOT is required}" +RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" +FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-t1-dash0-deadc4a}" +REPLAYSERVE_ROOT="${REPLAYSERVE_ROOT:-/home/admin/cpfs/wjh/replayserve}" +PYTHON_DEPS="${PYTHON_DEPS:-${VENV_ROOT}/lib/python3.12/site-packages}" +PROFILE_ROOT="${PROFILE_ROOT:-/home/admin/cpfs/wjh/aituner/aituner-graph-piecewise-bdc357d/runs/frontier-fidelity-envelope-v1/profiles/profile-v4-trace-final}" +KERNEL_PROFILE_ROOT="${KERNEL_PROFILE_ROOT:-/home/admin/cpfs/wjh/aituner/graph-piecewise-qwen30-20260717/full/frozen-kernel-only}" +ALLREDUCE_CSV="${ALLREDUCE_CSV:-/home/admin/cpfs/wjh/aituner/aituner-graph-piecewise-bdc357d/runs/frontier-fidelity-envelope-v1/profiles/measured-allreduce.csv}" +FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:?FLASHINFER_WORKSPACE_BASE is required}" +REQUESTS="${REQUESTS:-257}" +PER_GPU_RATE="${PER_GPU_RATE:-1.125}" + +[[ "${REQUESTS}" == "257" ]] || { echo 'ERROR: frozen campaign requires REQUESTS=257' >&2; exit 1; } +[[ "${PER_GPU_RATE}" == "1.125" ]] || { echo 'ERROR: frozen campaign requires PER_GPU_RATE=1.125' >&2; exit 1; } + +MATERIALIZER="${RUNNER_DIR}/prepare_qwen30_latency_case.py" +REAL_RUNNER="${RUNNER_DIR}/run_qwen30_latency_case_real_surface.sh" +SIM_RUNNER="${RUNNER_DIR}/run_frontier_qwen30_exact_trace_surface.py" +REAL_AUDITOR="${RUNNER_DIR}/audit_qwen30_latency_case.py" +COMPARATOR="${RUNNER_DIR}/analyze_qwen30_latency_case.py" + +mkdir -p "${CAMPAIGN_ROOT}/provenance" "${CAMPAIGN_ROOT}/traces" \ + "${CAMPAIGN_ROOT}/real" "${CAMPAIGN_ROOT}/sim" "${CAMPAIGN_ROOT}/analysis" +exec > >(tee -a "${CAMPAIGN_ROOT}/controller.log") 2>&1 + +printf '%s\n' "Q30_FIXED_PRESSURE_CAMPAIGN_LAUNCH_ECHO host=dash0 model=Qwen3-30B-A3B engine=vLLM-0.20.0+cu129 dtype=BF16 cases={fixed-pd:4096_to_256,fixed-po:4096_to_1} prefix=false requests=${REQUESTS} per_gpu_rate=${PER_GPU_RATE} global_rates={TP1:1.125,TP2:2.25,TP4:4.5} surface_per_case=TP{1,2,4}xMNS{8,16,32,64} real_trials=3 fresh_server=true simulator=Frontier-deadc4a_piecewise_graph-kernel-only metrics=mean,p90(TTFT,TPOT-if-PD,E2E) SLO=not_scored flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} expected_wall=2-5h expected_cost=12-24_H20-GPUh output=${CAMPAIGN_ROOT}" +date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ + +sha256sum "${BASH_SOURCE[0]}" "${MATERIALIZER}" "${REAL_RUNNER}" \ + "${SIM_RUNNER}" "${REAL_AUDITOR}" "${COMPARATOR}" \ + "${MODEL_ROOT}/config.json" "${PROFILE_ROOT}/manifest.json" \ + "${KERNEL_PROFILE_ROOT}/manifest.json" "${ALLREDUCE_CSV}" \ + > "${CAMPAIGN_ROOT}/provenance/input.sha256" +git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${CAMPAIGN_ROOT}/provenance/frontier.commit" +git -C "${RUNNER_DIR}" rev-parse HEAD > "${CAMPAIGN_ROOT}/provenance/aituner.commit" +"${VENV_ROOT}/bin/vllm" --version > "${CAMPAIGN_ROOT}/provenance/vllm.version" +nvidia-smi --query-gpu=index,name,uuid,driver_version,memory.total --format=csv,noheader \ + > "${CAMPAIGN_ROOT}/provenance/gpus.before.csv" + +prepare_case() { + local case_name="$1" output_tokens="$2" + local tp + for tp in 1 2 4; do + "${VENV_ROOT}/bin/python" "${MATERIALIZER}" fixed \ + --model "${MODEL_ROOT}" --input-tokens 4096 --output-tokens "${output_tokens}" \ + --requests "${REQUESTS}" --per-gpu-rate "${PER_GPU_RATE}" --tp "${tp}" \ + --output-root "${CAMPAIGN_ROOT}/traces/${case_name}/tp${tp}" + done +} + +run_real_case() { + local case_name="$1" base_port="$2" + CASE_NAME="${case_name}" PREFIX_CACHING=false \ + TRACE_ROOT="${CAMPAIGN_ROOT}/traces/${case_name}" \ + OUTPUT_ROOT="${CAMPAIGN_ROOT}/real/${case_name}" \ + RUNNER_DIR="${RUNNER_DIR}" VENV_ROOT="${VENV_ROOT}" MODEL_ROOT="${MODEL_ROOT}" \ + FLASHINFER_SHARED_WORKSPACE="${FLASHINFER_WORKSPACE_BASE}" \ + BASE_PORT="${base_port}" RESUME_VALID_CELLS=true \ + bash "${REAL_RUNNER}" +} + +run_sim_tp() { + local case_name="$1" tp="$2" + local sim_root="${CAMPAIGN_ROOT}/sim/${case_name}" + local -a configs=() + local mns + for mns in 8 16 32 64; do + configs+=(--config "tp${tp}_mns${mns}") + done + /usr/bin/python3 "${SIM_RUNNER}" \ + --frontier-source "${FRONTIER_SOURCE}" --replayserve-root "${REPLAYSERVE_ROOT}" \ + --profile-root "${PROFILE_ROOT}" --kernel-profile-root "${KERNEL_PROFILE_ROOT}" \ + --python-deps "${PYTHON_DEPS}" --output-root "${sim_root}" \ + --trace "tp${tp}=${CAMPAIGN_ROOT}/traces/${case_name}/tp${tp}/public/frontier.csv" \ + "${configs[@]}" --rate-contract uniform-spacing --no-prefix-caching \ + --cc-backend vidur --allreduce-csv "${ALLREDUCE_CSV}" \ + --timeout-seconds 3600 --predictor-training-job-threads 4 \ + --decode-cuda-graph-mode piecewise --align-real-graph-runtime \ + --fresh-predictor-cache --resume --continue-on-failure +} + +run_sim_case() { + local case_name="$1" failed=0 pid + local -a pids=() + mkdir -p "${CAMPAIGN_ROOT}/sim/${case_name}" + for tp in 1 2 4; do + run_sim_tp "${case_name}" "${tp}" \ + > "${CAMPAIGN_ROOT}/sim/${case_name}/launcher-tp${tp}.log" 2>&1 & + pids+=("$!") + done + for pid in "${pids[@]}"; do + wait "${pid}" || failed=1 + done + [[ "${failed}" -eq 0 ]] || return 1 +} + +analyze_case() { + local case_name="$1" + local analysis_root="${CAMPAIGN_ROOT}/analysis/${case_name}" + mkdir -p "${analysis_root}" + "${VENV_ROOT}/bin/python" "${REAL_AUDITOR}" \ + --case-root "${CAMPAIGN_ROOT}/real/${case_name}" \ + --traces-root "${CAMPAIGN_ROOT}/traces/${case_name}" \ + --json-output "${analysis_root}/real-audit.json" \ + --markdown-output "${analysis_root}/real-audit.md" + "${VENV_ROOT}/bin/python" "${COMPARATOR}" \ + --sim-root "${CAMPAIGN_ROOT}/sim/${case_name}" \ + --real-audit "${analysis_root}/real-audit.json" \ + --json-output "${analysis_root}/comparison.json" \ + --markdown-output "${analysis_root}/comparison.md" +} + +prepare_case fixed-pd 256 +prepare_case fixed-po 1 + +run_real_case fixed-pd 9000 +run_real_case fixed-po 9100 + +run_sim_case fixed-pd +analyze_case fixed-pd +run_sim_case fixed-po +analyze_case fixed-po + +nvidia-smi --query-gpu=index,name,uuid,driver_version,memory.total --format=csv,noheader \ + > "${CAMPAIGN_ROOT}/provenance/gpus.after.csv" +find "${CAMPAIGN_ROOT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 \ + | sort -z | xargs -0 sha256sum > "${CAMPAIGN_ROOT}/provenance/artifacts.sha256" +date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ +printf '%s\n' 'Q30_FIXED_PRESSURE_CAMPAIGN_COMPLETE' diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh index 7de86fc..e6a68ff 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_latency_case_real_surface.sh @@ -21,6 +21,14 @@ SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-300}" IDLE_GPU_MEMORY_TOLERANCE_MIB="${IDLE_GPU_MEMORY_TOLERANCE_MIB:-16}" RESUME_VALID_CELLS="${RESUME_VALID_CELLS:-false}" PORT="${BASE_PORT:-8300}" +REQUEST_COUNT="$(wc -l < "${TRACE_ROOT}/tp1/private/real_requests.jsonl")" + +for tp in 2 4; do + [[ "$(wc -l < "${TRACE_ROOT}/tp${tp}/private/real_requests.jsonl")" == "${REQUEST_COUNT}" ]] || { + echo "ERROR: TP-specific request counts differ" >&2 + exit 1 + } +done case "${PREFIX_CACHING}" in true|false) ;; @@ -186,7 +194,7 @@ run_tp_wave() { } { - printf '%s\n' "REAL_LAUNCH_ECHO host=dash0 model=Qwen3-30B-A3B engine=vLLM-0.20.0+cu129 dtype=BF16 case=${CASE_NAME} traces=${TRACE_ROOT}/tp{1,2,4}/private/real_requests.jsonl prefix=${PREFIX_CACHING} requests=129 transform=t_prime=t/TP surface=TP{1,2,4}xMNS{8,16,32,64} trials=3 fresh_server=true resume_valid_cells=${RESUME_VALID_CELLS} idle_gpu_memory_tolerance_mib=${IDLE_GPU_MEMORY_TOLERANCE_MIB} metrics=mean,p90(TTFT,TPOT-if-OSL-gt-1,E2E) expected_cost=13_H20-GPUh_nominal__41_H20-GPUh_max output=${OUT}/real" + printf '%s\n' "REAL_LAUNCH_ECHO host=dash0 model=Qwen3-30B-A3B engine=vLLM-0.20.0+cu129 dtype=BF16 case=${CASE_NAME} traces=${TRACE_ROOT}/tp{1,2,4}/private/real_requests.jsonl prefix=${PREFIX_CACHING} requests=${REQUEST_COUNT} transform=t_prime=t/TP surface=TP{1,2,4}xMNS{8,16,32,64} trials=3 fresh_server=true resume_valid_cells=${RESUME_VALID_CELLS} idle_gpu_memory_tolerance_mib=${IDLE_GPU_MEMORY_TOLERANCE_MIB} metrics=mean,p90(TTFT,TPOT-if-OSL-gt-1,E2E) expected_cost=13_H20-GPUh_nominal__41_H20-GPUh_max output=${OUT}/real" date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ mkdir -p "${OUT}/provenance" "${FLASHINFER_SHARED_WORKSPACE}" sha256sum "${BASH_SOURCE[0]}" "${RUNNER}" "${CLIENT}" "${PREFILL_CLIENT}" \ From f4813cf5371cbcd8bc70d4aecc5131cb535248b4 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sun, 19 Jul 2026 02:39:16 +0800 Subject: [PATCH 139/148] Add Qwen235 vLLM 0.20 real surface runner --- .../run_qwen235_v020_real_surface.sh | 119 ++++++++++++++++++ .../run_qwen30_exact_trace_real_anchor.sh | 29 ++++- 2 files changed, 144 insertions(+), 4 deletions(-) create mode 100644 runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh new file mode 100644 index 0000000..5d04bb2 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash + +# Three fresh-server trials for the four-cell Qwen235 vLLM 0.20 surface. + +set -euo pipefail + +OUT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}" +TRACE_ROOT="${TRACE_ROOT:?TRACE_ROOT is required}" +CASE_NAME="${CASE_NAME:?CASE_NAME is required}" +PREFIX_CACHING="${PREFIX_CACHING:?PREFIX_CACHING is required}" +RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +RUNNER="${RUNNER_DIR}/run_qwen30_exact_trace_real_anchor.sh" +CLIENT="${RUNNER_DIR}/qwen30_exact_trace_client.py" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}" +SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-600}" +IDLE_GPU_MEMORY_TOLERANCE_MIB="${IDLE_GPU_MEMORY_TOLERANCE_MIB:-16}" +RESUME_VALID_CELLS="${RESUME_VALID_CELLS:-true}" +PORT="${BASE_PORT:-9300}" +REQUEST_COUNT="$(wc -l < "${TRACE_ROOT}/tp4/private/real_requests.jsonl")" + +[[ "$(wc -l < "${TRACE_ROOT}/tp8/private/real_requests.jsonl")" == "${REQUEST_COUNT}" ]] || { + echo 'ERROR: TP-specific request counts differ' >&2 + exit 1 +} +case "${CASE_NAME}" in fixed-pd|fixed-po|trace-pd|trace-po) ;; *) exit 2 ;; esac +case "${PREFIX_CACHING}" in true|false) ;; *) exit 2 ;; esac + +mkdir -p "${OUT}/provenance" +declare -a WAVE_PIDS=() + +has_valid_result() { + local result="$1" + [[ -s "${result}" ]] || return 1 + "${VENV_ROOT}/bin/python" - "${result}" "${REQUEST_COUNT}" <<'PY' +import json, sys +s = json.load(open(sys.argv[1])).get("summary", {}) +raise SystemExit(0 if s.get("completed") == int(sys.argv[2]) and s.get("failed") == 0 else 1) +PY +} + +preflight_gpus() { + nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu --format=csv,noheader \ + | tee -a "${OUT}/controller.log" + nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits \ + | awk -v tolerance="${IDLE_GPU_MEMORY_TOLERANCE_MIB}" '$1 > tolerance {exit 1}' +} + +assert_no_server() { + ! pgrep -fa 'qwen3-235b-v020-eval' > "${OUT}/provenance/unexpected_server_processes.txt" +} + +wait_for_wave() { + local failed=0 pid + for pid in "${WAVE_PIDS[@]}"; do wait "${pid}" || failed=1; done + WAVE_PIDS=() + [[ "${failed}" -eq 0 ]] +} + +launch_config() { + local trial="$1" tp="$2" mns="$3" gpus="$4" ep=false + [[ "${tp}" == 8 ]] && ep=true + local config="tp${tp}_ep${ep}_mns${mns}" run_out="${OUT}/real/${config}/trial${trial}" + local requests="${TRACE_ROOT}/tp${tp}/private/real_requests.jsonl" port="${PORT}" + PORT=$((PORT + 1)) + if [[ "${RESUME_VALID_CELLS}" == true ]] && has_valid_result "${run_out}/results/result.json"; then + printf 'CONFIG_REUSED_VALID case=%s trial=%s config=%s\n' "${CASE_NAME}" "${trial}" "${config}" + return 0 + fi + mkdir -p "${run_out}" + ( + cd "${RUNNER_DIR}" + set +e + env HOME=/tmp/wjh XDG_CACHE_HOME=/tmp/wjh/.cache \ + VLLM_CACHE_ROOT=/tmp/wjh/.cache/vllm CUDA_VISIBLE_DEVICES="${gpus}" \ + TP="${tp}" MNS="${mns}" TRACE_LABEL="${CASE_NAME}/tp${tp}-normalized" \ + PREFIX_CACHING="${PREFIX_CACHING}" ENABLE_EXPERT_PARALLEL="${ep}" \ + MODEL_QUANTIZATION=fp8 DISABLE_CUSTOM_ALL_REDUCE=true GPU_MEMORY_UTILIZATION=0.80 \ + SERVED_MODEL=qwen3-235b-v020-eval SERVER_PORT="${port}" OUTPUT_ROOT="${run_out}" \ + REQUESTS_FILE="${requests}" VENV_ROOT="${VENV_ROOT}" MODEL_ROOT="${MODEL_ROOT}" \ + SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS}" CLIENT_TIMEOUT_SECONDS=6000 \ + EXACT_TRACE_CLIENT="${CLIENT}" \ + timeout --signal=TERM --kill-after=60s 7800 bash "${RUNNER}" \ + > "${run_out}/launcher.stdout.log" 2> "${run_out}/launcher.stderr.log" + status="$?" + printf '%s\n' "${status}" > "${run_out}/launcher.exit_code" + exit "${status}" + ) & + WAVE_PIDS+=("$!") +} + +run_trial() { + local trial="$1" order="$2" + IFS=',' read -r -a mnss <<< "${order}" + preflight_gpus && assert_no_server + printf 'WAVE_START case=%s trial=%s tp=4 mns=%s,%s\n' \ + "${CASE_NAME}" "${trial}" "${mnss[0]}" "${mnss[1]}" | tee -a "${OUT}/controller.log" + launch_config "${trial}" 4 "${mnss[0]}" '0,1,2,3' + launch_config "${trial}" 4 "${mnss[1]}" '4,5,6,7' + wait_for_wave && preflight_gpus && assert_no_server + for mns in "${mnss[@]}"; do + printf 'WAVE_START case=%s trial=%s tp=8 mns=%s\n' \ + "${CASE_NAME}" "${trial}" "${mns}" | tee -a "${OUT}/controller.log" + launch_config "${trial}" 8 "${mns}" '0,1,2,3,4,5,6,7' + wait_for_wave && preflight_gpus && assert_no_server + done +} + +{ + printf '%s\n' "Q235_REAL_LAUNCH_ECHO host=dash0 model=Qwen3-235B-A22B-FP8 engine=vLLM-0.20.0+cu129 checkpoint=FP8 cases=${CASE_NAME} trace=${TRACE_ROOT}/tp{4,8} requests=${REQUEST_COUNT} transform=t_prime=t/TP prefix=${PREFIX_CACHING} surface={TP4/EP1,TP8/EP8}xMNS{64,128} MBT=8192 trials=3 fresh_server=true metrics=mean,p90(TTFT,TPOT-if-PD,E2E) SLO=not_scored output=${OUT}/real" + date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ + sha256sum "${BASH_SOURCE[0]}" "${RUNNER}" "${CLIENT}" "${MODEL_ROOT}/config.json" \ + > "${OUT}/provenance/real-input.sha256" + run_trial 1 '64,128' + run_trial 2 '128,64' + run_trial 3 '64,128' + date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ + printf '%s\n' 'Q235_REAL_SURFACE_COMPLETE' +} 2>&1 | tee -a "${OUT}/controller.log" diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh index 8d64119..3ef6c21 100755 --- a/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh @@ -9,11 +9,16 @@ MNS="${MNS:?MNS is required}" TRACE_LABEL="${TRACE_LABEL:?TRACE_LABEL is required}" SERVER_PORT="${SERVER_PORT:?SERVER_PORT is required}" PREFIX_CACHING="${PREFIX_CACHING:-true}" +ENABLE_EXPERT_PARALLEL="${ENABLE_EXPERT_PARALLEL:-false}" +MODEL_QUANTIZATION="${MODEL_QUANTIZATION:-}" +DISABLE_CUSTOM_ALL_REDUCE="${DISABLE_CUSTOM_ALL_REDUCE:-false}" +GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.92}" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" FLASHINFER_WORKSPACE_BASE="${FLASHINFER_WORKSPACE_BASE:-/tmp/wjh/flashinfer-workspace-vllm020-profiler-v1}" SERVER_READY_ATTEMPTS="${SERVER_READY_ATTEMPTS:-120}" -SERVED_MODEL="qwen3-30b-exact-trace" +CLIENT_TIMEOUT_SECONDS="${CLIENT_TIMEOUT_SECONDS:-1800}" +SERVED_MODEL="${SERVED_MODEL:-qwen3-30b-exact-trace}" EXACT_TRACE_CLIENT="${EXACT_TRACE_CLIENT:-qwen30_exact_trace_client.py}" SERVER_PID="" @@ -53,8 +58,23 @@ case "${PREFIX_CACHING}" in ;; esac +case "${ENABLE_EXPERT_PARALLEL}" in + true) EP_FLAG=(--enable-expert-parallel) ;; + false) EP_FLAG=() ;; + *) echo "ERROR: ENABLE_EXPERT_PARALLEL must be true or false" >&2; exit 1 ;; +esac +case "${DISABLE_CUSTOM_ALL_REDUCE}" in + true) CUSTOM_AR_FLAG=(--disable-custom-all-reduce) ;; + false) CUSTOM_AR_FLAG=() ;; + *) echo "ERROR: DISABLE_CUSTOM_ALL_REDUCE must be true or false" >&2; exit 1 ;; +esac +QUANT_FLAG=() +if [[ -n "${MODEL_QUANTIZATION}" ]]; then + QUANT_FLAG=(--quantization "${MODEL_QUANTIZATION}") +fi + REQUEST_COUNT="$(wc -l < "${REQUESTS_FILE}")" -echo "QWEN30_EXACT_TRACE_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 config=TP${TP}_MNS${MNS}_MBT8192 trace=${TRACE_LABEL} requests=${REQUEST_COUNT} source=${REQUESTS_FILE} arrivals=manifest prefix=${PREFIX_CACHING} block=16 metrics=TTFT,TPOT-if-OSL-gt-1,E2E flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} output=${OUTPUT_ROOT} ready_budget_s=$((SERVER_READY_ATTEMPTS * 3)) hard_wall=3600s" +echo "EXACT_TRACE_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 quantization=${MODEL_QUANTIZATION:-none} config=TP${TP}_EP${ENABLE_EXPERT_PARALLEL}_MNS${MNS}_MBT8192 trace=${TRACE_LABEL} requests=${REQUEST_COUNT} source=${REQUESTS_FILE} arrivals=manifest prefix=${PREFIX_CACHING} block=16 metrics=TTFT,TPOT-if-OSL-gt-1,E2E flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} output=${OUTPUT_ROOT} ready_budget_s=$((SERVER_READY_ATTEMPTS * 3)) client_timeout_s=${CLIENT_TIMEOUT_SECONDS}" date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" sha256sum qwen30_exact_trace_client.py run_qwen30_exact_trace_real_anchor.sh \ ../frontier-phase-factorial-v0/qwen30_prefill_client.py \ @@ -74,9 +94,10 @@ ulimit -n 65536 setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \ --host 127.0.0.1 --port "${SERVER_PORT}" --served-model-name "${SERVED_MODEL}" \ - --tensor-parallel-size "${TP}" --gpu-memory-utilization 0.92 \ + --tensor-parallel-size "${TP}" --gpu-memory-utilization "${GPU_MEMORY_UTILIZATION}" \ --max-model-len 40960 --max-num-batched-tokens 8192 --max-num-seqs "${MNS}" \ "${PREFIX_CACHING_FLAG}" --enable-chunked-prefill --no-enable-log-requests \ + "${CUSTOM_AR_FLAG[@]}" "${QUANT_FLAG[@]}" "${EP_FLAG[@]}" \ > "${OUTPUT_ROOT}/logs/server.log" 2>&1 & SERVER_PID=$! READY=0 @@ -110,7 +131,7 @@ fi --port "${SERVER_PORT}" --requests-file "${REQUESTS_FILE}" \ --served-model "${SERVED_MODEL}" \ --output "${OUTPUT_ROOT}/results/result.json" --tpot-slo-ms 150 \ - --timeout-seconds 1800 + --timeout-seconds "${CLIENT_TIMEOUT_SECONDS}" cleanup find "${OUTPUT_ROOT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 \ From a3c8cb5808bbba58b97b35f01fb55237a33b21f9 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sun, 19 Jul 2026 02:42:28 +0800 Subject: [PATCH 140/148] Add Qwen235 Frontier MoE portability smoke --- .../smoke_qwen235_v020_frontier_moe.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 runs/frontier-fidelity-envelope-v1/smoke_qwen235_v020_frontier_moe.py diff --git a/runs/frontier-fidelity-envelope-v1/smoke_qwen235_v020_frontier_moe.py b/runs/frontier-fidelity-envelope-v1/smoke_qwen235_v020_frontier_moe.py new file mode 100644 index 0000000..5157422 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/smoke_qwen235_v020_frontier_moe.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""One-cell gate for the two Qwen235 vLLM 0.20 MoE runtime backends.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +import torch + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--frontier-source", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def routing(tokens: int, experts: int = 128, topk: int = 8): + ids = torch.arange(tokens * topk, device="cuda", dtype=torch.int64) + ids = (ids % experts).view(tokens, topk) + weights = torch.full((tokens, topk), 1.0 / topk, device="cuda") + return weights, ids + + +def main() -> None: + args = parse_args() + sys.path.insert(0, str(args.frontier_source.resolve())) + from frontier.profiling.moe.moe_vllm_kernel import profile_fused_moe_kernel + + weights, ids = routing(8) + cells = [] + cells.append( + { + "name": "tp4_ep1_triton", + "stats": profile_fused_moe_kernel( + num_tokens=8, + num_experts=128, + hidden_dim=4096, + expert_hidden_dim=1536, + top_k=8, + topk_weights=weights, + topk_ids=ids, + tensor_parallel_size=4, + use_fp8=True, + block_shape=[128, 128], + warmup_steps=1, + active_steps=2, + ), + } + ) + expert_map = torch.full((128,), -1, device="cuda", dtype=torch.int32) + expert_map[:16] = torch.arange(16, device="cuda", dtype=torch.int32) + cells.append( + { + "name": "tp1_ep8_flashinfer_cutlass", + "stats": profile_fused_moe_kernel( + num_tokens=8, + num_experts=16, + hidden_dim=4096, + expert_hidden_dim=1536, + top_k=8, + topk_weights=weights, + topk_ids=ids, + tensor_parallel_size=1, + use_fp8=True, + block_shape=[128, 128], + warmup_steps=1, + active_steps=2, + global_num_experts=128, + expert_map=expert_map, + ), + } + ) + payload = {"schema": "qwen235-v020-frontier-moe-smoke-v1", "cells": cells} + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + print(json.dumps(payload, sort_keys=True)) + + +if __name__ == "__main__": + main() From e631f6a269b83adc0acf817587ebbc4b5a49e135 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sun, 19 Jul 2026 02:44:55 +0800 Subject: [PATCH 141/148] Add Qwen235 Frontier latency surface runner --- .../run_frontier_qwen235_v020_surface.py | 233 ++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py diff --git a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py new file mode 100644 index 0000000..6f7be30 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Run Frontier on the four-cell Qwen235 vLLM 0.20 latency surface.""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import subprocess +import sys +import time +from dataclasses import asdict, dataclass +from pathlib import Path + + +HERE = Path(__file__).resolve().parent +Q30_RUNNER = HERE / "run_frontier_qwen30_exact_trace_surface.py" + + +def load_q30(): + spec = importlib.util.spec_from_file_location("q30_exact_surface", Q30_RUNNER) + if spec is None or spec.loader is None: + raise ImportError(Q30_RUNNER) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +Q30 = load_q30() +BASE = Q30.BASE +MODEL = "Qwen3-235B-A22B" + + +@dataclass(frozen=True) +class Config: + tp: int + mns: int + moe_tp: int + moe_ep: int + + @property + def name(self) -> str: + return f"tp{self.tp}_ep{self.moe_ep}_mns{self.mns}" + + +GRID = tuple( + Config(tp, mns, 4 if tp == 4 else 1, 1 if tp == 4 else 8) + for tp in (4, 8) + for mns in (64, 128) +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--frontier-source", type=Path, required=True) + parser.add_argument("--replayserve-root", type=Path, required=True) + parser.add_argument("--profile-root", type=Path, required=True) + parser.add_argument("--python-deps", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument("--runtime-contract", type=Path, required=True) + parser.add_argument("--trace", action="append", required=True) + parser.add_argument("--config", action="append") + parser.add_argument("--prefix-caching", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--allreduce-csv", type=Path, required=True) + parser.add_argument("--timeout-seconds", type=float, default=7200) + parser.add_argument("--resume", action="store_true") + return parser.parse_args() + + +def profile_paths(root: Path) -> dict[str, Path]: + paths = { + "linear": root / "linear_op.csv", + "attention": root / "attention.csv", + "moe": root / "moe.csv", + "linear_kernel": root / "linear_op_kernel_only.csv", + "attention_kernel": root / "attention_kernel_only.csv", + "moe_kernel": root / "moe_kernel_only.csv", + "manifest": root / "manifest.json", + } + missing = [str(path) for path in paths.values() if not path.is_file()] + if missing: + raise FileNotFoundError(missing) + manifest = json.loads(paths["manifest"].read_text()) + for name, path in paths.items(): + if name == "manifest": + continue + expected = manifest.get("outputs", {}).get(path.name) + if expected != BASE.sha256(path): + raise ValueError(f"profile hash mismatch: {path}") + return paths + + +def knobs(config: Config, paths: dict[str, Path], contract: dict, cache: Path, prefix: bool): + resolved = contract[config.name] + return { + "simulation_mode": "online", + "sys_arch": "co-location", + "num_replicas": 1, + "cluster_scheduler": "sticky_round_robin", + "model_name": MODEL, + "device": "h20", + "network_device": "h20_dgx", + "attn_tensor_parallel_size": config.tp, + "attn_data_parallel_size": 1, + "moe_tensor_parallel_size": config.moe_tp, + "moe_expert_parallel_size": config.moe_ep, + "num_pipeline_stages": 1, + "replica_scheduler": "vllm_v1", + "decode_cuda_graph_mode": "piecewise", + "batch_size_cap": config.mns, + "max_tokens_in_batch": 8192, + "long_prefill_token_threshold": 0, + "block_size": 16, + "num_blocks_mode": "explicit", + "num_blocks": int(resolved["num_gpu_blocks"]), + "gpu_memory_utilization": 0.80, + "non_kv_cache_overhead_bytes": 0, + "trace_max_tokens": 40960, + "cache_dir": str(cache / config.name), + "enable_prefix_caching": prefix, + "enable_dummy_mode": False, + "linear_op_input_file": str(paths["linear"]), + "atten_input_file": str(paths["attention"]), + "moe_input_file": str(paths["moe"]), + "linear_op_kernel_only_input_file": str(paths["linear_kernel"]), + "atten_kernel_only_input_file": str(paths["attention_kernel"]), + "moe_kernel_only_input_file": str(paths["moe_kernel"]), + "prediction_max_prefill_chunk_size": 8192, + "prediction_max_tokens_per_request": 40960, + "prediction_max_batch_size": max(int(v) for v in resolved["capture_sizes"]), + "no_cache": True, + } + + +def main() -> None: + args = parse_args() + for name in ("frontier_source", "replayserve_root", "profile_root", "python_deps", "output_root", "runtime_contract", "allreduce_csv"): + setattr(args, name, getattr(args, name).resolve()) + paths = profile_paths(args.profile_root) + contract = json.loads(args.runtime_contract.read_text())["configs"] + traces = [ + Q30.parse_trace(spec, rate_contract="trace-window", prefix_caching=args.prefix_caching) + for spec in args.trace + ] + selected = list(GRID) + if args.config: + wanted = set(args.config) + selected = [config for config in GRID if config.name in wanted] + if {config.name for config in selected} != wanted: + raise ValueError(f"unknown configs: {wanted - {config.name for config in selected}}") + + builder = BASE.load_module( + "qwen235_frontier_builder", args.replayserve_root / "tools/run_frontier_sweep.py" + ) + environment = os.environ.copy() + environment.update( + { + "PYTHONPATH": ":".join([str(args.python_deps), str(args.frontier_source)]), + "CUDA_VISIBLE_DEVICES": "", + "NVIDIA_VISIBLE_DEVICES": "void", + "WANDB_DISABLED": "true", + "VIDUR_DISABLE_WANDB": "1", + "FRONTIER_LOG_LEVEL": "WARNING", + "PYTHONDONTWRITEBYTECODE": "1", + } + ) + results = [] + for config in selected: + config_knobs = knobs(config, paths, contract, args.output_root / "cache", args.prefix_caching) + for trace in traces: + run_dir = args.output_root / "runs" / config.name / trace["label"] + result_path = run_dir / "result.json" + if args.resume and result_path.is_file(): + results.append(json.loads(result_path.read_text())) + continue + run_dir.mkdir(parents=True, exist_ok=True) + command = builder.build_frontier_command( + python_bin="/usr/bin/python3", + trace_file=trace["path"], + metrics_root=run_dir / "metrics", + run_id=f"qwen235_v020_{config.name}_{trace['label']}", + knobs=config_knobs, + ) + command.extend(["--cudagraph_capture_sizes", *(str(v) for v in contract[config.name]["capture_sizes"])]) + command = BASE.configure_cc_command( + command, + backend="vidur", + allreduce_csv=args.allreduce_csv, + cache=args.output_root / "cc-cache", + ) + BASE.write_json(run_dir / "command.json", command) + started = time.time() + with (run_dir / "stdout.log").open("w") as stdout, (run_dir / "stderr.log").open("w") as stderr: + completed = subprocess.run( + command, + cwd=args.frontier_source, + env=environment, + stdout=stdout, + stderr=stderr, + timeout=args.timeout_seconds, + check=False, + ) + result = { + "config": asdict(config) | {"name": config.name}, + "trace": Q30.trace_manifest_entry(trace), + "elapsed_seconds": time.time() - started, + "returncode": completed.returncode, + } + if completed.returncode == 0: + metrics = Q30.find_request_metrics(run_dir) + result.update(status="completed", metrics=Q30.score(metrics, trace["shapes"]), request_metrics_sha256=BASE.sha256(metrics)) + else: + stderr_text = (run_dir / "stderr.log").read_text(errors="replace") + result.update(status="failed", failure_class=Q30.classify_frontier_failure(stderr_text)) + BASE.write_json(result_path, result) + results.append(result) + print(json.dumps({"config": config.name, "trace": trace["label"], "status": result["status"]}, sort_keys=True), flush=True) + + manifest = { + "schema": "qwen235-v020-frontier-latency-surface-v1", + "frontier_commit": subprocess.check_output(["git", "-C", str(args.frontier_source), "rev-parse", "HEAD"], text=True).strip(), + "profiles": {name: BASE.sha256(path) for name, path in paths.items()}, + "runtime_contract_sha256": BASE.sha256(args.runtime_contract), + "prefix_caching": args.prefix_caching, + "results": results, + } + BASE.write_json(args.output_root / "frontier_surface.json", manifest) + + +if __name__ == "__main__": + main() From 40bac6dbf4c98f7288678e9d3e8c69739c296e6f Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sun, 19 Jul 2026 02:46:58 +0800 Subject: [PATCH 142/148] Add Qwen235 same-stack Frontier profiling campaign --- .../assemble_qwen235_v020_profiles.py | 93 +++++++++++++++++++ .../run_qwen235_v020_profiles.sh | 68 ++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 runs/frontier-fidelity-envelope-v1/assemble_qwen235_v020_profiles.py create mode 100644 runs/frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh diff --git a/runs/frontier-fidelity-envelope-v1/assemble_qwen235_v020_profiles.py b/runs/frontier-fidelity-envelope-v1/assemble_qwen235_v020_profiles.py new file mode 100644 index 0000000..e9fdb96 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/assemble_qwen235_v020_profiles.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Assemble immutable flat profiles for the Qwen235 Frontier runner.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import shutil +from pathlib import Path + + +MODEL = "Qwen3-235B-A22B" + + +def parse_args(): + parser = argparse.ArgumentParser() + for name in ("cuda_common", "cuda_moe_tp4", "cuda_moe_ep8", "kernel_common", "kernel_moe_tp4", "kernel_moe_ep8"): + parser.add_argument(f"--{name.replace('_', '-')}", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + return parser.parse_args() + + +def digest(path: Path) -> str: + value = hashlib.sha256() + value.update(path.read_bytes()) + return value.hexdigest() + + +def model_file(root: Path, name: str) -> Path: + path = root / "compute" / "h20" / MODEL / name + if not path.is_file(): + raise FileNotFoundError(path) + return path + + +def merge_csv(inputs: list[Path], output: Path) -> None: + fields = None + rows = [] + for path in inputs: + with path.open(newline="") as source: + reader = csv.DictReader(source) + if fields is None: + fields = reader.fieldnames + elif reader.fieldnames != fields: + raise ValueError(f"CSV schema mismatch: {path}") + rows.extend(reader) + if not fields or not rows: + raise ValueError("cannot merge empty profile CSV") + with output.open("w", newline="") as target: + writer = csv.DictWriter(target, fieldnames=fields, lineterminator="\n") + writer.writeheader() + writer.writerows(rows) + + +def main() -> None: + args = parse_args() + output = args.output_root.resolve() + if output.exists(): + raise FileExistsError(output) + output.mkdir(parents=True) + sources = { + "linear_op.csv": model_file(args.cuda_common, "linear_op.csv"), + "attention.csv": model_file(args.cuda_common, "attention.csv"), + "linear_op_kernel_only.csv": model_file(args.kernel_common, "linear_op_kernel_only.csv"), + "attention_kernel_only.csv": model_file(args.kernel_common, "attention_kernel_only.csv"), + } + for name, source in sources.items(): + shutil.copy2(source, output / name) + merge_csv( + [model_file(args.cuda_moe_tp4, "moe.csv"), model_file(args.cuda_moe_ep8, "moe.csv")], + output / "moe.csv", + ) + merge_csv( + [model_file(args.kernel_moe_tp4, "moe_kernel_only.csv"), model_file(args.kernel_moe_ep8, "moe_kernel_only.csv")], + output / "moe_kernel_only.csv", + ) + outputs = {path.name: digest(path) for path in sorted(output.glob("*.csv"))} + manifest = { + "schema": "qwen235-v020-frontier-profile-v1", + "model": MODEL, + "measurement_families": ["CUDA_EVENT", "KERNEL_ONLY"], + "moe_runtime_paths": {"tp4_ep1": "TRITON", "tp8_ep8": "FLASHINFER_CUTLASS"}, + "inputs": {name: str(getattr(args, name).resolve()) for name in ("cuda_common", "cuda_moe_tp4", "cuda_moe_ep8", "kernel_common", "kernel_moe_tp4", "kernel_moe_ep8")}, + "outputs": outputs, + } + (output / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + print(json.dumps(manifest, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh new file mode 100644 index 0000000..8e3d830 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +set -euo pipefail + +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}" +FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +MODEL=Qwen3-235B-A22B +TOKENS=(1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128 256 512 1024 2048 4096 8192) +BATCHES=(1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128) +KV=(128 1024 2048 4096 8192 16384 32768 40960) + +mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" +exec > >(tee -a "${OUTPUT_ROOT}/controller.log") 2>&1 + +echo "Q235_PROFILE_LAUNCH_ECHO host=dash0 model=${MODEL} vllm=0.20.0 frontier=$(git -C "${FRONTIER_SOURCE}" rev-parse HEAD) device=H20 methods={CUDA_EVENT,KERNEL_ONLY} attention_tp={4,8} moe_paths={TP4/EP1:Triton,TP1/EP8:FlashInfer-CUTLASS} token_points=${#TOKENS[@]} batch_points=${#BATCHES[@]} kv_points=${#KV[@]} parallel_gpus=6 expected_wall=20-75m expected_cost=2-7_H20-GPUh output=${OUTPUT_ROOT}" +date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ +nvidia-smi --query-gpu=index,memory.used --format=csv,noheader,nounits \ + | awk '$2 > 16 {exit 1}' +git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit" +"${VENV_ROOT}/bin/vllm" --version > "${OUTPUT_ROOT}/provenance/vllm.version" + +common_profile() { + local gpu="$1" method="$2" root="$3" + env CUDA_VISIBLE_DEVICES="${gpu}" PYTHONPATH="${FRONTIER_SOURCE}" \ + "${VENV_ROOT}/bin/python" -m frontier.profiling.linear_op.main \ + --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \ + --num_tensor_parallel_workers 1 4 8 --attn_tp 4 8 --ffn_tp 1 4 \ + --num_tokens_list "${TOKENS[@]}" --profile_method "${method}" --yes + env CUDA_VISIBLE_DEVICES="${gpu}" PYTHONPATH="${FRONTIER_SOURCE}" \ + "${VENV_ROOT}/bin/python" -m frontier.profiling.attention.main \ + --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \ + --num_tensor_parallel_workers 4 8 --max_model_len 40960 --max_seq_len 8192 \ + --batch_size_list "${BATCHES[@]}" --decode_kv_cache_size_list "${KV[@]}" \ + --fixed_chunked_prefill_size 8192 --attention_backend FLASHINFER \ + --profile_method "${method}" --yes +} + +moe_profile() { + local gpu="$1" method="$2" tp="$3" ep="$4" root="$5" + env CUDA_VISIBLE_DEVICES="${gpu}" PYTHONPATH="${FRONTIER_SOURCE}" \ + "${VENV_ROOT}/bin/python" -m frontier.profiling.moe.main \ + --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \ + --num_tensor_parallel_workers "${tp}" --expert_parallel_sizes "${ep}" \ + --num_tokens_list "${TOKENS[@]}" --load_distributions uniform \ + --num_samples_per_distribution 1 --profile_method "${method}" --yes +} + +declare -a pids=() +common_profile 0 cuda_event "${OUTPUT_ROOT}/cuda-common" > "${OUTPUT_ROOT}/logs/cuda-common.log" 2>&1 & pids+=("$!") +common_profile 1 record_function "${OUTPUT_ROOT}/kernel-common" > "${OUTPUT_ROOT}/logs/kernel-common.log" 2>&1 & pids+=("$!") +moe_profile 2 cuda_event 4 1 "${OUTPUT_ROOT}/cuda-moe-tp4" > "${OUTPUT_ROOT}/logs/cuda-moe-tp4.log" 2>&1 & pids+=("$!") +moe_profile 3 cuda_event 1 8 "${OUTPUT_ROOT}/cuda-moe-ep8" > "${OUTPUT_ROOT}/logs/cuda-moe-ep8.log" 2>&1 & pids+=("$!") +moe_profile 4 record_function 4 1 "${OUTPUT_ROOT}/kernel-moe-tp4" > "${OUTPUT_ROOT}/logs/kernel-moe-tp4.log" 2>&1 & pids+=("$!") +moe_profile 5 record_function 1 8 "${OUTPUT_ROOT}/kernel-moe-ep8" > "${OUTPUT_ROOT}/logs/kernel-moe-ep8.log" 2>&1 & pids+=("$!") +failed=0 +for pid in "${pids[@]}"; do wait "${pid}" || failed=1; done +[[ "${failed}" -eq 0 ]] || { tail -80 "${OUTPUT_ROOT}"/logs/*.log; exit 1; } + +"${VENV_ROOT}/bin/python" "${RUNNER_DIR}/assemble_qwen235_v020_profiles.py" \ + --cuda-common "${OUTPUT_ROOT}/cuda-common" \ + --cuda-moe-tp4 "${OUTPUT_ROOT}/cuda-moe-tp4" --cuda-moe-ep8 "${OUTPUT_ROOT}/cuda-moe-ep8" \ + --kernel-common "${OUTPUT_ROOT}/kernel-common" \ + --kernel-moe-tp4 "${OUTPUT_ROOT}/kernel-moe-tp4" --kernel-moe-ep8 "${OUTPUT_ROOT}/kernel-moe-ep8" \ + --output-root "${OUTPUT_ROOT}/frozen" +date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ +echo Q235_V020_PROFILES_COMPLETE From 3e32ea609fc4c553d918b8da758dd9f7ab491986 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sun, 19 Jul 2026 02:48:51 +0800 Subject: [PATCH 143/148] Cover Qwen235 MNS128 graph buckets --- .../run_qwen235_v020_profiles.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh index 8e3d830..20a93e1 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh @@ -7,8 +7,8 @@ FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" MODEL=Qwen3-235B-A22B -TOKENS=(1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128 256 512 1024 2048 4096 8192) -BATCHES=(1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128) +TOKENS=(1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128 136 144 152 160 168 176 184 192 200 208 216 224 232 240 248 256 512 1024 2048 4096 8192) +BATCHES=(1 2 4 8 16 24 32 40 48 56 64 72 80 88 96 104 112 120 128 136 144 152 160 168 176 184 192 200 208 216 224 232 240 248 256) KV=(128 1024 2048 4096 8192 16384 32768 40960) mkdir -p "${OUTPUT_ROOT}/logs" "${OUTPUT_ROOT}/provenance" From 80a067e3a5d04fb26b5e7ce2fec905f46b5c7c2d Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sun, 19 Jul 2026 02:52:54 +0800 Subject: [PATCH 144/148] Orchestrate Qwen235 four-case fidelity campaign --- .../analyze_qwen235_v020_campaign.py | 128 ++++++++++++++++++ .../extract_qwen235_v020_runtime_contract.py | 44 ++++++ .../run_frontier_qwen235_v020_surface.py | 19 ++- .../run_qwen235_v020_campaign.sh | 86 ++++++++++++ .../run_qwen235_v020_real_surface.sh | 6 +- 5 files changed, 274 insertions(+), 9 deletions(-) create mode 100644 runs/frontier-fidelity-envelope-v1/analyze_qwen235_v020_campaign.py create mode 100644 runs/frontier-fidelity-envelope-v1/extract_qwen235_v020_runtime_contract.py create mode 100644 runs/frontier-fidelity-envelope-v1/run_qwen235_v020_campaign.sh diff --git a/runs/frontier-fidelity-envelope-v1/analyze_qwen235_v020_campaign.py b/runs/frontier-fidelity-envelope-v1/analyze_qwen235_v020_campaign.py new file mode 100644 index 0000000..25853e6 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/analyze_qwen235_v020_campaign.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Compare pooled real and Frontier Qwen235 latency-selection surfaces.""" + +from __future__ import annotations + +import argparse +import itertools +import json +import math +from pathlib import Path + + +CONFIGS = ("tp4_ep1_mns64", "tp4_ep1_mns128", "tp8_ep8_mns64", "tp8_ep8_mns128") +CASES = ("fixed-pd", "fixed-po", "trace-pd", "trace-po") + + +def percentile(values: list[float], fraction: float): + if not values: + return None + return sorted(values)[math.ceil(len(values) * fraction) - 1] + + +def summarize(values: list[float]): + if not values: + return {"mean": None, "p90": None} + return {"mean": sum(values) / len(values), "p90": percentile(values, 0.9)} + + +def real_surface(root: Path, case: str): + surface = {} + for config in CONFIGS: + requests = [] + trials = [] + for trial in (1, 2, 3): + path = root / "real" / case / "real" / config / f"trial{trial}" / "results" / "result.json" + payload = json.loads(path.read_text()) + summary = payload["summary"] + if summary["failed"] or summary["completed"] != len(payload["requests"]): + raise ValueError(f"invalid real result: {path}") + if any(not row["success"] for row in payload["requests"]): + raise ValueError(f"failed request: {path}") + requests.extend(payload["requests"]) + trials.append(str(path.resolve())) + ttft = summarize([float(row["ttft_ms"]) for row in requests]) + tpot = summarize([float(row["tpot_ms"]) for row in requests if row["tpot_ms"] is not None]) + e2e = summarize([float(row["e2e_ms"]) for row in requests]) + surface[config] = { + "ttft_mean_ms": ttft["mean"], "ttft_p90_ms": ttft["p90"], + "tpot_mean_ms": tpot["mean"], "tpot_p90_ms": tpot["p90"], + "e2e_mean_ms": e2e["mean"], "e2e_p90_ms": e2e["p90"], + "request_samples": len(requests), "trials": trials, + } + return surface + + +def sim_surface(root: Path, case: str): + payload = json.loads((root / "sim" / case / "frontier_surface.json").read_text()) + surface = {} + for result in payload["results"]: + if result["status"] != "completed": + continue + config = result["config"]["name"] + if config in surface: + raise ValueError(f"duplicate simulator config: {config}") + surface[config] = {key: value for key, value in result["metrics"].items() if key.endswith("_ms")} + if set(surface) != set(CONFIGS): + raise ValueError(f"simulator coverage failure for {case}: {sorted(surface)}") + return surface + + +def compare_metric(real: dict, sim: dict, metric: str): + applicable = [config for config in CONFIGS if real[config][metric] is not None and sim[config][metric] is not None] + real_winner = min(applicable, key=lambda config: (real[config][metric], config)) + sim_winner = min(applicable, key=lambda config: (sim[config][metric], config)) + regret = real[sim_winner][metric] / real[real_winner][metric] - 1 + informative = agreement = 0 + reversals = [] + for left, right in itertools.combinations(applicable, 2): + real_direction = (real[left][metric] > real[right][metric]) - (real[left][metric] < real[right][metric]) + sim_direction = (sim[left][metric] > sim[right][metric]) - (sim[left][metric] < sim[right][metric]) + if real_direction and sim_direction: + informative += 1 + agreement += int(real_direction == sim_direction) + if real_direction != sim_direction: + reversals.append([left, right]) + return { + "real_winner": real_winner, + "sim_winner": sim_winner, + "winner_match": real_winner == sim_winner, + "selected_real_regret": regret, + "informative_pairs": informative, + "pair_direction_agreement": agreement / informative if informative else None, + "reversals": reversals, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--campaign-root", type=Path, required=True) + parser.add_argument("--json-output", type=Path, required=True) + parser.add_argument("--markdown-output", type=Path, required=True) + args = parser.parse_args() + cases = {} + lines = ["# Qwen235 vLLM 0.20 Frontier vs real", "", "| case | metric | Frontier winner | real winner | match | regret | pair agreement |", "|---|---|---|---|---:|---:|---:|"] + for case in CASES: + real = real_surface(args.campaign_root, case) + sim = sim_surface(args.campaign_root, case) + metrics = ["ttft_mean_ms", "ttft_p90_ms", "e2e_mean_ms", "e2e_p90_ms"] + if case.endswith("pd"): + metrics[2:2] = ["tpot_mean_ms", "tpot_p90_ms"] + comparisons = {} + for metric in metrics: + item = compare_metric(real, sim, metric) + comparisons[metric] = item + lines.append( + f"| {case} | {metric} | {item['sim_winner']} | {item['real_winner']} | " + f"{'yes' if item['winner_match'] else 'no'} | {item['selected_real_regret']:.1%} | " + f"{item['pair_direction_agreement']:.1%} |" + ) + cases[case] = {"real": real, "sim": sim, "comparison": comparisons} + payload = {"schema": "qwen235-v020-simulator-real-comparison-v1", "cases": cases} + args.json_output.parent.mkdir(parents=True, exist_ok=True) + args.json_output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + args.markdown_output.write_text("\n".join(lines) + "\n") + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-fidelity-envelope-v1/extract_qwen235_v020_runtime_contract.py b/runs/frontier-fidelity-envelope-v1/extract_qwen235_v020_runtime_contract.py new file mode 100644 index 0000000..1b7b4ee --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/extract_qwen235_v020_runtime_contract.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Extract simulator-visible graph/KV metadata from Qwen235 server starts.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + + +CONFIGS = ("tp4_ep1_mns64", "tp4_ep1_mns128", "tp8_ep8_mns64", "tp8_ep8_mns128") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--case-root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + configs = {} + for config in CONFIGS: + log = args.case_root / "real" / config / "trial1" / "logs" / "server.log" + text = log.read_text(errors="replace") + token_matches = re.findall(r"GPU KV cache size:\s*([0-9,]+) tokens", text) + capture_matches = re.findall(r"cudagraph_capture_sizes': \[([^]]+)\]", text) + if not token_matches or not capture_matches: + raise ValueError(f"missing runtime metadata in {log}") + tokens = int(token_matches[-1].replace(",", "")) + if tokens % 16: + raise ValueError(f"KV token count is not block aligned: {tokens}") + capture = [int(value.strip()) for value in capture_matches[-1].split(",")] + configs[config] = { + "num_gpu_blocks": tokens // 16, + "capture_sizes": capture, + "source_log": str(log.resolve()), + } + payload = {"schema": "qwen235-v020-runtime-contract-v1", "configs": configs} + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + print(json.dumps(payload, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py index 6f7be30..2f25871 100644 --- a/runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py +++ b/runs/frontier-fidelity-envelope-v1/run_frontier_qwen235_v020_surface.py @@ -60,7 +60,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--python-deps", type=Path, required=True) parser.add_argument("--output-root", type=Path, required=True) parser.add_argument("--runtime-contract", type=Path, required=True) - parser.add_argument("--trace", action="append", required=True) + parser.add_argument("--trace-tp", action="append", required=True, help="TP=PATH") parser.add_argument("--config", action="append") parser.add_argument("--prefix-caching", action=argparse.BooleanOptionalAction, default=True) parser.add_argument("--allreduce-csv", type=Path, required=True) @@ -140,10 +140,17 @@ def main() -> None: setattr(args, name, getattr(args, name).resolve()) paths = profile_paths(args.profile_root) contract = json.loads(args.runtime_contract.read_text())["configs"] - traces = [ - Q30.parse_trace(spec, rate_contract="trace-window", prefix_caching=args.prefix_caching) - for spec in args.trace - ] + trace_by_tp = {} + for specification in args.trace_tp: + raw_tp, separator, path = specification.partition("=") + if not separator: + raise ValueError(f"trace-tp must be TP=PATH: {specification}") + tp = int(raw_tp) + trace_by_tp[tp] = Q30.parse_trace( + f"eval={path}", rate_contract="trace-window", prefix_caching=args.prefix_caching + ) + if set(trace_by_tp) != {4, 8}: + raise ValueError("trace-tp must provide exactly TP4 and TP8") selected = list(GRID) if args.config: wanted = set(args.config) @@ -169,7 +176,7 @@ def main() -> None: results = [] for config in selected: config_knobs = knobs(config, paths, contract, args.output_root / "cache", args.prefix_caching) - for trace in traces: + for trace in (trace_by_tp[config.tp],): run_dir = args.output_root / "runs" / config.name / trace["label"] result_path = run_dir / "result.json" if args.resume and result_path.is_file(): diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_campaign.sh b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_campaign.sh new file mode 100644 index 0000000..0663035 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_campaign.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +set -euo pipefail + +CAMPAIGN_ROOT="${CAMPAIGN_ROOT:?CAMPAIGN_ROOT is required}" +PROFILE_ROOT="${PROFILE_ROOT:?PROFILE_ROOT is required}" +RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}" +REPLAYSERVE_ROOT="${REPLAYSERVE_ROOT:-/home/admin/cpfs/wjh/replayserve}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-235B-A22B-FP8}" +PYTHON_DEPS="${PYTHON_DEPS:-${VENV_ROOT}/lib/python3.12/site-packages}" +ALLREDUCE_CSV="${ALLREDUCE_CSV:-${RUNNER_DIR}/profiles/measured-allreduce.csv}" +BASE_PUBLIC=/home/admin/cpfs/wjh/aituner/fidelity-envelope-private/trace-exact-v1/public/u0p01/frontier.csv +BASE_PRIVATE=/home/admin/cpfs/wjh/aituner/fidelity-envelope-private/trace-exact-v1/private/u0p01/real_requests.jsonl +FIXED_PER_GPU_RATE="${FIXED_PER_GPU_RATE:-0.2}" +REQUESTS=129 + +mkdir -p "${CAMPAIGN_ROOT}"/{provenance,traces,real,sim,analysis} +exec > >(tee -a "${CAMPAIGN_ROOT}/controller.log") 2>&1 +echo "Q235_CAMPAIGN_LAUNCH_ECHO host=dash0 model=Qwen3-235B-A22B-FP8 engine=vLLM-0.20.0+cu129 simulator=Frontier-5b953f5-piecewise cases={Fixed-PD:4096x256@0.2rps/gpu,Fixed-PO:4096x1@0.2rps/gpu,Trace-PD:u0p01-original-OSL,Trace-PO:u0p01-OSL1} requests=129 transform=t_prime=t/TP configs={TP4/EP1,TP8/EP8}xMNS{64,128} MBT=8192 trials=3 fresh_server=true metrics=mean,p90(TTFT,TPOT-if-PD,E2E) SLO=not_scored expected_wall=10-30h expected_cost=90-220_H20-GPUh profile=${PROFILE_ROOT} output=${CAMPAIGN_ROOT}" +date -u +START_UTC=%Y-%m-%dT%H:%M:%SZ +sha256sum "${BASH_SOURCE[0]}" "${BASE_PUBLIC}" "${BASE_PRIVATE}" "${MODEL_ROOT}/config.json" \ + "${PROFILE_ROOT}/manifest.json" "${ALLREDUCE_CSV}" > "${CAMPAIGN_ROOT}/provenance/input.sha256" +git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${CAMPAIGN_ROOT}/provenance/frontier.commit" +git -C "${RUNNER_DIR}" rev-parse HEAD > "${CAMPAIGN_ROOT}/provenance/aituner.commit" + +NORMALIZER="${RUNNER_DIR}/../simulator-tuning-latency-matrix-v0/materialize_qwen30_tp_normalized_trace.py" +MATERIALIZER="${RUNNER_DIR}/prepare_qwen30_latency_case.py" +for tp in 4 8; do + "${VENV_ROOT}/bin/python" "${NORMALIZER}" --base-public-csv "${BASE_PUBLIC}" \ + --base-private-jsonl "${BASE_PRIVATE}" --tp "${tp}" \ + --output-root "${CAMPAIGN_ROOT}/traces/trace-pd/tp${tp}" + "${VENV_ROOT}/bin/python" "${MATERIALIZER}" trace \ + --base-public "${CAMPAIGN_ROOT}/traces/trace-pd/tp${tp}/public/frontier.csv" \ + --base-private "${CAMPAIGN_ROOT}/traces/trace-pd/tp${tp}/private/real_requests.jsonl" \ + --output-tokens 1 --tp "${tp}" --output-root "${CAMPAIGN_ROOT}/traces/trace-po/tp${tp}" + for spec in fixed-pd:256 fixed-po:1; do + case_name="${spec%%:*}"; osl="${spec##*:}" + "${VENV_ROOT}/bin/python" "${MATERIALIZER}" fixed --model "${MODEL_ROOT}" \ + --input-tokens 4096 --output-tokens "${osl}" --requests "${REQUESTS}" \ + --per-gpu-rate "${FIXED_PER_GPU_RATE}" --tp "${tp}" \ + --output-root "${CAMPAIGN_ROOT}/traces/${case_name}/tp${tp}" + done +done + +run_real() { + local case_name="$1" prefix="$2" port="$3" + CASE_NAME="${case_name}" PREFIX_CACHING="${prefix}" \ + TRACE_ROOT="${CAMPAIGN_ROOT}/traces/${case_name}" OUTPUT_ROOT="${CAMPAIGN_ROOT}/real/${case_name}" \ + RUNNER_DIR="${RUNNER_DIR}" VENV_ROOT="${VENV_ROOT}" MODEL_ROOT="${MODEL_ROOT}" \ + BASE_PORT="${port}" bash "${RUNNER_DIR}/run_qwen235_v020_real_surface.sh" +} +run_real fixed-pd false 9300 +run_real fixed-po false 9400 +run_real trace-pd true 9500 +run_real trace-po true 9600 + +"${VENV_ROOT}/bin/python" "${RUNNER_DIR}/extract_qwen235_v020_runtime_contract.py" \ + --case-root "${CAMPAIGN_ROOT}/real/fixed-pd" \ + --output "${CAMPAIGN_ROOT}/provenance/runtime-contract.json" + +run_sim() { + local case_name="$1" prefix_flag="$2" + "${VENV_ROOT}/bin/python" "${RUNNER_DIR}/run_frontier_qwen235_v020_surface.py" \ + --frontier-source "${FRONTIER_SOURCE}" --replayserve-root "${REPLAYSERVE_ROOT}" \ + --profile-root "${PROFILE_ROOT}" --python-deps "${PYTHON_DEPS}" \ + --output-root "${CAMPAIGN_ROOT}/sim/${case_name}" \ + --runtime-contract "${CAMPAIGN_ROOT}/provenance/runtime-contract.json" \ + --trace-tp "4=${CAMPAIGN_ROOT}/traces/${case_name}/tp4/public/frontier.csv" \ + --trace-tp "8=${CAMPAIGN_ROOT}/traces/${case_name}/tp8/public/frontier.csv" \ + "${prefix_flag}" --allreduce-csv "${ALLREDUCE_CSV}" --resume +} +run_sim fixed-pd --no-prefix-caching +run_sim fixed-po --no-prefix-caching +run_sim trace-pd --prefix-caching +run_sim trace-po --prefix-caching + +"${VENV_ROOT}/bin/python" "${RUNNER_DIR}/analyze_qwen235_v020_campaign.py" \ + --campaign-root "${CAMPAIGN_ROOT}" \ + --json-output "${CAMPAIGN_ROOT}/analysis/comparison.json" \ + --markdown-output "${CAMPAIGN_ROOT}/analysis/comparison.md" +find "${CAMPAIGN_ROOT}" -type f ! -path '*/provenance/artifacts.sha256' -print0 \ + | sort -z | xargs -0 sha256sum > "${CAMPAIGN_ROOT}/provenance/artifacts.sha256" +date -u +END_UTC=%Y-%m-%dT%H:%M:%SZ +echo Q235_V020_CAMPAIGN_COMPLETE diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh index 5d04bb2..b7fd213 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_real_surface.sh @@ -58,9 +58,9 @@ wait_for_wave() { } launch_config() { - local trial="$1" tp="$2" mns="$3" gpus="$4" ep=false - [[ "${tp}" == 8 ]] && ep=true - local config="tp${tp}_ep${ep}_mns${mns}" run_out="${OUT}/real/${config}/trial${trial}" + local trial="$1" tp="$2" mns="$3" gpus="$4" ep=false ep_size=1 + if [[ "${tp}" == 8 ]]; then ep=true; ep_size=8; fi + local config="tp${tp}_ep${ep_size}_mns${mns}" run_out="${OUT}/real/${config}/trial${trial}" local requests="${TRACE_ROOT}/tp${tp}/private/real_requests.jsonl" port="${PORT}" PORT=$((PORT + 1)) if [[ "${RESUME_VALID_CELLS}" == true ]] && has_valid_result "${run_out}/results/result.json"; then From 79e9870975b662a8f037e1e42c2317e0723010b0 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sun, 19 Jul 2026 02:54:32 +0800 Subject: [PATCH 145/148] Defer Qwen235 campaign until Qwen30 completion --- .../run_q235_after_q30.sh | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 runs/frontier-fidelity-envelope-v1/run_q235_after_q30.sh diff --git a/runs/frontier-fidelity-envelope-v1/run_q235_after_q30.sh b/runs/frontier-fidelity-envelope-v1/run_q235_after_q30.sh new file mode 100644 index 0000000..a3e6a05 --- /dev/null +++ b/runs/frontier-fidelity-envelope-v1/run_q235_after_q30.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -euo pipefail + +Q30_ROOT="${Q30_ROOT:?Q30_ROOT is required}" +Q30_SESSION="${Q30_SESSION:-q30_fixed_pressure_20260719}" +OUTPUT_ROOT="${OUTPUT_ROOT:?OUTPUT_ROOT is required}" +PROFILE_ROOT="${PROFILE_ROOT:?PROFILE_ROOT is required}" +RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" +FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}" +VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" + +mkdir -p "${OUTPUT_ROOT}/supervisor" "${PROFILE_ROOT}" +exec > >(tee -a "${OUTPUT_ROOT}/supervisor/controller.log") 2>&1 +echo "Q235_DEFERRED_LAUNCH_ECHO dependency=${Q30_ROOT}:Q30_FIXED_PRESSURE_CAMPAIGN_COMPLETE profile_model=Qwen3-235B-A22B-FP8 profile_backends={TP4/EP1:Triton,TP8/EP8:FlashInfer-CUTLASS} profile_cost=2-7_H20-GPUh experiment_cases={Fixed-PD,Fixed-PO,Trace-PD,Trace-PO} configs={TP4/EP1,TP8/EP8}xMNS{64,128} requests=129 trials=3 expected_campaign_wall=10-30h expected_campaign_cost=90-220_H20-GPUh profile_output=${PROFILE_ROOT} campaign_output=${OUTPUT_ROOT}" +date -u +SUPERVISOR_START_UTC=%Y-%m-%dT%H:%M:%SZ + +while tmux has-session -t "${Q30_SESSION}" 2>/dev/null; do + printf 'WAIT_Q30_UTC=%s results=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + "$(find "${Q30_ROOT}/real" -path '*/results/result.json' 2>/dev/null | wc -l)" + sleep 30 +done +grep -q 'Q30_FIXED_PRESSURE_CAMPAIGN_COMPLETE' "${Q30_ROOT}/controller.log" || { + echo 'ERROR: Q30 session ended without completion marker; refusing Q235 launch' >&2 + exit 1 +} +nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | awk '$1 > 16 {exit 1}' + +mkdir -p "${OUTPUT_ROOT}/profile-smoke" +env CUDA_VISIBLE_DEVICES=0 PYTHONPATH="${FRONTIER_SOURCE}" \ + "${VENV_ROOT}/bin/python" "${RUNNER_DIR}/smoke_qwen235_v020_frontier_moe.py" \ + --frontier-source "${FRONTIER_SOURCE}" \ + --output "${OUTPUT_ROOT}/profile-smoke/moe.json" + +OUTPUT_ROOT="${PROFILE_ROOT}" FRONTIER_SOURCE="${FRONTIER_SOURCE}" \ + VENV_ROOT="${VENV_ROOT}" RUNNER_DIR="${RUNNER_DIR}" \ + bash "${RUNNER_DIR}/run_qwen235_v020_profiles.sh" + +CAMPAIGN_ROOT="${OUTPUT_ROOT}" PROFILE_ROOT="${PROFILE_ROOT}/frozen" \ + FRONTIER_SOURCE="${FRONTIER_SOURCE}" VENV_ROOT="${VENV_ROOT}" RUNNER_DIR="${RUNNER_DIR}" \ + bash "${RUNNER_DIR}/run_qwen235_v020_campaign.sh" +date -u +SUPERVISOR_END_UTC=%Y-%m-%dT%H:%M:%SZ +echo Q235_AFTER_Q30_COMPLETE From 39e4719b282f59930f8e1d601c54d66bda5cfb29 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sun, 19 Jul 2026 03:00:04 +0800 Subject: [PATCH 146/148] Run Qwen235 profilers from Frontier source --- .../frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh index 20a93e1..e939009 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh @@ -20,6 +20,9 @@ nvidia-smi --query-gpu=index,memory.used --format=csv,noheader,nounits \ | awk '$2 > 16 {exit 1}' git -C "${FRONTIER_SOURCE}" rev-parse HEAD > "${OUTPUT_ROOT}/provenance/frontier.commit" "${VENV_ROOT}/bin/vllm" --version > "${OUTPUT_ROOT}/provenance/vllm.version" +# Frontier's model-config loader resolves data/config/models from the current +# source checkout, so all profiling entrypoints must run from this directory. +cd "${FRONTIER_SOURCE}" common_profile() { local gpu="$1" method="$2" root="$3" From 5067bc2cb188336f5f8e5c0944ca03e63bd025b7 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sun, 19 Jul 2026 05:42:26 +0800 Subject: [PATCH 147/148] Expose nvcc to Q235 FlashInfer profiling --- runs/frontier-fidelity-envelope-v1/run_q235_after_q30.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/runs/frontier-fidelity-envelope-v1/run_q235_after_q30.sh b/runs/frontier-fidelity-envelope-v1/run_q235_after_q30.sh index a3e6a05..90fe528 100644 --- a/runs/frontier-fidelity-envelope-v1/run_q235_after_q30.sh +++ b/runs/frontier-fidelity-envelope-v1/run_q235_after_q30.sh @@ -10,6 +10,11 @@ RUNNER_DIR="${RUNNER_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}" FRONTIER_SOURCE="${FRONTIER_SOURCE:-/home/admin/cpfs/wjh/aituner/frontier-q235-v020-5b953f5}" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" +# FlashInfer's TensorRT-LLM MoE runtime invokes `nvcc` by name when it +# autotunes a previously unseen FP8 grouped-GEMM shape. +export PATH="/usr/local/cuda/bin:${PATH}" +command -v nvcc >/dev/null + mkdir -p "${OUTPUT_ROOT}/supervisor" "${PROFILE_ROOT}" exec > >(tee -a "${OUTPUT_ROOT}/supervisor/controller.log") 2>&1 echo "Q235_DEFERRED_LAUNCH_ECHO dependency=${Q30_ROOT}:Q30_FIXED_PRESSURE_CAMPAIGN_COMPLETE profile_model=Qwen3-235B-A22B-FP8 profile_backends={TP4/EP1:Triton,TP8/EP8:FlashInfer-CUTLASS} profile_cost=2-7_H20-GPUh experiment_cases={Fixed-PD,Fixed-PO,Trace-PD,Trace-PO} configs={TP4/EP1,TP8/EP8}xMNS{64,128} requests=129 trials=3 expected_campaign_wall=10-30h expected_campaign_cost=90-220_H20-GPUh profile_output=${PROFILE_ROOT} campaign_output=${OUTPUT_ROOT}" From a21382c8aa7c21e7b08978de72b4e4581e8dbcc8 Mon Sep 17 00:00:00 2001 From: Gahow Wang Date: Sun, 19 Jul 2026 05:45:37 +0800 Subject: [PATCH 148/148] Raise Q235 profiler token limit --- .../run_qwen235_v020_profiles.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh index e939009..ed775b3 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen235_v020_profiles.sh @@ -30,7 +30,8 @@ common_profile() { "${VENV_ROOT}/bin/python" -m frontier.profiling.linear_op.main \ --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \ --num_tensor_parallel_workers 1 4 8 --attn_tp 4 8 --ffn_tp 1 4 \ - --num_tokens_list "${TOKENS[@]}" --profile_method "${method}" --yes + --max_tokens 8192 --num_tokens_list "${TOKENS[@]}" \ + --profile_method "${method}" --yes env CUDA_VISIBLE_DEVICES="${gpu}" PYTHONPATH="${FRONTIER_SOURCE}" \ "${VENV_ROOT}/bin/python" -m frontier.profiling.attention.main \ --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \ @@ -46,7 +47,7 @@ moe_profile() { "${VENV_ROOT}/bin/python" -m frontier.profiling.moe.main \ --disable_ray --num_gpus 1 --device h20 --output_dir "${root}" --models "${MODEL}" \ --num_tensor_parallel_workers "${tp}" --expert_parallel_sizes "${ep}" \ - --num_tokens_list "${TOKENS[@]}" --load_distributions uniform \ + --max_tokens 8192 --num_tokens_list "${TOKENS[@]}" --load_distributions uniform \ --num_samples_per_distribution 1 --profile_method "${method}" --yes } @@ -59,7 +60,7 @@ moe_profile 4 record_function 4 1 "${OUTPUT_ROOT}/kernel-moe-tp4" > "${OUTPUT_RO moe_profile 5 record_function 1 8 "${OUTPUT_ROOT}/kernel-moe-ep8" > "${OUTPUT_ROOT}/logs/kernel-moe-ep8.log" 2>&1 & pids+=("$!") failed=0 for pid in "${pids[@]}"; do wait "${pid}" || failed=1; done -[[ "${failed}" -eq 0 ]] || { tail -80 "${OUTPUT_ROOT}"/logs/*.log; exit 1; } +[[ "${failed}" -eq 0 ]] || { tail -n 80 "${OUTPUT_ROOT}"/logs/*.log; exit 1; } "${VENV_ROOT}/bin/python" "${RUNNER_DIR}/assemble_qwen235_v020_profiles.py" \ --cuda-common "${OUTPUT_ROOT}/cuda-common" \