Add Frontier fidelity envelope campaign

runs/frontier-phase-factorial-v0/run_frontier_qwen30_prefill_surface.py

@@ -48,6 +48,10 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--requests", type=int, default=64)
     parser.add_argument("--rate", type=float, action="append")
     parser.add_argument("--config", action="append")
+    parser.add_argument(
+        "--cc-backend", choices=("analytical", "vidur"), default="analytical"
+    )
+    parser.add_argument("--allreduce-csv", type=Path)
     parser.add_argument("--timeout-seconds", type=float, default=900.0)
     parser.add_argument("--resume", action="store_true")
     return parser.parse_args()
@@ -182,6 +186,37 @@ def knobs(config: Config, paths: dict[str, Path], cache: Path) -> dict[str, Any]
     }
 
 
+def configure_cc_command(
+    command: list[str], *, backend: str, allreduce_csv: Path | None, cache: Path
+) -> list[str]:
+    configured = list(command)
+    option = "--cc_backend_config_type"
+    try:
+        index = configured.index(option)
+    except ValueError as error:
+        raise ValueError(f"Frontier command is missing {option}") from error
+    configured[index + 1] = backend
+    if backend == "analytical":
+        if allreduce_csv is not None:
+            raise ValueError("--allreduce-csv requires --cc-backend vidur")
+        return configured
+    if allreduce_csv is None:
+        raise ValueError("--cc-backend vidur requires --allreduce-csv")
+    configured.extend(
+        [
+            "--vidur_cc_backend_config_all_reduce_input_file",
+            str(allreduce_csv),
+            "--vidur_cc_backend_config_cache_dir",
+            str(cache),
+            "--vidur_cc_backend_config_k_fold_cv_splits",
+            "6",
+            "--vidur_cc_backend_config_num_training_job_threads",
+            "1",
+        ]
+    )
+    return configured
+
+
 def find_metrics(run_dir: Path) -> tuple[Path, Path]:
     systems = list((run_dir / "metrics").rglob("system_metrics.json"))
     requests = list((run_dir / "metrics").rglob("request_metrics.csv"))
@@ -231,6 +266,10 @@ def main() -> None:
     args.profile_root = args.profile_root.resolve()
     args.python_deps = args.python_deps.resolve()
     args.output_root = args.output_root.resolve()
+    if args.allreduce_csv is not None:
+        args.allreduce_csv = args.allreduce_csv.resolve()
+        if not args.allreduce_csv.is_file():
+            raise FileNotFoundError(args.allreduce_csv)
     rates = tuple(args.rate or RATES)
     selected = list(GRID)
     if args.config:
@@ -274,6 +313,12 @@ def main() -> None:
                 run_id=f"qwen30_prefill_{config.name}_r{rate:g}",
                 knobs=config_knobs,
             )
+            command = configure_cc_command(
+                command,
+                backend=args.cc_backend,
+                allreduce_csv=args.allreduce_csv,
+                cache=args.output_root / "cc-cache",
+            )
             write_json(run_dir / "command.json", command)
             environment = os.environ.copy()
             pythonpath = [str(args.python_deps), str(args.frontier_source)]
@@ -389,6 +434,15 @@ def main() -> None:
             "coverage": coverage,
             "sha256": {name: sha256(path) for name, path in paths.items()},
         },
+        "collective": {
+            "backend": args.cc_backend,
+            "allreduce_csv": (
+                str(args.allreduce_csv) if args.allreduce_csv is not None else None
+            ),
+            "allreduce_csv_sha256": (
+                sha256(args.allreduce_csv) if args.allreduce_csv is not None else None
+            ),
+        },
         "config_results": config_results,
         "capacity": capacities,
     }

runs/frontier-phase-factorial-v0/test_phase_factorial.py

@@ -43,3 +43,20 @@ def test_kendall_tau_b() -> None:
     analysis = load("analyze_qwen30_prefill_fidelity.py")
     assert analysis.kendall_tau_b([1, 2, 3], [1, 2, 3])["kendall_tau_b"] == 1
     assert analysis.kendall_tau_b([1, 2, 3], [3, 2, 1])["kendall_tau_b"] == -1
+
+
+def test_configure_cc_command(tmp_path: Path) -> None:
+    surface = load("run_frontier_qwen30_prefill_surface.py")
+    base = ["python", "--cc_backend_config_type", "analytical", "--other", "x"]
+    analytical = surface.configure_cc_command(
+        base, backend="analytical", allreduce_csv=None, cache=tmp_path
+    )
+    assert analytical == base
+    profile = tmp_path / "all_reduce.csv"
+    profile.write_text("header\n")
+    vidur = surface.configure_cc_command(
+        base, backend="vidur", allreduce_csv=profile, cache=tmp_path / "cache"
+    )
+    assert vidur[2] == "vidur"
+    assert "--vidur_cc_backend_config_all_reduce_input_file" in vidur
+    assert str(profile) in vidur