Launch frozen Qwen30 latency case surfaces
This commit is contained in:
@@ -22,6 +22,11 @@ TRIALS = (1, 2, 3)
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--case-root", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--traces-root",
|
||||
type=Path,
|
||||
help="Frozen TP-normalized traces; defaults to <case-root>/traces.",
|
||||
)
|
||||
parser.add_argument("--json-output", type=Path, required=True)
|
||||
parser.add_argument("--markdown-output", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
@@ -42,8 +47,8 @@ def finite(value: Any, field: str) -> float:
|
||||
return value
|
||||
|
||||
|
||||
def read_manifest(root: Path, tp: int) -> dict[str, Any]:
|
||||
path = root / "traces" / f"tp{tp}" / "public" / "manifest.json"
|
||||
def read_manifest(traces_root: Path, tp: int) -> dict[str, Any]:
|
||||
path = traces_root / f"tp{tp}" / "public" / "manifest.json"
|
||||
manifest = json.loads(path.read_text())
|
||||
if manifest.get("schema") != "qwen30-latency-case-v1":
|
||||
raise ValueError(f"unexpected trace schema in {path}")
|
||||
@@ -116,7 +121,8 @@ def validate_trial(path: Path, manifest: dict[str, Any]) -> tuple[dict[str, list
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
root = args.case_root.resolve()
|
||||
manifests = {tp: read_manifest(root, tp) for tp in (1, 2, 4)}
|
||||
traces_root = (args.traces_root or root / "traces").resolve()
|
||||
manifests = {tp: read_manifest(traces_root, tp) for tp in (1, 2, 4)}
|
||||
prefill_only = int(manifests[1]["output_tokens"][0]) == 1
|
||||
if any((int(manifest["output_tokens"][0]) == 1) != prefill_only for manifest in manifests.values()):
|
||||
raise ValueError("TP-specific output contracts differ")
|
||||
@@ -165,6 +171,7 @@ def main() -> None:
|
||||
payload = {
|
||||
"schema": "qwen30-latency-case-real-audit-v1",
|
||||
"case_root": str(root),
|
||||
"traces_root": str(traces_root),
|
||||
"prefill_only": prefill_only,
|
||||
"applicable_metrics": list(applicable),
|
||||
"trace_manifests": {f"tp{tp}": manifests[tp] for tp in manifests},
|
||||
|
||||
Reference in New Issue
Block a user