Launch frozen Qwen30 latency case surfaces

This commit is contained in:
2026-07-18 10:48:45 +08:00
parent 5fc0b48fa1
commit a01c1206ab
2 changed files with 190 additions and 3 deletions

View File

@@ -22,6 +22,11 @@ TRIALS = (1, 2, 3)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--case-root", type=Path, required=True)
parser.add_argument(
"--traces-root",
type=Path,
help="Frozen TP-normalized traces; defaults to <case-root>/traces.",
)
parser.add_argument("--json-output", type=Path, required=True)
parser.add_argument("--markdown-output", type=Path, required=True)
return parser.parse_args()
@@ -42,8 +47,8 @@ def finite(value: Any, field: str) -> float:
return value
def read_manifest(root: Path, tp: int) -> dict[str, Any]:
path = root / "traces" / f"tp{tp}" / "public" / "manifest.json"
def read_manifest(traces_root: Path, tp: int) -> dict[str, Any]:
path = traces_root / f"tp{tp}" / "public" / "manifest.json"
manifest = json.loads(path.read_text())
if manifest.get("schema") != "qwen30-latency-case-v1":
raise ValueError(f"unexpected trace schema in {path}")
@@ -116,7 +121,8 @@ def validate_trial(path: Path, manifest: dict[str, Any]) -> tuple[dict[str, list
def main() -> None:
args = parse_args()
root = args.case_root.resolve()
manifests = {tp: read_manifest(root, tp) for tp in (1, 2, 4)}
traces_root = (args.traces_root or root / "traces").resolve()
manifests = {tp: read_manifest(traces_root, tp) for tp in (1, 2, 4)}
prefill_only = int(manifests[1]["output_tokens"][0]) == 1
if any((int(manifest["output_tokens"][0]) == 1) != prefill_only for manifest in manifests.values()):
raise ValueError("TP-specific output contracts differ")
@@ -165,6 +171,7 @@ def main() -> None:
payload = {
"schema": "qwen30-latency-case-real-audit-v1",
"case_root": str(root),
"traces_root": str(traces_root),
"prefill_only": prefill_only,
"applicable_metrics": list(applicable),
"trace_manifests": {f"tp{tp}": manifests[tp] for tp in manifests},