diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen235_fixed_pd_state_replay.py b/runs/frontier-fidelity-envelope-v1/run_qwen235_fixed_pd_state_replay.py index 995b185..ef23da4 100644 --- a/runs/frontier-fidelity-envelope-v1/run_qwen235_fixed_pd_state_replay.py +++ b/runs/frontier-fidelity-envelope-v1/run_qwen235_fixed_pd_state_replay.py @@ -37,6 +37,7 @@ ALLOWED_FLAG_CHANGES = { "--metrics_config_store_frontier_stage_batch_ledger", "--no-metrics_config_keep_individual_batch_metrics", "--metrics_config_keep_individual_batch_metrics", + "--metrics_config_enable_op_level_tracing", } @@ -104,13 +105,18 @@ def _semantic_options(command: list[str]) -> dict[str, tuple[str, ...]]: def transform_command( - command: list[str], *, metrics_root: Path, run_id: str + command: list[str], *, metrics_root: Path, run_id: str, op_trace: bool = False ) -> list[str]: result = replace_option( command, "--metrics_config_output_dir", str(metrics_root.resolve()) ) result = replace_option(result, "--metrics_config_run_id", run_id) result = enable_state_outputs(result) + op_trace_flag = "--metrics_config_enable_op_level_tracing" + if op_trace: + if op_trace_flag in result: + raise ValueError(f"frozen command already enables {op_trace_flag}") + result.append(op_trace_flag) before = _semantic_options(command) after = _semantic_options(result) @@ -130,6 +136,8 @@ def transform_command( } if not required <= changed: raise ValueError(f"required controlled changes missing: {sorted(required - changed)}") + if op_trace and op_trace_flag not in changed: + raise ValueError("op-trace observation flag was not audited as a command change") return result @@ -158,6 +166,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--config", action="append", choices=CONFIGS) parser.add_argument("--timeout-seconds", type=float, default=1200) parser.add_argument("--resume", action="store_true") + parser.add_argument("--op-trace", action="store_true") return parser.parse_args() @@ -170,6 +179,7 @@ def run_cell( python_deps: Path, timeout_seconds: float, resume: bool, + op_trace: bool, q30: Any, ) -> dict[str, Any]: base_run = base_sim_root / "runs" / config / "eval" @@ -192,6 +202,7 @@ def run_cell( == sha256_file(base_command_path) and previous.get("inputs", {}).get("base_result_sha256") == sha256_file(base_result_path) + and bool(previous.get("op_trace_enabled")) == op_trace ): return previous if run_root.exists() and any(run_root.iterdir()): @@ -202,6 +213,7 @@ def run_cell( base_command, metrics_root=run_root / "frontier_metrics", run_id=f"qwen235_fixed_pd_state_{config}", + op_trace=op_trace, ) atomic_json(run_root / "command.json", command) manifest = { @@ -218,7 +230,7 @@ def run_cell( "metrics run id", "full Frontier stage/batch ledger enabled", "individual batch metrics enabled", - ], + ] + (["op-level tracing enabled"] if op_trace else []), "command_sha256": sha256_file(run_root / "command.json"), "frontier": { "source": str(frontier_source), @@ -232,6 +244,7 @@ def run_cell( "NVIDIA_VISIBLE_DEVICES": "void", "FRONTIER_LOG_LEVEL": "WARNING", }, + "op_trace_enabled": op_trace, } atomic_json(run_root / "run_manifest.json", manifest) @@ -281,6 +294,13 @@ def run_cell( ledger_path=paths["ledger"], ) atomic_json(run_root / "common-state.json", state) + if op_trace: + op_trace_matches = sorted(run_root.glob("frontier_metrics/**/op_traces.jsonl")) + if len(op_trace_matches) != 1: + raise ValueError( + f"expected one op trace for {config}, found {len(op_trace_matches)}" + ) + paths["op_trace"] = op_trace_matches[0] result = { "schema": "qwen235-fixed-pd-state-replay-result-v1", "status": "PASS", @@ -299,6 +319,7 @@ def run_cell( }, "common_state": state, "collective_fallback_evidence": fallback_evidence, + "op_trace_enabled": op_trace, } atomic_json(result_path, result) return result @@ -320,6 +341,7 @@ def main() -> None: python_deps=args.python_deps, timeout_seconds=args.timeout_seconds, resume=args.resume, + op_trace=args.op_trace, q30=q30, ) results.append(result) diff --git a/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh b/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh index 3ef6c21..cc3afa9 100755 --- a/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh +++ b/runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh @@ -12,6 +12,7 @@ PREFIX_CACHING="${PREFIX_CACHING:-true}" ENABLE_EXPERT_PARALLEL="${ENABLE_EXPERT_PARALLEL:-false}" MODEL_QUANTIZATION="${MODEL_QUANTIZATION:-}" DISABLE_CUSTOM_ALL_REDUCE="${DISABLE_CUSTOM_ALL_REDUCE:-false}" +ENABLE_LOGGING_ITERATION_DETAILS="${ENABLE_LOGGING_ITERATION_DETAILS:-false}" GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.92}" VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}" MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}" @@ -68,13 +69,18 @@ case "${DISABLE_CUSTOM_ALL_REDUCE}" in false) CUSTOM_AR_FLAG=() ;; *) echo "ERROR: DISABLE_CUSTOM_ALL_REDUCE must be true or false" >&2; exit 1 ;; esac +case "${ENABLE_LOGGING_ITERATION_DETAILS}" in + true) ITERATION_DETAILS_FLAG=(--enable-logging-iteration-details) ;; + false) ITERATION_DETAILS_FLAG=() ;; + *) echo "ERROR: ENABLE_LOGGING_ITERATION_DETAILS must be true or false" >&2; exit 1 ;; +esac QUANT_FLAG=() if [[ -n "${MODEL_QUANTIZATION}" ]]; then QUANT_FLAG=(--quantization "${MODEL_QUANTIZATION}") fi REQUEST_COUNT="$(wc -l < "${REQUESTS_FILE}")" -echo "EXACT_TRACE_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 quantization=${MODEL_QUANTIZATION:-none} config=TP${TP}_EP${ENABLE_EXPERT_PARALLEL}_MNS${MNS}_MBT8192 trace=${TRACE_LABEL} requests=${REQUEST_COUNT} source=${REQUESTS_FILE} arrivals=manifest prefix=${PREFIX_CACHING} block=16 metrics=TTFT,TPOT-if-OSL-gt-1,E2E flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} output=${OUTPUT_ROOT} ready_budget_s=$((SERVER_READY_ATTEMPTS * 3)) client_timeout_s=${CLIENT_TIMEOUT_SECONDS}" +echo "EXACT_TRACE_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 quantization=${MODEL_QUANTIZATION:-none} config=TP${TP}_EP${ENABLE_EXPERT_PARALLEL}_MNS${MNS}_MBT8192 trace=${TRACE_LABEL} requests=${REQUEST_COUNT} source=${REQUESTS_FILE} arrivals=manifest prefix=${PREFIX_CACHING} block=16 metrics=TTFT,TPOT-if-OSL-gt-1,E2E iteration_details=${ENABLE_LOGGING_ITERATION_DETAILS} flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} output=${OUTPUT_ROOT} ready_budget_s=$((SERVER_READY_ATTEMPTS * 3)) client_timeout_s=${CLIENT_TIMEOUT_SECONDS}" date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ" sha256sum qwen30_exact_trace_client.py run_qwen30_exact_trace_real_anchor.sh \ ../frontier-phase-factorial-v0/qwen30_prefill_client.py \ @@ -98,6 +104,7 @@ setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \ --max-model-len 40960 --max-num-batched-tokens 8192 --max-num-seqs "${MNS}" \ "${PREFIX_CACHING_FLAG}" --enable-chunked-prefill --no-enable-log-requests \ "${CUSTOM_AR_FLAG[@]}" "${QUANT_FLAG[@]}" "${EP_FLAG[@]}" \ + "${ITERATION_DETAILS_FLAG[@]}" \ > "${OUTPUT_ROOT}/logs/server.log" 2>&1 & SERVER_PID=$! READY=0 diff --git a/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py b/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py index 8a1de8c..4a784c0 100644 --- a/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py +++ b/runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py @@ -170,6 +170,14 @@ class FidelityEnvelopeTest(unittest.TestCase): self.assertNotIn( "--no-metrics_config_store_frontier_stage_batch_ledger", transformed ) + traced = module.transform_command( + base, metrics_root=Path("/new/traced-metrics"), run_id="traced", op_trace=True + ) + self.assertIn("--metrics_config_enable_op_level_tracing", traced) + self.assertEqual( + module.option_value(traced, "--communication_collective_profile_path"), + "/profiles/real-allreduce.csv", + ) def test_qwen235_state_replay_rejects_implicit_ledger_base(self) -> None: module = load("run_qwen235_fixed_pd_state_replay.py")