Add exact state observation modes

runs/frontier-fidelity-envelope-v1/run_qwen235_fixed_pd_state_replay.py

@@ -37,6 +37,7 @@ ALLOWED_FLAG_CHANGES = {
     "--metrics_config_store_frontier_stage_batch_ledger",
     "--no-metrics_config_keep_individual_batch_metrics",
     "--metrics_config_keep_individual_batch_metrics",
+    "--metrics_config_enable_op_level_tracing",
 }
 
 
@@ -104,13 +105,18 @@ def _semantic_options(command: list[str]) -> dict[str, tuple[str, ...]]:
 
 
 def transform_command(
-    command: list[str], *, metrics_root: Path, run_id: str
+    command: list[str], *, metrics_root: Path, run_id: str, op_trace: bool = False
 ) -> list[str]:
     result = replace_option(
         command, "--metrics_config_output_dir", str(metrics_root.resolve())
     )
     result = replace_option(result, "--metrics_config_run_id", run_id)
     result = enable_state_outputs(result)
+    op_trace_flag = "--metrics_config_enable_op_level_tracing"
+    if op_trace:
+        if op_trace_flag in result:
+            raise ValueError(f"frozen command already enables {op_trace_flag}")
+        result.append(op_trace_flag)
 
     before = _semantic_options(command)
     after = _semantic_options(result)
@@ -130,6 +136,8 @@ def transform_command(
     }
     if not required <= changed:
         raise ValueError(f"required controlled changes missing: {sorted(required - changed)}")
+    if op_trace and op_trace_flag not in changed:
+        raise ValueError("op-trace observation flag was not audited as a command change")
     return result
 
 
@@ -158,6 +166,7 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--config", action="append", choices=CONFIGS)
     parser.add_argument("--timeout-seconds", type=float, default=1200)
     parser.add_argument("--resume", action="store_true")
+    parser.add_argument("--op-trace", action="store_true")
     return parser.parse_args()
 
 
@@ -170,6 +179,7 @@ def run_cell(
     python_deps: Path,
     timeout_seconds: float,
     resume: bool,
+    op_trace: bool,
     q30: Any,
 ) -> dict[str, Any]:
     base_run = base_sim_root / "runs" / config / "eval"
@@ -192,6 +202,7 @@ def run_cell(
             == sha256_file(base_command_path)
             and previous.get("inputs", {}).get("base_result_sha256")
             == sha256_file(base_result_path)
+            and bool(previous.get("op_trace_enabled")) == op_trace
         ):
             return previous
     if run_root.exists() and any(run_root.iterdir()):
@@ -202,6 +213,7 @@ def run_cell(
         base_command,
         metrics_root=run_root / "frontier_metrics",
         run_id=f"qwen235_fixed_pd_state_{config}",
+        op_trace=op_trace,
     )
     atomic_json(run_root / "command.json", command)
     manifest = {
@@ -218,7 +230,7 @@ def run_cell(
             "metrics run id",
             "full Frontier stage/batch ledger enabled",
             "individual batch metrics enabled",
-        ],
+        ] + (["op-level tracing enabled"] if op_trace else []),
         "command_sha256": sha256_file(run_root / "command.json"),
         "frontier": {
             "source": str(frontier_source),
@@ -232,6 +244,7 @@ def run_cell(
             "NVIDIA_VISIBLE_DEVICES": "void",
             "FRONTIER_LOG_LEVEL": "WARNING",
         },
+        "op_trace_enabled": op_trace,
     }
     atomic_json(run_root / "run_manifest.json", manifest)
 
@@ -281,6 +294,13 @@ def run_cell(
         ledger_path=paths["ledger"],
     )
     atomic_json(run_root / "common-state.json", state)
+    if op_trace:
+        op_trace_matches = sorted(run_root.glob("frontier_metrics/**/op_traces.jsonl"))
+        if len(op_trace_matches) != 1:
+            raise ValueError(
+                f"expected one op trace for {config}, found {len(op_trace_matches)}"
+            )
+        paths["op_trace"] = op_trace_matches[0]
     result = {
         "schema": "qwen235-fixed-pd-state-replay-result-v1",
         "status": "PASS",
@@ -299,6 +319,7 @@ def run_cell(
         },
         "common_state": state,
         "collective_fallback_evidence": fallback_evidence,
+        "op_trace_enabled": op_trace,
     }
     atomic_json(result_path, result)
     return result
@@ -320,6 +341,7 @@ def main() -> None:
             python_deps=args.python_deps,
             timeout_seconds=args.timeout_seconds,
             resume=args.resume,
+            op_trace=args.op_trace,
             q30=q30,
         )
         results.append(result)

runs/frontier-fidelity-envelope-v1/run_qwen30_exact_trace_real_anchor.sh

@@ -12,6 +12,7 @@ PREFIX_CACHING="${PREFIX_CACHING:-true}"
 ENABLE_EXPERT_PARALLEL="${ENABLE_EXPERT_PARALLEL:-false}"
 MODEL_QUANTIZATION="${MODEL_QUANTIZATION:-}"
 DISABLE_CUSTOM_ALL_REDUCE="${DISABLE_CUSTOM_ALL_REDUCE:-false}"
+ENABLE_LOGGING_ITERATION_DETAILS="${ENABLE_LOGGING_ITERATION_DETAILS:-false}"
 GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.92}"
 VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
 MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}"
@@ -68,13 +69,18 @@ case "${DISABLE_CUSTOM_ALL_REDUCE}" in
   false) CUSTOM_AR_FLAG=() ;;
   *) echo "ERROR: DISABLE_CUSTOM_ALL_REDUCE must be true or false" >&2; exit 1 ;;
 esac
+case "${ENABLE_LOGGING_ITERATION_DETAILS}" in
+  true) ITERATION_DETAILS_FLAG=(--enable-logging-iteration-details) ;;
+  false) ITERATION_DETAILS_FLAG=() ;;
+  *) echo "ERROR: ENABLE_LOGGING_ITERATION_DETAILS must be true or false" >&2; exit 1 ;;
+esac
 QUANT_FLAG=()
 if [[ -n "${MODEL_QUANTIZATION}" ]]; then
   QUANT_FLAG=(--quantization "${MODEL_QUANTIZATION}")
 fi
 
 REQUEST_COUNT="$(wc -l < "${REQUESTS_FILE}")"
-echo "EXACT_TRACE_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 quantization=${MODEL_QUANTIZATION:-none} config=TP${TP}_EP${ENABLE_EXPERT_PARALLEL}_MNS${MNS}_MBT8192 trace=${TRACE_LABEL} requests=${REQUEST_COUNT} source=${REQUESTS_FILE} arrivals=manifest prefix=${PREFIX_CACHING} block=16 metrics=TTFT,TPOT-if-OSL-gt-1,E2E flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} output=${OUTPUT_ROOT} ready_budget_s=$((SERVER_READY_ATTEMPTS * 3)) client_timeout_s=${CLIENT_TIMEOUT_SECONDS}"
+echo "EXACT_TRACE_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 quantization=${MODEL_QUANTIZATION:-none} config=TP${TP}_EP${ENABLE_EXPERT_PARALLEL}_MNS${MNS}_MBT8192 trace=${TRACE_LABEL} requests=${REQUEST_COUNT} source=${REQUESTS_FILE} arrivals=manifest prefix=${PREFIX_CACHING} block=16 metrics=TTFT,TPOT-if-OSL-gt-1,E2E iteration_details=${ENABLE_LOGGING_ITERATION_DETAILS} flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} output=${OUTPUT_ROOT} ready_budget_s=$((SERVER_READY_ATTEMPTS * 3)) client_timeout_s=${CLIENT_TIMEOUT_SECONDS}"
 date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ"
 sha256sum qwen30_exact_trace_client.py run_qwen30_exact_trace_real_anchor.sh \
   ../frontier-phase-factorial-v0/qwen30_prefill_client.py \
@@ -98,6 +104,7 @@ setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \
   --max-model-len 40960 --max-num-batched-tokens 8192 --max-num-seqs "${MNS}" \
   "${PREFIX_CACHING_FLAG}" --enable-chunked-prefill --no-enable-log-requests \
   "${CUSTOM_AR_FLAG[@]}" "${QUANT_FLAG[@]}" "${EP_FLAG[@]}" \
+  "${ITERATION_DETAILS_FLAG[@]}" \
   > "${OUTPUT_ROOT}/logs/server.log" 2>&1 &
 SERVER_PID=$!
 READY=0

runs/frontier-fidelity-envelope-v1/test_fidelity_envelope.py

@@ -170,6 +170,14 @@ class FidelityEnvelopeTest(unittest.TestCase):
         self.assertNotIn(
             "--no-metrics_config_store_frontier_stage_batch_ledger", transformed
         )
+        traced = module.transform_command(
+            base, metrics_root=Path("/new/traced-metrics"), run_id="traced", op_trace=True
+        )
+        self.assertIn("--metrics_config_enable_op_level_tracing", traced)
+        self.assertEqual(
+            module.option_value(traced, "--communication_collective_profile_path"),
+            "/profiles/real-allreduce.csv",
+        )
 
     def test_qwen235_state_replay_rejects_implicit_ledger_base(self) -> None:
         module = load("run_qwen235_fixed_pd_state_replay.py")