Add exact state observation modes
This commit is contained in:
@@ -37,6 +37,7 @@ ALLOWED_FLAG_CHANGES = {
|
||||
"--metrics_config_store_frontier_stage_batch_ledger",
|
||||
"--no-metrics_config_keep_individual_batch_metrics",
|
||||
"--metrics_config_keep_individual_batch_metrics",
|
||||
"--metrics_config_enable_op_level_tracing",
|
||||
}
|
||||
|
||||
|
||||
@@ -104,13 +105,18 @@ def _semantic_options(command: list[str]) -> dict[str, tuple[str, ...]]:
|
||||
|
||||
|
||||
def transform_command(
|
||||
command: list[str], *, metrics_root: Path, run_id: str
|
||||
command: list[str], *, metrics_root: Path, run_id: str, op_trace: bool = False
|
||||
) -> list[str]:
|
||||
result = replace_option(
|
||||
command, "--metrics_config_output_dir", str(metrics_root.resolve())
|
||||
)
|
||||
result = replace_option(result, "--metrics_config_run_id", run_id)
|
||||
result = enable_state_outputs(result)
|
||||
op_trace_flag = "--metrics_config_enable_op_level_tracing"
|
||||
if op_trace:
|
||||
if op_trace_flag in result:
|
||||
raise ValueError(f"frozen command already enables {op_trace_flag}")
|
||||
result.append(op_trace_flag)
|
||||
|
||||
before = _semantic_options(command)
|
||||
after = _semantic_options(result)
|
||||
@@ -130,6 +136,8 @@ def transform_command(
|
||||
}
|
||||
if not required <= changed:
|
||||
raise ValueError(f"required controlled changes missing: {sorted(required - changed)}")
|
||||
if op_trace and op_trace_flag not in changed:
|
||||
raise ValueError("op-trace observation flag was not audited as a command change")
|
||||
return result
|
||||
|
||||
|
||||
@@ -158,6 +166,7 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--config", action="append", choices=CONFIGS)
|
||||
parser.add_argument("--timeout-seconds", type=float, default=1200)
|
||||
parser.add_argument("--resume", action="store_true")
|
||||
parser.add_argument("--op-trace", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -170,6 +179,7 @@ def run_cell(
|
||||
python_deps: Path,
|
||||
timeout_seconds: float,
|
||||
resume: bool,
|
||||
op_trace: bool,
|
||||
q30: Any,
|
||||
) -> dict[str, Any]:
|
||||
base_run = base_sim_root / "runs" / config / "eval"
|
||||
@@ -192,6 +202,7 @@ def run_cell(
|
||||
== sha256_file(base_command_path)
|
||||
and previous.get("inputs", {}).get("base_result_sha256")
|
||||
== sha256_file(base_result_path)
|
||||
and bool(previous.get("op_trace_enabled")) == op_trace
|
||||
):
|
||||
return previous
|
||||
if run_root.exists() and any(run_root.iterdir()):
|
||||
@@ -202,6 +213,7 @@ def run_cell(
|
||||
base_command,
|
||||
metrics_root=run_root / "frontier_metrics",
|
||||
run_id=f"qwen235_fixed_pd_state_{config}",
|
||||
op_trace=op_trace,
|
||||
)
|
||||
atomic_json(run_root / "command.json", command)
|
||||
manifest = {
|
||||
@@ -218,7 +230,7 @@ def run_cell(
|
||||
"metrics run id",
|
||||
"full Frontier stage/batch ledger enabled",
|
||||
"individual batch metrics enabled",
|
||||
],
|
||||
] + (["op-level tracing enabled"] if op_trace else []),
|
||||
"command_sha256": sha256_file(run_root / "command.json"),
|
||||
"frontier": {
|
||||
"source": str(frontier_source),
|
||||
@@ -232,6 +244,7 @@ def run_cell(
|
||||
"NVIDIA_VISIBLE_DEVICES": "void",
|
||||
"FRONTIER_LOG_LEVEL": "WARNING",
|
||||
},
|
||||
"op_trace_enabled": op_trace,
|
||||
}
|
||||
atomic_json(run_root / "run_manifest.json", manifest)
|
||||
|
||||
@@ -281,6 +294,13 @@ def run_cell(
|
||||
ledger_path=paths["ledger"],
|
||||
)
|
||||
atomic_json(run_root / "common-state.json", state)
|
||||
if op_trace:
|
||||
op_trace_matches = sorted(run_root.glob("frontier_metrics/**/op_traces.jsonl"))
|
||||
if len(op_trace_matches) != 1:
|
||||
raise ValueError(
|
||||
f"expected one op trace for {config}, found {len(op_trace_matches)}"
|
||||
)
|
||||
paths["op_trace"] = op_trace_matches[0]
|
||||
result = {
|
||||
"schema": "qwen235-fixed-pd-state-replay-result-v1",
|
||||
"status": "PASS",
|
||||
@@ -299,6 +319,7 @@ def run_cell(
|
||||
},
|
||||
"common_state": state,
|
||||
"collective_fallback_evidence": fallback_evidence,
|
||||
"op_trace_enabled": op_trace,
|
||||
}
|
||||
atomic_json(result_path, result)
|
||||
return result
|
||||
@@ -320,6 +341,7 @@ def main() -> None:
|
||||
python_deps=args.python_deps,
|
||||
timeout_seconds=args.timeout_seconds,
|
||||
resume=args.resume,
|
||||
op_trace=args.op_trace,
|
||||
q30=q30,
|
||||
)
|
||||
results.append(result)
|
||||
|
||||
@@ -12,6 +12,7 @@ PREFIX_CACHING="${PREFIX_CACHING:-true}"
|
||||
ENABLE_EXPERT_PARALLEL="${ENABLE_EXPERT_PARALLEL:-false}"
|
||||
MODEL_QUANTIZATION="${MODEL_QUANTIZATION:-}"
|
||||
DISABLE_CUSTOM_ALL_REDUCE="${DISABLE_CUSTOM_ALL_REDUCE:-false}"
|
||||
ENABLE_LOGGING_ITERATION_DETAILS="${ENABLE_LOGGING_ITERATION_DETAILS:-false}"
|
||||
GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.92}"
|
||||
VENV_ROOT="${VENV_ROOT:-/tmp/wjh/venvs/vllm-0.20.0-cu129-profiler-v1}"
|
||||
MODEL_ROOT="${MODEL_ROOT:-/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B}"
|
||||
@@ -68,13 +69,18 @@ case "${DISABLE_CUSTOM_ALL_REDUCE}" in
|
||||
false) CUSTOM_AR_FLAG=() ;;
|
||||
*) echo "ERROR: DISABLE_CUSTOM_ALL_REDUCE must be true or false" >&2; exit 1 ;;
|
||||
esac
|
||||
case "${ENABLE_LOGGING_ITERATION_DETAILS}" in
|
||||
true) ITERATION_DETAILS_FLAG=(--enable-logging-iteration-details) ;;
|
||||
false) ITERATION_DETAILS_FLAG=() ;;
|
||||
*) echo "ERROR: ENABLE_LOGGING_ITERATION_DETAILS must be true or false" >&2; exit 1 ;;
|
||||
esac
|
||||
QUANT_FLAG=()
|
||||
if [[ -n "${MODEL_QUANTIZATION}" ]]; then
|
||||
QUANT_FLAG=(--quantization "${MODEL_QUANTIZATION}")
|
||||
fi
|
||||
|
||||
REQUEST_COUNT="$(wc -l < "${REQUESTS_FILE}")"
|
||||
echo "EXACT_TRACE_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 quantization=${MODEL_QUANTIZATION:-none} config=TP${TP}_EP${ENABLE_EXPERT_PARALLEL}_MNS${MNS}_MBT8192 trace=${TRACE_LABEL} requests=${REQUEST_COUNT} source=${REQUESTS_FILE} arrivals=manifest prefix=${PREFIX_CACHING} block=16 metrics=TTFT,TPOT-if-OSL-gt-1,E2E flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} output=${OUTPUT_ROOT} ready_budget_s=$((SERVER_READY_ATTEMPTS * 3)) client_timeout_s=${CLIENT_TIMEOUT_SECONDS}"
|
||||
echo "EXACT_TRACE_REAL_LAUNCH_ECHO host=$(hostname) gpus=${CUDA_VISIBLE_DEVICES} model=${MODEL_ROOT} runtime=vLLM-0.20.0+cu129 dtype=BF16 quantization=${MODEL_QUANTIZATION:-none} config=TP${TP}_EP${ENABLE_EXPERT_PARALLEL}_MNS${MNS}_MBT8192 trace=${TRACE_LABEL} requests=${REQUEST_COUNT} source=${REQUESTS_FILE} arrivals=manifest prefix=${PREFIX_CACHING} block=16 metrics=TTFT,TPOT-if-OSL-gt-1,E2E iteration_details=${ENABLE_LOGGING_ITERATION_DETAILS} flashinfer_workspace=${FLASHINFER_WORKSPACE_BASE} output=${OUTPUT_ROOT} ready_budget_s=$((SERVER_READY_ATTEMPTS * 3)) client_timeout_s=${CLIENT_TIMEOUT_SECONDS}"
|
||||
date -u +"START_UTC=%Y-%m-%dT%H:%M:%SZ"
|
||||
sha256sum qwen30_exact_trace_client.py run_qwen30_exact_trace_real_anchor.sh \
|
||||
../frontier-phase-factorial-v0/qwen30_prefill_client.py \
|
||||
@@ -98,6 +104,7 @@ setsid "${VENV_ROOT}/bin/vllm" serve "${MODEL_ROOT}" \
|
||||
--max-model-len 40960 --max-num-batched-tokens 8192 --max-num-seqs "${MNS}" \
|
||||
"${PREFIX_CACHING_FLAG}" --enable-chunked-prefill --no-enable-log-requests \
|
||||
"${CUSTOM_AR_FLAG[@]}" "${QUANT_FLAG[@]}" "${EP_FLAG[@]}" \
|
||||
"${ITERATION_DETAILS_FLAG[@]}" \
|
||||
> "${OUTPUT_ROOT}/logs/server.log" 2>&1 &
|
||||
SERVER_PID=$!
|
||||
READY=0
|
||||
|
||||
@@ -170,6 +170,14 @@ class FidelityEnvelopeTest(unittest.TestCase):
|
||||
self.assertNotIn(
|
||||
"--no-metrics_config_store_frontier_stage_batch_ledger", transformed
|
||||
)
|
||||
traced = module.transform_command(
|
||||
base, metrics_root=Path("/new/traced-metrics"), run_id="traced", op_trace=True
|
||||
)
|
||||
self.assertIn("--metrics_config_enable_op_level_tracing", traced)
|
||||
self.assertEqual(
|
||||
module.option_value(traced, "--communication_collective_profile_path"),
|
||||
"/profiles/real-allreduce.csv",
|
||||
)
|
||||
|
||||
def test_qwen235_state_replay_rejects_implicit_ledger_base(self) -> None:
|
||||
module = load("run_qwen235_fixed_pd_state_replay.py")
|
||||
|
||||
Reference in New Issue
Block a user