gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add OpProf campaign: protocols, results, patches, run evidence (P0-P6)

Browse Source 
Workload-conditioned operator profiling on patched vLLM 0.24.0 +
Qwen3-30B-A3B/H20. H1b PASS (irregular patterns carry +23-45pp R64
raggedness, 8-45% token-efficiency loss vs rectangular controls);
mechanism decomposition kills the padding narrative and finds the
arrival-uniformization artifact (-12.9%); cross-version churn surface
shows TP2/MNS64 -29.4% across vLLM 0.20->0.24 while the argmax held.
Raw Layer-1 JSONL streams (507 MB) stay on disk, git-ignored; footer
sidecars and metrics are tracked.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-07-13 11:06:10 +08:00
parent 607e88da3c
commit d5b276180d
412 changed files with 125056 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
runs/opprof-phase3/phase3/access-blocker-20260712.json Normal file
View File

@@ -0,0 +1,47 @@
{
"schema": 1,
"status": "external_access_blocked",
"host": "dash0",
"last_reachability_probe_local": "2026-07-12T16:16:39+08:00",
"failure": "SSH connection timed out during banner exchange before any remote command executed",
"last_durable_remote_state": {
"observed_before_outage": true,
"controller_pid": 2237019,
"controller_status": "running",
"active_stage": "primary-02-saturation",
"active_stage_phase": "starting_servers",
"completed_measured_runs": 8,
"drain_quarantined_runs": 0,
"clean_window_failures": 0,
"missing_trace_files": 8,
"gpu_hours_total": 6.8156048206885655
},
"resume": {
"state": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/controller-state.json",
"log": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/phase3-matrix-controller.log",
"command": "cd /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712 && /tmp/wjh-opprof-phase2-dash0-20260711/.venv/bin/python scripts/opprof_phase3_matrix.py run --resume",
"rule": "inspect state/PID/GPU ownership first; do not relaunch a live controller"
},
"profile_control_repair": {
"reason": "P03/C01 saturated all data-plane connector slots, so /start_profile timed out before reaching the server",
"change": "dedicated aiohttp TCPConnector(limit=2) and ClientSession for profile control calls",
"old_client_sha256": "a87d92efecd5a8765b51067800b6382f9b174a2ede65f8933fcc9f846ff03d84",
"new_client_sha256": "ab937a5f28252559c2fd97e848a500f1094cef232823ce4b90da8c0ece7554a0",
"remote_record": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/repairs/repair-004-profile-control-connector.json",
"tests": "13/13 unittest PASS; ruff PASS; py_compile PASS"
},
"analysis_ready": {
"script_sha256": "205d9012d9462d84403a0f1435c8a449389aa4bef448d8d6dbced707a11559b0",
"test_sha256": "eac6f55042865c9a24811eef9c1a13c23a26dd1504b54348513d8e6a7ba2b939",
"tests": "4/4 unittest PASS; ruff PASS; py_compile PASS",
"executed_on_matrix": false
},
"not_claimed": [
"matrix completion",
"final quarantine count",
"H1a verdict",
"H1b verdict",
"final GPU hours",
"final GPU cleanup"
]
}

1
runs/opprof-phase3/phase3/analysis/analyze-ap37-final.log Normal file
View File

File diff suppressed because one or more lines are too long

1
runs/opprof-phase3/phase3/analysis/analyze-ap37.log Normal file
View File

File diff suppressed because one or more lines are too long

1
runs/opprof-phase3/phase3/analysis/launch-ap37.log Normal file
View File

@@ -0,0 +1 @@
CPU_ANALYSIS A-P3-7: accepted_runs=40, run_root=/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3, private_manifests=/home/admin/cpfs/wjh/opprof-phase3-private/manifests, bootstrap=100000 seed=20260714, output=runs/phase3/analysis/metrics-ap37.json, expected=10-30m CPU-only, GPU_cost=0

737
runs/opprof-phase3/phase3/controller-state-stop-ap36.json Normal file
View File

@@ -0,0 +1,737 @@
{
"clean_window_failures": 0,
"completed_burnins": 5,
"completed_measured_runs": 40,
"controller_pid": 2438791,
"created_at": 1783833885.960389,
"drain_quarantined_runs": 0,
"fingerprint": {
"cells": [
"P08-C00",
"P03-C10",
"P07-C00",
"P10-C01",
"P01-C10",
"P01-C01",
"P10-C10",
"P03-C01",
"P09-C00",
"P06-C01",
"P10-C11",
"P01-C00",
"P01-C11",
"P10-C00",
"P04-C00",
"P03-C00",
"P06-C10",
"P02-C00",
"P06-C00",
"P06-C11",
"P11-C00",
"P10-C00-TP2",
"P03-C11",
"P05-C00"
],
"client_sha256": "ab937a5f28252559c2fd97e848a500f1094cef232823ce4b90da8c0ece7554a0",
"common_controller_sha256": "95f7169a1771e385aab40fcaecd967dc5cff0c21ea67bdd139382774ba43f01f",
"controller_sha256": "6ac565ff35ead305f7b2e39e6a754389d03c27ea6511b2c9e8ebc0c868c9519f",
"cpu_map": {
"0": "0-19",
"1": "20-39",
"2": "40-59",
"3": "60-79",
"4": "80-99",
"5": "100-119",
"6": "120-139",
"7": "140-159"
},
"manifests": {
"P01": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P01.jsonl",
"rows": 32768,
"sha256": "13ffb226c83373f54c4a7afea6c78cb7cd29720f1858d56728826fc1367b31a4"
},
"P02": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P02.jsonl",
"rows": 32768,
"sha256": "0138ada3fccc98298daee66c26bd1952c987cb42ed8d5341d66b698a597417f9"
},
"P03": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P03.jsonl",
"rows": 32768,
"sha256": "432cfbc26d36f105c179c83f3bb0f3b24b8b3f205788263b4171797f0a4d6fa1"
},
"P04": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P04.jsonl",
"rows": 32768,
"sha256": "caf8d0941b093956a81e1413adc4a4ea9d92460f1b2ed0f6a9b118d0119d6247"
},
"P05": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P05.jsonl",
"rows": 32768,
"sha256": "192e213109f8cb429b99d9eb0f227bb4390fc03f63b02d5456569369bff5a3d7"
},
"P06": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P06.jsonl",
"rows": 32768,
"sha256": "65954bc6e47de9e7be07b8975f97f0bc4639979ef6d33e8c664557ded34b9f96"
},
"P07": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P07.jsonl",
"rows": 32768,
"sha256": "74df66e21a705cd583493199a875e226e93d2da7cfede9bca81dfd9bcb8c9cc6"
},
"P08": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P08.jsonl",
"rows": 32768,
"sha256": "0c4f835aae099265c3eb596d06c6c2fa7070dd280558eb289c602e8c3434dfe9"
},
"P09": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl",
"rows": 32768,
"sha256": "7af92ee3c27dc7d2cf895d6ff3a6e737ec4b6da13d6841ca59e1166f28a0ae1e"
},
"P10": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P10.jsonl",
"rows": 4011,
"sha256": "f51b7a1cc657d62b9ea81823c754408732326b06e03439452433cd8ed481bf33"
},
"P11": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P11.jsonl",
"rows": 32768,
"sha256": "7d196df38963528ff181cf72ce39c8ad913c8f61d40b1425410d3c6c30b6be18"
}
},
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
"source_commit": "4b253fd8619764b6971a7f2e3a3aa7545f6ace05",
"source_tree": "a3d536b287a724e60abbec68b45eed7e088a15d1"
},
"gpu_hours_this_stage": 0.7173924536837472,
"gpu_hours_total": 14.025875418755744,
"missing_trace_files": 8,
"repairs": [
{
"change": "dedicated aiohttp TCPConnector(limit=2) for profile endpoint session; request stream unchanged",
"evidence": "/start_profile connection acquisition timed out before server receipt while 256 data-plane connections were occupied",
"failed_run": "P03-C01-saturation",
"failed_stage": "primary-02-saturation",
"new_client_sha256": "ab937a5f28252559c2fd97e848a500f1094cef232823ce4b90da8c0ece7554a0",
"old_client_sha256": "a87d92efecd5a8765b51067800b6382f9b174a2ede65f8933fcc9f846ff03d84",
"repair": "profile-control-connector-isolation",
"retry": "one exact whole-wave retry; failed directories retained as .interrupted-*",
"schema": 1,
"tests": {
"py_compile": "PASS",
"ruff": "PASS",
"unittest": "13/13 PASS"
},
"timestamp": 1783837763.431071
},
{
"budget": {
"hard_cap": 16.0,
"projected_remaining_h20_hours": 6.3857560787267165,
"projected_total_h20_hours": 14.513826778670154,
"projected_total_with_15pct_remaining_contingency": 15.471690190479162,
"used_h20_hours": 8.128070699943438
},
"cause": "controller required failed_records==0 across excluded profile/recovery time although the registered hard gate is zero clean-window failures",
"change": "replace all_failures_zero with clean-window boundary validation plus failed-record accounting; excluded failures remain reported",
"evidence": {
"clean_completed": 5828,
"clean_failed": 0,
"error_kind": "ServerDisconnectedError",
"excluded_window_failures": 2,
"failure_completion_s": [
373.1505718010012,
373.9740898209857
],
"run": "P01-C01-moderate"
},
"failed_stage": "primary-02-moderate",
"gpu_rerun": false,
"new_controller_sha256": "becfe00889274b51023016b1e7edb866d10e477249504f2032859a4d621f295f",
"old_controller_sha256": "167e48f98f307e16ee44b321068a82a13813b3bf9a4d76882f774f15fe85e595",
"repair": "clean-window-failure-scope",
"schema": 1,
"tests": {
"py_compile": "PASS",
"ruff": "PASS",
"unittest": "14/14 PASS"
},
"timestamp": 1783847092.6920242
},
{
"cause": "single preflight NVML sample observed 4 MiB on GPU0-3 with 0% utilization and no compute apps immediately after a verified cleanup",
"change": "bounded 60s preflight requiring three consecutive samples with zero memory, zero utilization, and no compute apps; no nonzero-memory tolerance",
"failed_stage": "primary-06-saturation",
"gpu_hours_total_unchanged": 11.920448313262728,
"gpu_launch_before_failure": false,
"new_common_controller_sha256": "95f7169a1771e385aab40fcaecd967dc5cff0c21ea67bdd139382774ba43f01f",
"old_common_controller_sha256": "15ad254298a38c4a9318468db89ab32707b6196bb38d5e2a007f2267529397a5",
"repair": "stable-zero-preflight",
"schema": 1,
"tests": {
"py_compile": "PASS",
"ruff": "PASS",
"unittest": "15/15 PASS"
},
"timestamp": 1783850898.1496143
},
{
"code_change": false,
"gpu_hours_used": 12.606695837948058,
"hard_cap": 16.0,
"projected_remaining_h20_hours": 2.1455133807990285,
"projected_total_with_15pct_contingency": 15.07403622586694,
"protocol_action": "single exact whole-wave infrastructure retry under identical 4-way placement; gate unchanged",
"reason": "first TP2 P10 inference-shape cold start/autotune after readiness yielded 15 successful warmup completions against required 32; clean window and artifacts otherwise valid",
"repair": "tp2-warmup-exact-retry",
"run": "P10-C00-TP2-saturation",
"schema": 1,
"stage": "primary-06-saturation",
"timestamp": 1783851710.3100822,
"warmup_required": 32,
"warmup_success": 15
},
{
"accepted_runs_preserved": 40,
"amendment": "A-P3-6",
"changed_fingerprint_keys": [
"controller_sha256"
],
"created_at": 1783853258.3098204,
"failed_stage_preserved": "primary-06-saturation",
"new_controller_sha256": "6ac565ff35ead305f7b2e39e6a754389d03c27ea6511b2c9e8ebc0c868c9519f",
"old_controller_sha256": "becfe00889274b51023016b1e7edb866d10e477249504f2032859a4d621f295f",
"projected_remaining_h20_hours": 2.1,
"projected_total_h20_hours": 15.408482965071997,
"repair_id": "repair-008-ap36",
"retained_attempt_re_adjudicated": false,
"retained_attempt_reason": "21 completions but A-P3-6 normalized drift 2.3385345997286295 and bin step counts 13/9/44",
"schema": 1
}
],
"schema": 1,
"stages": {
"burnin-01": {
"assignments": [
{
"cell": "P06-C00",
"gpus": [
0
]
},
{
"cell": "P06-C10",
"gpus": [
1
]
},
{
"cell": "P06-C01",
"gpus": [
2
]
},
{
"cell": "P06-C11",
"gpus": [
3
]
}
],
"burnin": true,
"clients": {},
"completed_at": 1783834288.1238313,
"confirmation": false,
"gpu_hours": 0.30808498481909435,
"load_point": "saturation",
"profile": false,
"servers": {},
"started_at": 1783833886.4448946,
"status": "complete",
"validation_attempt1_failure": "RuntimeError(\"server config/backend failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/burnins/C01: {'triton_moe': True, 'chunked_mbt': False, 'tp_effective': True, 'drain_shutdown': True, 'mns_effective': True}\")"
},
"burnin-02": {
"assignments": [
{
"cell": "P06-C00-TP2",
"gpus": [
0,
1
]
}
],
"burnin": true,
"clients": {},
"completed_at": 1783834671.921921,
"confirmation": false,
"gpu_hours": 0.19903395679261948,
"load_point": "saturation",
"profile": false,
"servers": {},
"started_at": 1783834301.7813473,
"status": "complete"
},
"primary-01-moderate": {
"assignments": [
{
"cell": "P08-C00",
"gpus": [
0
]
},
{
"cell": "P03-C10",
"gpus": [
1
]
},
{
"cell": "P07-C00",
"gpus": [
2
]
},
{
"cell": "P10-C01",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {},
"completed_at": 1783836723.3639715,
"confirmation": false,
"gpu_hours": 0.5195140059126748,
"load_point": "moderate",
"profile": true,
"servers": {},
"started_at": 1783836150.2169476,
"status": "complete",
"validation_attempt1_failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P10-C01/moderate: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'all_failures_zero': True, 'manifest_no_wrap': True, 'warmup_completions': False, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]\")",
"warmup_feasibility": "P10 target=min(32,floor(0.445*60))=26; observed=27"
},
"primary-01-saturation": {
"assignments": [
{
"cell": "P08-C00",
"gpus": [
0
]
},
{
"cell": "P03-C10",
"gpus": [
1
]
},
{
"cell": "P07-C00",
"gpus": [
2
]
},
{
"cell": "P10-C01",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {},
"completed_at": 1783836130.8498049,
"confirmation": false,
"drain_readjudication": "P10/C01 288.619107924s <= 600s",
"gpu_hours": 0.8836704309119119,
"load_point": "saturation",
"missing_trace_files": 8,
"profile": true,
"servers": {},
"started_at": 1783834671.9563339,
"status": "complete",
"validation_attempt1_failure": "RuntimeError(\"client failures: {'P10-C01-saturation': 1}\")"
},
"primary-02-moderate": {
"assignments": [
{
"cell": "P01-C10",
"gpus": [
0
]
},
{
"cell": "P01-C01",
"gpus": [
1
]
},
{
"cell": "P10-C10",
"gpus": [
2
]
},
{
"cell": "P03-C01",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {},
"completed_at": 1783847092.7229948,
"confirmation": false,
"failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P01-C01/moderate: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'all_failures_zero': False, 'manifest_no_wrap': True, 'warmup_completions': True, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]\")",
"gpu_hours": 0.5151156750652525,
"load_point": "moderate",
"profile": true,
"readjudicated_without_gpu": true,
"servers": {},
"started_at": 1783838513.4472992,
"status": "complete",
"validation_attempt1_failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P01-C01/moderate: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'all_failures_zero': False, 'manifest_no_wrap': True, 'warmup_completions': True, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]\")"
},
"primary-02-saturation": {
"assignments": [
{
"cell": "P01-C10",
"gpus": [
0
]
},
{
"cell": "P01-C01",
"gpus": [
1
]
},
{
"cell": "P10-C10",
"gpus": [
2
]
},
{
"cell": "P03-C01",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {},
"completed_at": 1783838513.4027848,
"confirmation": false,
"gpu_hours": 0.7973502041896184,
"load_point": "saturation",
"profile": true,
"servers": {},
"started_at": 1783837773.6088765,
"status": "complete"
},
"primary-03-moderate": {
"assignments": [
{
"cell": "P09-C00",
"gpus": [
0
]
},
{
"cell": "P06-C01",
"gpus": [
1
]
},
{
"cell": "P10-C11",
"gpus": [
2
]
},
{
"cell": "P01-C00",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {},
"completed_at": 1783848369.3507695,
"confirmation": false,
"gpu_hours": 0.5205460974905226,
"load_point": "moderate",
"profile": true,
"servers": {},
"started_at": 1783847881.06146,
"status": "complete"
},
"primary-03-saturation": {
"assignments": [
{
"cell": "P09-C00",
"gpus": [
0
]
},
{
"cell": "P06-C01",
"gpus": [
1
]
},
{
"cell": "P10-C11",
"gpus": [
2
]
},
{
"cell": "P01-C00",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {},
"completed_at": 1783847881.0197804,
"confirmation": false,
"gpu_hours": 0.8273645816908942,
"load_point": "saturation",
"profile": true,
"servers": {},
"started_at": 1783847109.5683346,
"status": "complete"
},
"primary-04-moderate": {
"assignments": [
{
"cell": "P01-C11",
"gpus": [
0
]
},
{
"cell": "P10-C00",
"gpus": [
1
]
},
{
"cell": "P04-C00",
"gpus": [
2
]
},
{
"cell": "P03-C00",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {},
"completed_at": 1783849583.5228572,
"confirmation": false,
"gpu_hours": 0.5205216948853598,
"load_point": "moderate",
"profile": true,
"servers": {},
"started_at": 1783849096.4363403,
"status": "complete"
},
"primary-04-saturation": {
"assignments": [
{
"cell": "P01-C11",
"gpus": [
0
]
},
{
"cell": "P10-C00",
"gpus": [
1
]
},
{
"cell": "P04-C00",
"gpus": [
2
]
},
{
"cell": "P03-C00",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {},
"completed_at": 1783849096.3948114,
"confirmation": false,
"gpu_hours": 0.7872003297011058,
"load_point": "saturation",
"profile": true,
"servers": {},
"started_at": 1783848369.390857,
"status": "complete"
},
"primary-05-moderate": {
"assignments": [
{
"cell": "P06-C10",
"gpus": [
0
]
},
{
"cell": "P02-C00",
"gpus": [
1
]
},
{
"cell": "P06-C00",
"gpus": [
2
]
},
{
"cell": "P06-C11",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {},
"completed_at": 1783850641.6781306,
"confirmation": false,
"gpu_hours": 0.5138198028008143,
"load_point": "moderate",
"profile": true,
"servers": {},
"started_at": 1783850163.1211865,
"status": "complete"
},
"primary-05-saturation": {
"assignments": [
{
"cell": "P06-C10",
"gpus": [
0
]
},
{
"cell": "P02-C00",
"gpus": [
1
]
},
{
"cell": "P06-C00",
"gpus": [
2
]
},
{
"cell": "P06-C11",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {},
"completed_at": 1783850163.0472832,
"confirmation": false,
"gpu_hours": 0.6229251067505942,
"load_point": "saturation",
"profile": true,
"servers": {},
"started_at": 1783849583.5652869,
"status": "complete"
},
"primary-06-saturation": {
"assignments": [
{
"cell": "P11-C00",
"gpus": [
0
]
},
{
"cell": "P10-C00-TP2",
"gpus": [
1,
2
]
},
{
"cell": "P03-C11",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {
"P03-C11-saturation": {
"pgid": 2440965,
"pid": 2440965
},
"P10-C00-TP2-saturation": {
"pgid": 2440964,
"pid": 2440964
},
"P11-C00-saturation": {
"pgid": 2440962,
"pid": 2440962
}
},
"confirmation": false,
"failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P10-C00-TP2/saturation: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'failed_records_accounted': True, 'manifest_no_wrap': True, 'warmup_completions': False, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]; warmup_completions=17; warmup_gate_branch=failed; warmup_stability={'passed': False, 'reason': 'A-P3-6 stabilization criterion not met', 'window_seconds': [45.0, 60.0], 'bin_seconds': 5.0, 'step_counts': [11, 10, 16], 'scheduled_tokens': [90112, 81920, 70313], 'scheduled_token_throughput': [18022.4, 16384.0, 14062.6], 'mean_scheduled_token_throughput': 16156.333333333334, 'slope_tokens_per_second_squared': -395.98000000000013, 'normalized_drift': 0.367639109533929, 'normalized_drift_limit': 0.1, 'step_indices_continuous': True}\")",
"gpu_hours": 0.7173924536837472,
"load_point": "saturation",
"profile": true,
"servers": {
"P03-C11-saturation": {
"gpus": [
3
],
"pgid": 2438868,
"pid": 2438868
},
"P10-C00-TP2-saturation": {
"gpus": [
1,
2
],
"pgid": 2438867,
"pid": 2438867
},
"P11-C00-saturation": {
"gpus": [
0
],
"pgid": 2438866,
"pid": 2438866
}
},
"started_at": 1783853279.9695158,
"status": "failed"
}
},
"status": "failed",
"updated_at": 1783853946.066117
}

39015
runs/opprof-phase3/phase3/metrics.json Normal file
View File

File diff suppressed because it is too large Load Diff

103
runs/opprof-phase3/phase3/remote-evidence/P03-C10/result.json Normal file
View File

@@ -0,0 +1,103 @@
{
"admission_stop_s": 417.3262625900097,
"arrival": "steady",
"clean": {
"admitted": 559,
"completed": 559,
"completed_throughput_rps": 2.3291666666666666,
"duration_s": 240.0,
"end_s": 300.0,
"failed": 0,
"input_tokens": 3429894,
"offered_rps": 2.3291666666666666,
"output_tokens": 35776,
"start_s": 60.0
},
"clean_segment_seconds": 80.0,
"drain_seconds": 91.29178713698639,
"elapsed_seconds": 508.6180497269961,
"failed_records": 0,
"load_point": "saturation",
"manifest_admitted": 1094,
"manifest_exhausted": false,
"manifest_rows": 32768,
"manifest_sha256": "432cfbc26d36f105c179c83f3bb0f3b24b8b3f205788263b4171797f0a4d6fa1",
"manifest_wrapped": false,
"max_in_flight": 256,
"num_clean_segments": 3,
"profiles": [
{
"start_call_s": 300.0043962339987,
"start_return_s": 301.3819164079905,
"start_status": 200,
"stop_call_s": 321.8697677869932,
"stop_return_s": 326.31530767300865,
"stop_status": 200,
"trace_file": "dp0_pp0_tp0_dcp0_ep0_rank0.1783835059474707213.pt.trace.json.gz",
"trace_ready_s": 321.86976291501196,
"trace_sha256": "527ddb81b1d54a12a463508bc163d85a2c443d94c8a897959a39735e030b90a2",
"window": 1
},
{
"start_call_s": 356.32926020701416,
"start_return_s": 361.34978177401354,
"start_status": 200,
"stop_call_s": 381.577829301008,
"stop_return_s": 387.31277802900877,
"stop_status": 200,
"trace_file": "dp0_pp0_tp0_dcp0_ep0_rank0.1783835119198018909.pt.trace.json.gz",
"trace_ready_s": 381.57782580400817,
"trace_sha256": "0882d9705f1208580d5e693f94ba64d89ab12ef0aae2731a82d460a46a3e4cb1",
"window": 2
}
],
"rate_fraction": null,
"records": 1094,
"request_rate": "inf",
"schema": 1,
"segments": [
{
"admitted": 191,
"completed": 191,
"completed_throughput_rps": 2.3875,
"duration_s": 80.0,
"end_s": 140.0,
"failed": 0,
"input_tokens": 1154628,
"name": "A",
"offered_rps": 2.3875,
"output_tokens": 12224,
"start_s": 60.0
},
{
"admitted": 184,
"completed": 184,
"completed_throughput_rps": 2.3,
"duration_s": 80.0,
"end_s": 220.0,
"failed": 0,
"input_tokens": 1135217,
"name": "B",
"offered_rps": 2.3,
"output_tokens": 11776,
"start_s": 140.0
},
{
"admitted": 184,
"completed": 184,
"completed_throughput_rps": 2.3,
"duration_s": 80.0,
"end_s": 300.0,
"failed": 0,
"input_tokens": 1140049,
"name": "C",
"offered_rps": 2.3,
"output_tokens": 11776,
"start_s": 220.0
}
],
"successful_records": 1094,
"t0_mono_ns": 166323993366724,
"t0_wall_ns": 1783834751119796663,
"warmup_seconds": 60.0
}

65
runs/opprof-phase3/phase3/remote-evidence/P03-C10/sanity.json Normal file
View File

@@ -0,0 +1,65 @@
{
"invariants": {
"clean_duration_exact": true,
"clean_failures_zero": true,
"concurrency_bounded": true,
"drain_within_timeout": true,
"manifest_no_wrap": true,
"manifest_not_exhausted": true,
"output_tokens_exact": true,
"profile_count_exact": true,
"profile_status_ok": true,
"segment_count_exact": true
},
"numeric": {
"actual_output_tokens": {
"distinct_n": 1,
"finite_n": 1094,
"max": 64.0,
"min": 64.0,
"missing_n": 0,
"n": 1094
},
"admitted_s": {
"distinct_n": 1094,
"finite_n": 1094,
"max": 416.7779763180006,
"min": 0.0008771540015004575,
"missing_n": 0,
"n": 1094
},
"completed_s": {
"distinct_n": 1094,
"finite_n": 1094,
"max": 508.61568894199445,
"min": 22.18080371801625,
"missing_n": 0,
"n": 1094
},
"input_tokens": {
"distinct_n": 964,
"finite_n": 1094,
"max": 8190.0,
"min": 4097.0,
"missing_n": 0,
"n": 1094
},
"requested_output_tokens": {
"distinct_n": 1,
"finite_n": 1094,
"max": 64.0,
"min": 64.0,
"missing_n": 0,
"n": 1094
},
"scheduled_s": {
"distinct_n": 1094,
"finite_n": 1094,
"max": 416.7779758319957,
"min": 0.0008756940078455955,
"missing_n": 0,
"n": 1094
}
},
"schema": 1
}

103
runs/opprof-phase3/phase3/remote-evidence/P07-C00/result.json Normal file
View File

@@ -0,0 +1,103 @@
{
"admission_stop_s": 417.66287675101194,
"arrival": "burst:8",
"clean": {
"admitted": 1226,
"completed": 1226,
"completed_throughput_rps": 5.108333333333333,
"duration_s": 240.0,
"end_s": 300.0,
"failed": 0,
"input_tokens": 1569280,
"offered_rps": 5.108333333333333,
"output_tokens": 627712,
"start_s": 60.0
},
"clean_segment_seconds": 80.0,
"drain_seconds": 26.098238860984566,
"elapsed_seconds": 443.7611156119965,
"failed_records": 0,
"load_point": "saturation",
"manifest_admitted": 2078,
"manifest_exhausted": false,
"manifest_rows": 32768,
"manifest_sha256": "74df66e21a705cd583493199a875e226e93d2da7cfede9bca81dfd9bcb8c9cc6",
"manifest_wrapped": false,
"max_in_flight": 256,
"num_clean_segments": 3,
"profiles": [
{
"start_call_s": 300.0039349270228,
"start_return_s": 300.9924320260179,
"start_status": 200,
"stop_call_s": 323.90002606701455,
"stop_return_s": 328.078528626007,
"stop_status": 200,
"trace_file": "dp0_pp0_tp0_dcp0_ep0_rank0.1783835058771060551.pt.trace.json.gz",
"trace_ready_s": 323.9000201699964,
"trace_sha256": "2d12038b03c6f9c28d1a488abd880ff2763e1f3c0ab502cfdd2507b649de2af7",
"window": 1
},
{
"start_call_s": 358.0940292410087,
"start_return_s": 358.8905565890018,
"start_status": 200,
"stop_call_s": 382.04741394601297,
"stop_return_s": 387.64796055300394,
"stop_status": 200,
"trace_file": "dp0_pp0_tp0_dcp0_ep0_rank0.1783835116964699401.pt.trace.json.gz",
"trace_ready_s": 382.04741024502437,
"trace_sha256": "2184d4f83ea22c0e5455dd3914922548ce8d3d5eba34f89562df69326e3832b5",
"window": 2
}
],
"rate_fraction": null,
"records": 2078,
"request_rate": "inf",
"schema": 1,
"segments": [
{
"admitted": 394,
"completed": 394,
"completed_throughput_rps": 4.925,
"duration_s": 80.0,
"end_s": 140.0,
"failed": 0,
"input_tokens": 504320,
"name": "A",
"offered_rps": 4.925,
"output_tokens": 201728,
"start_s": 60.0
},
{
"admitted": 397,
"completed": 397,
"completed_throughput_rps": 4.9625,
"duration_s": 80.0,
"end_s": 220.0,
"failed": 0,
"input_tokens": 508160,
"name": "B",
"offered_rps": 4.9625,
"output_tokens": 203264,
"start_s": 140.0
},
{
"admitted": 435,
"completed": 435,
"completed_throughput_rps": 5.4375,
"duration_s": 80.0,
"end_s": 300.0,
"failed": 0,
"input_tokens": 556800,
"name": "C",
"offered_rps": 5.4375,
"output_tokens": 222720,
"start_s": 220.0
}
],
"successful_records": 2078,
"t0_mono_ns": 166323999932716,
"t0_wall_ns": 1783834751126361482,
"warmup_seconds": 60.0
}

65
runs/opprof-phase3/phase3/remote-evidence/P07-C00/sanity.json Normal file
View File

@@ -0,0 +1,65 @@
{
"invariants": {
"clean_duration_exact": true,
"clean_failures_zero": true,
"concurrency_bounded": true,
"drain_within_timeout": true,
"manifest_no_wrap": true,
"manifest_not_exhausted": true,
"output_tokens_exact": true,
"profile_count_exact": true,
"profile_status_ok": true,
"segment_count_exact": true
},
"numeric": {
"actual_output_tokens": {
"distinct_n": 1,
"finite_n": 2078,
"max": 512.0,
"min": 512.0,
"missing_n": 0,
"n": 2078
},
"admitted_s": {
"distinct_n": 2078,
"finite_n": 2078,
"max": 417.40899016702315,
"min": 0.0008179180149454623,
"missing_n": 0,
"n": 2078
},
"completed_s": {
"distinct_n": 2078,
"finite_n": 2078,
"max": 443.75592052700813,
"min": 35.15151289102505,
"missing_n": 0,
"n": 2078
},
"input_tokens": {
"distinct_n": 1,
"finite_n": 2078,
"max": 1280.0,
"min": 1280.0,
"missing_n": 0,
"n": 2078
},
"requested_output_tokens": {
"distinct_n": 1,
"finite_n": 2078,
"max": 512.0,
"min": 512.0,
"missing_n": 0,
"n": 2078
},
"scheduled_s": {
"distinct_n": 2078,
"finite_n": 2078,
"max": 417.4089898300008,
"min": 0.0008165129984263331,
"missing_n": 0,
"n": 2078
}
},
"schema": 1
}

103
runs/opprof-phase3/phase3/remote-evidence/P08-C00/result.json Normal file
View File

@@ -0,0 +1,103 @@
{
"admission_stop_s": 493.2576180040196,
"arrival": "burst:8",
"clean": {
"admitted": 2233,
"completed": 2233,
"completed_throughput_rps": 9.304166666666667,
"duration_s": 240.0,
"end_s": 300.0,
"failed": 0,
"input_tokens": 2858240,
"offered_rps": 9.304166666666667,
"output_tokens": 1143296,
"start_s": 60.0
},
"clean_segment_seconds": 80.0,
"drain_seconds": 14.923095890000695,
"elapsed_seconds": 508.1807138940203,
"failed_records": 0,
"load_point": "saturation",
"manifest_admitted": 3941,
"manifest_exhausted": false,
"manifest_rows": 32768,
"manifest_sha256": "0c4f835aae099265c3eb596d06c6c2fa7070dd280558eb289c602e8c3434dfe9",
"manifest_wrapped": false,
"max_in_flight": 256,
"num_clean_segments": 3,
"profiles": [
{
"start_call_s": 300.0054247789958,
"start_return_s": 323.45228312400286,
"start_status": 200,
"stop_call_s": 354.6066670610162,
"stop_return_s": 381.28106322701205,
"stop_status": 200,
"trace_file": "dp0_pp0_tp0_dcp0_ep0_rank0.1783835082061801278.pt.trace.json.gz",
"trace_ready_s": 354.6066622030048,
"trace_sha256": "53d36238e82e5d031d8ed7841094c56d715b7cd5009368b64e380bf89a0f0dec",
"window": 1
},
{
"start_call_s": 411.30330635100836,
"start_return_s": 411.4401313569979,
"start_status": 200,
"stop_call_s": 436.0742275560042,
"stop_return_s": 463.23921855099616,
"stop_status": 200,
"trace_file": "dp0_pp0_tp0_dcp0_ep0_rank0.1783835169024306574.pt.trace.json.gz",
"trace_ready_s": 436.0742245099973,
"trace_sha256": "e088bd800a4e43446effbbcd4fce7d9ba1defaceb378a6e0db04d64780e1d09a",
"window": 2
}
],
"rate_fraction": null,
"records": 3941,
"request_rate": "inf",
"schema": 1,
"segments": [
{
"admitted": 697,
"completed": 697,
"completed_throughput_rps": 8.7125,
"duration_s": 80.0,
"end_s": 140.0,
"failed": 0,
"input_tokens": 892160,
"name": "A",
"offered_rps": 8.7125,
"output_tokens": 356864,
"start_s": 60.0
},
{
"admitted": 768,
"completed": 768,
"completed_throughput_rps": 9.6,
"duration_s": 80.0,
"end_s": 220.0,
"failed": 0,
"input_tokens": 983040,
"name": "B",
"offered_rps": 9.6,
"output_tokens": 393216,
"start_s": 140.0
},
{
"admitted": 768,
"completed": 768,
"completed_throughput_rps": 9.6,
"duration_s": 80.0,
"end_s": 300.0,
"failed": 0,
"input_tokens": 983040,
"name": "C",
"offered_rps": 9.6,
"output_tokens": 393216,
"start_s": 220.0
}
],
"successful_records": 3941,
"t0_mono_ns": 166324009670273,
"t0_wall_ns": 1783834751136099481,
"warmup_seconds": 60.0
}

65
runs/opprof-phase3/phase3/remote-evidence/P08-C00/sanity.json Normal file
View File

@@ -0,0 +1,65 @@
{
"invariants": {
"clean_duration_exact": true,
"clean_failures_zero": true,
"concurrency_bounded": true,
"drain_within_timeout": true,
"manifest_no_wrap": true,
"manifest_not_exhausted": true,
"output_tokens_exact": true,
"profile_count_exact": true,
"profile_status_ok": true,
"segment_count_exact": true
},
"numeric": {
"actual_output_tokens": {
"distinct_n": 1,
"finite_n": 3941,
"max": 512.0,
"min": 512.0,
"missing_n": 0,
"n": 3941
},
"admitted_s": {
"distinct_n": 3941,
"finite_n": 3941,
"max": 493.25117621201207,
"min": 0.0009094980196096003,
"missing_n": 0,
"n": 3941
},
"completed_s": {
"distinct_n": 3941,
"finite_n": 3941,
"max": 508.1755032700021,
"min": 30.030000179016497,
"missing_n": 0,
"n": 3941
},
"input_tokens": {
"distinct_n": 1,
"finite_n": 3941,
"max": 1280.0,
"min": 1280.0,
"missing_n": 0,
"n": 3941
},
"requested_output_tokens": {
"distinct_n": 1,
"finite_n": 3941,
"max": 512.0,
"min": 512.0,
"missing_n": 0,
"n": 3941
},
"scheduled_s": {
"distinct_n": 3941,
"finite_n": 3941,
"max": 493.25117589501315,
"min": 0.000907983019715175,
"missing_n": 0,
"n": 3941
}
},
"schema": 1
}

103
runs/opprof-phase3/phase3/remote-evidence/P10-C01/result.json Normal file
View File

@@ -0,0 +1,103 @@
{
"admission_stop_s": 420.6437278209778,
"arrival": "steady",
"clean": {
"admitted": 178,
"completed": 178,
"completed_throughput_rps": 0.7416666666666667,
"duration_s": 240.0,
"end_s": 300.0,
"failed": 0,
"input_tokens": 1755158,
"offered_rps": 0.7416666666666667,
"output_tokens": 40658,
"start_s": 60.0
},
"clean_segment_seconds": 80.0,
"drain_seconds": 288.6191079240234,
"elapsed_seconds": 709.2628357450012,
"failed_records": 0,
"load_point": "saturation",
"manifest_admitted": 558,
"manifest_exhausted": false,
"manifest_rows": 4011,
"manifest_sha256": "f51b7a1cc657d62b9ea81823c754408732326b06e03439452433cd8ed481bf33",
"manifest_wrapped": false,
"max_in_flight": 256,
"num_clean_segments": 3,
"profiles": [
{
"start_call_s": 300.0035294489935,
"start_return_s": 310.3520467719936,
"start_status": 200,
"stop_call_s": 325.0655262139917,
"stop_return_s": 338.54296391498065,
"stop_status": 200,
"trace_file": "dp0_pp0_tp0_dcp0_ep0_rank0.1783835066783986388.pt.trace.json.gz",
"trace_ready_s": 325.0655210709956,
"trace_sha256": "d457af416fa91f44e617c7259167fca467bea8466a3e4c4b916f811fb5899c73",
"window": 1
},
{
"start_call_s": 368.55468072599615,
"start_return_s": 372.31818850699347,
"start_status": 200,
"stop_call_s": 386.0271748309897,
"stop_return_s": 390.6334540609969,
"stop_status": 200,
"trace_file": "dp0_pp0_tp0_dcp0_ep0_rank0.1783835128341345321.pt.trace.json.gz",
"trace_ready_s": 386.02717122397735,
"trace_sha256": "41e8b34a0464cfd92d16b21c6242113b23594081697f5d1e88cdc3647542a96a",
"window": 2
}
],
"rate_fraction": null,
"records": 558,
"request_rate": "inf",
"schema": 1,
"segments": [
{
"admitted": 48,
"completed": 48,
"completed_throughput_rps": 0.6,
"duration_s": 80.0,
"end_s": 140.0,
"failed": 0,
"input_tokens": 548329,
"name": "A",
"offered_rps": 0.6,
"output_tokens": 11027,
"start_s": 60.0
},
{
"admitted": 70,
"completed": 70,
"completed_throughput_rps": 0.875,
"duration_s": 80.0,
"end_s": 220.0,
"failed": 0,
"input_tokens": 574251,
"name": "B",
"offered_rps": 0.875,
"output_tokens": 16298,
"start_s": 140.0
},
{
"admitted": 60,
"completed": 60,
"completed_throughput_rps": 0.75,
"duration_s": 80.0,
"end_s": 300.0,
"failed": 0,
"input_tokens": 632578,
"name": "C",
"offered_rps": 0.75,
"output_tokens": 13333,
"start_s": 220.0
}
],
"successful_records": 558,
"t0_mono_ns": 166325149689000,
"t0_wall_ns": 1783834752276118250,
"warmup_seconds": 60.0
}

65
runs/opprof-phase3/phase3/remote-evidence/P10-C01/sanity.json Normal file
View File

@@ -0,0 +1,65 @@
{
"invariants": {
"clean_duration_exact": true,
"clean_failures_zero": true,
"concurrency_bounded": true,
"drain_within_timeout": false,
"manifest_no_wrap": true,
"manifest_not_exhausted": true,
"output_tokens_exact": true,
"profile_count_exact": true,
"profile_status_ok": true,
"segment_count_exact": true
},
"numeric": {
"actual_output_tokens": {
"distinct_n": 92,
"finite_n": 558,
"max": 256.0,
"min": 6.0,
"missing_n": 0,
"n": 558
},
"admitted_s": {
"distinct_n": 558,
"finite_n": 558,
"max": 420.08466253298684,
"min": 0.000850101001560688,
"missing_n": 0,
"n": 558
},
"completed_s": {
"distinct_n": 558,
"finite_n": 558,
"max": 709.2613093179825,
"min": 14.001143716974184,
"missing_n": 0,
"n": 558
},
"input_tokens": {
"distinct_n": 535,
"finite_n": 558,
"max": 32527.0,
"min": 70.0,
"missing_n": 0,
"n": 558
},
"requested_output_tokens": {
"distinct_n": 92,
"finite_n": 558,
"max": 256.0,
"min": 6.0,
"missing_n": 0,
"n": 558
},
"scheduled_s": {
"distinct_n": 558,
"finite_n": 558,
"max": 420.08466206499725,
"min": 0.0008485929865855724,
"missing_n": 0,
"n": 558
}
},
"schema": 1
}

258
runs/opprof-phase3/phase3/remote-evidence/controller-state.json Normal file
View File

@@ -0,0 +1,258 @@
{
"completed_burnins": 5,
"completed_measured_runs": 0,
"controller_pid": 2187729,
"created_at": 1783833885.960389,
"fingerprint": {
"cells": [
"P08-C00",
"P03-C10",
"P07-C00",
"P10-C01",
"P01-C10",
"P01-C01",
"P10-C10",
"P03-C01",
"P09-C00",
"P06-C01",
"P10-C11",
"P01-C00",
"P01-C11",
"P10-C00",
"P04-C00",
"P03-C00",
"P06-C10",
"P02-C00",
"P06-C00",
"P06-C11",
"P11-C00",
"P10-C00-TP2",
"P03-C11",
"P05-C00"
],
"client_sha256": "a87d92efecd5a8765b51067800b6382f9b174a2ede65f8933fcc9f846ff03d84",
"common_controller_sha256": "15ad254298a38c4a9318468db89ab32707b6196bb38d5e2a007f2267529397a5",
"controller_sha256": "c2d3232fb99c66cea55e30e6cb1aa6a84d3a816434ed8155a309f41df2837f73",
"cpu_map": {
"0": "0-19",
"1": "20-39",
"2": "40-59",
"3": "60-79",
"4": "80-99",
"5": "100-119",
"6": "120-139",
"7": "140-159"
},
"manifests": {
"P01": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P01.jsonl",
"rows": 32768,
"sha256": "13ffb226c83373f54c4a7afea6c78cb7cd29720f1858d56728826fc1367b31a4"
},
"P02": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P02.jsonl",
"rows": 32768,
"sha256": "0138ada3fccc98298daee66c26bd1952c987cb42ed8d5341d66b698a597417f9"
},
"P03": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P03.jsonl",
"rows": 32768,
"sha256": "432cfbc26d36f105c179c83f3bb0f3b24b8b3f205788263b4171797f0a4d6fa1"
},
"P04": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P04.jsonl",
"rows": 32768,
"sha256": "caf8d0941b093956a81e1413adc4a4ea9d92460f1b2ed0f6a9b118d0119d6247"
},
"P05": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P05.jsonl",
"rows": 32768,
"sha256": "192e213109f8cb429b99d9eb0f227bb4390fc03f63b02d5456569369bff5a3d7"
},
"P06": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P06.jsonl",
"rows": 32768,
"sha256": "65954bc6e47de9e7be07b8975f97f0bc4639979ef6d33e8c664557ded34b9f96"
},
"P07": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P07.jsonl",
"rows": 32768,
"sha256": "74df66e21a705cd583493199a875e226e93d2da7cfede9bca81dfd9bcb8c9cc6"
},
"P08": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P08.jsonl",
"rows": 32768,
"sha256": "0c4f835aae099265c3eb596d06c6c2fa7070dd280558eb289c602e8c3434dfe9"
},
"P09": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl",
"rows": 32768,
"sha256": "7af92ee3c27dc7d2cf895d6ff3a6e737ec4b6da13d6841ca59e1166f28a0ae1e"
},
"P10": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P10.jsonl",
"rows": 4011,
"sha256": "f51b7a1cc657d62b9ea81823c754408732326b06e03439452433cd8ed481bf33"
},
"P11": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P11.jsonl",
"rows": 32768,
"sha256": "7d196df38963528ff181cf72ce39c8ad913c8f61d40b1425410d3c6c30b6be18"
}
},
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
"source_commit": "4b253fd8619764b6971a7f2e3a3aa7545f6ace05",
"source_tree": "a3d536b287a724e60abbec68b45eed7e088a15d1"
},
"gpu_hours_this_stage": 0.8836704309119119,
"gpu_hours_total": 5.473521912336349,
"schema": 1,
"stages": {
"burnin-01": {
"assignments": [
{
"cell": "P06-C00",
"gpus": [
0
]
},
{
"cell": "P06-C10",
"gpus": [
1
]
},
{
"cell": "P06-C01",
"gpus": [
2
]
},
{
"cell": "P06-C11",
"gpus": [
3
]
}
],
"burnin": true,
"clients": {},
"completed_at": 1783834288.1238313,
"confirmation": false,
"gpu_hours": 0.30808498481909435,
"load_point": "saturation",
"profile": false,
"servers": {},
"started_at": 1783833886.4448946,
"status": "complete",
"validation_attempt1_failure": "RuntimeError(\"server config/backend failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/burnins/C01: {'triton_moe': True, 'chunked_mbt': False, 'tp_effective': True, 'drain_shutdown': True, 'mns_effective': True}\")"
},
"burnin-02": {
"assignments": [
{
"cell": "P06-C00-TP2",
"gpus": [
0,
1
]
}
],
"burnin": true,
"clients": {},
"completed_at": 1783834671.921921,
"confirmation": false,
"gpu_hours": 0.19903395679261948,
"load_point": "saturation",
"profile": false,
"servers": {},
"started_at": 1783834301.7813473,
"status": "complete"
},
"primary-01-saturation": {
"assignments": [
{
"cell": "P08-C00",
"gpus": [
0
]
},
{
"cell": "P03-C10",
"gpus": [
1
]
},
{
"cell": "P07-C00",
"gpus": [
2
]
},
{
"cell": "P10-C01",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {
"P03-C10-saturation": {
"pgid": 2195599,
"pid": 2195599
},
"P07-C00-saturation": {
"pgid": 2195600,
"pid": 2195600
},
"P08-C00-saturation": {
"pgid": 2195597,
"pid": 2195597
},
"P10-C01-saturation": {
"pgid": 2195601,
"pid": 2195601
}
},
"confirmation": false,
"failure": "RuntimeError(\"client failures: {'P10-C01-saturation': 1}\")",
"gpu_hours": 0.8836704309119119,
"load_point": "saturation",
"profile": true,
"servers": {
"P03-C10-saturation": {
"gpus": [
1
],
"pgid": 2193680,
"pid": 2193680
},
"P07-C00-saturation": {
"gpus": [
2
],
"pgid": 2193681,
"pid": 2193681
},
"P08-C00-saturation": {
"gpus": [
0
],
"pgid": 2193679,
"pid": 2193679
},
"P10-C01-saturation": {
"gpus": [
3
],
"pgid": 2193682,
"pid": 2193682
}
},
"started_at": 1783834671.9563339,
"status": "failed"
}
},
"status": "failed",
"updated_at": 1783835479.1651883
}

16
runs/opprof-phase3/phase3/remote-evidence/repair-001-explicit-mbt-log.json Normal file
View File

@@ -0,0 +1,16 @@
{
"at": 1783834288.13871,
"finding": "Explicit MBT=2048 is logged in non-default args; scheduler.py emits the separate Chunked-prefill line only for default MBT.",
"gpu_rerun": false,
"kind": "validator-only",
"new_controller_sha256": "c2d3232fb99c66cea55e30e6cb1aa6a84d3a816434ed8155a309f41df2837f73",
"old_controller_sha256": "74859614a8da3341bf827bb75b667db7d219f059d1d29c1182b9a77c2ee59e92",
"protocol_or_workload_changed": false,
"revalidated_runs": [
"P06-C00-burnin",
"P06-C10-burnin",
"P06-C01-burnin",
"P06-C11-burnin"
],
"schema": 1
}

355
runs/opprof-phase3/phase3/remote-evidence/sidecar-verification-state.json Normal file
View File

@@ -0,0 +1,355 @@
{
"gpu_seconds": 427.43714332580566,
"results": {
"graceful": {
"accounting": {
"bytes": 1462585,
"checkpoint_age_seconds": -0.382308841,
"counters": {
"dropped_records": 0,
"encoded_records": 1598,
"written_records": 1598
},
"footer_count": 1,
"invariants": {
"all_schema_1": true,
"encoded_balanced": true,
"final_sidecar": true,
"footer_sidecar_agree": true,
"last_step_matches": true,
"one_footer_last": true,
"steps_contiguous": true,
"written_matches_records": true,
"zero_drops": true
},
"last_step_index": 1597,
"records": 1598,
"sidecar": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/e-b-sidecar-verification/graceful/opprof/opprof-v1-dp0-pid2163417-1783832631507578426.jsonl.footer.json",
"stream": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/e-b-sidecar-verification/graceful/opprof/opprof-v1-dp0-pid2163417-1783832631507578426.jsonl"
},
"clean": {
"admitted": 5202,
"completed": 5202,
"completed_throughput_rps": 43.35,
"duration_s": 120.0,
"end_s": 140.0,
"failed": 0,
"input_tokens": 1662118,
"offered_rps": 43.35,
"output_tokens": 332928,
"start_s": 20.0
},
"failed_records": 0,
"gpu_seconds": 219.92383646965027,
"gpu_zero_samples": [
[
{
"index": 0,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 1,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 2,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 3,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 4,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 5,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 6,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 7,
"memory_mib": 0,
"utilization_pct": 0
}
],
[
{
"index": 0,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 1,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 2,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 3,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 4,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 5,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 6,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 7,
"memory_mib": 0,
"utilization_pct": 0
}
],
[
{
"index": 0,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 1,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 2,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 3,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 4,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 5,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 6,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 7,
"memory_mib": 0,
"utilization_pct": 0
}
]
],
"mode": "graceful",
"schema": 1,
"server_returncode": 0,
"status": "pass",
"termination_wall_ns": 1783832775735239240
},
"hard-kill": {
"accounting": {
"bytes": 1460860,
"checkpoint_age_seconds": 0.02436319,
"counters": {
"dropped_records": 0,
"encoded_records": 1596,
"written_records": 1596
},
"footer_count": 0,
"invariants": {
"all_schema_1": true,
"checkpoint_sidecar": true,
"checkpoint_within_bound": true,
"encoded_balanced": true,
"last_step_matches": true,
"no_in_stream_footer": true,
"steps_contiguous": true,
"written_matches_records": true,
"zero_drops": true
},
"last_step_index": 1595,
"records": 1596,
"sidecar": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/e-b-sidecar-verification/hard-kill/opprof/opprof-v1-dp0-pid2166360-1783832841962667333.jsonl.footer.json",
"stream": "/home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/e-b-sidecar-verification/hard-kill/opprof/opprof-v1-dp0-pid2166360-1783832841962667333.jsonl"
},
"clean": {
"admitted": 5179,
"completed": 5179,
"completed_throughput_rps": 43.15833333333333,
"duration_s": 120.0,
"end_s": 140.0,
"failed": 0,
"input_tokens": 1655384,
"offered_rps": 43.15833333333333,
"output_tokens": 331456,
"start_s": 20.0
},
"failed_records": 0,
"gpu_seconds": 207.5133068561554,
"gpu_zero_samples": [
[
{
"index": 0,
"memory_mib": 0,
"utilization_pct": 1
},
{
"index": 1,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 2,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 3,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 4,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 5,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 6,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 7,
"memory_mib": 0,
"utilization_pct": 0
}
],
[
{
"index": 0,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 1,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 2,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 3,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 4,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 5,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 6,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 7,
"memory_mib": 0,
"utilization_pct": 0
}
],
[
{
"index": 0,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 1,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 2,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 3,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 4,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 5,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 6,
"memory_mib": 0,
"utilization_pct": 0
},
{
"index": 7,
"memory_mib": 0,
"utilization_pct": 0
}
]
],
"mode": "hard-kill",
"schema": 1,
"server_returncode": -9,
"status": "pass",
"termination_wall_ns": 1783832986410202706
}
},
"schema": 1,
"status": "complete"
}

17
runs/opprof-phase3/phase3/repair-008-ap36.json Normal file
View File

@@ -0,0 +1,17 @@
{
"accepted_runs_preserved": 40,
"amendment": "A-P3-6",
"changed_fingerprint_keys": [
"controller_sha256"
],
"created_at": 1783853258.3098204,
"failed_stage_preserved": "primary-06-saturation",
"new_controller_sha256": "6ac565ff35ead305f7b2e39e6a754389d03c27ea6511b2c9e8ebc0c868c9519f",
"old_controller_sha256": "becfe00889274b51023016b1e7edb866d10e477249504f2032859a4d621f295f",
"projected_remaining_h20_hours": 2.1,
"projected_total_h20_hours": 15.408482965071997,
"repair_id": "repair-008-ap36",
"retained_attempt_re_adjudicated": false,
"retained_attempt_reason": "21 completions but A-P3-6 normalized drift 2.3385345997286295 and bin step counts 13/9/44",
"schema": 1
}