gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
9b3a2eab80faa79bf06596d3559df5a3e216ac39
aituner/runs/opprof-phase3/phase3/controller-state-stop-ap36.json
Gahow Wang d5b276180d Add OpProf campaign: protocols, results, patches, run evidence (P0-P6) 
Workload-conditioned operator profiling on patched vLLM 0.24.0 +
Qwen3-30B-A3B/H20. H1b PASS (irregular patterns carry +23-45pp R64
raggedness, 8-45% token-efficiency loss vs rectangular controls);
mechanism decomposition kills the padding narrative and finds the
arrival-uniformization artifact (-12.9%); cross-version churn surface
shows TP2/MNS64 -29.4% across vLLM 0.20->0.24 while the argmax held.
Raw Layer-1 JSONL streams (507 MB) stay on disk, git-ignored; footer
sidecars and metrics are tracked.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-13 11:06:10 +08:00

738 lines
21 KiB
JSON
Raw Blame History

{
"clean_window_failures": 0,
"completed_burnins": 5,
"completed_measured_runs": 40,
"controller_pid": 2438791,
"created_at": 1783833885.960389,
"drain_quarantined_runs": 0,
"fingerprint": {
"cells": [
"P08-C00",
"P03-C10",
"P07-C00",
"P10-C01",
"P01-C10",
"P01-C01",
"P10-C10",
"P03-C01",
"P09-C00",
"P06-C01",
"P10-C11",
"P01-C00",
"P01-C11",
"P10-C00",
"P04-C00",
"P03-C00",
"P06-C10",
"P02-C00",
"P06-C00",
"P06-C11",
"P11-C00",
"P10-C00-TP2",
"P03-C11",
"P05-C00"
],
"client_sha256": "ab937a5f28252559c2fd97e848a500f1094cef232823ce4b90da8c0ece7554a0",
"common_controller_sha256": "95f7169a1771e385aab40fcaecd967dc5cff0c21ea67bdd139382774ba43f01f",
"controller_sha256": "6ac565ff35ead305f7b2e39e6a754389d03c27ea6511b2c9e8ebc0c868c9519f",
"cpu_map": {
"0": "0-19",
"1": "20-39",
"2": "40-59",
"3": "60-79",
"4": "80-99",
"5": "100-119",
"6": "120-139",
"7": "140-159"
},
"manifests": {
"P01": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P01.jsonl",
"rows": 32768,
"sha256": "13ffb226c83373f54c4a7afea6c78cb7cd29720f1858d56728826fc1367b31a4"
},
"P02": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P02.jsonl",
"rows": 32768,
"sha256": "0138ada3fccc98298daee66c26bd1952c987cb42ed8d5341d66b698a597417f9"
},
"P03": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P03.jsonl",
"rows": 32768,
"sha256": "432cfbc26d36f105c179c83f3bb0f3b24b8b3f205788263b4171797f0a4d6fa1"
},
"P04": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P04.jsonl",
"rows": 32768,
"sha256": "caf8d0941b093956a81e1413adc4a4ea9d92460f1b2ed0f6a9b118d0119d6247"
},
"P05": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P05.jsonl",
"rows": 32768,
"sha256": "192e213109f8cb429b99d9eb0f227bb4390fc03f63b02d5456569369bff5a3d7"
},
"P06": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P06.jsonl",
"rows": 32768,
"sha256": "65954bc6e47de9e7be07b8975f97f0bc4639979ef6d33e8c664557ded34b9f96"
},
"P07": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P07.jsonl",
"rows": 32768,
"sha256": "74df66e21a705cd583493199a875e226e93d2da7cfede9bca81dfd9bcb8c9cc6"
},
"P08": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P08.jsonl",
"rows": 32768,
"sha256": "0c4f835aae099265c3eb596d06c6c2fa7070dd280558eb289c602e8c3434dfe9"
},
"P09": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P09.jsonl",
"rows": 32768,
"sha256": "7af92ee3c27dc7d2cf895d6ff3a6e737ec4b6da13d6841ca59e1166f28a0ae1e"
},
"P10": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P10.jsonl",
"rows": 4011,
"sha256": "f51b7a1cc657d62b9ea81823c754408732326b06e03439452433cd8ed481bf33"
},
"P11": {
"path": "/home/admin/cpfs/wjh/opprof-phase3-private/manifests/P11.jsonl",
"rows": 32768,
"sha256": "7d196df38963528ff181cf72ce39c8ad913c8f61d40b1425410d3c6c30b6be18"
}
},
"model": "/home/admin/cpfs/wjh/models/Qwen/Qwen3-30B-A3B",
"source_commit": "4b253fd8619764b6971a7f2e3a3aa7545f6ace05",
"source_tree": "a3d536b287a724e60abbec68b45eed7e088a15d1"
},
"gpu_hours_this_stage": 0.7173924536837472,
"gpu_hours_total": 14.025875418755744,
"missing_trace_files": 8,
"repairs": [
{
"change": "dedicated aiohttp TCPConnector(limit=2) for profile endpoint session; request stream unchanged",
"evidence": "/start_profile connection acquisition timed out before server receipt while 256 data-plane connections were occupied",
"failed_run": "P03-C01-saturation",
"failed_stage": "primary-02-saturation",
"new_client_sha256": "ab937a5f28252559c2fd97e848a500f1094cef232823ce4b90da8c0ece7554a0",
"old_client_sha256": "a87d92efecd5a8765b51067800b6382f9b174a2ede65f8933fcc9f846ff03d84",
"repair": "profile-control-connector-isolation",
"retry": "one exact whole-wave retry; failed directories retained as .interrupted-*",
"schema": 1,
"tests": {
"py_compile": "PASS",
"ruff": "PASS",
"unittest": "13/13 PASS"
},
"timestamp": 1783837763.431071
},
{
"budget": {
"hard_cap": 16.0,
"projected_remaining_h20_hours": 6.3857560787267165,
"projected_total_h20_hours": 14.513826778670154,
"projected_total_with_15pct_remaining_contingency": 15.471690190479162,
"used_h20_hours": 8.128070699943438
},
"cause": "controller required failed_records==0 across excluded profile/recovery time although the registered hard gate is zero clean-window failures",
"change": "replace all_failures_zero with clean-window boundary validation plus failed-record accounting; excluded failures remain reported",
"evidence": {
"clean_completed": 5828,
"clean_failed": 0,
"error_kind": "ServerDisconnectedError",
"excluded_window_failures": 2,
"failure_completion_s": [
373.1505718010012,
373.9740898209857
],
"run": "P01-C01-moderate"
},
"failed_stage": "primary-02-moderate",
"gpu_rerun": false,
"new_controller_sha256": "becfe00889274b51023016b1e7edb866d10e477249504f2032859a4d621f295f",
"old_controller_sha256": "167e48f98f307e16ee44b321068a82a13813b3bf9a4d76882f774f15fe85e595",
"repair": "clean-window-failure-scope",
"schema": 1,
"tests": {
"py_compile": "PASS",
"ruff": "PASS",
"unittest": "14/14 PASS"
},
"timestamp": 1783847092.6920242
},
{
"cause": "single preflight NVML sample observed 4 MiB on GPU0-3 with 0% utilization and no compute apps immediately after a verified cleanup",
"change": "bounded 60s preflight requiring three consecutive samples with zero memory, zero utilization, and no compute apps; no nonzero-memory tolerance",
"failed_stage": "primary-06-saturation",
"gpu_hours_total_unchanged": 11.920448313262728,
"gpu_launch_before_failure": false,
"new_common_controller_sha256": "95f7169a1771e385aab40fcaecd967dc5cff0c21ea67bdd139382774ba43f01f",
"old_common_controller_sha256": "15ad254298a38c4a9318468db89ab32707b6196bb38d5e2a007f2267529397a5",
"repair": "stable-zero-preflight",
"schema": 1,
"tests": {
"py_compile": "PASS",
"ruff": "PASS",
"unittest": "15/15 PASS"
},
"timestamp": 1783850898.1496143
},
{
"code_change": false,
"gpu_hours_used": 12.606695837948058,
"hard_cap": 16.0,
"projected_remaining_h20_hours": 2.1455133807990285,
"projected_total_with_15pct_contingency": 15.07403622586694,
"protocol_action": "single exact whole-wave infrastructure retry under identical 4-way placement; gate unchanged",
"reason": "first TP2 P10 inference-shape cold start/autotune after readiness yielded 15 successful warmup completions against required 32; clean window and artifacts otherwise valid",
"repair": "tp2-warmup-exact-retry",
"run": "P10-C00-TP2-saturation",
"schema": 1,
"stage": "primary-06-saturation",
"timestamp": 1783851710.3100822,
"warmup_required": 32,
"warmup_success": 15
},
{
"accepted_runs_preserved": 40,
"amendment": "A-P3-6",
"changed_fingerprint_keys": [
"controller_sha256"
],
"created_at": 1783853258.3098204,
"failed_stage_preserved": "primary-06-saturation",
"new_controller_sha256": "6ac565ff35ead305f7b2e39e6a754389d03c27ea6511b2c9e8ebc0c868c9519f",
"old_controller_sha256": "becfe00889274b51023016b1e7edb866d10e477249504f2032859a4d621f295f",
"projected_remaining_h20_hours": 2.1,
"projected_total_h20_hours": 15.408482965071997,
"repair_id": "repair-008-ap36",
"retained_attempt_re_adjudicated": false,
"retained_attempt_reason": "21 completions but A-P3-6 normalized drift 2.3385345997286295 and bin step counts 13/9/44",
"schema": 1
}
],
"schema": 1,
"stages": {
"burnin-01": {
"assignments": [
{
"cell": "P06-C00",
"gpus": [
0
]
},
{
"cell": "P06-C10",
"gpus": [
1
]
},
{
"cell": "P06-C01",
"gpus": [
2
]
},
{
"cell": "P06-C11",
"gpus": [
3
]
}
],
"burnin": true,
"clients": {},
"completed_at": 1783834288.1238313,
"confirmation": false,
"gpu_hours": 0.30808498481909435,
"load_point": "saturation",
"profile": false,
"servers": {},
"started_at": 1783833886.4448946,
"status": "complete",
"validation_attempt1_failure": "RuntimeError(\"server config/backend failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/burnins/C01: {'triton_moe': True, 'chunked_mbt': False, 'tp_effective': True, 'drain_shutdown': True, 'mns_effective': True}\")"
},
"burnin-02": {
"assignments": [
{
"cell": "P06-C00-TP2",
"gpus": [
0,
1
]
}
],
"burnin": true,
"clients": {},
"completed_at": 1783834671.921921,
"confirmation": false,
"gpu_hours": 0.19903395679261948,
"load_point": "saturation",
"profile": false,
"servers": {},
"started_at": 1783834301.7813473,
"status": "complete"
},
"primary-01-moderate": {
"assignments": [
{
"cell": "P08-C00",
"gpus": [
0
]
},
{
"cell": "P03-C10",
"gpus": [
1
]
},
{
"cell": "P07-C00",
"gpus": [
2
]
},
{
"cell": "P10-C01",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {},
"completed_at": 1783836723.3639715,
"confirmation": false,
"gpu_hours": 0.5195140059126748,
"load_point": "moderate",
"profile": true,
"servers": {},
"started_at": 1783836150.2169476,
"status": "complete",
"validation_attempt1_failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P10-C01/moderate: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'all_failures_zero': True, 'manifest_no_wrap': True, 'warmup_completions': False, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]\")",
"warmup_feasibility": "P10 target=min(32,floor(0.445*60))=26; observed=27"
},
"primary-01-saturation": {
"assignments": [
{
"cell": "P08-C00",
"gpus": [
0
]
},
{
"cell": "P03-C10",
"gpus": [
1
]
},
{
"cell": "P07-C00",
"gpus": [
2
]
},
{
"cell": "P10-C01",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {},
"completed_at": 1783836130.8498049,
"confirmation": false,
"drain_readjudication": "P10/C01 288.619107924s <= 600s",
"gpu_hours": 0.8836704309119119,
"load_point": "saturation",
"missing_trace_files": 8,
"profile": true,
"servers": {},
"started_at": 1783834671.9563339,
"status": "complete",
"validation_attempt1_failure": "RuntimeError(\"client failures: {'P10-C01-saturation': 1}\")"
},
"primary-02-moderate": {
"assignments": [
{
"cell": "P01-C10",
"gpus": [
0
]
},
{
"cell": "P01-C01",
"gpus": [
1
]
},
{
"cell": "P10-C10",
"gpus": [
2
]
},
{
"cell": "P03-C01",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {},
"completed_at": 1783847092.7229948,
"confirmation": false,
"failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P01-C01/moderate: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'all_failures_zero': False, 'manifest_no_wrap': True, 'warmup_completions': True, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]\")",
"gpu_hours": 0.5151156750652525,
"load_point": "moderate",
"profile": true,
"readjudicated_without_gpu": true,
"servers": {},
"started_at": 1783838513.4472992,
"status": "complete",
"validation_attempt1_failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P01-C01/moderate: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'all_failures_zero': False, 'manifest_no_wrap': True, 'warmup_completions': True, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]\")"
},
"primary-02-saturation": {
"assignments": [
{
"cell": "P01-C10",
"gpus": [
0
]
},
{
"cell": "P01-C01",
"gpus": [
1
]
},
{
"cell": "P10-C10",
"gpus": [
2
]
},
{
"cell": "P03-C01",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {},
"completed_at": 1783838513.4027848,
"confirmation": false,
"gpu_hours": 0.7973502041896184,
"load_point": "saturation",
"profile": true,
"servers": {},
"started_at": 1783837773.6088765,
"status": "complete"
},
"primary-03-moderate": {
"assignments": [
{
"cell": "P09-C00",
"gpus": [
0
]
},
{
"cell": "P06-C01",
"gpus": [
1
]
},
{
"cell": "P10-C11",
"gpus": [
2
]
},
{
"cell": "P01-C00",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {},
"completed_at": 1783848369.3507695,
"confirmation": false,
"gpu_hours": 0.5205460974905226,
"load_point": "moderate",
"profile": true,
"servers": {},
"started_at": 1783847881.06146,
"status": "complete"
},
"primary-03-saturation": {
"assignments": [
{
"cell": "P09-C00",
"gpus": [
0
]
},
{
"cell": "P06-C01",
"gpus": [
1
]
},
{
"cell": "P10-C11",
"gpus": [
2
]
},
{
"cell": "P01-C00",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {},
"completed_at": 1783847881.0197804,
"confirmation": false,
"gpu_hours": 0.8273645816908942,
"load_point": "saturation",
"profile": true,
"servers": {},
"started_at": 1783847109.5683346,
"status": "complete"
},
"primary-04-moderate": {
"assignments": [
{
"cell": "P01-C11",
"gpus": [
0
]
},
{
"cell": "P10-C00",
"gpus": [
1
]
},
{
"cell": "P04-C00",
"gpus": [
2
]
},
{
"cell": "P03-C00",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {},
"completed_at": 1783849583.5228572,
"confirmation": false,
"gpu_hours": 0.5205216948853598,
"load_point": "moderate",
"profile": true,
"servers": {},
"started_at": 1783849096.4363403,
"status": "complete"
},
"primary-04-saturation": {
"assignments": [
{
"cell": "P01-C11",
"gpus": [
0
]
},
{
"cell": "P10-C00",
"gpus": [
1
]
},
{
"cell": "P04-C00",
"gpus": [
2
]
},
{
"cell": "P03-C00",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {},
"completed_at": 1783849096.3948114,
"confirmation": false,
"gpu_hours": 0.7872003297011058,
"load_point": "saturation",
"profile": true,
"servers": {},
"started_at": 1783848369.390857,
"status": "complete"
},
"primary-05-moderate": {
"assignments": [
{
"cell": "P06-C10",
"gpus": [
0
]
},
{
"cell": "P02-C00",
"gpus": [
1
]
},
{
"cell": "P06-C00",
"gpus": [
2
]
},
{
"cell": "P06-C11",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {},
"completed_at": 1783850641.6781306,
"confirmation": false,
"gpu_hours": 0.5138198028008143,
"load_point": "moderate",
"profile": true,
"servers": {},
"started_at": 1783850163.1211865,
"status": "complete"
},
"primary-05-saturation": {
"assignments": [
{
"cell": "P06-C10",
"gpus": [
0
]
},
{
"cell": "P02-C00",
"gpus": [
1
]
},
{
"cell": "P06-C00",
"gpus": [
2
]
},
{
"cell": "P06-C11",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {},
"completed_at": 1783850163.0472832,
"confirmation": false,
"gpu_hours": 0.6229251067505942,
"load_point": "saturation",
"profile": true,
"servers": {},
"started_at": 1783849583.5652869,
"status": "complete"
},
"primary-06-saturation": {
"assignments": [
{
"cell": "P11-C00",
"gpus": [
0
]
},
{
"cell": "P10-C00-TP2",
"gpus": [
1,
2
]
},
{
"cell": "P03-C11",
"gpus": [
3
]
}
],
"burnin": false,
"clients": {
"P03-C11-saturation": {
"pgid": 2440965,
"pid": 2440965
},
"P10-C00-TP2-saturation": {
"pgid": 2440964,
"pid": 2440964
},
"P11-C00-saturation": {
"pgid": 2440962,
"pid": 2440962
}
},
"confirmation": false,
"failure": "RuntimeError(\"client invariant failure: /home/admin/cpfs/wjh/opprof-phase3-dash0-20260712/runs/phase3/primary/P10-C00-TP2/saturation: {'client_sanity': True, 'clean_duration': True, 'clean_failures_zero': True, 'failed_records_accounted': True, 'manifest_no_wrap': True, 'warmup_completions': False, 'profile_count': True, 'profile_after_clean': True, 'drain_re_adjudicated': True}; failed=[]; warmup_completions=17; warmup_gate_branch=failed; warmup_stability={'passed': False, 'reason': 'A-P3-6 stabilization criterion not met', 'window_seconds': [45.0, 60.0], 'bin_seconds': 5.0, 'step_counts': [11, 10, 16], 'scheduled_tokens': [90112, 81920, 70313], 'scheduled_token_throughput': [18022.4, 16384.0, 14062.6], 'mean_scheduled_token_throughput': 16156.333333333334, 'slope_tokens_per_second_squared': -395.98000000000013, 'normalized_drift': 0.367639109533929, 'normalized_drift_limit': 0.1, 'step_indices_continuous': True}\")",
"gpu_hours": 0.7173924536837472,
"load_point": "saturation",
"profile": true,
"servers": {
"P03-C11-saturation": {
"gpus": [
3
],
"pgid": 2438868,
"pid": 2438868
},
"P10-C00-TP2-saturation": {
"gpus": [
1,
2
],
"pgid": 2438867,
"pid": 2438867
},
"P11-C00-saturation": {
"gpus": [
0
],
"pgid": 2438866,
"pid": 2438866
}
},
"started_at": 1783853279.9695158,
"status": "failed"
}
},
"status": "failed",
"updated_at": 1783853946.066117
}
Reference in New Issue View Git Blame Copy Permalink