Add prospective active intervention experiment
This commit is contained in:
@@ -453,11 +453,21 @@ def execute_session(
|
||||
)
|
||||
if float(state["gpu_hours_total"]) + projection > base.GPU_LIMIT:
|
||||
raise RuntimeError(f"projected cost exceeds cap before {name}")
|
||||
load_values = {
|
||||
float(item["selection"]["offered_req_s_per_gpu"])
|
||||
for item in manifest["repetitions"].values()
|
||||
}
|
||||
load_text = (
|
||||
f"{next(iter(load_values)):.6g}"
|
||||
if len(load_values) == 1
|
||||
else ",".join(f"{value:.6g}" for value in sorted(load_values))
|
||||
)
|
||||
echo = (
|
||||
f"ACTION_AWARE_SESSION_ECHO host=dash0 config={name} tp=4 "
|
||||
f"mns={config['mns']} mbbt={config['mbbt']} gpus=0-3 "
|
||||
f"workload={manifest['source']['window_id']} load_per_gpu=2.125 "
|
||||
f"duration_s=300 repetitions={','.join(map(str, config['repetition_order']))} "
|
||||
f"workload={manifest['source']['window_id']} load_per_gpu={load_text} "
|
||||
f"duration_s={manifest['engine']['duration_s']} "
|
||||
f"repetitions={','.join(map(str, config['repetition_order']))} "
|
||||
f"source={args.manifest} output={args.run_root / 'sessions' / name} "
|
||||
f"spent_h20h={state['gpu_hours_total']:.6f} "
|
||||
f"remaining_projection_h20h={projection:.3f} cap_h20h={base.GPU_LIMIT:.1f}"
|
||||
|
||||
Reference in New Issue
Block a user