#!/usr/bin/env bash
# Full engine-state ablation sweep on dash1 (sequential; shared-venv patch
# can't be parallelized). Waits for the in-flight champion es1 to finish, then
# runs the remaining policies as matched es0/es1 pairs. Each es1 cell gets a
# freshness sampler (age_s = now - state.ts) written to cpfs.
#
# Champion: es0 already done (v3trace_unified_AB_es0_20260528_1633),
#           es1 in flight (launch_es1.sh) -> reused, not re-run here.
set -uo pipefail
PROJ=/home/admin/cpfs/wjh/agentic-kv
LWDIR=$PROJ/microbench/connector_tax/layerwise
R=$LWDIR/run_v3_trace.sh
SAMPLER=$LWDIR/fresh_sampler.py
PY=$PROJ/.venv/bin/python
AB="--overload-factor 1.3 --lmetric-decode-weight 0.01"
PROG=$PROJ/outputs/abl_full.progress
MASTERDONE=$PROJ/outputs/abl_full.done
rm -f "$MASTERDONE"

echo "[driver] $(date) waiting for champion es1 (abl_unified_AB_es1.done) ..." >> "$PROG"
while [ ! -f "$PROJ/outputs/abl_unified_AB_es1.done" ]; do sleep 30; done
echo "[driver] $(date) champion es1 done -> starting sweep" >> "$PROG"

# TAG | POLICY | MODE | AB(yes=AB) | ES
CONFIGS=(
 "lmetric_es0|lmetric|baseline||0"
 "lmetric_es1|lmetric|baseline||1"
 "load_only_es0|load_only|baseline||0"
 "load_only_es1|load_only|baseline||1"
 "sticky_es0|sticky|baseline||0"
 "sticky_es1|sticky|baseline||1"
 "v3_AB_lw_es0|unified_v3|layerwise|AB|0"
 "v3_AB_lw_es1|unified_v3|layerwise|AB|1"
 "ukvboth_AB_es0|unified_kv_both|baseline|AB|0"
 "ukvboth_AB_es1|unified_kv_both|baseline|AB|1"
)

for cfg in "${CONFIGS[@]}"; do
  IFS='|' read -r tag policy mode ab es <<< "$cfg"
  abf=""; [ "$ab" = "AB" ] && abf="$AB"
  esdir=/dev/shm/agentic_engine_state_${tag}
  fresh=$PROJ/outputs/abl_${tag}.freshness.jsonl
  rl=$PROJ/outputs/abl_${tag}.runlog
  celldone=$PROJ/outputs/abl_${tag}.celldone
  rm -f "$fresh" "$celldone"
  echo "[driver] $(date) START $tag (policy=$policy mode=$mode ab=$ab es=$es)" >> "$PROG"

  sp=""
  if [ "$es" = "1" ]; then
    nohup "$PY" "$SAMPLER" "$esdir" "$fresh" "$celldone" >/dev/null 2>&1 &
    sp=$!
  fi

  TAG="$tag" POLICY="$policy" MODE="$mode" AB_FLAGS="$abf" ES="$es" \
    bash "$R" > "$rl" 2>&1
  ec=$?

  echo "done $ec $(date)" > "$celldone"
  [ -n "$sp" ] && { kill "$sp" 2>/dev/null || true; }
  echo "[driver] $(date) END   $tag exit=$ec" >> "$PROG"
  sleep 10
done

echo "ALL DONE $(date)" > "$MASTERDONE"
echo "[driver] $(date) SWEEP COMPLETE" >> "$PROG"