feat(kvc): session migration with reset-on-success + direct-append threshold tuning

KVC v2 beats 4DP at ts=1 same-scale on 7/8 metrics: TTFT mean -24%, p50 -54%, p90 -64%; lat mean -0.8%, p50 -12.6%, p90 -0.7%. Direct-to-D rate jumped 42.8% -> 91.7%. REFACTOR_PLAN_V1 scenario C achieved. Two-knob fix: - reset-on-success blacklist decay: clear (sess, D) reject counter on successful direct-to-D path. Eliminates v1 thrashing where session 6880 was stable on decode-1 for 70 turns then collapsed to 75 D-changes after cumulative transient pressure tripped the permanent blacklist. - bump --kvcache-direct-max-uncached-tokens default 2048 -> 8192 via CLI flag. 41% of v1 fallbacks were 'real-large-append' (>2048 token append); raising the threshold lets these go through the direct-to-D fast path. Code: - policies.py: RoutingState.session_d_rejects counter + KvAwarePolicy migration_reject_threshold; degenerate fallback picks least-rejected D. - replay.py: record_admission_reject + reset-on-success in _run_request; _fallthrough_reason classifies turn-2+ fall-throughs as session-not-resident / real-large-append / etc, replacing misleading 'large-append' suffix (TEAM_REPORT §2.7). - cli.py + benchmark.py: --kvcache-migration-reject-threshold flag wiring. Docs: - REFACTOR_PLAN_V1_ZH.md: forward-looking plan after ts=1 validation. - MIGRATION_V1_FINDINGS_ZH.md: v1 thrashing root-cause analysis. - V2_RESULTS_ZH.md: v2 results, scenario C achievement, attribution. - TEAM_REPORT_AGENTIC_PD_HYBRID_ZH.md: comprehensive team report. Scripts: - sweep_ts1_kvc_n3_plus_dp.sh: ts=1 baseline (KVC 1P3D N=3 + 4DP CA). - sweep_ts1_migration_v1.sh / v2.sh: validation runs. - analyze_ts1_validation.py: 4-way comparison analyzer. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-09 01:18:13 +08:00
parent 1d51704dad
commit 2ec0debef4
12 changed files with 2350 additions and 12 deletions
--- a/scripts/analysis/analyze_ts1_validation.py
+++ b/scripts/analysis/analyze_ts1_validation.py
@@ -0,0 +1,316 @@
+#!/usr/bin/env python3
+"""TS=1 validation analysis: KVC 1P3D × N=3 + 4DP × 1.
+
+Reads metrics from outputs/qwen3-30b-tp1-ts1-validation/{kvc_1p3d_run{1,2,3},dp4}_metrics.jsonl
+and reports per the structural claims in docs/AGENTIC_FIT_ANALYSIS_ZH.md and TEAM_REPORT.
+
+Sections:
+  1. Headline summary table (errors, latency p50/p90/p99, TTFT p50)
+  2. §1 (session pinning): distinct-D-per-session distribution + direct-to-D bimodal
+  3. §1 (cross-run consistency): sessions consistently starved across all 3 runs + size ratio
+  4. §2 (LRU): KVTransferError counts per D + peak token_usage from worker logs
+  5. §7 (ts=1 vs ts=10): direct-to-D rate, fallback rate, per-D load balance
+  6. KVC vs DP same-scale comparison
+
+Usage: python scripts/analysis/analyze_ts1_validation.py [--root PATH]
+"""
+import argparse
+import json
+import re
+from collections import Counter, defaultdict
+from pathlib import Path
+
+import numpy as np
+
+
+def load_metrics(path):
+    rows = []
+    with open(path) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            rows.append(json.loads(line))
+    return rows
+
+
+def load_summary(path):
+    with open(path) as f:
+        return json.load(f)
+
+
+def pct(arr, p):
+    if not arr:
+        return float("nan")
+    return float(np.percentile(arr, p))
+
+
+def summarize_run(label, rows, summary):
+    ok = [r for r in rows if r.get("error") is None]
+    err = [r for r in rows if r.get("error") is not None]
+    lats = [r["latency_s"] for r in ok if r.get("latency_s") is not None]
+    ttfts = [r["ttft_s"] for r in ok if r.get("ttft_s") is not None]
+    return {
+        "label": label,
+        "n": len(rows),
+        "ok": len(ok),
+        "err": len(err),
+        "lat_mean": float(np.mean(lats)) if lats else float("nan"),
+        "lat_p50": pct(lats, 50),
+        "lat_p90": pct(lats, 90),
+        "lat_p99": pct(lats, 99),
+        "ttft_mean": float(np.mean(ttfts)) if ttfts else float("nan"),
+        "ttft_p50": pct(ttfts, 50),
+        "summary": summary,
+    }
+
+
+def headline_table(stats):
+    print("\n" + "=" * 110)
+    print("HEADLINE: same trace, same scale, same ts=1")
+    print("=" * 110)
+    cols = ["label", "ok/n", "err", "lat_mean", "lat_p50", "lat_p90", "lat_p99", "ttft_mean", "ttft_p50"]
+    print(f"{cols[0]:<22}{cols[1]:>12}{cols[2]:>6}{cols[3]:>10}{cols[4]:>10}{cols[5]:>10}{cols[6]:>10}{cols[7]:>10}{cols[8]:>10}")
+    for s in stats:
+        ok_n = f"{s['ok']}/{s['n']}"
+        print(f"{s['label']:<22}{ok_n:>12}{s['err']:>6}"
+              f"{s['lat_mean']:>9.3f}s{s['lat_p50']:>9.3f}s{s['lat_p90']:>9.3f}s{s['lat_p99']:>9.3f}s"
+              f"{s['ttft_mean']:>9.3f}s{s['ttft_p50']:>9.3f}s")
+
+
+def session_pinning(rows, label):
+    """§1: distinct D per session — should be ~1.0 if pin behavior persists."""
+    sess_d = defaultdict(set)
+    for r in rows:
+        sid = r.get("session_id")
+        d = r.get("assigned_decode_node") or r.get("decode_node")
+        if sid is not None and d is not None:
+            sess_d[sid].add(d)
+    if not sess_d:
+        return None
+    distinct = [len(s) for s in sess_d.values()]
+    return {
+        "label": label,
+        "n_sessions": len(sess_d),
+        "avg_distinct_D": float(np.mean(distinct)),
+        "max_distinct_D": max(distinct),
+        "sess_d": {sid: sorted(ds) for sid, ds in sess_d.items()},
+    }
+
+
+def direct_to_d_distribution(rows, label):
+    """§1: per-session direct-to-D rate; check for bimodal."""
+    sess_total = Counter()
+    sess_direct = Counter()
+    for r in rows:
+        sid = r.get("session_id")
+        if sid is None:
+            continue
+        sess_total[sid] += 1
+        mode = r.get("execution_mode", "")
+        if mode == "kvcache-direct-to-d-session":
+            sess_direct[sid] += 1
+    rates = []
+    for sid in sess_total:
+        rate = sess_direct[sid] / sess_total[sid]
+        rates.append((sid, rate, sess_total[sid]))
+    bins = [0, 0.2, 0.4, 0.6, 0.8, 1.01]
+    bin_labels = ["0-20%", "20-40%", "40-60%", "60-80%", "80-100%"]
+    counts = [0] * 5
+    for _, r, _ in rates:
+        for i in range(5):
+            if bins[i] <= r < bins[i + 1]:
+                counts[i] += 1
+                break
+    print(f"\n  [{label}] direct-to-D rate distribution (n={len(rates)} sessions):")
+    for lbl, cnt in zip(bin_labels, counts):
+        bar = "█" * cnt
+        print(f"    {lbl:<10}: {cnt:>3}  {bar}")
+    return rates
+
+
+def starved_cross_run(per_run_rates, threshold=0.20):
+    """§1: sessions starved (<threshold direct-to-D) in ALL runs."""
+    if len(per_run_rates) < 2:
+        return None
+    sess_starved = defaultdict(int)
+    sess_lucky = defaultdict(int)
+    for rates in per_run_rates:
+        for sid, rate, _ in rates:
+            if rate < threshold:
+                sess_starved[sid] += 1
+            elif rate > 0.80:
+                sess_lucky[sid] += 1
+    n_runs = len(per_run_rates)
+    consistently_starved = [sid for sid, c in sess_starved.items() if c == n_runs]
+    consistently_lucky = [sid for sid, c in sess_lucky.items() if c == n_runs]
+    return {
+        "n_runs": n_runs,
+        "consistently_starved": consistently_starved,
+        "consistently_lucky": consistently_lucky,
+    }
+
+
+def session_size_comparison(rows, sids_a, sids_b, label_a="A", label_b="B"):
+    """Compare peak input_length of two session groups."""
+    sess_max_input = defaultdict(int)
+    for r in rows:
+        sid = r.get("session_id")
+        ilen = r.get("input_length") or 0
+        if sid is not None and ilen > sess_max_input[sid]:
+            sess_max_input[sid] = ilen
+    a_inputs = [sess_max_input[s] for s in sids_a if s in sess_max_input]
+    b_inputs = [sess_max_input[s] for s in sids_b if s in sess_max_input]
+    if a_inputs and b_inputs:
+        ratio = np.mean(a_inputs) / np.mean(b_inputs)
+        print(f"\n  Cross-run starvation correlates with session size?")
+        print(f"    consistently {label_a} (n={len(a_inputs)}): peak_input mean = {np.mean(a_inputs):.0f}")
+        print(f"    consistently {label_b} (n={len(b_inputs)}): peak_input mean = {np.mean(b_inputs):.0f}")
+        print(f"    {label_a}/{label_b} ratio = {ratio:.2f}x (ts=10 baseline was 1.98x)")
+
+
+def per_d_balance(rows, label):
+    """§7: per-D load balance."""
+    per_d = Counter()
+    for r in rows:
+        d = r.get("assigned_decode_node") or r.get("decode_node")
+        if d:
+            per_d[d] += 1
+    if not per_d:
+        return
+    counts = list(per_d.values())
+    spread = (max(counts) - min(counts)) / max(np.mean(counts), 1)
+    print(f"\n  [{label}] per-D load: {dict(sorted(per_d.items()))}")
+    print(f"    spread (max-min)/mean = {spread*100:.1f}%   "
+          f"(ts=10 KVC 2P6D = ±26%, 8DP CA = ±10%)")
+
+
+def execution_modes_table(rows, label):
+    """Show top execution modes."""
+    ok = [r for r in rows if r.get("error") is None]
+    if not ok:
+        return
+    modes = Counter(r["execution_mode"] for r in ok)
+    print(f"\n  [{label}] execution modes (n_ok={len(ok)}):")
+    for mode, cnt in modes.most_common(8):
+        mode_rows = [r for r in ok if r["execution_mode"] == mode]
+        lats = [r["latency_s"] for r in mode_rows if r.get("latency_s") is not None]
+        ttfts = [r["ttft_s"] for r in mode_rows if r.get("ttft_s") is not None]
+        if lats:
+            print(f"    {mode:<55} {cnt:>5}  ({cnt/len(ok)*100:>4.1f}%)  "
+                  f"lat p50={pct(lats,50):.3f}s p90={pct(lats,90):.3f}s  ttft p50={pct(ttfts,50):.3f}s")
+
+
+def lru_vs_errors(run_dir, label):
+    """§2: trim events vs KVTransferError per worker."""
+    log_dir = run_dir / "logs"
+    if not log_dir.exists():
+        return
+    print(f"\n  [{label}] D-side LRU vs errors (from worker logs):")
+    print(f"    {'worker':<14}{'trim':>8}{'KVTransferError':>20}{'peak_token_usage':>20}")
+    for log_file in sorted(log_dir.glob("decode-*.log")):
+        worker = log_file.stem
+        text = log_file.read_text(errors="ignore")
+        trim_count = len(re.findall(r"Trimmed decode session cache", text))
+        err_count = len(re.findall(r"KVTransferError", text))
+        usages = re.findall(r"token usage: ([\d.]+)", text)
+        peak = max((float(u) for u in usages), default=0.0)
+        print(f"    {worker:<14}{trim_count:>8}{err_count:>20}{peak:>20.3f}")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--root", default="outputs/qwen3-30b-tp1-ts1-validation",
+                        help="Sweep output root")
+    args = parser.parse_args()
+
+    root = Path(args.root)
+    if not root.is_absolute():
+        root = Path("/mnt/kzlin/workflow/pd-hybrid/agentic-pd-hybrid") / root
+
+    # Load all available runs
+    stats = []
+    rows_by_run = {}
+    for label in ("kvc_1p3d_run1", "kvc_1p3d_run2", "kvc_1p3d_run3", "dp4"):
+        m = root / f"{label}_metrics.jsonl"
+        s = root / f"{label}_summary.json"
+        if not m.exists() or not s.exists():
+            print(f"  [{label}] not yet available ({m.name})")
+            continue
+        rows = load_metrics(m)
+        summary = load_summary(s)
+        rows_by_run[label] = rows
+        stats.append(summarize_run(label, rows, summary))
+
+    if not stats:
+        print("No runs available yet.")
+        return
+
+    # 1. Headline table
+    headline_table(stats)
+
+    # 2. §1 session pinning per KVC run + per-D balance + execution modes
+    print("\n" + "=" * 110)
+    print("§1 / §7: SESSION PINNING + LOAD BALANCE")
+    print("=" * 110)
+    per_run_rates = []
+    for label, rows in rows_by_run.items():
+        if not label.startswith("kvc_"):
+            continue
+        pin = session_pinning(rows, label)
+        if pin:
+            print(f"\n  [{label}] sessions={pin['n_sessions']}  "
+                  f"avg_distinct_D={pin['avg_distinct_D']:.2f}  "
+                  f"max_distinct_D={pin['max_distinct_D']}  "
+                  f"(ts=10 baseline avg=1.00 → 100% pin)")
+        rates = direct_to_d_distribution(rows, label)
+        per_run_rates.append(rates)
+        per_d_balance(rows, label)
+        execution_modes_table(rows, label)
+
+    # 3. §1 cross-run starvation
+    if len(per_run_rates) >= 2:
+        print("\n" + "=" * 110)
+        print(f"§1 CROSS-RUN STARVATION (across {len(per_run_rates)} KVC runs)")
+        print("=" * 110)
+        cross = starved_cross_run(per_run_rates)
+        if cross:
+            n_starved = len(cross["consistently_starved"])
+            n_lucky = len(cross["consistently_lucky"])
+            print(f"\n  Sessions starved (<20% direct-to-D) in all {cross['n_runs']} runs: {n_starved}")
+            print(f"  Sessions lucky (>80% direct-to-D) in all {cross['n_runs']} runs: {n_lucky}")
+            print(f"  (ts=10 baseline: 13/52 starved, 14/52 lucky — extreme bimodal)")
+            # session size comparison from run 1
+            if "kvc_1p3d_run1" in rows_by_run and n_starved and n_lucky:
+                session_size_comparison(rows_by_run["kvc_1p3d_run1"],
+                                        cross["consistently_starved"],
+                                        cross["consistently_lucky"],
+                                        "starved", "lucky")
+
+    # 4. §2 D-side LRU vs errors from raw logs
+    print("\n" + "=" * 110)
+    print("§2: D-SIDE LRU TRIM vs KVTransferError (from worker logs)")
+    print("=" * 110)
+    for label in rows_by_run:
+        if not label.startswith("kvc_"):
+            continue
+        # find the matching raw run dir
+        run_dirs = sorted(root.glob("kvcache-centric-*/"))
+        if not run_dirs:
+            continue
+        # naive: index matches run order; could be wrong if dirs got reordered
+        idx = int(label.split("run")[-1]) - 1
+        if idx < len(run_dirs):
+            lru_vs_errors(run_dirs[idx], label)
+
+    # 5. DP-only inspection
+    if "dp4" in rows_by_run:
+        print("\n" + "=" * 110)
+        print("4DP CA SANITY")
+        print("=" * 110)
+        per_d_balance(rows_by_run["dp4"], "dp4")
+        execution_modes_table(rows_by_run["dp4"], "dp4")
+
+
+if __name__ == "__main__":
+    main()
--- a/scripts/sweep_ts1_kvc_n3_plus_dp.sh
+++ b/scripts/sweep_ts1_kvc_n3_plus_dp.sh
@@ -0,0 +1,146 @@
+#!/bin/bash
+# Time-scale=1 validation sweep, downscaled to 4 GPUs:
+#   - KVC v5 1P3D × N=3   (new data, validates §1/§2 structural claims at real timing)
+#   - 4-way DP cache-aware × 1 (sanity baseline at same scale + ts=1)
+#
+# Goal: per docs/AGENTIC_FIT_ANALYSIS_ZH.md §7 / TEAM_REPORT §2.6 — all v3-v6 KVC
+# data was at time-scale=10 (inter-turn gap p50 = 0.25s, vs real 2.5s). This run
+# tests whether the gap structurally reverses any conclusion.
+#
+# CONFIG NOTE: Original experiments used 8 GPUs (2P6D / 8-way DP). This host has
+# only 4 H100s available, so we downscale proportionally to 1P3D / 4-way DP.
+# Cross-compare against existing 2P6D ts=10 data is confounded by *both*
+# time-scale and capacity. Internal comparison (1P3D KVC vs 4DP) at ts=1 is the
+# clean signal. §5 (P-side imbalance) is NOT testable here — only 1 P.
+#
+# Capacity ratio: 3D × ~92K tok = 276K KV pool vs 52 sessions × ~50K peak input
+#   working set ≈ 1.5M → ~5.4× overload (vs 2.7× in original 2P6D).
+#   Pressure is HIGHER than original; partly offset by ts=1 letting D drain between turns.
+#
+# Output:
+#   outputs/qwen3-30b-tp1-ts1-validation/
+#     ├── kvc_1p3d_run{1,2,3}_summary.json
+#     ├── kvc_1p3d_run{1,2,3}_metrics.jsonl
+#     ├── dp4_summary.json
+#     ├── dp4_metrics.jsonl
+#     └── kvcache-centric-... / pd-colo-kv-aware-...    (raw run dirs)
+#
+# Estimated GPU time: KVC ts=1 ≈ 100-180 min/run × 3 = 5-9h
+#                     DP ts=1  ≈ 100-120 min × 1     = ~2h
+#                     Total                           = 7-11h
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+MODEL=/mnt/kzlin/workflow/pd-hybrid/simm-swe-bench/models/Qwen3-30B-A3B-Instruct-2507
+TRACE=outputs/qwen35-swebench-50sess.jsonl
+OUTPUT=outputs/qwen3-30b-tp1-ts1-validation
+VENV_PYTHON=.venv/bin/python
+RESULTS_FILE=$OUTPUT/sweep_results.txt
+
+mkdir -p $OUTPUT
+
+log() {
+  echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a $RESULTS_FILE
+}
+
+run_kvc_1p3d() {
+  local run_idx=$1
+  local label="kvc_1p3d_run${run_idx}"
+  log ""
+  log "=== [KVC ${run_idx}/3] 1P3D KVC kv-aware Option D, time-scale=1 ==="
+  PYTHONPATH=src:third_party/sglang/python \
+  $VENV_PYTHON -m agentic_pd_hybrid.cli benchmark-live \
+    --trace $TRACE \
+    --output-root $OUTPUT \
+    --mechanism kvcache-centric \
+    --policy kv-aware \
+    --model-path $MODEL \
+    --prefill-workers 1 --decode-workers 3 \
+    --prefill-tp-size 1 --decode-tp-size 1 \
+    --prefill-gpu-ids 0 --decode-gpu-ids 1,2,3 \
+    --transfer-backend mooncake \
+    --gpu-budget 4 \
+    --time-scale 1 \
+    --session-sample-rate 1.0 \
+    --target-duration-s 100000 \
+    --concurrency-limit 32 \
+    --timeout-s 900 \
+    --request-timeout-s 300 \
+    --kvcache-admission-mode worker \
+    --kvcache-seed-min-turn-id 1 \
+    --kvcache-seed-max-inflight-decode -1 \
+    --kvcache-prefill-backup-policy release-after-transfer \
+    --kvcache-prefill-priority-eviction
+
+  local run_dir=$(ls -td $OUTPUT/kvcache-centric-*/ 2>/dev/null | head -1)
+  log "=== [KVC ${run_idx}/3] $label COMPLETED ==="
+  if [ -f "$run_dir/request-metrics.jsonl.summary.json" ]; then
+    cp "$run_dir/request-metrics.jsonl.summary.json" "$OUTPUT/${label}_summary.json"
+    cp "$run_dir/request-metrics.jsonl" "$OUTPUT/${label}_metrics.jsonl"
+    local errs=$($VENV_PYTHON -c "import json; d=json.load(open('$OUTPUT/${label}_summary.json')); print(d.get('error_count',0))")
+    log "  errors = $errs"
+    cat "$run_dir/request-metrics.jsonl.summary.json" >> $RESULTS_FILE
+    echo "" >> $RESULTS_FILE
+  else
+    log "WARNING: no summary file in $run_dir"
+  fi
+}
+
+run_dp4_sanity() {
+  local label="dp4"
+  log ""
+  log "=== [DP] 4-way DP cache-aware sanity, time-scale=1 ==="
+  PYTHONPATH=src:third_party/sglang/python \
+  $VENV_PYTHON -m agentic_pd_hybrid.cli benchmark-live \
+    --trace $TRACE \
+    --output-root $OUTPUT \
+    --mechanism pd-colo \
+    --policy kv-aware \
+    --model-path $MODEL \
+    --prefill-workers 0 --decode-workers 0 \
+    --direct-workers 4 --direct-tp-size 1 \
+    --direct-gpu-ids 0,1,2,3 \
+    --gpu-budget 4 \
+    --time-scale 1 \
+    --session-sample-rate 1.0 \
+    --target-duration-s 100000 \
+    --concurrency-limit 32 \
+    --timeout-s 900 \
+    --request-timeout-s 300
+
+  local run_dir=$(ls -td $OUTPUT/pd-colo-kv-aware-*/ 2>/dev/null | head -1)
+  log "=== [DP] $label COMPLETED ==="
+  if [ -f "$run_dir/request-metrics.jsonl.summary.json" ]; then
+    cp "$run_dir/request-metrics.jsonl.summary.json" "$OUTPUT/${label}_summary.json"
+    cp "$run_dir/request-metrics.jsonl" "$OUTPUT/${label}_metrics.jsonl"
+    local errs=$($VENV_PYTHON -c "import json; d=json.load(open('$OUTPUT/${label}_summary.json')); print(d.get('error_count',0))")
+    log "  errors = $errs"
+    cat "$run_dir/request-metrics.jsonl.summary.json" >> $RESULTS_FILE
+    echo "" >> $RESULTS_FILE
+  else
+    log "WARNING: no summary file in $run_dir"
+  fi
+}
+
+log "=== TS=1 VALIDATION (4-GPU): KVC 1P3D × N=3 + 4DP × 1 ==="
+log "Model: $MODEL"
+log "Trace: $TRACE (4449 requests, 52 sessions)"
+log "Goal: validate whether ts=10 was the main distortion in v3-v6 KVC vs DP"
+
+# KVC × 3 first (the new data we need); DP last (cheaper sanity at end)
+for i in 1 2 3; do
+  run_kvc_1p3d $i
+done
+
+run_dp4_sanity
+
+log ""
+log "=== TS=1 SUMMARY ==="
+for label in kvc_1p3d_run1 kvc_1p3d_run2 kvc_1p3d_run3 dp4; do
+  if [ -f "$OUTPUT/${label}_summary.json" ]; then
+    e=$($VENV_PYTHON -c "import json; d=json.load(open('$OUTPUT/${label}_summary.json')); print(d.get('error_count',0))")
+    p50=$($VENV_PYTHON -c "import json; d=json.load(open('$OUTPUT/${label}_summary.json')); print(d.get('latency_stats_s',{}).get('p50','n/a'))")
+    log "  ${label}: errors=$e  lat_p50=${p50}s"
+  fi
+done
+log "=== TS=1 ALL DONE ==="
--- a/scripts/sweep_ts1_migration_v1.sh
+++ b/scripts/sweep_ts1_migration_v1.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+# Migration v1 validation: KVC 1P3D ts=1 with --kvcache-migration-reject-threshold=3
+# Compare against baseline outputs/qwen3-30b-tp1-ts1-validation/kvc_1p3d_run{1,2,3}
+# (all of which had no migration — runs were structurally identical).
+#
+# Goal: verify §1 fix changes the categorical outcome — direct-to-D % up,
+# fallback-session-not-resident % down, lat mean down.
+#
+# ts=1 is deterministic at the categorical level, so N=1 is sufficient
+# (TEAM_REPORT §2.8 revised).
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+MODEL=/mnt/kzlin/workflow/pd-hybrid/simm-swe-bench/models/Qwen3-30B-A3B-Instruct-2507
+TRACE=outputs/qwen35-swebench-50sess.jsonl
+OUTPUT=outputs/qwen3-30b-tp1-ts1-migration-v1
+VENV_PYTHON=.venv/bin/python
+RESULTS_FILE=$OUTPUT/sweep_results.txt
+
+mkdir -p $OUTPUT
+
+log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a $RESULTS_FILE; }
+
+log "=== TS=1 MIGRATION v1: KVC 1P3D --kvcache-migration-reject-threshold=3 ==="
+log "Baseline reference: outputs/qwen3-30b-tp1-ts1-validation/kvc_1p3d_run1 (errors=5, lat mean=1.574s, direct-to-D=42.8%)"
+
+label=kvc_1p3d_migration_run1
+log ""
+log "=== [migration v1] starting ==="
+PYTHONPATH=src:third_party/sglang/python \
+$VENV_PYTHON -m agentic_pd_hybrid.cli benchmark-live \
+  --trace $TRACE \
+  --output-root $OUTPUT \
+  --mechanism kvcache-centric \
+  --policy kv-aware \
+  --model-path $MODEL \
+  --prefill-workers 1 --decode-workers 3 \
+  --prefill-tp-size 1 --decode-tp-size 1 \
+  --prefill-gpu-ids 0 --decode-gpu-ids 1,2,3 \
+  --transfer-backend mooncake \
+  --gpu-budget 4 \
+  --time-scale 1 \
+  --session-sample-rate 1.0 \
+  --target-duration-s 100000 \
+  --concurrency-limit 32 \
+  --timeout-s 900 \
+  --request-timeout-s 300 \
+  --kvcache-admission-mode worker \
+  --kvcache-seed-min-turn-id 1 \
+  --kvcache-seed-max-inflight-decode -1 \
+  --kvcache-prefill-backup-policy release-after-transfer \
+  --kvcache-prefill-priority-eviction \
+  --kvcache-migration-reject-threshold 3
+
+run_dir=$(ls -td $OUTPUT/kvcache-centric-*/ 2>/dev/null | head -1)
+log "=== [migration v1] $label COMPLETED ==="
+if [ -f "$run_dir/request-metrics.jsonl.summary.json" ]; then
+  cp "$run_dir/request-metrics.jsonl.summary.json" "$OUTPUT/${label}_summary.json"
+  cp "$run_dir/request-metrics.jsonl" "$OUTPUT/${label}_metrics.jsonl"
+  errs=$($VENV_PYTHON -c "import json; d=json.load(open('$OUTPUT/${label}_summary.json')); print(d.get('error_count',0))")
+  p50=$($VENV_PYTHON -c "import json; d=json.load(open('$OUTPUT/${label}_summary.json')); print(d.get('latency_stats_s',{}).get('p50',0))")
+  log "  errors=$errs lat_p50=${p50}s"
+  cat "$run_dir/request-metrics.jsonl.summary.json" >> $RESULTS_FILE
+fi
+log "=== migration v1 DONE ==="
--- a/scripts/sweep_ts1_migration_v2.sh
+++ b/scripts/sweep_ts1_migration_v2.sh
@@ -0,0 +1,76 @@
+#!/bin/bash
+# Migration v2 validation: KVC 1P3D ts=1 with BOTH:
+#   (1) reset-on-success blacklist decay (replay.py code change)
+#   (2) --kvcache-direct-max-uncached-tokens 8192 (was 2048 default)
+#
+# v1 results (kvc_1p3d_migration_run1) showed:
+#   - lat mean WORSE +11.7%, TTFT mean WORSE +71.3% — thrashing tax
+#   - direct-to-D rate UP +10.5pp (42.8 → 53.3%)
+#   - Fallback breakdown surprise: 41.3% are 'real-large-append' (>2048 tok),
+#     NOT 'session-not-resident' as we hypothesized
+#
+# v2 design (REFACTOR_PLAN_V1 + MIGRATION_V1_FINDINGS):
+#   (1) reset-on-success: clear (sess,D) reject counter on successful direct-to-D
+#       — eliminates blacklist-permanence bug → kills thrashing
+#   (2) bump direct-append threshold 2048 → 8192: lets more large-append turns
+#       go direct-to-D instead of fall through to seed (which often rejects)
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+MODEL=/mnt/kzlin/workflow/pd-hybrid/simm-swe-bench/models/Qwen3-30B-A3B-Instruct-2507
+TRACE=outputs/qwen35-swebench-50sess.jsonl
+OUTPUT=outputs/qwen3-30b-tp1-ts1-migration-v2
+VENV_PYTHON=.venv/bin/python
+RESULTS_FILE=$OUTPUT/sweep_results.txt
+
+mkdir -p $OUTPUT
+
+log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a $RESULTS_FILE; }
+
+log "=== TS=1 MIGRATION v2: reset-on-success + threshold=8192 ==="
+log "Baselines:"
+log "  baseline (no migration):        kvc_1p3d_run1 errors=5 lat_p50=0.811s ttft_p50=0.124s direct=42.8%"
+log "  v1 (migration permanent):       kvc_1p3d_migration_run1 errors=6 lat_p50=0.773s ttft_p50=0.057s direct=53.3% lat_mean=1.758s"
+log "  4DP ts=1:                       errors=0 lat_p50=0.659s ttft_p50=0.090s lat_mean=1.443s"
+log "Goal: kill thrashing tax (lat_mean ≤ 1.5s, p99 ≤ 9s) while preserving v1's direct-to-D gains."
+
+label=kvc_1p3d_migration_v2_run1
+log ""
+log "=== [migration v2] starting ==="
+PYTHONPATH=src:third_party/sglang/python \
+$VENV_PYTHON -m agentic_pd_hybrid.cli benchmark-live \
+  --trace $TRACE \
+  --output-root $OUTPUT \
+  --mechanism kvcache-centric \
+  --policy kv-aware \
+  --model-path $MODEL \
+  --prefill-workers 1 --decode-workers 3 \
+  --prefill-tp-size 1 --decode-tp-size 1 \
+  --prefill-gpu-ids 0 --decode-gpu-ids 1,2,3 \
+  --transfer-backend mooncake \
+  --gpu-budget 4 \
+  --time-scale 1 \
+  --session-sample-rate 1.0 \
+  --target-duration-s 100000 \
+  --concurrency-limit 32 \
+  --timeout-s 900 \
+  --request-timeout-s 300 \
+  --kvcache-admission-mode worker \
+  --kvcache-seed-min-turn-id 1 \
+  --kvcache-seed-max-inflight-decode -1 \
+  --kvcache-prefill-backup-policy release-after-transfer \
+  --kvcache-prefill-priority-eviction \
+  --kvcache-migration-reject-threshold 3 \
+  --kvcache-direct-max-uncached-tokens 8192
+
+run_dir=$(ls -td $OUTPUT/kvcache-centric-*/ 2>/dev/null | head -1)
+log "=== [migration v2] $label COMPLETED ==="
+if [ -f "$run_dir/request-metrics.jsonl.summary.json" ]; then
+  cp "$run_dir/request-metrics.jsonl.summary.json" "$OUTPUT/${label}_summary.json"
+  cp "$run_dir/request-metrics.jsonl" "$OUTPUT/${label}_metrics.jsonl"
+  errs=$($VENV_PYTHON -c "import json; d=json.load(open('$OUTPUT/${label}_summary.json')); print(d.get('error_count',0))")
+  p50=$($VENV_PYTHON -c "import json; d=json.load(open('$OUTPUT/${label}_summary.json')); print(d.get('latency_stats_s',{}).get('p50',0))")
+  log "  errors=$errs lat_p50=${p50}s"
+  cat "$run_dir/request-metrics.jsonl.summary.json" >> $RESULTS_FILE
+fi
+log "=== migration v2 DONE ==="