Production-realistic baseline: APC 67.5%, TPOT +139% from interference

Updated methodology:
- Window+thin sampling preserves cross-session sharing (48% vs 16%)
- --max-single-turn-ratio 0.3 boosts multi-turn to 70%
- --window-seconds 600 for 10-min contiguous window
- Trace-driven replay (no session limit, no time compression)
- Daily config: --requests 850 (~13 min, APC~76%)

Key result: TPOT p90=0.175s (vs 0.073s in legacy 1-req/GPU setup),
confirming prefill-decode interference is real at production concurrency.
APC 67.5% (vs 44%) from better KV reuse preservation.

Also fixed KV reuse breakdown: 62% intra-session / 38% cross-session
(was incorrectly reported as 91% / 9%).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

scripts/sample_trace.py

@@ -70,13 +70,15 @@ def sample_sessions(
     sample_ratio: float | None = None,
     target_requests: int | None = None,
     max_single_turn_ratio: float | None = None,
+    window_seconds: float | None = None,
     seed: int,
 ) -> list[str]:
     """Sample sessions preserving KV cache reuse."""
     rng = random.Random(seed)
 
     if sample_ratio is not None:
-        selected = _sample_window_then_thin(rows_by_session, sample_ratio, rng)
+        selected = _sample_window_then_thin(rows_by_session, sample_ratio,
+                                            window_seconds, rng)
     elif target_requests is not None:
         all_sids = list(rows_by_session.keys())
         rng.shuffle(all_sids)
@@ -121,17 +123,18 @@ def _cap_single_turn(
 def _sample_window_then_thin(
     rows_by_session: dict[str, list[dict]],
     ratio: float,
+    window_seconds: float | None,
     rng: random.Random,
 ) -> list[str]:
     """Window + thin sampling that preserves cross-session sharing.
 
     1. Compute first-request timestamp for each session.
-    2. Pick a contiguous time window sized so that
-       window_sessions * thin_ratio ≈ total_sessions * ratio.
-       thin_ratio is kept >= 0.5 to preserve cross-session sharing.
-    3. Randomly drop (1 - thin_ratio) of sessions within the window.
+    2. Pick a contiguous time window:
+       - If --window-seconds given: use that duration, thin by ratio within it.
+       - Otherwise: auto-size so window_sessions * thin_ratio ≈ target.
+    3. Keep all sessions whose first request falls within the window.
+    4. Randomly thin sessions within the window to hit target count.
     """
-    # Session start times (timestamp of first request)
     session_starts: list[tuple[float, str]] = []
     for sid, rows in rows_by_session.items():
         t0 = min(float(r["timestamp"]) for r in rows)
@@ -140,35 +143,44 @@ def _sample_window_then_thin(
 
     total_sessions = len(session_starts)
     target_n = max(1, int(total_sessions * ratio))
+    trace_start = session_starts[0][0]
+    trace_end = session_starts[-1][0]
+    trace_duration = trace_end - trace_start
 
-    # Determine thin_ratio and window size
-    # thin_ratio >= 0.5 to preserve sharing; prefer 1.0 if window fits
-    # window_sessions = target_n / thin_ratio
+    if window_seconds is not None:
+        # Fixed window: pick a random start, thin to hit target ratio
+        max_start_t = trace_end - window_seconds
+        if max_start_t <= trace_start:
+            win_start_t = trace_start
+        else:
+            win_start_t = trace_start + rng.random() * (max_start_t - trace_start)
+        win_end_t = win_start_t + window_seconds
+
+        window_sids = [sid for t, sid in session_starts
+                       if win_start_t <= t <= win_end_t]
+        # Thin to target
+        if len(window_sids) > target_n:
+            thin_ratio = target_n / len(window_sids)
+            window_sids = [s for s in window_sids if rng.random() < thin_ratio]
+        return window_sids
+
+    # Auto-size window
     thin_ratio = min(1.0, max(0.5, ratio * 10))
-    window_sessions = int(target_n / thin_ratio)
-    window_sessions = min(window_sessions, total_sessions)
+    window_sessions = min(int(target_n / thin_ratio), total_sessions)
 
-    # Pick window start: random position in the trace
     max_start = total_sessions - window_sessions
-    if max_start <= 0:
-        window_start = 0
-    else:
-        window_start = rng.randint(0, max_start)
+    window_start = rng.randint(0, max_start) if max_start > 0 else 0
+    window_sids = [sid for _, sid in
+                   session_starts[window_start:window_start + window_sessions]]
 
-    window_sids = [sid for _, sid in session_starts[window_start:window_start + window_sessions]]
+    if thin_ratio < 1.0:
+        window_sids = [s for s in window_sids if rng.random() < thin_ratio]
 
-    # Thin within window
-    if thin_ratio >= 1.0:
-        selected = window_sids
-    else:
-        selected = [sid for sid in window_sids if rng.random() < thin_ratio]
+    if len(window_sids) > target_n * 1.2:
+        rng.shuffle(window_sids)
+        window_sids = window_sids[:int(target_n * 1.1)]
 
-    # Ensure we don't overshoot target by too much
-    if len(selected) > target_n * 1.2:
-        rng.shuffle(selected)
-        selected = selected[:int(target_n * 1.1)]
-
-    return selected
+    return window_sids
 
 
 def build_output(
@@ -245,6 +257,8 @@ def main() -> None:
                    help="Target number of requests (legacy, no sharing preservation)")
     p.add_argument("--max-single-turn-ratio", type=float, default=None,
                    help="Cap single-turn sessions to this fraction of total (e.g. 0.3)")
+    p.add_argument("--window-seconds", type=float, default=None,
+                   help="Time window duration in seconds (e.g. 600 for 10 min)")
     p.add_argument("--seed", type=int, default=42)
     args = p.parse_args()
 
@@ -262,6 +276,7 @@ def main() -> None:
         sample_ratio=args.sample_ratio,
         target_requests=args.target_requests,
         max_single_turn_ratio=args.max_single_turn_ratio,
+        window_seconds=args.window_seconds,
         seed=args.seed,
     )