Fix multi-turn replay fidelity: track realized output tokens across all components
The replayer and proxy were building multi-turn prompts from trace tokens, but the model generates different output tokens. Subsequent turns had wrong prefix tokens, causing cache misses and invalid experimental measurements. - replay.py: min_tokens=max_tokens for deterministic length, return_token_ids to capture actual output, _apply_realized_prefix for next-turn correction - proxy: extract output token_ids from SSE, record prompt+output as realized prefix in shadow cache, extract _handle_local_request to deduplicate - bench.sh/launch_elastic_p2p.sh: default elastic mode to unified policy - mooncake_connector: only send prompt blocks (not stale output blocks), track failed_recving_block_ids for error recovery Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -26,7 +26,8 @@ TRACE="${TRACE:-$PROJECT_DIR/traces/w600_r0.0015_st30.jsonl}"
|
||||
# Defaults
|
||||
TAG=""
|
||||
MODE="baseline" # baseline | elastic
|
||||
POLICY="linear" # linear | lmetric
|
||||
POLICY="linear" # linear | lmetric | unified
|
||||
POLICY_SET=false
|
||||
N_INSTANCES=8
|
||||
BASE_PORT=8000
|
||||
PROXY_PORT=9090
|
||||
@@ -44,7 +45,7 @@ while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--tag) TAG="$2"; shift 2 ;;
|
||||
--mode) MODE="$2"; shift 2 ;;
|
||||
--policy) POLICY="$2"; shift 2 ;;
|
||||
--policy) POLICY="$2"; POLICY_SET=true; shift 2 ;;
|
||||
--instances) N_INSTANCES="$2"; shift 2 ;;
|
||||
--requests) REQUESTS="$2"; shift 2 ;;
|
||||
--trace) TRACE="$2"; shift 2 ;;
|
||||
@@ -60,11 +61,15 @@ while [[ $# -gt 0 ]]; do
|
||||
done
|
||||
|
||||
if [ -z "$TAG" ]; then
|
||||
echo "Usage: bench.sh --tag NAME --mode {baseline|elastic} [--instances N] [--policy {linear|lmetric}] [--requests N]"
|
||||
echo "Usage: bench.sh --tag NAME --mode {baseline|elastic} [--instances N] [--policy {linear|lmetric|unified}] [--requests N]"
|
||||
echo " Trace QPS is controlled by sample_trace.py --sample-ratio, not by bench.sh."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$MODE" = "elastic" ] && [ "$POLICY_SET" = "false" ]; then
|
||||
POLICY="unified"
|
||||
fi
|
||||
|
||||
OUTDIR="$PROJECT_DIR/outputs/$TAG"
|
||||
if [ -d "$OUTDIR" ] && [ -f "$OUTDIR/metrics.jsonl" ]; then
|
||||
echo "[ERROR] Output directory $OUTDIR already exists with data. Use a different --tag."
|
||||
|
||||
Reference in New Issue
Block a user