diff --git a/scripts/cache_aware_proxy.py b/scripts/cache_aware_proxy.py index 629ce49..4a49072 100644 --- a/scripts/cache_aware_proxy.py +++ b/scripts/cache_aware_proxy.py @@ -426,16 +426,16 @@ async def _handle_combined(api, req_data, token_ids, input_length, session_id, h try: for attempt in range(MAX_STREAM_RETRIES): try: - async with inst.client.stream("POST", api, json=req_data, headers=headers) as resp: + async with chosen.client.stream("POST", api, json=req_data, headers=headers) as resp: resp.raise_for_status() async for chunk in resp.aiter_bytes(): if not prefill_done: - inst.pending_prefill_tokens -= estimated_new - inst.ongoing_decode_tokens += input_length + chosen.pending_prefill_tokens -= estimated_new + chosen.ongoing_decode_tokens += input_length breakdown["t_first_token"] = _time.monotonic() prefill_done = True yield chunk - inst.record_prefix(token_ids) + chosen.record_prefix(token_ids) break except (httpx.ConnectError, httpx.RemoteProtocolError): if prefill_done or attempt >= MAX_STREAM_RETRIES - 1: @@ -443,11 +443,11 @@ async def _handle_combined(api, req_data, token_ids, input_length, session_id, h await asyncio.sleep(RETRY_DELAY_S) finally: if not prefill_done: - inst.pending_prefill_tokens -= estimated_new + chosen.pending_prefill_tokens -= estimated_new else: - inst.ongoing_decode_tokens -= input_length - inst.ongoing_tokens -= input_length - inst.num_requests -= 1 + chosen.ongoing_decode_tokens -= input_length + chosen.ongoing_tokens -= input_length + chosen.num_requests -= 1 breakdown["t_done"] = _time.monotonic() _breakdown_log.append(breakdown)