gahow/agentic-kvc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix: rename inst->chosen in generate function (NameError crash)

Browse Source 
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-05-24 02:55:01 +08:00
parent 5892739159
commit 97f4fe5164
1 changed files with 8 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
scripts/cache_aware_proxy.py
View File

@@ -426,16 +426,16 @@ async def _handle_combined(api, req_data, token_ids, input_length, session_id, h
try: try:
for attempt in range(MAX_STREAM_RETRIES): for attempt in range(MAX_STREAM_RETRIES):
try: try:
async with inst.client.stream("POST", api, json=req_data, headers=headers) as resp: async with chosen.client.stream("POST", api, json=req_data, headers=headers) as resp:
resp.raise_for_status() resp.raise_for_status()
async for chunk in resp.aiter_bytes(): async for chunk in resp.aiter_bytes():
if not prefill_done: if not prefill_done:
inst.pending_prefill_tokens -= estimated_new chosen.pending_prefill_tokens -= estimated_new
inst.ongoing_decode_tokens += input_length chosen.ongoing_decode_tokens += input_length
breakdown["t_first_token"] = _time.monotonic() breakdown["t_first_token"] = _time.monotonic()
prefill_done = True prefill_done = True
yield chunk yield chunk
inst.record_prefix(token_ids) chosen.record_prefix(token_ids)
break break
except (httpx.ConnectError, httpx.RemoteProtocolError): except (httpx.ConnectError, httpx.RemoteProtocolError):
if prefill_done or attempt >= MAX_STREAM_RETRIES - 1: if prefill_done or attempt >= MAX_STREAM_RETRIES - 1:
@@ -443,11 +443,11 @@ async def _handle_combined(api, req_data, token_ids, input_length, session_id, h
await asyncio.sleep(RETRY_DELAY_S) await asyncio.sleep(RETRY_DELAY_S)
finally: finally:
if not prefill_done: if not prefill_done:
inst.pending_prefill_tokens -= estimated_new chosen.pending_prefill_tokens -= estimated_new
else: else:
inst.ongoing_decode_tokens -= input_length chosen.ongoing_decode_tokens -= input_length
inst.ongoing_tokens -= input_length chosen.ongoing_tokens -= input_length
inst.num_requests -= 1 chosen.num_requests -= 1
breakdown["t_done"] = _time.monotonic() breakdown["t_done"] = _time.monotonic()
_breakdown_log.append(breakdown) _breakdown_log.append(breakdown)