gahow/agentic-kvc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix: rename inst->chosen in generate function (NameError crash)

Browse Source 
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Gahow Wang
2026-05-24 02:55:01 +08:00
parent 5892739159
commit 97f4fe5164
1 changed files with 8 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
scripts/cache_aware_proxy.py
View File

@@ -426,16 +426,16 @@ async def _handle_combined(api, req_data, token_ids, input_length, session_id, h
try:
for attempt in range(MAX_STREAM_RETRIES):
try:
async with inst.client.stream("POST", api, json=req_data, headers=headers) as resp:
async with chosen.client.stream("POST", api, json=req_data, headers=headers) as resp:
resp.raise_for_status()
async for chunk in resp.aiter_bytes():
if not prefill_done:
inst.pending_prefill_tokens -= estimated_new
inst.ongoing_decode_tokens += input_length
chosen.pending_prefill_tokens -= estimated_new
chosen.ongoing_decode_tokens += input_length
breakdown["t_first_token"] = _time.monotonic()
prefill_done = True
yield chunk
inst.record_prefix(token_ids)
chosen.record_prefix(token_ids)
break
except (httpx.ConnectError, httpx.RemoteProtocolError):
if prefill_done or attempt >= MAX_STREAM_RETRIES - 1:
@@ -443,11 +443,11 @@ async def _handle_combined(api, req_data, token_ids, input_length, session_id, h
await asyncio.sleep(RETRY_DELAY_S)
finally:
if not prefill_done:
inst.pending_prefill_tokens -= estimated_new
chosen.pending_prefill_tokens -= estimated_new
else:
inst.ongoing_decode_tokens -= input_length
inst.ongoing_tokens -= input_length
inst.num_requests -= 1
chosen.ongoing_decode_tokens -= input_length
chosen.ongoing_tokens -= input_length
chosen.num_requests -= 1
breakdown["t_done"] = _time.monotonic()
_breakdown_log.append(breakdown)