fix(formatter): align release hash blocks to provider input length

Local tokenization does not always agree with the input_length the provider
reports, which left the release row carrying a block count derived from our own
count. Truncate or pad digest_hexes to the block count implied by the provider
value. Padding digests are derived per request, so a padded block can never
collide with another request and be scored as shared prefix reuse.
This commit is contained in:
2026-09-11 06:30:57 +00:00
parent 5ee98f9e4d
commit 03d4828818
2 changed files with 117 additions and 0 deletions

View File

@@ -539,6 +539,45 @@ def _block_digest(block: list[int]) -> bytes:
return digest.digest()
def _prompt_block_count_for_input_length(input_length: int, block_size: int) -> int:
if input_length <= 0:
return 0
return (input_length + block_size - 1) // block_size
def _padding_block_digest_hex(*, request_identity: str, input_length: int, block_index: int) -> str:
digest = hashlib.blake2b(digest_size=16)
digest.update(b"trace_formatter.provider_input_length_padding\x00")
digest.update(request_identity.encode("utf-8", "ignore"))
digest.update(b"\x00")
digest.update(str(input_length).encode("ascii"))
digest.update(b"\x00")
digest.update(str(block_index).encode("ascii"))
return digest.hexdigest()
def _align_digest_hexes_to_provider_input_length(
digest_hexes: list[str],
*,
input_length: int,
block_size: int,
request_identity: str,
) -> list[str]:
target_block_count = _prompt_block_count_for_input_length(input_length, block_size)
if len(digest_hexes) >= target_block_count:
return digest_hexes[:target_block_count]
aligned = list(digest_hexes)
for block_index in range(len(aligned), target_block_count):
aligned.append(
_padding_block_digest_hex(
request_identity=request_identity,
input_length=input_length,
block_index=block_index,
)
)
return aligned
def _load_release_tokenizer(model_family: str) -> Tokenizer:
resolved = Path(resolve_tokenizer_path(model_family=model_family))
tokenizer_file = resolved / "tokenizer.json" if resolved.is_dir() else resolved
@@ -637,6 +676,16 @@ def _build_release_shard(
digest_hexes.append(_block_digest(block).hex())
core = _compute_release_row_core(row, base_ms=base_ms)
request_identity = str(
meta.get("request_id")
or f"{core.get('chat_id')}:{core.get('timestamp')}:{core.get('turn')}:{shard_index}:{row_count}"
)
digest_hexes = _align_digest_hexes_to_provider_input_length(
digest_hexes,
input_length=safe_int(core.get("input_length")),
block_size=block_size,
request_identity=request_identity,
)
destination.write(json.dumps(core, ensure_ascii=False, separators=(",", ":")))
destination.write("\t")
destination.write(",".join(digest_hexes))