fix(formatter): align release hash blocks to provider input length
Local tokenization does not always agree with the input_length the provider reports, which left the release row carrying a block count derived from our own count. Truncate or pad digest_hexes to the block count implied by the provider value. Padding digests are derived per request, so a padded block can never collide with another request and be scored as shared prefix reuse.
This commit is contained in:
@@ -817,6 +817,74 @@ class AliTracePipelineTest(unittest.TestCase):
|
||||
|
||||
self.assertEqual(Path(captured["dir"]), release_output_dir)
|
||||
|
||||
def test_export_release_ready_trace_aligns_hash_blocks_to_provider_input_length(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
root = Path(temp_dir)
|
||||
raw_input_path = root / "trace-raw.jsonl"
|
||||
release_output_path = root / "trace.jsonl"
|
||||
rows = [
|
||||
{
|
||||
"schema_version": "2026.04.21",
|
||||
"sort_time_ms": utc_ms("2026-04-17 15:00:01.000"),
|
||||
"meta": {
|
||||
"request_id": "req-pad",
|
||||
"model_family": "glm5",
|
||||
"request_ready_time_ms": utc_ms("2026-04-17 15:00:01.000"),
|
||||
"chat_id": 0,
|
||||
"parent_chat_id": -1,
|
||||
"turn": 1,
|
||||
},
|
||||
"canonical_prompt": "hello",
|
||||
"usage": {"input_tokens": 17, "output_tokens": 1},
|
||||
},
|
||||
{
|
||||
"schema_version": "2026.04.21",
|
||||
"sort_time_ms": utc_ms("2026-04-17 15:00:02.000"),
|
||||
"meta": {
|
||||
"request_id": "req-trim",
|
||||
"model_family": "glm5",
|
||||
"request_ready_time_ms": utc_ms("2026-04-17 15:00:02.000"),
|
||||
"chat_id": 1,
|
||||
"parent_chat_id": -1,
|
||||
"turn": 1,
|
||||
},
|
||||
"canonical_prompt": " ".join(str(index) for index in range(100)),
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1},
|
||||
},
|
||||
{
|
||||
"schema_version": "2026.04.21",
|
||||
"sort_time_ms": utc_ms("2026-04-17 15:00:03.000"),
|
||||
"meta": {
|
||||
"request_id": "req-zero",
|
||||
"model_family": "glm5",
|
||||
"request_ready_time_ms": utc_ms("2026-04-17 15:00:03.000"),
|
||||
"chat_id": 2,
|
||||
"parent_chat_id": -1,
|
||||
"turn": 1,
|
||||
},
|
||||
"canonical_prompt": "hello",
|
||||
"usage": {"input_tokens": 0, "output_tokens": 1},
|
||||
},
|
||||
]
|
||||
raw_input_path.write_text(
|
||||
"".join(json.dumps(row) + "\n" for row in rows),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
export_release_ready_trace(
|
||||
raw_input_path=raw_input_path,
|
||||
release_output_path=release_output_path,
|
||||
jobs=1,
|
||||
block_size=8,
|
||||
)
|
||||
|
||||
release_rows = [json.loads(line) for line in release_output_path.read_text(encoding="utf-8").splitlines()]
|
||||
self.assertEqual([len(row["hash_ids"]) for row in release_rows], [3, 1, 0])
|
||||
for row in release_rows:
|
||||
input_length = row["input_length"]
|
||||
expected_blocks = 0 if input_length <= 0 else (input_length + 7) // 8
|
||||
self.assertEqual(len(row["hash_ids"]), expected_blocks)
|
||||
|
||||
def test_format_and_sort_trace_infers_window_in_ready_time_scale_when_wall_clock_has_timezone_offset(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
root = Path(temp_dir)
|
||||
|
||||
Reference in New Issue
Block a user