gahow/ali-trace-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
Gahow Wang
2026-04-21 15:44:47 +00:00
commit bce3fe1395
40 changed files with 1758724 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tests/conftest.py Normal file
View File

@@ -0,0 +1,7 @@
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))

728
tests/test_ali_trace_pipeline.py Normal file
View File

@@ -0,0 +1,728 @@
import json
import tempfile
import unittest
from datetime import datetime, timezone
from pathlib import Path
from trace_analyzer.cli import main as analyzer_main
from trace_analyzer.parser import load_records
from trace_formatter.cli import main as formatter_main
from trace_formatter.formatting import build_unified_row, discover_source_files, format_and_sort_trace
def utc_ms(value: str) -> int:
return int(datetime.strptime(value, "%Y-%m-%d %H:%M:%S.%f").replace(tzinfo=timezone.utc).timestamp() * 1000)
def wall_clock_ms_with_offset(value: str, offset_hours: int) -> int:
return utc_ms(value) - offset_hours * 60 * 60 * 1000
def make_raw_row(
request_id: str,
ready_ms: int,
tool_role: bool = False,
*,
raw_session_id: str = "sess-1",
user_id: str = "user-1",
messages: list[dict] | None = None,
time_text: str = "2026-04-17 15:00:00.000",
) -> dict:
if messages is None:
messages = [{"role": "user", "content": "hello"}]
if tool_role:
messages.extend(
[
{"role": "assistant", "content": "calling"},
{"role": "tool", "content": "tool-output"},
]
)
return {
"__time__": str(ready_ms // 1000),
"request_id": request_id,
"session_id": raw_session_id,
"request_model": "glm-5",
"time": time_text,
"status_code": "1000",
"status_name": "ok",
"total_cost_time": "250",
"request_params": json.dumps(
{
"header": {
"attributes": {
"x-dashscope-inner-requestreadytime": str(ready_ms),
"user_id": user_id,
}
},
"payload": {
"input": {"messages": messages},
"parameters": {
"tools": [{"type": "function", "function": {"name": "read_file"}}],
},
},
}
),
"response_params": json.dumps(
{
"header": {
"attributes": {
"x-ds-backend-first-request-time": str(ready_ms - 100),
"x-ds-backend-first-response-time": str(ready_ms + 150),
}
},
"payload": {
"output": {
"choices": [
{
"message": {
"role": "assistant",
"content": "done",
}
}
]
},
"usage": {
"input_tokens": 20,
"output_tokens": 5,
"total_tokens": 25,
"output_tokens_details": {"reasoning_tokens": 1},
"prompt_tokens_details": {"cached_tokens": 10},
},
}
}
),
}
def make_qwen_raw_row(
request_id: str,
ready_ms: int,
*,
raw_session_id: str = "qwen-sess-1",
user_id: str = "qwen-user-1",
time_text: str = "2026-04-19 15:00:00.000",
) -> dict:
return {
"__time__": str(ready_ms // 1000),
"request_id": request_id,
"session_id": raw_session_id,
"request_model": "qwen3-coder-plus-2025-09-23",
"time": time_text,
"status_code": "200",
"status_name": "OK",
"total_cost_time": "800",
"request_params": json.dumps(
{
"header": {
"attributes": {
"x-dashscope-inner-requestreadytime": str(ready_ms),
"user_id": user_id,
}
},
"payload": {
"input": {
"messages": [
{"role": "system", "content": "You are Qwen."},
{"role": "user", "content": "Write a function."},
]
},
"parameters": {
"tools": [
{
"type": "function",
"function": {
"name": "run_command",
"parameters": {
"type": "object",
"properties": {"cmd": {"type": "string"}},
},
},
}
]
},
},
},
ensure_ascii=False,
),
"response_params": json.dumps(
{
"header": {
"attributes": {
"x-ds-backend-first-request-time": str(ready_ms - 200),
"x-ds-backend-first-response-time": str(ready_ms + 300),
}
},
"payload": {
"output": {
"choices": [
{
"delta": {"role": "assistant", "content": "Sure."},
"finish_reason": "stop",
}
],
"usage": {
"prompt_tokens": 128,
"completion_tokens": 16,
"total_tokens": 144,
"prompt_tokens_details": {"cached_tokens": 32},
},
}
}
},
ensure_ascii=False,
),
}
class AliTracePipelineTest(unittest.TestCase):
def test_build_unified_row_keeps_analysis_fields(self):
ready_ms = utc_ms("2026-04-17 15:00:00.321")
row = build_unified_row(make_raw_row("req-1", ready_ms), source_file="a.jsonl", source_line=3)
self.assertEqual(row["meta"]["request_id"], "req-1")
self.assertEqual(row["sort_time_ms"], ready_ms)
self.assertEqual(row["usage"]["cached_tokens"], 10)
self.assertEqual(row["meta"]["raw_session_id"], "sess-1")
self.assertEqual(row["meta"]["user_id"], "user-1")
self.assertEqual(row["meta"]["model_family"], "glm5")
self.assertEqual(row["meta"]["backend_first_request_time_ms"], ready_ms - 100)
self.assertEqual(row["meta"]["backend_first_response_time_ms"], ready_ms + 150)
self.assertEqual(row["meta"]["total_cost_time_ms"], 250)
self.assertEqual(row["declared_tools"][0]["name"], "read_file")
self.assertEqual(row["message_events"][0]["role"], "user")
self.assertEqual(row["raw_messages"][0]["content"], "hello")
self.assertIn("[gMASK]<sop>", row["canonical_prompt"])
def test_build_unified_row_supports_qwen_raw_trace_defaults(self):
ready_ms = utc_ms("2026-04-19 15:00:00.321")
row = build_unified_row(make_qwen_raw_row("qwen-req-1", ready_ms), source_file="qwen.jsonl", source_line=5)
self.assertEqual(row["meta"]["model_family"], "qwen3-coder")
self.assertEqual(row["usage"]["input_tokens"], 128)
self.assertEqual(row["usage"]["output_tokens"], 16)
self.assertEqual(row["usage"]["cached_tokens"], 32)
self.assertEqual(row["meta"]["backend_first_request_time_ms"], ready_ms - 200)
self.assertEqual(row["meta"]["backend_first_response_time_ms"], ready_ms + 300)
self.assertEqual(row["declared_tools"][0]["name"], "run_command")
self.assertIn("<|im_start|>system", row["canonical_prompt"])
def test_format_and_sort_trace_outputs_time_sorted_unified_rows(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
input_dir = root / "raw"
input_dir.mkdir()
output_path = root / "formatted.jsonl"
rows_by_file = {
"0417-1530-1600.jsonl": [
make_raw_row(
"req-late",
utc_ms("2026-04-17 15:00:03.000"),
time_text="2026-04-17 15:30:30.000",
),
make_raw_row(
"req-middle",
utc_ms("2026-04-17 15:00:02.000"),
tool_role=True,
time_text="2026-04-17 15:30:20.000",
),
],
"0417-1500-1530.jsonl": [
make_raw_row(
"req-first",
utc_ms("2026-04-17 15:00:01.000"),
time_text="2026-04-17 15:00:10.000",
),
],
}
for filename, rows in rows_by_file.items():
with (input_dir / filename).open("w", encoding="utf-8") as handle:
for row in rows:
handle.write(json.dumps(row) + "\n")
discovered = discover_source_files(input_dir)
self.assertEqual([path.name for path in discovered], sorted(rows_by_file))
stats = format_and_sort_trace(input_dir=input_dir, output_path=output_path, chunk_bytes=512)
self.assertEqual(stats["row_count"], 3)
records = load_records(output_path)
self.assertEqual([record.meta.request_id for record in records], ["req-first", "req-middle", "req-late"])
self.assertEqual(records[1].usage.cached_tokens, 10)
def test_format_and_sort_trace_reconstructs_logical_sessions_from_message_history(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
input_dir = root / "raw"
input_dir.mkdir()
output_path = root / "formatted.jsonl"
rows = [
make_raw_row(
"req-root-a",
utc_ms("2026-04-17 15:00:01.000"),
raw_session_id="111",
user_id="user-a",
messages=[{"role": "user", "content": "hello"}],
),
make_raw_row(
"req-root-b",
utc_ms("2026-04-17 15:00:01.500"),
raw_session_id="111",
user_id="user-b",
messages=[{"role": "user", "content": "hello"}],
),
make_raw_row(
"req-turn-2-a",
utc_ms("2026-04-17 15:00:02.000"),
raw_session_id="999",
user_id="user-a",
messages=[
{"role": "user", "content": "hello"},
{"role": "assistant", "content": "hi"},
{"role": "user", "content": "continue"},
],
),
]
with (input_dir / "0417-1500-1530.jsonl").open("w", encoding="utf-8") as handle:
for row in rows:
handle.write(json.dumps(row) + "\n")
format_and_sort_trace(input_dir=input_dir, output_path=output_path, chunk_bytes=256)
formatted_rows = [json.loads(line) for line in output_path.read_text(encoding="utf-8").splitlines()]
self.assertEqual(formatted_rows[0]["meta"]["turn"], 1)
self.assertEqual(formatted_rows[1]["meta"]["turn"], 1)
self.assertEqual(formatted_rows[2]["meta"]["turn"], 2)
self.assertNotEqual(formatted_rows[0]["meta"]["session_id"], formatted_rows[1]["meta"]["session_id"])
self.assertEqual(formatted_rows[0]["meta"]["session_id"], formatted_rows[2]["meta"]["session_id"])
self.assertEqual(formatted_rows[2]["meta"]["parent_request_id"], "req-root-a")
def test_format_and_sort_trace_does_not_merge_sessions_on_shared_system_prompt_only(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
input_dir = root / "raw"
input_dir.mkdir()
output_path = root / "formatted.jsonl"
rows = [
make_raw_row(
"req-a",
utc_ms("2026-04-17 15:00:01.000"),
user_id="user-a",
messages=[
{"role": "system", "content": "shared system prompt"},
{"role": "user", "content": "question a"},
],
),
make_raw_row(
"req-b",
utc_ms("2026-04-17 15:00:02.000"),
user_id="user-a",
messages=[
{"role": "system", "content": "shared system prompt"},
{"role": "user", "content": "question b"},
],
),
]
with (input_dir / "0417-1500-1530.jsonl").open("w", encoding="utf-8") as handle:
for row in rows:
handle.write(json.dumps(row) + "\n")
format_and_sort_trace(input_dir=input_dir, output_path=output_path, chunk_bytes=256)
formatted_rows = [json.loads(line) for line in output_path.read_text(encoding="utf-8").splitlines()]
self.assertEqual([row["meta"]["turn"] for row in formatted_rows], [1, 1])
self.assertNotEqual(formatted_rows[0]["meta"]["session_id"], formatted_rows[1]["meta"]["session_id"])
self.assertEqual([row["meta"]["parent_request_id"] for row in formatted_rows], ["", ""])
def test_format_and_sort_trace_reconstructs_sessions_when_child_drops_parent_suffix(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
input_dir = root / "raw"
input_dir.mkdir()
output_path = root / "formatted.jsonl"
rows = [
make_raw_row(
"req-root",
utc_ms("2026-04-17 15:00:01.000"),
raw_session_id="111",
user_id="user-a",
messages=[
{"role": "user", "content": "hello"},
{"role": "assistant", "content": "draft answer"},
{"role": "user", "content": "continue"},
{"role": "assistant", "content": "long hidden reasoning that may be evicted"},
],
),
make_raw_row(
"req-turn-2",
utc_ms("2026-04-17 15:00:02.000"),
raw_session_id="999",
user_id="user-a",
messages=[
{"role": "user", "content": "hello"},
{"role": "assistant", "content": "draft answer"},
{"role": "user", "content": "continue"},
],
),
]
with (input_dir / "0417-1500-1530.jsonl").open("w", encoding="utf-8") as handle:
for row in rows:
handle.write(json.dumps(row) + "\n")
format_and_sort_trace(input_dir=input_dir, output_path=output_path, chunk_bytes=256)
formatted_rows = [json.loads(line) for line in output_path.read_text(encoding="utf-8").splitlines()]
self.assertEqual(formatted_rows[0]["meta"]["turn"], 1)
self.assertEqual(formatted_rows[1]["meta"]["turn"], 2)
self.assertEqual(formatted_rows[0]["meta"]["session_id"], formatted_rows[1]["meta"]["session_id"])
self.assertEqual(formatted_rows[1]["meta"]["parent_request_id"], "req-root")
def test_format_and_sort_trace_reconstructs_sessions_when_last_non_user_messages_change(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
input_dir = root / "raw"
input_dir.mkdir()
output_path = root / "formatted.jsonl"
rows = [
make_raw_row(
"req-root",
utc_ms("2026-04-17 15:00:01.000"),
user_id="user-a",
messages=[
{"role": "user", "content": "hello"},
{"role": "assistant", "content": "draft answer"},
],
),
make_raw_row(
"req-regenerated",
utc_ms("2026-04-17 15:00:02.000"),
user_id="user-a",
messages=[
{"role": "user", "content": "hello"},
{"role": "assistant", "content": "regenerated answer"},
],
),
]
with (input_dir / "0417-1500-1530.jsonl").open("w", encoding="utf-8") as handle:
for row in rows:
handle.write(json.dumps(row) + "\n")
format_and_sort_trace(input_dir=input_dir, output_path=output_path, chunk_bytes=256)
formatted_rows = [json.loads(line) for line in output_path.read_text(encoding="utf-8").splitlines()]
self.assertEqual([row["meta"]["turn"] for row in formatted_rows], [1, 2])
self.assertEqual(formatted_rows[0]["meta"]["session_id"], formatted_rows[1]["meta"]["session_id"])
self.assertEqual(formatted_rows[1]["meta"]["parent_request_id"], "req-root")
def test_format_and_sort_trace_truncates_requests_before_inferred_window_start_by_ready_time(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
input_dir = root / "raw"
input_dir.mkdir()
output_path = root / "formatted.jsonl"
rows = [
make_raw_row(
"req-too-early",
utc_ms("2026-04-17 14:59:59.900"),
time_text="2026-04-17 15:00:00.100",
),
make_raw_row(
"req-kept",
utc_ms("2026-04-17 15:00:00.100"),
time_text="2026-04-17 15:00:00.100",
),
]
with (input_dir / "0417-1500-1530.jsonl").open("w", encoding="utf-8") as handle:
for row in rows:
handle.write(json.dumps(row) + "\n")
stats = format_and_sort_trace(input_dir=input_dir, output_path=output_path, chunk_bytes=256)
self.assertEqual(stats["row_count"], 1)
self.assertEqual(stats["truncated_row_count"], 1)
formatted_rows = [json.loads(line) for line in output_path.read_text(encoding="utf-8").splitlines()]
self.assertEqual([row["meta"]["request_id"] for row in formatted_rows], ["req-kept"])
def test_format_and_sort_trace_filters_empty_messages_and_empty_response_params(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
input_dir = root / "raw"
input_dir.mkdir()
output_path = root / "formatted.jsonl"
empty_messages_row = make_raw_row("req-empty-messages", utc_ms("2026-04-17 15:00:01.000"))
empty_messages_request = json.loads(empty_messages_row["request_params"])
empty_messages_request["payload"]["input"]["messages"] = []
empty_messages_row["request_params"] = json.dumps(empty_messages_request)
empty_response_row = make_raw_row("req-empty-response", utc_ms("2026-04-17 15:00:02.000"))
empty_response_row["response_params"] = None
kept_row = make_raw_row("req-kept", utc_ms("2026-04-17 15:00:03.000"))
with (input_dir / "0417-1500-1530.jsonl").open("w", encoding="utf-8") as handle:
for row in [empty_messages_row, empty_response_row, kept_row]:
handle.write(json.dumps(row) + "\n")
stats = format_and_sort_trace(input_dir=input_dir, output_path=output_path, chunk_bytes=256)
self.assertEqual(stats["row_count"], 1)
self.assertEqual(stats["filtered_row_count"], 2)
self.assertEqual(stats["filtered_empty_messages_row_count"], 1)
self.assertEqual(stats["filtered_empty_response_params_row_count"], 1)
formatted_rows = [json.loads(line) for line in output_path.read_text(encoding="utf-8").splitlines()]
self.assertEqual([row["meta"]["request_id"] for row in formatted_rows], ["req-kept"])
def test_trace_formatter_cli_formats_one_raw_jsonl_file(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
raw_path = root / "trace-glm5.jsonl"
output_root = root / "outputs" / "formatted"
with raw_path.open("w", encoding="utf-8") as handle:
handle.write(json.dumps(make_raw_row("req-1", utc_ms("2026-04-17 15:00:01.000"))) + "\n")
handle.write(json.dumps(make_raw_row("req-2", utc_ms("2026-04-17 15:00:02.000"))) + "\n")
exit_code = formatter_main(
["format", str(raw_path), "--output-root", str(output_root), "--chunk-bytes", "256"]
)
self.assertEqual(exit_code, 0)
raw_formatted_path = output_root / "trace-glm5-raw.jsonl"
self.assertTrue(raw_formatted_path.exists())
records = load_records(raw_formatted_path)
self.assertEqual([record.meta.request_id for record in records], ["req-1", "req-2"])
def test_trace_formatter_cli_defaults_to_trace_formatted_dir_for_trace_file(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
trace_dir = root / "trace-demo"
trace_dir.mkdir()
raw_path = trace_dir / "0417-1500-1530.jsonl"
with raw_path.open("w", encoding="utf-8") as handle:
handle.write(json.dumps(make_raw_row("req-1", utc_ms("2026-04-17 15:00:01.000"))) + "\n")
exit_code = formatter_main(["format", str(raw_path)])
self.assertEqual(exit_code, 0)
raw_formatted_path = root / "trace-demo-formatted" / "041715-041715-raw.jsonl"
self.assertTrue(raw_formatted_path.exists())
records = load_records(raw_formatted_path)
self.assertEqual([record.meta.request_id for record in records], ["req-1"])
def test_trace_formatter_cli_build_release_from_raw_in_second_stage(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
raw_path = root / "trace-glm5.jsonl"
output_root = root / "outputs" / "formatted"
with raw_path.open("w", encoding="utf-8") as handle:
handle.write(json.dumps(make_raw_row("req-1", utc_ms("2026-04-17 15:00:01.000"))) + "\n")
handle.write(json.dumps(make_raw_row("req-2", utc_ms("2026-04-17 15:00:02.000"))) + "\n")
format_exit_code = formatter_main(
["format", str(raw_path), "--output-root", str(output_root), "--chunk-bytes", "256"]
)
self.assertEqual(format_exit_code, 0)
raw_formatted_path = output_root / "trace-glm5-raw.jsonl"
release_formatted_path = output_root / "trace-glm5.jsonl"
release_exit_code = formatter_main(
["build-release", str(raw_formatted_path), "--jobs", "2", "--block-size", "8"]
)
self.assertEqual(release_exit_code, 0)
self.assertTrue(release_formatted_path.exists())
release_rows = [json.loads(line) for line in release_formatted_path.read_text(encoding="utf-8").splitlines()]
self.assertEqual(len(release_rows), 2)
self.assertEqual(sorted(release_rows[0]), ["chat_id", "hash_ids", "input_length", "output_length", "parent_chat_id", "timestamp", "turn", "type"])
self.assertEqual([row["chat_id"] for row in release_rows], [0, 1])
def test_trace_formatter_cli_can_write_progress_log(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
raw_path = root / "trace-glm5.jsonl"
output_root = root / "formatted"
log_path = root / "tmp" / "format.log"
with raw_path.open("w", encoding="utf-8") as handle:
handle.write(json.dumps(make_raw_row("req-1", utc_ms("2026-04-17 15:00:01.000"))) + "\n")
exit_code = formatter_main(
[
"format",
str(raw_path),
"--output-root",
str(output_root),
"--chunk-bytes",
"256",
"--log-file",
str(log_path),
]
)
self.assertEqual(exit_code, 0)
self.assertTrue(log_path.exists())
self.assertIn("Scan raw trace", log_path.read_text(encoding="utf-8"))
def test_format_and_sort_trace_infers_window_in_ready_time_scale_when_wall_clock_has_timezone_offset(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
input_dir = root / "raw"
input_dir.mkdir()
output_path = root / "formatted.jsonl"
rows = [
make_raw_row(
"req-too-early",
wall_clock_ms_with_offset("2026-04-19 14:59:59.900", 8),
time_text="2026-04-19 14:59:59.900",
),
make_raw_row(
"req-kept-start",
wall_clock_ms_with_offset("2026-04-19 15:00:00.100", 8),
time_text="2026-04-19 15:00:00.100",
),
make_raw_row(
"req-kept-end",
wall_clock_ms_with_offset("2026-04-19 16:59:59.900", 8),
time_text="2026-04-19 16:59:59.900",
),
make_raw_row(
"req-too-late",
wall_clock_ms_with_offset("2026-04-19 17:00:00.000", 8),
time_text="2026-04-19 17:00:00.000",
),
]
with (input_dir / "0419-1500-1700.jsonl").open("w", encoding="utf-8") as handle:
for row in rows:
handle.write(json.dumps(row) + "\n")
stats = format_and_sort_trace(input_dir=input_dir, output_path=output_path, chunk_bytes=256)
self.assertEqual(stats["row_count"], 2)
self.assertEqual(stats["truncated_row_count"], 2)
formatted_rows = [json.loads(line) for line in output_path.read_text(encoding="utf-8").splitlines()]
self.assertEqual([row["meta"]["request_id"] for row in formatted_rows], ["req-kept-start", "req-kept-end"])
def test_format_and_sort_trace_normalizes_mixed_ready_time_scales_before_sorting(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
input_dir = root / "raw"
input_dir.mkdir()
output_path = root / "formatted.jsonl"
standard_row = make_raw_row(
"req-standard",
wall_clock_ms_with_offset("2026-04-19 15:01:00.000", 8),
time_text="2026-04-19 15:01:00.000",
)
anomalous_row = make_raw_row(
"req-anomalous",
utc_ms("2026-04-19 15:00:10.000"),
time_text="2026-04-19 15:00:10.000",
)
with (input_dir / "0419-1500-1700.jsonl").open("w", encoding="utf-8") as handle:
handle.write(json.dumps(standard_row) + "\n")
handle.write(json.dumps(anomalous_row) + "\n")
stats = format_and_sort_trace(input_dir=input_dir, output_path=output_path, chunk_bytes=256)
self.assertEqual(stats["row_count"], 1)
self.assertEqual(stats["truncated_row_count"], 1)
formatted_rows = [json.loads(line) for line in output_path.read_text(encoding="utf-8").splitlines()]
self.assertEqual([row["meta"]["request_id"] for row in formatted_rows], ["req-standard"])
def test_trace_analyzer_analyze_writes_reports_and_figures_under_outputs(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
raw_dir = root / "trace-glm5"
raw_dir.mkdir()
formatted_root = root / "outputs" / "formatted"
analysis_root = root / "outputs" / "analysis"
raw_path = raw_dir / "0417-1500-1530.jsonl"
with raw_path.open("w", encoding="utf-8") as handle:
handle.write(
json.dumps(
make_raw_row(
"req-1",
utc_ms("2026-04-17 15:00:01.000"),
user_id="user-1",
)
)
+ "\n"
)
handle.write(
json.dumps(
make_raw_row(
"req-2",
utc_ms("2026-04-17 15:00:02.000"),
user_id="user-1",
messages=[
{"role": "user", "content": "hello"},
{"role": "assistant", "content": "hi"},
{"role": "user", "content": "continue"},
],
)
)
+ "\n"
)
formatter_exit_code = formatter_main(["format", str(raw_dir), "--output-root", str(formatted_root)])
self.assertEqual(formatter_exit_code, 0)
formatted_path = formatted_root / "041715-041715-raw.jsonl"
release_exit_code = formatter_main(["build-release", str(formatted_path), "--jobs", "1", "--block-size", "8"])
self.assertEqual(release_exit_code, 0)
analyzer_exit_code = analyzer_main(
[
"analyze",
str(formatted_path),
"--output-root",
str(analysis_root),
"--segment-mode",
"bytes",
"--block-size",
"8",
]
)
self.assertEqual(analyzer_exit_code, 0)
analysis_dir = analysis_root / "041715-041715"
self.assertFalse((analysis_dir / "normalized.jsonl").exists())
self.assertTrue((analysis_dir / "features.csv").exists())
self.assertTrue((analysis_dir / "summary.json").exists())
self.assertTrue((analysis_dir / "report.md").exists())
self.assertTrue((analysis_dir / "details" / "details_summary.json").exists())
self.assertFalse(any((analysis_dir / "details").glob("*.sqlite*")))
self.assertTrue((analysis_dir / "details" / "request_metrics.csv").exists())
self.assertTrue((analysis_dir / "details" / "theoretical_block_reuse_gaps.csv").exists())
self.assertTrue((analysis_dir / "details" / "session_bucket_boundary_miss.csv").exists())
self.assertTrue((analysis_dir / "details" / "theoretical_alive_block_timeline.csv").exists())
self.assertTrue((analysis_dir / "figures" / "01_input_output_length_cdf.png").exists())
self.assertTrue((analysis_dir / "figures" / "02_session_turns_cdf.png").exists())
self.assertTrue((analysis_dir / "figures" / "13_session_cross_bucket_kvcache_miss.png").exists())
self.assertTrue((analysis_dir / "figures" / "manifest.json").exists())
self.assertFalse((root / "figs").exists())
features_mtime_ns = (analysis_dir / "features.csv").stat().st_mtime_ns
details_summary_mtime_ns = (analysis_dir / "details" / "details_summary.json").stat().st_mtime_ns
analyzer_exit_code = analyzer_main(
[
"analyze",
str(formatted_path),
"--output-root",
str(analysis_root),
"--segment-mode",
"bytes",
"--block-size",
"8",
]
)
self.assertEqual(analyzer_exit_code, 0)
self.assertEqual((analysis_dir / "features.csv").stat().st_mtime_ns, features_mtime_ns)
self.assertEqual(
(analysis_dir / "details" / "details_summary.json").stat().st_mtime_ns,
details_summary_mtime_ns,
)

772
tests/test_trace_analyzer.py Normal file
View File

@@ -0,0 +1,772 @@
import csv
import json
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from trace_analyzer.features import compute_features
from trace_analyzer.parser import default_output_dir, infer_analysis_dataset_name, load_records
from trace_analyzer.report import build_summary
from trace_analyzer.study import (
build_alive_block_timeline,
build_input_length_bucket_defs,
compute_theoretical_cache,
parse_input_length_bucket_thresholds,
summarize_cache_reuse_by_input_length_bucket,
summarize_session_bucket_boundary_miss,
)
from trace_formatter.formatting import format_and_sort_trace
def make_record(
request_id,
session_id,
messages,
tools,
usage,
total_cost_time,
status_code="1000",
model="glm-5",
):
return {
"request_id": request_id,
"session_id": session_id,
"request_model": model,
"time": "2026-04-09 09:00:00.000",
"status_code": status_code,
"status_name": "ok",
"total_cost_time": str(total_cost_time),
"request_params": json.dumps(
{
"payload": {
"input": {"messages": messages},
"parameters": {"tools": tools},
}
},
ensure_ascii=False,
),
"response_params": json.dumps(
{
"header": {
"attributes": {
"x-ds-backend-first-request-time": "123",
"x-ds-backend-first-response-time": "456",
}
},
"payload": {
"usage": usage,
}
},
ensure_ascii=False,
),
}
def make_qwen_record(request_id="qwen-1", session_id="sess-qwen-1", total_cost_time=900):
return {
"request_id": request_id,
"session_id": session_id,
"request_model": "qwen3-coder-plus-2025-09-23",
"time": "2026-04-19 15:00:00.000",
"status_code": "200",
"status_name": "OK",
"total_cost_time": str(total_cost_time),
"request_params": json.dumps(
{
"header": {
"attributes": {
"x-dashscope-inner-requestreadytime": "1776582000000",
}
},
"payload": {
"input": {
"messages": [
{"role": "system", "content": "You are Qwen."},
{"role": "user", "content": "List files"},
]
},
"parameters": {
"tools": [
{
"type": "function",
"function": {
"name": "run_command",
"parameters": {
"type": "object",
"properties": {"cmd": {"type": "string"}},
},
},
}
]
},
},
},
ensure_ascii=False,
),
"response_params": json.dumps(
{
"header": {
"attributes": {
"x-ds-backend-first-request-time": "1776581999083",
"x-ds-backend-first-response-time": "1776581999918",
}
},
"payload": {
"output": {
"choices": [
{
"delta": {"role": "assistant", "content": "ls"},
"finish_reason": "stop",
}
],
"usage": {
"prompt_tokens": 90,
"completion_tokens": 12,
"total_tokens": 102,
"prompt_tokens_details": {"cached_tokens": 24},
},
}
}
},
ensure_ascii=False,
),
}
class TraceAnalyzerTest(unittest.TestCase):
def write_raw_fixture(self, rows):
temp_dir = tempfile.TemporaryDirectory()
path = Path(temp_dir.name) / "trace.jsonl"
with open(path, "w", encoding="utf-8") as handle:
for row in rows:
handle.write(json.dumps(row, ensure_ascii=False) + "\n")
self.addCleanup(temp_dir.cleanup)
return path
def format_fixture(self, rows):
raw_path = self.write_raw_fixture(rows)
formatted_path = raw_path.parent / "trace-raw.jsonl"
format_and_sort_trace(
input_dir=raw_path,
output_path=formatted_path,
chunk_bytes=256,
truncate_to_window=False,
)
return formatted_path
def test_infer_analysis_dataset_name_includes_model_slug_from_formatted_parent(self):
path = Path("trace-qwen3-coder-formatted/041915-041917-raw.jsonl")
self.assertEqual(
infer_analysis_dataset_name(path),
"qwen3-coder-041915-041917",
)
self.assertEqual(
default_output_dir(path),
Path("outputs/analysis/qwen3-coder-041915-041917"),
)
def test_load_records_parses_formatter_output(self):
path = self.format_fixture(
[
make_record(
"req-1",
"sess-1",
[
{
"role": "system",
"content": [
{
"text": "sys",
"cache_control": {"type": "ephemeral"},
}
],
},
{"role": "user", "content": "hello"},
{"role": "assistant", "content": "working"},
{"role": "tool", "content": "tool-output"},
],
[{"type": "function", "function": {"name": "read"}}],
{
"input_tokens": 100,
"output_tokens": 20,
"total_tokens": 120,
"output_tokens_details": {"reasoning_tokens": 7},
"prompt_tokens_details": {"cached_tokens": 60},
},
total_cost_time=500,
)
]
)
records = load_records(path)
self.assertEqual(len(records), 1)
record = records[0]
self.assertEqual(record.meta.request_id, "req-1")
self.assertEqual(record.meta.line_number, 1)
self.assertEqual(record.messages[0].has_cache_control, True)
self.assertEqual(record.declared_tools[0].name, "read")
self.assertEqual(record.usage.cached_tokens, 60)
self.assertEqual(record.usage.reasoning_tokens, 7)
self.assertEqual(record.meta.backend_first_request_time_ms, 123)
self.assertEqual(record.meta.backend_first_response_time_ms, 456)
self.assertEqual(record.meta.total_cost_time_ms, 500)
self.assertEqual(record.raw_messages[0]["role"], "system")
self.assertTrue(record.canonical_prompt)
self.assertIn("[gMASK]<sop>", record.canonical_prompt)
self.assertIn("<|assistant|><think>", record.canonical_prompt)
self.assertIn("<tools>", record.canonical_prompt)
def test_load_records_keeps_formatter_normalized_tool_calls_and_textual_tool_content(self):
path = self.format_fixture(
[
make_record(
"req-tool-shape",
"sess-tool-shape",
[
{"role": "user", "content": "hello"},
{
"role": "assistant",
"content": "calling tool",
"tool_calls": [
{
"id": "call-1",
"type": "function",
"name": "read_file",
"arguments": "{\"path\": \"/tmp/a.txt\"}",
}
],
},
{
"role": "tool",
"content": [
{
"text": "tool-output",
"cache_control": {"type": "ephemeral"},
}
],
},
],
[{"type": "function", "function": {"name": "read_file"}}],
{
"input_tokens": 50,
"output_tokens": 10,
"total_tokens": 60,
"prompt_tokens_details": {"cached_tokens": 0},
},
total_cost_time=100,
)
]
)
record = load_records(path)[0]
self.assertIn("<tool_call>read_file", record.canonical_prompt)
self.assertIn("<arg_key>path</arg_key>", record.canonical_prompt)
self.assertIn("<tool_response>tool-output</tool_response>", record.canonical_prompt)
def test_load_records_supports_qwen_formatter_output(self):
path = self.format_fixture([make_qwen_record()])
record = load_records(path)[0]
self.assertEqual(record.meta.provider, "qwen3-coder")
self.assertEqual(record.usage.input_tokens, 90)
self.assertEqual(record.usage.output_tokens, 12)
self.assertEqual(record.usage.cached_tokens, 24)
self.assertIn("<|im_start|>system", record.canonical_prompt)
self.assertIn("run_command", record.canonical_prompt)
def test_qwen_formatter_serializes_assistant_tool_calls_with_model_tool_parser_shape(self):
row = make_qwen_record()
request_params = json.loads(row["request_params"])
request_params["payload"]["input"]["messages"].append(
{
"role": "assistant",
"content": "calling tool",
"tool_calls": [
{
"id": "call-1",
"type": "function",
"function": {
"name": "run_command",
"arguments": "{\"cmd\": \"ls -la\"}",
},
}
],
}
)
row["request_params"] = json.dumps(request_params, ensure_ascii=False)
path = self.format_fixture([row])
record = load_records(path)[0]
self.assertIn("<tool_call>", record.canonical_prompt)
self.assertIn("<function=run_command>", record.canonical_prompt)
self.assertIn("<parameter=cmd>", record.canonical_prompt)
self.assertIn("ls -la", record.canonical_prompt)
def test_load_records_rejects_provider_raw_trace(self):
path = self.write_raw_fixture([make_qwen_record()])
with self.assertRaisesRegex(ValueError, r"formatter-generated \*-raw\.jsonl"):
load_records(path)
def test_input_length_bucket_cache_reuse_summary(self):
summary, bucket_rows = summarize_cache_reuse_by_input_length_bucket(
[
{
"input_tokens": 100,
"cached_tokens": 25,
"cache_hit_ratio": 0.25,
"theoretical_prompt_unit_length": 120,
"theoretical_prefix_hit_units": 60,
"theoretical_prefix_hit_ratio": 0.5,
},
{
"input_tokens": 40000,
"cached_tokens": 20000,
"cache_hit_ratio": 0.5,
"theoretical_prompt_unit_length": 42000,
"theoretical_prefix_hit_units": 31500,
"theoretical_prefix_hit_ratio": 0.75,
},
]
)
self.assertEqual(summary["request_count"], 2)
bucket_by_name = {row["bucket"]: row for row in bucket_rows}
self.assertEqual(bucket_by_name["0-32Ki"]["request_count"], 1)
self.assertEqual(bucket_by_name["32-85Ki"]["request_count"], 1)
self.assertAlmostEqual(bucket_by_name["0-32Ki"]["weighted_actual_cache_hit_ratio"], 0.25)
self.assertAlmostEqual(bucket_by_name["32-85Ki"]["weighted_theoretical_cache_hit_ratio"], 0.75)
def test_input_length_bucket_cache_reuse_summary_supports_custom_buckets(self):
summary, bucket_rows = summarize_cache_reuse_by_input_length_bucket(
[
{
"input_tokens": 40,
"cached_tokens": 10,
"cache_hit_ratio": 0.25,
"theoretical_prompt_unit_length": 50,
"theoretical_prefix_hit_units": 20,
"theoretical_prefix_hit_ratio": 0.4,
},
{
"input_tokens": 60,
"cached_tokens": 30,
"cache_hit_ratio": 0.5,
"theoretical_prompt_unit_length": 70,
"theoretical_prefix_hit_units": 35,
"theoretical_prefix_hit_ratio": 0.5,
},
],
bucket_defs=build_input_length_bucket_defs([50]),
)
self.assertEqual(
summary["bucket_definition"]["buckets"],
[
{
"bucket": "0-50",
"input_tokens_min_inclusive": 0,
"input_tokens_max_exclusive": 50,
},
{
"bucket": "50+",
"input_tokens_min_inclusive": 50,
"input_tokens_max_exclusive": None,
},
],
)
bucket_by_name = {row["bucket"]: row for row in bucket_rows}
self.assertEqual(bucket_by_name["0-50"]["request_count"], 1)
self.assertEqual(bucket_by_name["50+"]["request_count"], 1)
def test_input_length_bucket_cache_reuse_summary_tracks_bucketed_theoretical_upper_bound(self):
summary, bucket_rows = summarize_cache_reuse_by_input_length_bucket(
[
{
"input_tokens": 100,
"cached_tokens": 20,
"cache_hit_ratio": 0.2,
"theoretical_prompt_unit_length": 100,
"theoretical_prefix_hit_units": 60,
"theoretical_prefix_hit_ratio": 0.6,
"bucketed_theoretical_prefix_hit_units": 40,
"bucketed_theoretical_prefix_hit_ratio": 0.4,
}
],
bucket_defs=build_input_length_bucket_defs([200]),
)
self.assertEqual(summary["request_count"], 1)
row = bucket_rows[0]
self.assertEqual(row["bucket"], "0-200")
self.assertAlmostEqual(row["weighted_theoretical_cache_hit_ratio"], 0.6)
self.assertAlmostEqual(row["weighted_bucketed_theoretical_cache_hit_ratio"], 0.4)
self.assertAlmostEqual(row["weighted_bucket_boundary_loss_ratio"], 0.2)
self.assertAlmostEqual(row["bucketed_theoretical_reused_request_fraction"], 1.0)
def test_session_bucket_boundary_miss_summary_counts_cross_bucket_shared_prefix_loss(self):
summary, bucket_rows = summarize_session_bucket_boundary_miss(
[
{
"child_bucket": "0-32Ki",
"child_input_tokens": 100,
"shared_prefix_units": 8,
"is_cross_bucket": 1,
},
{
"child_bucket": "0-32Ki",
"child_input_tokens": 120,
"shared_prefix_units": 4,
"is_cross_bucket": 0,
},
],
bucket_defs=build_input_length_bucket_defs(),
)
self.assertEqual(summary["edge_count"], 2)
self.assertEqual(summary["cross_bucket_edge_count"], 1)
self.assertAlmostEqual(summary["cross_bucket_shared_prefix_unit_fraction"], 8 / 12)
bucket_by_name = {row["bucket"]: row for row in bucket_rows}
self.assertEqual(bucket_by_name["0-32Ki"]["edge_count"], 2)
self.assertAlmostEqual(
bucket_by_name["0-32Ki"]["cross_bucket_shared_prefix_unit_fraction"],
8 / 12,
)
def test_build_alive_block_timeline_counts_live_blocks_from_first_seen_to_last_reuse(self):
summary, rows = build_alive_block_timeline(
[
{"first_seen_ms": 10, "span_end_ms": 20},
{"first_seen_ms": 15, "span_end_ms": 15},
]
)
self.assertEqual(summary["peak_alive_blocks"], 2)
self.assertEqual(rows[0]["timestamp_ms"], 10)
self.assertEqual(rows[0]["alive_block_count"], 1)
row_by_ts = {row["timestamp_ms"]: row for row in rows}
self.assertEqual(row_by_ts[15]["alive_block_count"], 2)
self.assertEqual(row_by_ts[16]["alive_block_count"], 1)
self.assertEqual(row_by_ts[21]["alive_block_count"], 0)
def test_parse_input_length_bucket_thresholds_supports_ki_units(self):
self.assertEqual(
parse_input_length_bucket_thresholds("32Ki;85Ki;128Ki"),
[32 * 1024, 85 * 1024, 128 * 1024],
)
def test_compute_features_detects_bursts_and_cache(self):
path = self.format_fixture(
[
make_record(
"req-2",
"sess-2",
[
{"role": "user", "content": "u"},
{"role": "assistant", "content": "a"},
{"role": "tool", "content": "t1"},
{"role": "tool", "content": "t2"},
{"role": "tool", "content": "t3"},
{"role": "assistant", "content": "done"},
],
[{"type": "function", "function": {"name": "exec"}}],
{
"input_tokens": 40000,
"output_tokens": 200,
"total_tokens": 40200,
"prompt_tokens_details": {"cached_tokens": 1000},
},
total_cost_time=9000,
)
]
)
features = compute_features(load_records(path))
feature = features[0]
self.assertEqual(feature.assistant_to_tool_count, 1)
self.assertEqual(feature.tool_to_tool_count, 2)
self.assertEqual(feature.max_consecutive_tool_msgs, 3)
self.assertAlmostEqual(feature.cache_hit_ratio, 0.025)
self.assertIn("cache-cold", feature.pattern_labels)
self.assertIn("long-context-no-cache", feature.pattern_labels)
def test_compute_theoretical_cache_detects_prefix_reuse(self):
rows = [
make_record(
"req-a",
"sess-a",
[{"role": "user", "content": "prefix shared"}],
[],
{
"input_tokens": 10,
"output_tokens": 1,
"total_tokens": 11,
"prompt_tokens_details": {"cached_tokens": 0},
},
total_cost_time=10,
),
make_record(
"req-b",
"sess-a",
[{"role": "user", "content": "prefix shared and more"}],
[],
{
"input_tokens": 20,
"output_tokens": 1,
"total_tokens": 21,
"prompt_tokens_details": {"cached_tokens": 0},
},
total_cost_time=10,
),
]
first = json.loads(rows[0]["request_params"])
second = json.loads(rows[1]["request_params"])
first["header"] = {"attributes": {"x-dashscope-inner-requestreadytime": "1000"}}
second["header"] = {"attributes": {"x-dashscope-inner-requestreadytime": "2000"}}
rows[0]["request_params"] = json.dumps(first, ensure_ascii=False)
rows[1]["request_params"] = json.dumps(second, ensure_ascii=False)
theoretical = compute_theoretical_cache(
load_records(self.format_fixture(rows)),
block_size=8,
segment_mode="bytes",
)
request_rows = {row["request_id"]: row for row in theoretical["request_rows"]}
self.assertEqual(request_rows["req-a"]["theoretical_prefix_hit_ratio"], 0.0)
self.assertGreater(request_rows["req-b"]["theoretical_prefix_hit_ratio"], 0.0)
self.assertTrue(theoretical["reuse_gap_rows"])
reused_blocks = [row for row in theoretical["block_rows"] if row["reuse_count"] > 0]
self.assertTrue(reused_blocks)
self.assertIn("last_reuse_ms", reused_blocks[0])
self.assertIn("span_ms", reused_blocks[0])
self.assertGreaterEqual(reused_blocks[0]["lifetime_ms"], 0)
def test_report_cli_writes_outputs(self):
path = self.format_fixture(
[
make_record(
"req-3",
"sess-3",
[
{"role": "user", "content": "u"},
{"role": "assistant", "content": "a"},
],
[],
{
"input_tokens": 20,
"output_tokens": 5,
"total_tokens": 25,
"prompt_tokens_details": {"cached_tokens": 0},
},
total_cost_time=30,
),
make_record(
"req-4",
"sess-3",
[
{"role": "user", "content": "u"},
{"role": "assistant", "content": "a"},
{"role": "tool", "content": "t"},
{"role": "assistant", "content": "done"},
],
[{"type": "function", "function": {"name": "read"}}],
{
"input_tokens": 200,
"output_tokens": 50,
"total_tokens": 250,
"prompt_tokens_details": {"cached_tokens": 150},
},
total_cost_time=300,
),
]
)
with tempfile.TemporaryDirectory() as temp_dir:
completed = subprocess.run(
[
sys.executable,
"-m",
"trace_analyzer",
"report",
str(path),
"--output-dir",
temp_dir,
"--limit",
"2",
],
cwd=Path(__file__).resolve().parents[1],
check=True,
capture_output=True,
text=True,
)
self.assertIn("report.md", completed.stdout)
summary_path = Path(temp_dir) / "summary.json"
report_path = Path(temp_dir) / "report.md"
features_path = Path(temp_dir) / "features.csv"
self.assertTrue(summary_path.exists())
self.assertTrue(report_path.exists())
self.assertTrue(features_path.exists())
summary = json.loads(summary_path.read_text(encoding="utf-8"))
self.assertIn("tool_patterns", summary)
self.assertIn("cache_patterns", summary)
with open(features_path, "r", encoding="utf-8") as handle:
rows = list(csv.DictReader(handle))
self.assertEqual(len(rows), 2)
def test_study_cli_writes_advanced_outputs(self):
raw_rows = [
make_record(
"req-6",
"sess-6",
[{"role": "user", "content": "hello world"}],
[{"type": "function", "function": {"name": "read"}}],
{
"input_tokens": 40,
"output_tokens": 2,
"total_tokens": 42,
"prompt_tokens_details": {"cached_tokens": 10},
},
total_cost_time=100,
),
make_record(
"req-7",
"sess-6",
[
{"role": "assistant", "content": "a"},
{"role": "tool", "content": "result"},
{"role": "user", "content": "hello world again"},
],
[{"type": "function", "function": {"name": "read"}}],
{
"input_tokens": 60,
"output_tokens": 3,
"total_tokens": 63,
"prompt_tokens_details": {"cached_tokens": 20},
},
total_cost_time=150,
),
]
path = self.format_fixture(raw_rows)
with tempfile.TemporaryDirectory() as temp_dir:
subprocess.run(
[
sys.executable,
"-m",
"trace_analyzer",
"study",
str(path),
"--output-dir",
temp_dir,
"--block-size",
"8",
"--segment-mode",
"bytes",
"--input-length-buckets",
"50",
],
cwd=Path(__file__).resolve().parents[1],
check=True,
capture_output=True,
text=True,
)
self.assertTrue((Path(temp_dir) / "details" / "request_metrics.csv").exists())
self.assertTrue((Path(temp_dir) / "details" / "cdf_lengths.png").exists())
self.assertTrue((Path(temp_dir) / "details" / "tools_catalog.csv").exists())
bucket_summary = json.loads(
(Path(temp_dir) / "details" / "input_length_bucket_cache_reuse_summary.json").read_text(
encoding="utf-8"
)
)
self.assertEqual(
[row["bucket"] for row in bucket_summary["bucket_definition"]["buckets"]],
["0-50", "50+"],
)
def test_study_cli_reuses_existing_base_outputs(self):
raw_rows = [
make_record(
"req-8",
"sess-8",
[{"role": "user", "content": "prefix shared"}],
[{"type": "function", "function": {"name": "read"}}],
{
"input_tokens": 20,
"output_tokens": 2,
"total_tokens": 22,
"prompt_tokens_details": {"cached_tokens": 0},
},
total_cost_time=80,
),
make_record(
"req-9",
"sess-8",
[{"role": "user", "content": "prefix shared again"}],
[{"type": "function", "function": {"name": "read"}}],
{
"input_tokens": 30,
"output_tokens": 3,
"total_tokens": 33,
"prompt_tokens_details": {"cached_tokens": 10},
},
total_cost_time=120,
),
]
path = self.format_fixture(raw_rows)
with tempfile.TemporaryDirectory() as temp_dir:
subprocess.run(
[
sys.executable,
"-m",
"trace_analyzer",
"report",
str(path),
"--output-dir",
temp_dir,
],
cwd=Path(__file__).resolve().parents[1],
check=True,
capture_output=True,
text=True,
)
completed = subprocess.run(
[
sys.executable,
"-m",
"trace_analyzer",
"study",
str(path),
"--output-dir",
temp_dir,
"--block-size",
"8",
"--segment-mode",
"bytes",
],
cwd=Path(__file__).resolve().parents[1],
check=True,
capture_output=True,
text=True,
)
self.assertIn("details_summary.json", completed.stdout)
self.assertTrue((Path(temp_dir) / "details" / "progress.json").exists())
report_text = (Path(temp_dir) / "report.md").read_text(encoding="utf-8")
self.assertIn("Study Outputs", report_text)
def test_build_summary_contains_expected_keys(self):
path = self.format_fixture(
[
make_record(
"req-5",
"sess-5",
[{"role": "user", "content": "u"}],
[],
{
"input_tokens": 10,
"output_tokens": 1,
"total_tokens": 11,
"prompt_tokens_details": {"cached_tokens": 0},
},
total_cost_time=5,
)
]
)
records = load_records(path)
features = compute_features(records)
summary = build_summary(records, features)
self.assertIn("record_count", summary)
self.assertIn("tool_patterns", summary)
self.assertIn("cache_patterns", summary)
self.assertIn("anomalies", summary)
if __name__ == "__main__":
unittest.main()