Initial commit
This commit is contained in:
228
trace_analyzer/reporting.py
Normal file
228
trace_analyzer/reporting.py
Normal file
@@ -0,0 +1,228 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from trace_analyzer.helpers import safe_float, safe_int, series_stats
|
||||
from trace_analyzer.layout import resolve_details_summary_path
|
||||
from trace_analyzer.report import build_markdown_report
|
||||
|
||||
|
||||
def _iter_feature_rows(features_path: str | Path):
|
||||
with Path(features_path).open("r", encoding="utf-8") as handle:
|
||||
for row in csv.DictReader(handle):
|
||||
row["message_count"] = safe_int(row.get("message_count"))
|
||||
row["conversation_depth"] = safe_int(row.get("conversation_depth"))
|
||||
row["declared_tool_count"] = safe_int(row.get("declared_tool_count"))
|
||||
row["assistant_msg_count"] = safe_int(row.get("assistant_msg_count"))
|
||||
row["tool_msg_count"] = safe_int(row.get("tool_msg_count"))
|
||||
row["user_msg_count"] = safe_int(row.get("user_msg_count"))
|
||||
row["system_msg_count"] = safe_int(row.get("system_msg_count"))
|
||||
row["assistant_to_tool_count"] = safe_int(row.get("assistant_to_tool_count"))
|
||||
row["tool_to_assistant_count"] = safe_int(row.get("tool_to_assistant_count"))
|
||||
row["tool_to_tool_count"] = safe_int(row.get("tool_to_tool_count"))
|
||||
row["assistant_to_user_count"] = safe_int(row.get("assistant_to_user_count"))
|
||||
row["user_to_assistant_count"] = safe_int(row.get("user_to_assistant_count"))
|
||||
row["max_consecutive_tool_msgs"] = safe_int(row.get("max_consecutive_tool_msgs"))
|
||||
row["avg_tool_burst_len"] = safe_float(row.get("avg_tool_burst_len"))
|
||||
row["has_tool_loop"] = safe_int(row.get("has_tool_loop"))
|
||||
row["input_tokens"] = safe_int(row.get("input_tokens"))
|
||||
row["output_tokens"] = safe_int(row.get("output_tokens"))
|
||||
row["total_tokens"] = safe_int(row.get("total_tokens"))
|
||||
row["reasoning_tokens"] = safe_int(row.get("reasoning_tokens"))
|
||||
row["cached_tokens"] = safe_int(row.get("cached_tokens"))
|
||||
row["cache_hit_ratio"] = safe_float(row.get("cache_hit_ratio"))
|
||||
row["uncached_prompt_tokens"] = safe_int(row.get("uncached_prompt_tokens"))
|
||||
row["output_input_ratio"] = safe_float(row.get("output_input_ratio"))
|
||||
row["latency_ms"] = safe_int(row.get("latency_ms"))
|
||||
row["ms_per_input_token"] = safe_float(row.get("ms_per_input_token"))
|
||||
row["ms_per_output_token"] = safe_float(row.get("ms_per_output_token"))
|
||||
row["long_context"] = safe_int(row.get("long_context"))
|
||||
row["high_cache"] = safe_int(row.get("high_cache"))
|
||||
row["tool_burst_alert"] = safe_int(row.get("tool_burst_alert"))
|
||||
row["tool_loop_alert"] = safe_int(row.get("tool_loop_alert"))
|
||||
row["slow_request"] = safe_int(row.get("slow_request"))
|
||||
row["pattern_labels"] = [label for label in str(row.get("pattern_labels", "")).split(";") if label]
|
||||
yield row
|
||||
|
||||
|
||||
def build_summary_from_features(features_path: str | Path) -> dict:
|
||||
model_counts = Counter()
|
||||
status_code_counts = Counter()
|
||||
role_transition_counts = Counter()
|
||||
session_ids: set[str] = set()
|
||||
latencies: list[int] = []
|
||||
cache_ratios: list[float] = []
|
||||
cached_tokens_list: list[int] = []
|
||||
declared_tool_counts: list[int] = []
|
||||
burst_values: list[int] = []
|
||||
record_count = 0
|
||||
success_count = 0
|
||||
high_burst_requests: list[dict] = []
|
||||
slow_despite_cache: list[dict] = []
|
||||
long_context_no_cache: list[dict] = []
|
||||
tool_burst_alert_count = 0
|
||||
tool_loop_alert_count = 0
|
||||
cache_bucket_input = {
|
||||
"lt_0_2": {"latencies": [], "ratios": [], "count": 0},
|
||||
"0_2_to_0_8": {"latencies": [], "ratios": [], "count": 0},
|
||||
"ge_0_8": {"latencies": [], "ratios": [], "count": 0},
|
||||
}
|
||||
|
||||
for row in _iter_feature_rows(features_path):
|
||||
record_count += 1
|
||||
model_counts[row.get("model") or "unknown"] += 1
|
||||
status_code_counts[row.get("status_code") or "unknown"] += 1
|
||||
if row.get("session_id"):
|
||||
session_ids.add(row["session_id"])
|
||||
if row.get("status_code") in {"1000", "200"}:
|
||||
success_count += 1
|
||||
role_transition_counts["assistant->tool"] += row["assistant_to_tool_count"]
|
||||
role_transition_counts["tool->assistant"] += row["tool_to_assistant_count"]
|
||||
role_transition_counts["tool->tool"] += row["tool_to_tool_count"]
|
||||
role_transition_counts["assistant->user"] += row["assistant_to_user_count"]
|
||||
role_transition_counts["user->assistant"] += row["user_to_assistant_count"]
|
||||
latencies.append(row["latency_ms"])
|
||||
cache_ratios.append(row["cache_hit_ratio"])
|
||||
cached_tokens_list.append(row["cached_tokens"])
|
||||
declared_tool_counts.append(row["declared_tool_count"])
|
||||
burst_values.append(row["max_consecutive_tool_msgs"])
|
||||
tool_burst_alert_count += row["tool_burst_alert"]
|
||||
tool_loop_alert_count += row["tool_loop_alert"]
|
||||
|
||||
if row["tool_burst_alert"]:
|
||||
high_burst_requests.append(
|
||||
{
|
||||
"request_id": row["request_id"],
|
||||
"session_id": row["session_id"],
|
||||
"max_consecutive_tool_msgs": row["max_consecutive_tool_msgs"],
|
||||
"tool_to_tool_count": row["tool_to_tool_count"],
|
||||
}
|
||||
)
|
||||
high_burst_requests.sort(
|
||||
key=lambda item: (item["max_consecutive_tool_msgs"], item["tool_to_tool_count"]),
|
||||
reverse=True,
|
||||
)
|
||||
del high_burst_requests[10:]
|
||||
|
||||
if "slow-despite-cache" in row["pattern_labels"]:
|
||||
slow_despite_cache.append(
|
||||
{
|
||||
"request_id": row["request_id"],
|
||||
"session_id": row["session_id"],
|
||||
"latency_ms": row["latency_ms"],
|
||||
"cache_hit_ratio": row["cache_hit_ratio"],
|
||||
}
|
||||
)
|
||||
slow_despite_cache.sort(key=lambda item: item["latency_ms"], reverse=True)
|
||||
del slow_despite_cache[10:]
|
||||
|
||||
if "long-context-no-cache" in row["pattern_labels"]:
|
||||
long_context_no_cache.append(
|
||||
{
|
||||
"request_id": row["request_id"],
|
||||
"session_id": row["session_id"],
|
||||
"input_tokens": row["input_tokens"],
|
||||
"cache_hit_ratio": row["cache_hit_ratio"],
|
||||
}
|
||||
)
|
||||
long_context_no_cache.sort(key=lambda item: item["input_tokens"], reverse=True)
|
||||
del long_context_no_cache[10:]
|
||||
|
||||
ratio = row["cache_hit_ratio"]
|
||||
if ratio < 0.2:
|
||||
bucket_name = "lt_0_2"
|
||||
elif ratio < 0.8:
|
||||
bucket_name = "0_2_to_0_8"
|
||||
else:
|
||||
bucket_name = "ge_0_8"
|
||||
cache_bucket_input[bucket_name]["count"] += 1
|
||||
cache_bucket_input[bucket_name]["latencies"].append(row["latency_ms"])
|
||||
cache_bucket_input[bucket_name]["ratios"].append(row["cache_hit_ratio"])
|
||||
|
||||
latency_stats = series_stats(latencies)
|
||||
cache_ratio_stats = series_stats(cache_ratios)
|
||||
cached_token_stats = series_stats(cached_tokens_list)
|
||||
declared_tool_stats = series_stats(declared_tool_counts)
|
||||
burst_stats = series_stats(burst_values)
|
||||
|
||||
cache_buckets = []
|
||||
for label in ["lt_0_2", "0_2_to_0_8", "ge_0_8"]:
|
||||
bucket = cache_bucket_input[label]
|
||||
cache_buckets.append(
|
||||
{
|
||||
"bucket": label,
|
||||
"count": bucket["count"],
|
||||
"avg_latency_ms": series_stats(bucket["latencies"])["mean"],
|
||||
"avg_cache_hit_ratio": series_stats(bucket["ratios"])["mean"],
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"record_count": record_count,
|
||||
"success_count": success_count,
|
||||
"session_count": len(session_ids),
|
||||
"model_counts": dict(model_counts),
|
||||
"status_code_counts": dict(status_code_counts),
|
||||
"thresholds": {
|
||||
"long_context": 32000,
|
||||
"high_cache": 0.8,
|
||||
"tool_burst_alert": 4,
|
||||
"tool_loop_alert": 3,
|
||||
"slow_request_p90_latency_ms": latency_stats["p90"],
|
||||
},
|
||||
"tool_patterns": {
|
||||
"role_transitions": dict(role_transition_counts),
|
||||
"declared_tool_count": declared_tool_stats,
|
||||
"max_consecutive_tool_msgs": burst_stats,
|
||||
"tool_burst_alert_count": tool_burst_alert_count,
|
||||
"tool_loop_alert_count": tool_loop_alert_count,
|
||||
"high_burst_requests": high_burst_requests,
|
||||
},
|
||||
"cache_patterns": {
|
||||
"cached_tokens": cached_token_stats,
|
||||
"cache_hit_ratio": cache_ratio_stats,
|
||||
"latency_ms": latency_stats,
|
||||
"cache_buckets": cache_buckets,
|
||||
},
|
||||
"anomalies": {
|
||||
"slow_despite_cache": slow_despite_cache,
|
||||
"long_context_no_cache": long_context_no_cache,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def write_reports(
|
||||
*,
|
||||
features_path: str | Path,
|
||||
output_dir: str | Path,
|
||||
pipeline_summary: dict | None = None,
|
||||
) -> dict:
|
||||
output_root = Path(output_dir)
|
||||
output_root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
summary = build_summary_from_features(features_path)
|
||||
summary_path = output_root / "summary.json"
|
||||
summary_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
report_path = output_root / "report.md"
|
||||
report_path.write_text(build_markdown_report(summary), encoding="utf-8")
|
||||
|
||||
combined = {
|
||||
"summary": summary,
|
||||
"pipeline": pipeline_summary or {},
|
||||
}
|
||||
|
||||
details_summary_path = resolve_details_summary_path(output_root)
|
||||
if details_summary_path is not None:
|
||||
combined["details_summary"] = json.loads(details_summary_path.read_text(encoding="utf-8"))
|
||||
|
||||
combined_path = output_root / "analysis_snapshot.json"
|
||||
combined_path.write_text(json.dumps(combined, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return {
|
||||
"summary_path": str(summary_path),
|
||||
"report_path": str(report_path),
|
||||
"analysis_snapshot_path": str(combined_path),
|
||||
}
|
||||
Reference in New Issue
Block a user