gahow/aituner
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Handle zero-token source rows in code trace audit

Browse Source
This commit is contained in:
Gahow Wang
2026-07-23 23:52:03 +08:00
parent fbaa909723
commit 1d9182f305
5 changed files with 50 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
runs/frontier-code-trace-v0/README.md
View File

@@ -34,6 +34,9 @@ selected.selected_window_stats.max_model_len_coverage[推荐值].coverage = 1.0
```
旧记录预计 source block size 为 512、推荐 max model len 为 131072，但禁止把这两个值写死为实验事实。
审计会单独记录并排除 `input_length<=0` 或 `output_length<=0` 的 source
行；这些行只有在 raw trace 同样显示 zero usage/empty response 时才按
“未发生模型执行”处理，不能无记录过滤。
## 2. 物化稳定窗口

33
runs/frontier-code-trace-v0/audit_code_trace.py
View File

@@ -131,6 +131,9 @@ def choose_window(
def scan_source(path: Path, args: argparse.Namespace) -> dict[str, Any]:
rows = 0
source_rows = 0
invalid_zero_token_rows = 0
invalid_zero_token_examples: list[dict[str, Any]] = []
first_timestamp = None
last_timestamp = None
previous_timestamp = None
@@ -145,6 +148,7 @@ def scan_source(path: Path, args: argparse.Namespace) -> dict[str, Any]:
sampling_rows = 0
schema_keys: Counter[str] = Counter()
for line_number, row in iter_jsonl(path):
source_rows += 1
missing = [
key
for key in ("timestamp", "input_length", "output_length")
@@ -153,6 +157,22 @@ def scan_source(path: Path, args: argparse.Namespace) -> dict[str, Any]:
if missing:
raise ValueError(f"{path}:{line_number}: missing required fields {missing}")
timestamp = float(row["timestamp"])
input_tokens = int(row["input_length"])
output_tokens = int(row["output_length"])
schema_keys.update(row.keys())
if input_tokens <= 0 or output_tokens <= 0:
invalid_zero_token_rows += 1
if len(invalid_zero_token_examples) < 20:
invalid_zero_token_examples.append(
{
"line_number": line_number,
"chat_id": row.get("chat_id"),
"timestamp": timestamp,
"input_length": input_tokens,
"output_length": output_tokens,
}
)
continue
if first_timestamp is None:
first_timestamp = timestamp
if previous_timestamp is not None and timestamp < previous_timestamp:
@@ -169,10 +189,6 @@ def scan_source(path: Path, args: argparse.Namespace) -> dict[str, Any]:
max_gaps.get(previous_bin, 0.0),
timestamp - previous_timestamp,
)
input_tokens = int(row["input_length"])
output_tokens = max(1, int(row["output_length"]))
if input_tokens <= 0:
raise ValueError(f"{path}:{line_number}: input_length must be positive")
input_lengths.append(input_tokens)
output_lengths.append(output_tokens)
total_lengths.append(input_tokens + output_tokens)
@@ -186,7 +202,6 @@ def scan_source(path: Path, args: argparse.Namespace) -> dict[str, Any]:
isinstance(row.get("prompt"), (str, list)) and bool(row.get("prompt"))
)
sampling_rows += int("sampling_u" in row)
schema_keys.update(row.keys())
rows += 1
previous_timestamp = timestamp
last_timestamp = timestamp
@@ -205,6 +220,10 @@ def scan_source(path: Path, args: argparse.Namespace) -> dict[str, Any]:
return {
"source": str(path.resolve()),
"rows": rows,
"source_rows": source_rows,
"invalid_zero_token_rows": invalid_zero_token_rows,
"invalid_zero_token_fraction": invalid_zero_token_rows / source_rows,
"invalid_zero_token_examples": invalid_zero_token_examples,
"first_timestamp": first_timestamp,
"last_timestamp": last_timestamp,
"span_s": last_timestamp - first_timestamp,
@@ -253,7 +272,9 @@ def scan_window(source: Path, window: dict[str, Any]) -> dict[str, Any]:
if timestamp >= end:
break
input_tokens = int(row["input_length"])
output_tokens = max(1, int(row["output_length"]))
output_tokens = int(row["output_length"])
if input_tokens <= 0 or output_tokens <= 0:
continue
inputs.append(input_tokens)
outputs.append(output_tokens)
totals.append(input_tokens + output_tokens)

1
runs/frontier-code-trace-v0/experiment-card.md
View File

@@ -58,6 +58,7 @@ strict decode-only 必须同时具备：
- timestamp 单调，存在 60–75min 连续稳定窗口；
- `timestamp/input_length/output_length` 全行存在；
- `input_length<=0` 或 `output_length<=0` 的行不进入 replay，但必须计数并保留样例。已抽查的两条 0→0 行在 raw trace 中同时满足 `usage.total_tokens=0`、`response_message={}`，属于未发生模型执行的 source request，不是 full-cache decode；
- `hash_ids` 数量与某个 source block size 在全行严格满足
`ceil(ISL/source_block_size)`；预计值 512，但以审计结果为准；
- 记录 ISL/OSL/ISL+OSL 的 p50/p90/p95/p99/max、gap、request rate、prompt/sampling 字段覆盖。

2
runs/frontier-code-trace-v0/prepare_code_window.py
View File

@@ -63,6 +63,8 @@ def main() -> None:
continue
if timestamp >= end:
break
if int(row["input_length"]) <= 0 or int(row["output_length"]) <= 0:
continue
chat = row.get("chat_id", source_index)
parent = row.get("parent_chat_id")
has_parent = parent not in (None, "", -1, "-1")

17
runs/frontier-code-trace-v0/test_code_trace_preflight.py
View File

@@ -20,6 +20,21 @@ class CodeTracePreflightTest(unittest.TestCase):
source = root / "051315-051317.jsonl"
with source.open("w") as stream:
for index in range(4501):
if index == 100:
stream.write(
json.dumps(
{
"chat_id": index,
"parent_chat_id": -1,
"timestamp": float(index),
"input_length": 0,
"output_length": 0,
"hash_ids": [],
}
)
+ "\n"
)
continue
input_tokens = 513 if index % 2 else 512
stream.write(
json.dumps(
@@ -53,6 +68,7 @@ class CodeTracePreflightTest(unittest.TestCase):
self.assertEqual(
payload["selected"]["hash_contract"]["exact_source_block_size"], 512
)
self.assertEqual(payload["selected"]["invalid_zero_token_rows"], 1)
self.assertEqual(payload["max_model_len_recommendation"], 40960)
output = root / "window"
subprocess.run(
@@ -74,6 +90,7 @@ class CodeTracePreflightTest(unittest.TestCase):
self.assertEqual(manifest["requests"], len(rows))
self.assertGreaterEqual(manifest["duration_s"], 3600)
self.assertEqual(rows[0]["sampling_u"], rows[1]["sampling_u"])
self.assertNotIn(100, {row["source_index"] for row in rows})
if __name__ == "__main__":