feat(formatter): resumable chunk merge with bounded fan-in
Collapse chunk_*.jsonl in rounds with a bounded merge fan-in so the number of simultaneously open chunk iterators stays flat regardless of how many chunks a trace produces, and write a resume manifest so --resume-tmp-dir can continue from an existing formatter temp directory without rescanning the raw trace.
This commit is contained in:
@@ -64,6 +64,11 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
help="Base directory used when --output is omitted. Defaults to a sibling trace-*-formatted directory.",
|
||||
)
|
||||
format_parser.add_argument("--tmp-dir", default=None)
|
||||
format_parser.add_argument(
|
||||
"--resume-tmp-dir",
|
||||
default=None,
|
||||
help="Resume from an existing formatter temp directory containing chunk_*.jsonl and skip raw trace scanning.",
|
||||
)
|
||||
format_parser.add_argument("--chunk-bytes", type=int, default=128 * 1024 * 1024)
|
||||
format_parser.add_argument(
|
||||
"--log-file",
|
||||
@@ -146,6 +151,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
input_dir=args.input,
|
||||
output_path=raw_output_path,
|
||||
tmp_dir=args.tmp_dir,
|
||||
resume_tmp_dir=args.resume_tmp_dir,
|
||||
chunk_bytes=args.chunk_bytes,
|
||||
start_time=args.start_time,
|
||||
end_time=args.end_time,
|
||||
|
||||
Reference in New Issue
Block a user