Give long replay bands stable request identities
This commit is contained in:
@@ -114,18 +114,34 @@ def resolve_role_trace(base: dict[str, Any], role: str) -> Path:
|
|||||||
return trace
|
return trace
|
||||||
|
|
||||||
|
|
||||||
|
def private_request_id(
|
||||||
|
*, source_sha256: str, line_number: int, original_id: str
|
||||||
|
) -> str:
|
||||||
|
payload = f"{source_sha256}:{line_number}:{original_id}".encode()
|
||||||
|
return f"phase-v2-{hashlib.sha256(payload).hexdigest()}"
|
||||||
|
|
||||||
|
|
||||||
def merge_role_traces(sources: tuple[Path, Path], target: Path) -> dict[str, Any]:
|
def merge_role_traces(sources: tuple[Path, Path], target: Path) -> dict[str, Any]:
|
||||||
target.parent.mkdir(parents=True, exist_ok=True)
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
temporary = target.with_suffix(target.suffix + ".tmp")
|
temporary = target.with_suffix(target.suffix + ".tmp")
|
||||||
rows = 0
|
rows = 0
|
||||||
with temporary.open("w", encoding="utf-8") as output:
|
with temporary.open("w", encoding="utf-8") as output:
|
||||||
for source in sources:
|
for source in sources:
|
||||||
|
source_digest = sha256_file(source)
|
||||||
with source.open(encoding="utf-8") as input_file:
|
with source.open(encoding="utf-8") as input_file:
|
||||||
for line in input_file:
|
for line_number, line in enumerate(input_file, start=1):
|
||||||
if not line.strip():
|
if not line.strip():
|
||||||
continue
|
continue
|
||||||
json.loads(line)
|
row = json.loads(line)
|
||||||
output.write(line if line.endswith("\n") else line + "\n")
|
original_id = str(
|
||||||
|
row.get("request_id") or row.get("id") or line_number
|
||||||
|
)
|
||||||
|
row["request_id"] = private_request_id(
|
||||||
|
source_sha256=source_digest,
|
||||||
|
line_number=line_number,
|
||||||
|
original_id=original_id,
|
||||||
|
)
|
||||||
|
output.write(json.dumps(row, ensure_ascii=False) + "\n")
|
||||||
rows += 1
|
rows += 1
|
||||||
os.replace(temporary, target)
|
os.replace(temporary, target)
|
||||||
return {
|
return {
|
||||||
@@ -135,6 +151,7 @@ def merge_role_traces(sources: tuple[Path, Path], target: Path) -> dict[str, Any
|
|||||||
"rows": rows,
|
"rows": rows,
|
||||||
"sources": [str(source) for source in sources],
|
"sources": [str(source) for source in sources],
|
||||||
"source_sha256": [sha256_file(source) for source in sources],
|
"source_sha256": [sha256_file(source) for source in sources],
|
||||||
|
"request_id_scheme": "sha256(source_sha256:line_number:original_id)",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -85,6 +85,15 @@ def main() -> None:
|
|||||||
assert record["offered_req_s_per_gpu"] == 0.25
|
assert record["offered_req_s_per_gpu"] == 0.25
|
||||||
assert len(prepare.SESSION_ORDER) == 6
|
assert len(prepare.SESSION_ORDER) == 6
|
||||||
assert {mns for _replicate, mns in prepare.SESSION_ORDER} == {16, 64}
|
assert {mns for _replicate, mns in prepare.SESSION_ORDER} == {16, 64}
|
||||||
|
first_id = prepare.private_request_id(
|
||||||
|
source_sha256="a" * 64, line_number=1, original_id="1"
|
||||||
|
)
|
||||||
|
assert first_id == prepare.private_request_id(
|
||||||
|
source_sha256="a" * 64, line_number=1, original_id="1"
|
||||||
|
)
|
||||||
|
assert first_id != prepare.private_request_id(
|
||||||
|
source_sha256="b" * 64, line_number=1, original_id="1"
|
||||||
|
)
|
||||||
controller = load_controller_module()
|
controller = load_controller_module()
|
||||||
assert math.isclose(controller.remaining_projection(6, 0), 7.7)
|
assert math.isclose(controller.remaining_projection(6, 0), 7.7)
|
||||||
assert math.isclose(controller.remaining_projection(6, 5), 1.45)
|
assert math.isclose(controller.remaining_projection(6, 5), 1.45)
|
||||||
|
|||||||
Reference in New Issue
Block a user