Avoid duplicate TP accounting in P0 padding summary
This commit is contained in:
@@ -328,11 +328,29 @@ def target_execution_agreement(worker_events: list[dict[str, Any]]) -> dict[str,
|
|||||||
}
|
}
|
||||||
|
|
||||||
metadata_valid_count = 0
|
metadata_valid_count = 0
|
||||||
|
anchor_local_ranks: dict[int, int] = {}
|
||||||
|
for event in target_events:
|
||||||
|
dp_rank = integer(event.get("data_parallel_rank"))
|
||||||
|
local_rank = integer(event.get("local_rank"))
|
||||||
|
if dp_rank is None or local_rank is None:
|
||||||
|
continue
|
||||||
|
previous = anchor_local_ranks.get(dp_rank)
|
||||||
|
if previous is None or local_rank < previous:
|
||||||
|
anchor_local_ranks[dp_rank] = local_rank
|
||||||
|
anchor_events = [
|
||||||
|
event
|
||||||
|
for event in target_events
|
||||||
|
if integer(event.get("data_parallel_rank")) in anchor_local_ranks
|
||||||
|
and integer(event.get("local_rank"))
|
||||||
|
== anchor_local_ranks[integer(event.get("data_parallel_rank"))]
|
||||||
|
]
|
||||||
metadata_padding: list[int] = []
|
metadata_padding: list[int] = []
|
||||||
raw_unequal_count = 0
|
raw_unequal_count = 0
|
||||||
for event in target_events:
|
for event in target_events:
|
||||||
valid, padding, raw_unequal = _target_metadata_check(event)
|
valid, _, _ = _target_metadata_check(event)
|
||||||
metadata_valid_count += valid
|
metadata_valid_count += valid
|
||||||
|
for event in anchor_events:
|
||||||
|
_, padding, raw_unequal = _target_metadata_check(event)
|
||||||
if padding is not None:
|
if padding is not None:
|
||||||
metadata_padding.append(padding)
|
metadata_padding.append(padding)
|
||||||
raw_unequal_count += bool(raw_unequal)
|
raw_unequal_count += bool(raw_unequal)
|
||||||
@@ -352,8 +370,9 @@ def target_execution_agreement(worker_events: list[dict[str, Any]]) -> dict[str,
|
|||||||
"valid_record_count": metadata_valid_count,
|
"valid_record_count": metadata_valid_count,
|
||||||
"all_records_valid": len(target_events) > 0
|
"all_records_valid": len(target_events) > 0
|
||||||
and metadata_valid_count == len(target_events),
|
and metadata_valid_count == len(target_events),
|
||||||
"raw_dp_counts_unequal_record_count": raw_unequal_count,
|
"anchor_record_count": len(anchor_events),
|
||||||
"post_coordinate_minus_local_raw_rows": {
|
"raw_dp_counts_unequal_anchor_record_count": raw_unequal_count,
|
||||||
|
"per_dp_replica_anchor_post_coordinate_minus_local_raw_rows": {
|
||||||
"n": len(metadata_padding),
|
"n": len(metadata_padding),
|
||||||
"min": min(metadata_padding) if metadata_padding else None,
|
"min": min(metadata_padding) if metadata_padding else None,
|
||||||
"max": max(metadata_padding) if metadata_padding else None,
|
"max": max(metadata_padding) if metadata_padding else None,
|
||||||
|
|||||||
Reference in New Issue
Block a user