diff --git a/analysis/characterization/data/chatbot_inter_turn_gap.json b/analysis/characterization/data/chatbot_inter_turn_gap.json new file mode 100644 index 0000000..4228cac --- /dev/null +++ b/analysis/characterization/data/chatbot_inter_turn_gap.json @@ -0,0 +1 @@ +{"trace": "chatbot", "n_gaps": 42228, "n_sessions": 32262, "negative_dropped": 89, "matched_formatted_to_raw": 339437, "unmatched_formatted": 13, "stats_s": {"min": 0.024, "max": 2497.758, "mean": 10.094598868049633, "p1": 0.533, "p5": 1.6293499999999999, "p25": 4.84675, "p50": 7.18, "p75": 8.22, "p90": 15.048600000000006, "p95": 19.377650000000003, "p99": 43.23037000000022}, "fraction_below": {"0.1s": 7.104290991759022e-05, "0.5s": 0.007625272331154684, "1.0s": 0.037392251586624986, "2.0s": 0.054111016387231224, "5.0s": 0.293288813109785, "10.0s": 0.7763332386094535, "30.0s": 0.978331912475135, "60.0s": 0.9931325187079663, "300.0s": 0.998389694041868, "1800.0s": 0.9998815951501373}, "cdf_samples": [{"rank_pct": 0.0023680969972530077, "gap_s": 0.024}, {"rank_pct": 0.004736193994506015, "gap_s": 0.091}, {"rank_pct": 0.007104290991759022, "gap_s": 0.096}, {"rank_pct": 0.00947238798901203, "gap_s": 0.104}, {"rank_pct": 0.011840484986265037, "gap_s": 0.115}, {"rank_pct": 0.014208581983518044, "gap_s": 0.129}, {"rank_pct": 0.016576678980771052, "gap_s": 0.147}, {"rank_pct": 0.01894477597802406, "gap_s": 0.16}, {"rank_pct": 0.021312872975277068, "gap_s": 0.166}, {"rank_pct": 0.023680969972530073, "gap_s": 0.169}, {"rank_pct": 0.026049066969783083, "gap_s": 0.17}, {"rank_pct": 0.02841716396703609, "gap_s": 0.171}, {"rank_pct": 0.0307852609642891, "gap_s": 0.172}, {"rank_pct": 0.033153357961542104, "gap_s": 0.173}, {"rank_pct": 0.035521454958795114, "gap_s": 0.174}, {"rank_pct": 0.03788955195604812, "gap_s": 0.176}, {"rank_pct": 0.040257648953301126, "gap_s": 0.177}, {"rank_pct": 0.042625745950554135, "gap_s": 0.177}, {"rank_pct": 0.044993842947807144, "gap_s": 0.178}, {"rank_pct": 0.04736193994506015, "gap_s": 0.18}, {"rank_pct": 0.04973003694231316, "gap_s": 0.18}, {"rank_pct": 0.052098133939566166, "gap_s": 0.181}, {"rank_pct": 0.054466230936819175, "gap_s": 0.181}, {"rank_pct": 0.05683432793407218, "gap_s": 0.183}, {"rank_pct": 0.059202424931325194, "gap_s": 0.185}, {"rank_pct": 0.0615705219285782, "gap_s": 0.186}, {"rank_pct": 0.0639386189258312, "gap_s": 0.188}, {"rank_pct": 0.06630671592308421, "gap_s": 0.189}, {"rank_pct": 0.06867481292033722, "gap_s": 0.189}, {"rank_pct": 0.07104290991759023, "gap_s": 0.19}, {"rank_pct": 0.07341100691484324, "gap_s": 0.19}, {"rank_pct": 0.07577910391209625, "gap_s": 0.19}, {"rank_pct": 0.07814720090934925, "gap_s": 0.191}, {"rank_pct": 0.08051529790660225, "gap_s": 0.192}, {"rank_pct": 0.08288339490385527, "gap_s": 0.192}, {"rank_pct": 0.08525149190110827, "gap_s": 0.192}, {"rank_pct": 0.08761958889836129, "gap_s": 0.193}, {"rank_pct": 0.08998768589561429, "gap_s": 0.193}, {"rank_pct": 0.09235578289286729, "gap_s": 0.194}, {"rank_pct": 0.09709197688737331, "gap_s": 0.195}, {"rank_pct": 0.09946007388462633, "gap_s": 0.195}, {"rank_pct": 0.10182817088187932, "gap_s": 0.195}, {"rank_pct": 0.10656436487638535, "gap_s": 0.196}, {"rank_pct": 0.10893246187363835, "gap_s": 0.196}, {"rank_pct": 0.11130055887089135, "gap_s": 0.197}, {"rank_pct": 0.11603675286539737, "gap_s": 0.198}, {"rank_pct": 0.11840484986265039, "gap_s": 0.199}, {"rank_pct": 0.1231410438571564, "gap_s": 0.201}, {"rank_pct": 0.1255091408544094, "gap_s": 0.201}, {"rank_pct": 0.1302453348489154, "gap_s": 0.203}, {"rank_pct": 0.13498152884342143, "gap_s": 0.204}, {"rank_pct": 0.13971772283792744, "gap_s": 0.206}, {"rank_pct": 0.14208581983518045, "gap_s": 0.209}, {"rank_pct": 0.1468220138296865, "gap_s": 0.215}, {"rank_pct": 0.1515582078241925, "gap_s": 0.221}, {"rank_pct": 0.1562944018186985, "gap_s": 0.228}, {"rank_pct": 0.1610305958132045, "gap_s": 0.23}, {"rank_pct": 0.16576678980771054, "gap_s": 0.231}, {"rank_pct": 0.17050298380221654, "gap_s": 0.235}, {"rank_pct": 0.17760727479397556, "gap_s": 0.238}, {"rank_pct": 0.1823434687884816, "gap_s": 0.241}, {"rank_pct": 0.1870796627829876, "gap_s": 0.243}, {"rank_pct": 0.19418395377474662, "gap_s": 0.248}, {"rank_pct": 0.19892014776925265, "gap_s": 0.252}, {"rank_pct": 0.20602443876101165, "gap_s": 0.256}, {"rank_pct": 0.2131287297527707, "gap_s": 0.262}, {"rank_pct": 0.2178649237472767, "gap_s": 0.262}, {"rank_pct": 0.2249692147390357, "gap_s": 0.27}, {"rank_pct": 0.23207350573079474, "gap_s": 0.275}, {"rank_pct": 0.23917779672255374, "gap_s": 0.279}, {"rank_pct": 0.2462820877143128, "gap_s": 0.286}, {"rank_pct": 0.2557544757033248, "gap_s": 0.289}, {"rank_pct": 0.26285876669508385, "gap_s": 0.293}, {"rank_pct": 0.26996305768684287, "gap_s": 0.3}, {"rank_pct": 0.2794354456758549, "gap_s": 0.306}, {"rank_pct": 0.2865397366676139, "gap_s": 0.307}, {"rank_pct": 0.2960121246566259, "gap_s": 0.316}, {"rank_pct": 0.305484512645638, "gap_s": 0.327}, {"rank_pct": 0.31495690063465, "gap_s": 0.338}, {"rank_pct": 0.32442928862366205, "gap_s": 0.349}, {"rank_pct": 0.33390167661267406, "gap_s": 0.358}, {"rank_pct": 0.3457421615989391, "gap_s": 0.371}, {"rank_pct": 0.3552145495879511, "gap_s": 0.374}, {"rank_pct": 0.3670550345742162, "gap_s": 0.382}, {"rank_pct": 0.3788955195604812, "gap_s": 0.385}, {"rank_pct": 0.3907360045467463, "gap_s": 0.392}, {"rank_pct": 0.40257648953301123, "gap_s": 0.4}, {"rank_pct": 0.4144169745192763, "gap_s": 0.403}, {"rank_pct": 0.4262574595055414, "gap_s": 0.407}, {"rank_pct": 0.4404660414890594, "gap_s": 0.413}, {"rank_pct": 0.4546746234725774, "gap_s": 0.42}, {"rank_pct": 0.4665151084588425, "gap_s": 0.423}, {"rank_pct": 0.4830917874396135, "gap_s": 0.426}, {"rank_pct": 0.4973003694231316, "gap_s": 0.433}, {"rank_pct": 0.5115089514066496, "gap_s": 0.437}, {"rank_pct": 0.5280856303874206, "gap_s": 0.441}, {"rank_pct": 0.5446623093681917, "gap_s": 0.445}, {"rank_pct": 0.5612389883489628, "gap_s": 0.451}, {"rank_pct": 0.5778156673297339, "gap_s": 0.454}, {"rank_pct": 0.5967604433077579, "gap_s": 0.46}, {"rank_pct": 0.6157052192857819, "gap_s": 0.471}, {"rank_pct": 0.6346499952638059, "gap_s": 0.476}, {"rank_pct": 0.6535947712418301, "gap_s": 0.48}, {"rank_pct": 0.6725395472198541, "gap_s": 0.485}, {"rank_pct": 0.6938524201951313, "gap_s": 0.488}, {"rank_pct": 0.7151652931704082, "gap_s": 0.491}, {"rank_pct": 0.7364781661456854, "gap_s": 0.495}, {"rank_pct": 0.7601591361182154, "gap_s": 0.499}, {"rank_pct": 0.7838401060907454, "gap_s": 0.502}, {"rank_pct": 0.8075210760632756, "gap_s": 0.504}, {"rank_pct": 0.8335701430330587, "gap_s": 0.51}, {"rank_pct": 0.8596192100028417, "gap_s": 0.513}, {"rank_pct": 0.8856682769726247, "gap_s": 0.516}, {"rank_pct": 0.9117173439424078, "gap_s": 0.519}, {"rank_pct": 0.940134507909444, "gap_s": 0.524}, {"rank_pct": 0.9685516718764801, "gap_s": 0.528}, {"rank_pct": 0.9993369328407692, "gap_s": 0.533}, {"rank_pct": 1.0017050298380221, "gap_s": 0.533}, {"rank_pct": 1.3308705124561901, "gap_s": 0.575}, {"rank_pct": 1.6624040920716114, "gap_s": 0.617}, {"rank_pct": 1.993937671687032, "gap_s": 0.657}, {"rank_pct": 2.3254712513024534, "gap_s": 0.698}, {"rank_pct": 2.657004830917874, "gap_s": 0.742}, {"rank_pct": 2.9861703135360425, "gap_s": 0.797}, {"rank_pct": 3.3177038931514637, "gap_s": 0.863}, {"rank_pct": 3.649237472766884, "gap_s": 0.964}, {"rank_pct": 3.980771052382306, "gap_s": 1.108}, {"rank_pct": 4.312304631997726, "gap_s": 1.243}, {"rank_pct": 4.6414701146158945, "gap_s": 1.403}, {"rank_pct": 4.973003694231315, "gap_s": 1.604}, {"rank_pct": 5.304537273846737, "gap_s": 1.924}, {"rank_pct": 5.636070853462158, "gap_s": 2.215}, {"rank_pct": 5.967604433077579, "gap_s": 2.526}, {"rank_pct": 6.299138012693, "gap_s": 2.771}, {"rank_pct": 6.628303495311168, "gap_s": 3.091}, {"rank_pct": 6.95983707492659, "gap_s": 3.459}, {"rank_pct": 7.2913706545420105, "gap_s": 3.776}, {"rank_pct": 7.6229042341574305, "gap_s": 3.972}, {"rank_pct": 7.954437813772852, "gap_s": 4.019}, {"rank_pct": 8.283603296391021, "gap_s": 4.052}, {"rank_pct": 8.61513687600644, "gap_s": 4.078}, {"rank_pct": 8.946670455621861, "gap_s": 4.099}, {"rank_pct": 9.278204035237284, "gap_s": 4.117}, {"rank_pct": 9.609737614852705, "gap_s": 4.136}, {"rank_pct": 9.941271194468126, "gap_s": 4.153}, {"rank_pct": 10.270436677086293, "gap_s": 4.17}, {"rank_pct": 10.601970256701716, "gap_s": 4.187}, {"rank_pct": 10.933503836317135, "gap_s": 4.201}, {"rank_pct": 11.265037415932557, "gap_s": 4.216}, {"rank_pct": 11.596570995547978, "gap_s": 4.233}, {"rank_pct": 11.925736478166145, "gap_s": 4.251}, {"rank_pct": 12.257270057781566, "gap_s": 4.268}, {"rank_pct": 12.588803637396989, "gap_s": 4.287}, {"rank_pct": 12.92033721701241, "gap_s": 4.306}, {"rank_pct": 13.251870796627829, "gap_s": 4.329}, {"rank_pct": 13.583404376243251, "gap_s": 4.357}, {"rank_pct": 13.912569858861417, "gap_s": 4.394}, {"rank_pct": 14.244103438476841, "gap_s": 4.438}, {"rank_pct": 14.57563701809226, "gap_s": 4.482}, {"rank_pct": 14.907170597707683, "gap_s": 4.525}, {"rank_pct": 15.238704177323104, "gap_s": 4.555}, {"rank_pct": 15.567869659941271, "gap_s": 4.578}, {"rank_pct": 15.899403239556692, "gap_s": 4.598}, {"rank_pct": 16.23093681917211, "gap_s": 4.615}, {"rank_pct": 16.562470398787536, "gap_s": 4.627}, {"rank_pct": 16.894003978402957, "gap_s": 4.641}, {"rank_pct": 17.223169461021122, "gap_s": 4.653}, {"rank_pct": 17.554703040636543, "gap_s": 4.664}, {"rank_pct": 17.886236620251967, "gap_s": 4.674}, {"rank_pct": 18.217770199867385, "gap_s": 4.683}, {"rank_pct": 18.549303779482806, "gap_s": 4.692}, {"rank_pct": 18.88083735909823, "gap_s": 4.702}, {"rank_pct": 19.2100028417164, "gap_s": 4.709}, {"rank_pct": 19.541536421331816, "gap_s": 4.718}, {"rank_pct": 19.873070000947237, "gap_s": 4.726}, {"rank_pct": 20.20460358056266, "gap_s": 4.735}, {"rank_pct": 20.53613716017808, "gap_s": 4.743}, {"rank_pct": 20.865302642796248, "gap_s": 4.751}, {"rank_pct": 21.19683622241167, "gap_s": 4.758}, {"rank_pct": 21.528369802027093, "gap_s": 4.765}, {"rank_pct": 21.85990338164251, "gap_s": 4.772}, {"rank_pct": 22.19143696125793, "gap_s": 4.779}, {"rank_pct": 22.522970540873356, "gap_s": 4.786}, {"rank_pct": 22.852136023491525, "gap_s": 4.794}, {"rank_pct": 23.183669603106942, "gap_s": 4.802}, {"rank_pct": 23.515203182722363, "gap_s": 4.81}, {"rank_pct": 23.846736762337787, "gap_s": 4.818}, {"rank_pct": 24.178270341953205, "gap_s": 4.827}, {"rank_pct": 24.507435824571374, "gap_s": 4.835}, {"rank_pct": 24.838969404186795, "gap_s": 4.842}, {"rank_pct": 25.17050298380222, "gap_s": 4.85}, {"rank_pct": 25.502036563417636, "gap_s": 4.858}, {"rank_pct": 25.833570143033057, "gap_s": 4.869}, {"rank_pct": 26.16510372264848, "gap_s": 4.878}, {"rank_pct": 26.494269205266647, "gap_s": 4.887}, {"rank_pct": 26.825802784882068, "gap_s": 4.897}, {"rank_pct": 27.157336364497493, "gap_s": 4.906}, {"rank_pct": 27.48886994411291, "gap_s": 4.918}, {"rank_pct": 27.82040352372833, "gap_s": 4.929}, {"rank_pct": 28.149569006346496, "gap_s": 4.941}, {"rank_pct": 28.48110258596192, "gap_s": 4.956}, {"rank_pct": 28.812636165577345, "gap_s": 4.97}, {"rank_pct": 29.144169745192762, "gap_s": 4.988}, {"rank_pct": 29.475703324808183, "gap_s": 5.008}, {"rank_pct": 29.804868807426356, "gap_s": 5.034}, {"rank_pct": 30.136402387041773, "gap_s": 5.058}, {"rank_pct": 30.467935966657194, "gap_s": 5.092}, {"rank_pct": 30.79946954627262, "gap_s": 5.144}, {"rank_pct": 31.131003125888036, "gap_s": 5.227}, {"rank_pct": 31.462536705503457, "gap_s": 5.332}, {"rank_pct": 31.791702188121622, "gap_s": 5.452}, {"rank_pct": 32.12323576773704, "gap_s": 5.584}, {"rank_pct": 32.45476934735247, "gap_s": 5.776}, {"rank_pct": 32.786302926967885, "gap_s": 6.052}, {"rank_pct": 33.11783650658331, "gap_s": 6.25}, {"rank_pct": 33.44700198920148, "gap_s": 6.401}, {"rank_pct": 33.7785355688169, "gap_s": 6.472}, {"rank_pct": 34.11006914843232, "gap_s": 6.535}, {"rank_pct": 34.44160272804774, "gap_s": 6.591}, {"rank_pct": 34.77313630766316, "gap_s": 6.642}, {"rank_pct": 35.10466988727858, "gap_s": 6.684}, {"rank_pct": 35.43383536989675, "gap_s": 6.721}, {"rank_pct": 35.76536894951217, "gap_s": 6.758}, {"rank_pct": 36.0969025291276, "gap_s": 6.788}, {"rank_pct": 36.428436108743014, "gap_s": 6.815}, {"rank_pct": 36.75996968835843, "gap_s": 6.839}, {"rank_pct": 37.089135170976604, "gap_s": 6.858}, {"rank_pct": 37.42066875059202, "gap_s": 6.874}, {"rank_pct": 37.752202330207446, "gap_s": 6.893}, {"rank_pct": 38.08373590982287, "gap_s": 6.906}, {"rank_pct": 38.41526948943829, "gap_s": 6.918}, {"rank_pct": 38.74680306905371, "gap_s": 6.93}, {"rank_pct": 39.07596855167188, "gap_s": 6.94}, {"rank_pct": 39.407502131287295, "gap_s": 6.949}, {"rank_pct": 39.73903571090272, "gap_s": 6.959}, {"rank_pct": 40.07056929051814, "gap_s": 6.968}, {"rank_pct": 40.40210287013356, "gap_s": 6.977}, {"rank_pct": 40.73126835275173, "gap_s": 6.986}, {"rank_pct": 41.06280193236715, "gap_s": 6.994}, {"rank_pct": 41.394335511982575, "gap_s": 7.002}, {"rank_pct": 41.72586909159799, "gap_s": 7.011}, {"rank_pct": 42.05740267121341, "gap_s": 7.019}, {"rank_pct": 42.388936250828834, "gap_s": 7.027}, {"rank_pct": 42.718101733447, "gap_s": 7.034}, {"rank_pct": 43.049635313062424, "gap_s": 7.042}, {"rank_pct": 43.38116889267785, "gap_s": 7.049}, {"rank_pct": 43.712702472293266, "gap_s": 7.055}, {"rank_pct": 44.04423605190868, "gap_s": 7.061}, {"rank_pct": 44.37340153452685, "gap_s": 7.068}, {"rank_pct": 44.70493511414227, "gap_s": 7.074}, {"rank_pct": 45.0364686937577, "gap_s": 7.081}, {"rank_pct": 45.368002273373115, "gap_s": 7.086}, {"rank_pct": 45.69953585298854, "gap_s": 7.093}, {"rank_pct": 46.02870133560671, "gap_s": 7.101}, {"rank_pct": 46.36023491522213, "gap_s": 7.107}, {"rank_pct": 46.69176849483755, "gap_s": 7.114}, {"rank_pct": 47.02330207445297, "gap_s": 7.121}, {"rank_pct": 47.35483565406839, "gap_s": 7.127}, {"rank_pct": 47.68636923368381, "gap_s": 7.133}, {"rank_pct": 48.01553471630198, "gap_s": 7.139}, {"rank_pct": 48.3470682959174, "gap_s": 7.145}, {"rank_pct": 48.67860187553282, "gap_s": 7.152}, {"rank_pct": 49.01013545514824, "gap_s": 7.159}, {"rank_pct": 49.34166903476366, "gap_s": 7.165}, {"rank_pct": 49.670834517381834, "gap_s": 7.173}, {"rank_pct": 50.00236809699725, "gap_s": 7.18}, {"rank_pct": 50.33390167661267, "gap_s": 7.188}, {"rank_pct": 50.6654352562281, "gap_s": 7.195}, {"rank_pct": 50.99696883584352, "gap_s": 7.204}, {"rank_pct": 51.328502415458935, "gap_s": 7.212}, {"rank_pct": 51.65766789807711, "gap_s": 7.219}, {"rank_pct": 51.989201477692525, "gap_s": 7.227}, {"rank_pct": 52.32073505730794, "gap_s": 7.236}, {"rank_pct": 52.652268636923374, "gap_s": 7.244}, {"rank_pct": 52.98380221653879, "gap_s": 7.253}, {"rank_pct": 53.31296769915696, "gap_s": 7.263}, {"rank_pct": 53.64450127877238, "gap_s": 7.273}, {"rank_pct": 53.9760348583878, "gap_s": 7.283}, {"rank_pct": 54.307568438003216, "gap_s": 7.292}, {"rank_pct": 54.63910201761865, "gap_s": 7.301}, {"rank_pct": 54.970635597234065, "gap_s": 7.311}, {"rank_pct": 55.29980107985223, "gap_s": 7.32}, {"rank_pct": 55.631334659467655, "gap_s": 7.331}, {"rank_pct": 55.96286823908307, "gap_s": 7.342}, {"rank_pct": 56.29440181869849, "gap_s": 7.353}, {"rank_pct": 56.62593539831392, "gap_s": 7.365}, {"rank_pct": 56.955100880932086, "gap_s": 7.375}, {"rank_pct": 57.2866344605475, "gap_s": 7.385}, {"rank_pct": 57.61816804016292, "gap_s": 7.395}, {"rank_pct": 57.94970161977835, "gap_s": 7.407}, {"rank_pct": 58.28123519939376, "gap_s": 7.417}, {"rank_pct": 58.61040068201193, "gap_s": 7.429}, {"rank_pct": 58.94193426162736, "gap_s": 7.438}, {"rank_pct": 59.27346784124278, "gap_s": 7.447}, {"rank_pct": 59.605001420858194, "gap_s": 7.456}, {"rank_pct": 59.936535000473626, "gap_s": 7.466}, {"rank_pct": 60.26806858008904, "gap_s": 7.474}, {"rank_pct": 60.59723406270721, "gap_s": 7.483}, {"rank_pct": 60.92876764232263, "gap_s": 7.492}, {"rank_pct": 61.26030122193805, "gap_s": 7.5}, {"rank_pct": 61.59183480155347, "gap_s": 7.508}, {"rank_pct": 61.9233683811689, "gap_s": 7.516}, {"rank_pct": 62.252533863787065, "gap_s": 7.524}, {"rank_pct": 62.58406744340248, "gap_s": 7.531}, {"rank_pct": 62.9156010230179, "gap_s": 7.538}, {"rank_pct": 63.247134602633324, "gap_s": 7.546}, {"rank_pct": 63.57866818224874, "gap_s": 7.553}, {"rank_pct": 63.91020176186416, "gap_s": 7.56}, {"rank_pct": 64.23936724448234, "gap_s": 7.568}, {"rank_pct": 64.57090082409775, "gap_s": 7.575}, {"rank_pct": 64.90243440371317, "gap_s": 7.583}, {"rank_pct": 65.2339679833286, "gap_s": 7.589}, {"rank_pct": 65.56550156294402, "gap_s": 7.596}, {"rank_pct": 65.89466704556219, "gap_s": 7.603}, {"rank_pct": 66.22620062517761, "gap_s": 7.612}, {"rank_pct": 66.55773420479304, "gap_s": 7.619}, {"rank_pct": 66.88926778440845, "gap_s": 7.625}, {"rank_pct": 67.22080136402387, "gap_s": 7.634}, {"rank_pct": 67.5523349436393, "gap_s": 7.641}, {"rank_pct": 67.88150042625746, "gap_s": 7.65}, {"rank_pct": 68.21303400587288, "gap_s": 7.657}, {"rank_pct": 68.54456758548831, "gap_s": 7.666}, {"rank_pct": 68.87610116510372, "gap_s": 7.674}, {"rank_pct": 69.20763474471914, "gap_s": 7.683}, {"rank_pct": 69.53680022733731, "gap_s": 7.694}, {"rank_pct": 69.86833380695273, "gap_s": 7.703}, {"rank_pct": 70.19986738656814, "gap_s": 7.714}, {"rank_pct": 70.53140096618358, "gap_s": 7.725}, {"rank_pct": 70.86293454579899, "gap_s": 7.737}, {"rank_pct": 71.19446812541442, "gap_s": 7.75}, {"rank_pct": 71.52363360803258, "gap_s": 7.766}, {"rank_pct": 71.85516718764801, "gap_s": 7.783}, {"rank_pct": 72.18670076726342, "gap_s": 7.803}, {"rank_pct": 72.51823434687886, "gap_s": 7.823}, {"rank_pct": 72.84976792649427, "gap_s": 7.844}, {"rank_pct": 73.17893340911243, "gap_s": 7.871}, {"rank_pct": 73.51046698872786, "gap_s": 7.902}, {"rank_pct": 73.84200056834328, "gap_s": 7.948}, {"rank_pct": 74.17353414795869, "gap_s": 8.006}, {"rank_pct": 74.50506772757413, "gap_s": 8.08}, {"rank_pct": 74.8342332101923, "gap_s": 8.176}, {"rank_pct": 75.1657667898077, "gap_s": 8.259}, {"rank_pct": 75.49730036942314, "gap_s": 8.365}, {"rank_pct": 75.82883394903855, "gap_s": 8.502}, {"rank_pct": 76.16036752865398, "gap_s": 8.785}, {"rank_pct": 76.4919011082694, "gap_s": 9.067}, {"rank_pct": 76.82106659088757, "gap_s": 9.446}, {"rank_pct": 77.15260017050298, "gap_s": 9.737}, {"rank_pct": 77.4841337501184, "gap_s": 9.93}, {"rank_pct": 77.81566732973383, "gap_s": 10.073}, {"rank_pct": 78.14720090934925, "gap_s": 10.207}, {"rank_pct": 78.47636639196742, "gap_s": 10.366}, {"rank_pct": 78.80789997158284, "gap_s": 10.491}, {"rank_pct": 79.13943355119825, "gap_s": 10.585}, {"rank_pct": 79.47096713081368, "gap_s": 10.661}, {"rank_pct": 79.8025007104291, "gap_s": 10.727}, {"rank_pct": 80.13403429004453, "gap_s": 10.802}, {"rank_pct": 80.46319977266269, "gap_s": 10.875}, {"rank_pct": 80.79473335227812, "gap_s": 10.94}, {"rank_pct": 81.12626693189353, "gap_s": 11.015}, {"rank_pct": 81.45780051150895, "gap_s": 11.094}, {"rank_pct": 81.78933409112437, "gap_s": 11.184}, {"rank_pct": 82.11849957374254, "gap_s": 11.272}, {"rank_pct": 82.45003315335796, "gap_s": 11.345}, {"rank_pct": 82.78156673297337, "gap_s": 11.426}, {"rank_pct": 83.11310031258881, "gap_s": 11.51}, {"rank_pct": 83.44463389220422, "gap_s": 11.589}, {"rank_pct": 83.77616747181963, "gap_s": 11.681}, {"rank_pct": 84.10533295443781, "gap_s": 11.771}, {"rank_pct": 84.43686653405324, "gap_s": 11.867}, {"rank_pct": 84.76840011366865, "gap_s": 11.97}, {"rank_pct": 85.09993369328409, "gap_s": 12.083}, {"rank_pct": 85.4314672728995, "gap_s": 12.197}, {"rank_pct": 85.76063275551766, "gap_s": 12.308}, {"rank_pct": 86.09216633513309, "gap_s": 12.433}, {"rank_pct": 86.42369991474851, "gap_s": 12.57}, {"rank_pct": 86.75523349436392, "gap_s": 12.706}, {"rank_pct": 87.08676707397936, "gap_s": 12.879}, {"rank_pct": 87.41593255659753, "gap_s": 13.059}, {"rank_pct": 87.74746613621294, "gap_s": 13.207}, {"rank_pct": 88.07899971582836, "gap_s": 13.398}, {"rank_pct": 88.41053329544378, "gap_s": 13.598}, {"rank_pct": 88.7420668750592, "gap_s": 13.816}, {"rank_pct": 89.07360045467463, "gap_s": 14.075}, {"rank_pct": 89.4027659372928, "gap_s": 14.354}, {"rank_pct": 89.73429951690821, "gap_s": 14.756}, {"rank_pct": 90.06583309652363, "gap_s": 15.125}, {"rank_pct": 90.39736667613906, "gap_s": 15.204}, {"rank_pct": 90.72890025575447, "gap_s": 15.222}, {"rank_pct": 91.05806573837263, "gap_s": 15.247}, {"rank_pct": 91.38959931798807, "gap_s": 15.311}, {"rank_pct": 91.72113289760348, "gap_s": 15.36}, {"rank_pct": 92.0526664772189, "gap_s": 15.386}, {"rank_pct": 92.38420005683433, "gap_s": 15.424}, {"rank_pct": 92.71573363644976, "gap_s": 15.616}, {"rank_pct": 93.04489911906792, "gap_s": 16.063}, {"rank_pct": 93.37643269868335, "gap_s": 16.574}, {"rank_pct": 93.70796627829876, "gap_s": 17.047}, {"rank_pct": 94.03949985791418, "gap_s": 17.619}, {"rank_pct": 94.3710334375296, "gap_s": 18.254}, {"rank_pct": 94.70019892014777, "gap_s": 18.768}, {"rank_pct": 95.0317324997632, "gap_s": 19.452}, {"rank_pct": 95.36326607937862, "gap_s": 20.127}, {"rank_pct": 95.69479965899403, "gap_s": 21.03}, {"rank_pct": 96.02633323860945, "gap_s": 22.194}, {"rank_pct": 96.35786681822488, "gap_s": 23.272}, {"rank_pct": 96.68703230084304, "gap_s": 24.393}, {"rank_pct": 97.01856588045847, "gap_s": 25.813}, {"rank_pct": 97.35009946007388, "gap_s": 27.235}, {"rank_pct": 97.6816330396893, "gap_s": 29.034}, {"rank_pct": 98.01316661930473, "gap_s": 31.167}, {"rank_pct": 98.34233210192289, "gap_s": 33.766}, {"rank_pct": 98.67386568153832, "gap_s": 37.308}, {"rank_pct": 99.00539926115374, "gap_s": 43.357}, {"rank_pct": 99.33693284076915, "gap_s": 63.312}, {"rank_pct": 99.66846642038459, "gap_s": 128.766}, {"rank_pct": 100.0, "gap_s": 2497.758}]} \ No newline at end of file diff --git a/figs/f3a_inter_turn_gap.png b/figs/f3a_inter_turn_gap.png index b772e75..d2e722e 100644 Binary files a/figs/f3a_inter_turn_gap.png and b/figs/f3a_inter_turn_gap.png differ diff --git a/scripts/compute_inter_turn_gap_chatbot.py b/scripts/compute_inter_turn_gap_chatbot.py new file mode 100644 index 0000000..a33e705 --- /dev/null +++ b/scripts/compute_inter_turn_gap_chatbot.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""Compute inter-turn T_external on the chatbot trace, v2. + +Uses formatted's parent_chat_id chains for sessions, and matches each +formatted record to a raw input/output pair by (timestamp, input_length) +rather than by index (index-by-index drifted up to 120s in v1). + +Run on dash0. Writes /tmp/chatbot_inter_turn_gap.json. +""" +import json +import bisect +from collections import defaultdict +import numpy as np +from datetime import datetime + +RAW_IN = "/home/admin/cpfs/wjh/bailian-trace/qwen-trace-260321-260327/qwen3-max-input-032309-032311.jsonl" +RAW_OUT = "/home/admin/cpfs/wjh/bailian-trace/qwen-trace-260321-260327/qwen3-max-output-032109-032711.jsonl" +FMT = "/home/admin/cpfs/wjh/bailian-trace/qwen-trace-260321-260327-formatted/qwen_chat_blksz_64_032309-032311.jsonl" +OUT_JSON = "/tmp/chatbot_inter_turn_gap.json" + +def parse_time_str_to_ms(s): + try: + if "." in s: + dt = datetime.strptime(s, "%Y-%m-%d %H:%M:%S.%f") + else: + dt = datetime.strptime(s, "%Y-%m-%d %H:%M:%S") + return int(dt.timestamp() * 1000) + except Exception: + return None + +print("Reading raw output (joining by request_id)...") +# In this trace prompt_token_num is anonymized to '0'; use generate_token_num +# as the matching key (matches formatted output_length). For end_ms we use +# time_to_finish_token (ms duration from request start) — the "time" string +# field is log-write time, not request completion time. +out_info = {} # request_id -> (ttf_ms, generate_token_num) +n_out_seen = 0 +with open(RAW_OUT) as f: + for line in f: + try: + d = json.loads(line) + except: continue + rid = d.get("request_id") + if rid is None: + continue + n_out_seen += 1 + gtn = d.get("generate_token_num") + ttf = d.get("time_to_finish_token") + if ttf is None: + continue + try: + ttf_ms = int(float(ttf)) + except: continue + try: + gtn = int(gtn) if gtn is not None else None + except: gtn = None + # Keep the largest ttf_ms if duplicates (multiple log lines per request) + prev = out_info.get(rid) + if prev is None or ttf_ms > prev[0]: + out_info[rid] = (ttf_ms, gtn) +print(f" scanned: {n_out_seen}, unique req with ttf: {len(out_info)}") + +print("Reading raw input + joining...") +joined = [] # list of (start_ms, end_ms, input_length, request_id) +n_in_seen = 0 +seen_rids = set() +with open(RAW_IN) as f: + for line in f: + try: + d = json.loads(line) + except: continue + n_in_seen += 1 + rid = d.get("request_id") + ts = d.get("timestamp") + if rid is None or ts is None or rid in seen_rids: + continue + seen_rids.add(rid) + info = out_info.get(rid) + if info is None: + continue + ttf_ms, gtn = info + try: ts = int(ts) + except: continue + end_ms = ts + ttf_ms + joined.append((ts, end_ms, gtn, rid)) +print(f" input scanned: {n_in_seen}, joined start+end+gtn: {len(joined)}") + +joined.sort(key=lambda x: x[0]) +starts = [j[0] for j in joined] +gtns = [j[2] for j in joined] # generate_token_num (output_length-equivalent) +ends = [j[1] for j in joined] +print(f"start_ms range: [{starts[0]}, {starts[-1]}], duration {(starts[-1]-starts[0])/1000:.0f}s") + +print("Reading formatted...") +fmt_rows = [] +with open(FMT) as f: + for line in f: + try: + d = json.loads(line) + except: continue + fmt_rows.append(( + int(d["chat_id"]), + int(d["parent_chat_id"]), + float(d["timestamp"]), + int(d.get("input_length", 0)), + int(d.get("output_length", 0)), + )) +print(f" formatted records: {len(fmt_rows)}") +print(f"fmt timestamp range: [{fmt_rows[0][2]}, {fmt_rows[-1][2]}]s " + f"(duration {fmt_rows[-1][2] - fmt_rows[0][2]:.0f}s)") + +# Calibrate T0 by matching first few formatted records with raw records. +# We use output_length (formatted) vs generate_token_num (raw output) as the +# matching key — prompt_token_num is anonymized to 0. +print("Calibrating T0 (raw_ms anchor for formatted ts=0)...") +T0_candidates = [] +for chat_id, _pcid, ts_rel, _il, ol in fmt_rows[:200]: + for k in range(min(2000, len(joined))): + if gtns[k] == ol: + T0_candidates.append(starts[k] - int(ts_rel * 1000)) + break +T0_candidates.sort() +T0 = T0_candidates[len(T0_candidates) // 2] if T0_candidates else starts[0] +print(f" T0 from {len(T0_candidates)} candidates -> {T0} ms") +print(f" candidate T0 distribution: min={min(T0_candidates) if T0_candidates else 'n/a'} " + f"max={max(T0_candidates) if T0_candidates else 'n/a'}") + +print("Matching formatted -> raw by (ts_rel, output_length)...") +TOLERANCE_MS = 60_000 # ±60 s window +fmt_to_timing = {} +matched = 0 +ambiguous = 0 +unmatched = 0 +for chat_id, _pcid, ts_rel, _il, ol in fmt_rows: + target_ms = T0 + int(ts_rel * 1000) + lo = bisect.bisect_left(starts, target_ms - TOLERANCE_MS) + hi = bisect.bisect_right(starts, target_ms + TOLERANCE_MS) + best = None + best_drift = None + for k in range(lo, hi): + if gtns[k] == ol: + d = abs(starts[k] - target_ms) + if best_drift is None or d < best_drift: + best_drift = d + best = k + if best is None: + unmatched += 1 + continue + fmt_to_timing[chat_id] = (starts[best], ends[best]) + matched += 1 +print(f" matched: {matched}, unmatched: {unmatched}, ambiguous: {ambiguous}") +print(f" match rate: {matched/len(fmt_rows)*100:.1f}%") + +# Build session structure from parent_chat_id chains +chat_to_session = {} +for chat_id, pcid, _ts, _il, _ol in fmt_rows: + if pcid < 0: + sid = chat_id + else: + sid = chat_to_session.get(pcid, pcid) + chat_to_session[chat_id] = sid + +sessions = defaultdict(list) +for chat_id, _pcid, _ts, _il, _ol in fmt_rows: + timing = fmt_to_timing.get(chat_id) + if timing is None: + continue + sid = chat_to_session[chat_id] + sessions[sid].append(timing) + +gaps_ms = [] +neg = 0 +multi = 0 +for sid, turns in sessions.items(): + if len(turns) < 2: + continue + multi += 1 + turns.sort(key=lambda x: x[0]) + for i in range(len(turns) - 1): + g = turns[i + 1][0] - turns[i][1] + if g < 0: + neg += 1 + continue + gaps_ms.append(g) + +gaps = np.array(gaps_ms, dtype=np.float64) / 1000.0 +print(f"multi_turn_sessions: {multi}, gaps_kept: {len(gaps)}, neg_dropped: {neg}") + +if len(gaps) == 0: + print("No gaps to summarize.") +else: + pcts = [1, 5, 25, 50, 75, 90, 95, 99] + ps = {f"p{p}": float(np.percentile(gaps, p)) for p in pcts} + print(f"stats_s: min={gaps.min():.3f} mean={gaps.mean():.3f} max={gaps.max():.3f} {ps}") + for thr in [0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0, 300.0, 1800.0]: + pct = (gaps < thr).sum() / len(gaps) * 100 + print(f"frac < {thr:7.1f}s : {pct:5.1f}%") + + n = len(gaps) + arr = np.sort(gaps) + idx_top = np.unique(np.round(np.geomspace(1, max(1, n // 100), 200)).astype(int)) - 1 + idx_rest = np.unique(np.linspace(n // 100, n - 1, 300).astype(int)) + idx = np.unique(np.concatenate([[0], idx_top, idx_rest, [n - 1]])) + idx = idx[idx < n] + samples = [{"rank_pct": float((i + 1) / n * 100), "gap_s": float(arr[i])} for i in idx] + out = { + "trace": "chatbot", + "n_gaps": n, + "n_sessions": multi, + "negative_dropped": neg, + "matched_formatted_to_raw": matched, + "unmatched_formatted": unmatched, + "stats_s": {**{"min": float(gaps.min()), "max": float(gaps.max()), + "mean": float(gaps.mean())}, **ps}, + "fraction_below": {f"{thr}s": float((gaps < thr).sum() / n) + for thr in [0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0, 300.0, 1800.0]}, + "cdf_samples": samples, + } + open(OUT_JSON, "w").write(json.dumps(out)) + print(f"wrote {OUT_JSON}") diff --git a/scripts/plot_inter_turn_gap.py b/scripts/plot_inter_turn_gap.py index 10e0d9e..5f31493 100644 --- a/scripts/plot_inter_turn_gap.py +++ b/scripts/plot_inter_turn_gap.py @@ -30,52 +30,65 @@ def load(cache_path: Path) -> tuple[np.ndarray, np.ndarray, dict]: def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( - "--data", + "--agentic-data", default="analysis/characterization/data/agentic_inter_turn_gap.json", ) + parser.add_argument( + "--chatbot-data", + default="analysis/characterization/data/chatbot_inter_turn_gap.json", + ) parser.add_argument("--out", default="figs/f3a_inter_turn_gap.png") args = parser.parse_args() - xs, ys, d = load(Path(args.data)) + a_xs, a_ys, a_d = load(Path(args.agentic_data)) + c_xs, c_ys, c_d = load(Path(args.chatbot_data)) - fig, ax = plt.subplots(figsize=(9, 5.2)) - ax.plot(xs, ys, color="#1f77b4", lw=2.2, - label=f"agentic trace (n={d['n_gaps']:,} gaps, " - f"{d['n_sessions']:,} multi-turn sessions)") + fig, ax = plt.subplots(figsize=(10, 5.5)) + ax.plot(a_xs, a_ys, color="#1f77b4", lw=2.4, + label=f"agentic (n={a_d['n_gaps']:,} gaps, " + f"{a_d['n_sessions']:,} multi-turn sessions, Qwen3-Coder)") + ax.plot(c_xs, c_ys, color="#c44e52", lw=2.4, + label=f"chatbot (n={c_d['n_gaps']:,} gaps, " + f"{c_d['n_sessions']:,} multi-turn sessions, qwen3-max)") - p = d["stats_s"] - for pct, key in [(25, "p25"), (50, "p50"), (75, "p75"), (90, "p90")]: - v = p[key] - ax.scatter([v], [pct], color="#c44e52", s=55, zorder=5) - ax.annotate(f"p{pct} = {v:.2g}s", - xy=(v, pct), xytext=(8, -4), - textcoords="offset points", - fontsize=10, color="#7a1d1d") + for d, color, side in [(a_d, "#1f4e79", "left"), (c_d, "#7a1d1d", "right")]: + for pct, key in [(50, "p50")]: + v = d["stats_s"][key] + ax.scatter([v], [pct], color=color, s=55, zorder=5) + xt = 8 if side == "left" else -110 + yt = -10 if side == "left" else 8 + ax.annotate(f"p50 = {v:.2g}s", + xy=(v, pct), xytext=(xt, yt), + textcoords="offset points", + fontsize=10, color=color) - # Reference vertical lines: scheduler W_turn (TTFT p90 from our window_1 runs) refs = [ ("lmetric TTFT p90 = 15.7s", 15.7, "#888"), ("unified TTFT p90 = 7.3s", 7.3, "#444"), ] for label, v, color in refs: - ax.axvline(v, color=color, ls=":", lw=1.2, alpha=0.85) - ax.text(v * 1.05, 8, label, fontsize=8.5, color=color, + ax.axvline(v, color=color, ls=":", lw=1.2, alpha=0.7) + ax.text(v * 1.05, 5, label, fontsize=8.5, color=color, rotation=90, va="bottom") ax.set_xscale("log") - ax.set_xlim(0.05, 2000) + ax.set_xlim(0.05, 5000) ax.set_ylim(0, 102) ax.set_xlabel( - "Inter-turn gap T_external (s, log scale) " - "— next_turn.ready − prev_turn.end" + "Inter-turn gap T_external (s, log scale) — next.ready − prev.end" ) ax.set_ylabel("Cumulative % of inter-turn intervals") + ap = a_d["stats_s"] + cp = c_d["stats_s"] + af = a_d["fraction_below"] + cf = c_d["fraction_below"] ax.set_title( - "Inter-turn external gap CDF — production agentic trace\n" - f"median T_external = {p['p50']:.2g}s; " - f"{int(d['fraction_below']['1.0s']*100)}% gaps < 1s, " - f"{int(d['fraction_below']['5.0s']*100)}% < 5s, " - f"{int(d['fraction_below']['30.0s']*100)}% < 30s" + f"Agentic vs chatbot inter-turn external gap — agentic has a " + f"sub-second tool-call mode chatbot lacks\n" + f"agentic p50={ap['p50']:.2g}s, frac<1s={af['1.0s']*100:.0f}%, " + f"frac<5s={af['5.0s']*100:.0f}% · " + f"chatbot p50={cp['p50']:.2g}s, frac<1s={cf['1.0s']*100:.0f}%, " + f"frac<5s={cf['5.0s']*100:.0f}%" ) ax.grid(True, which="both", alpha=0.3) ax.legend(loc="lower right", framealpha=0.92, fontsize=9)