{
  "comparisons": [
    {
      "batch_spec": "q1ks8k",
      "first_mean_s": 0.0011425888061523438,
      "pass": true,
      "relative_difference": 0.0011153009347107706,
      "second_mean_s": 0.001141315186023712,
      "tp": 1
    },
    {
      "batch_spec": "q2ks66k",
      "first_mean_s": 0.01678766403198242,
      "pass": true,
      "relative_difference": 0.0007022802901221452,
      "second_mean_s": 0.016775878524780276,
      "tp": 1
    },
    {
      "batch_spec": "q4ks100k",
      "first_mean_s": 0.05035054740905761,
      "pass": true,
      "relative_difference": 0.000388526368934946,
      "second_mean_s": 0.0503309886932373,
      "tp": 1
    },
    {
      "batch_spec": "q512s128k",
      "first_mean_s": 0.009489968109130859,
      "pass": true,
      "relative_difference": 0.005183356056904404,
      "second_mean_s": 0.009440905380249024,
      "tp": 1
    },
    {
      "batch_spec": "q512s4k",
      "first_mean_s": 0.0003366623997688293,
      "pass": true,
      "relative_difference": 0.0014267930704769689,
      "second_mean_s": 0.0003361823946237564,
      "tp": 1
    },
    {
      "batch_spec": "q6ks134k",
      "first_mean_s": 0.0961898666381836,
      "pass": true,
      "relative_difference": 0.0005591912667149224,
      "second_mean_s": 0.09613609313964844,
      "tp": 1
    },
    {
      "batch_spec": "q8ks112k",
      "first_mean_s": 0.1069560287475586,
      "pass": true,
      "relative_difference": 0.00031876960426212737,
      "second_mean_s": 0.10692193984985351,
      "tp": 1
    },
    {
      "batch_spec": "q8ks128k",
      "first_mean_s": 0.12283342437744141,
      "pass": true,
      "relative_difference": 0.0004249343895711351,
      "second_mean_s": 0.12278123931884766,
      "tp": 1
    },
    {
      "batch_spec": "q8ks136k",
      "first_mean_s": 0.13079229431152345,
      "pass": true,
      "relative_difference": 0.0005674643646600315,
      "second_mean_s": 0.1307180953979492,
      "tp": 1
    },
    {
      "batch_spec": "q8ks48k",
      "first_mean_s": 0.04368390693664551,
      "pass": true,
      "relative_difference": 0.0004927566415802605,
      "second_mean_s": 0.043662386703491214,
      "tp": 1
    },
    {
      "batch_spec": "q8ks64k",
      "first_mean_s": 0.0595021312713623,
      "pass": true,
      "relative_difference": 0.0004229478941120101,
      "second_mean_s": 0.0594769702911377,
      "tp": 1
    },
    {
      "batch_spec": "q8ks80k",
      "first_mean_s": 0.07538758392333984,
      "pass": true,
      "relative_difference": 0.0012547739275033378,
      "second_mean_s": 0.07529304885864258,
      "tp": 1
    },
    {
      "batch_spec": "q8ks96k",
      "first_mean_s": 0.09124225158691406,
      "pass": true,
      "relative_difference": 0.0015351127502189047,
      "second_mean_s": 0.09110229187011719,
      "tp": 1
    },
    {
      "batch_spec": "q1ks8k",
      "first_mean_s": 0.0006064576089382171,
      "pass": true,
      "relative_difference": 0.04648308449369336,
      "second_mean_s": 0.0006353183984756471,
      "tp": 2
    },
    {
      "batch_spec": "q2ks66k",
      "first_mean_s": 0.00959802885055542,
      "pass": true,
      "relative_difference": 0.003767189056248553,
      "second_mean_s": 0.009561939239501951,
      "tp": 2
    },
    {
      "batch_spec": "q4ks100k",
      "first_mean_s": 0.025250255966186526,
      "pass": true,
      "relative_difference": 0.001402319427652538,
      "second_mean_s": 0.0252856897354126,
      "tp": 2
    },
    {
      "batch_spec": "q512s128k",
      "first_mean_s": 0.004790303993225097,
      "pass": true,
      "relative_difference": 0.006859924072500961,
      "second_mean_s": 0.0047575551986694335,
      "tp": 2
    },
    {
      "batch_spec": "q512s4k",
      "first_mean_s": 0.00020788159817457198,
      "pass": true,
      "relative_difference": 0.008626540222728182,
      "second_mean_s": 0.00020609600096940998,
      "tp": 2
    },
    {
      "batch_spec": "q6ks134k",
      "first_mean_s": 0.048172192001342776,
      "pass": true,
      "relative_difference": 0.0005976896073522018,
      "second_mean_s": 0.048143408584594725,
      "tp": 2
    },
    {
      "batch_spec": "q8ks112k",
      "first_mean_s": 0.05542207336425782,
      "pass": true,
      "relative_difference": 0.0006067719206058191,
      "second_mean_s": 0.055455712127685554,
      "tp": 2
    },
    {
      "batch_spec": "q8ks128k",
      "first_mean_s": 0.06362084197998047,
      "pass": true,
      "relative_difference": 6.421681207537411e-06,
      "second_mean_s": 0.06362125053405762,
      "tp": 2
    },
    {
      "batch_spec": "q8ks136k",
      "first_mean_s": 0.06770619888305665,
      "pass": true,
      "relative_difference": 0.00015497334523802088,
      "second_mean_s": 0.06771669235229492,
      "tp": 2
    },
    {
      "batch_spec": "q8ks48k",
      "first_mean_s": 0.02263068161010742,
      "pass": true,
      "relative_difference": 0.003557316307676039,
      "second_mean_s": 0.022550320053100585,
      "tp": 2
    },
    {
      "batch_spec": "q8ks64k",
      "first_mean_s": 0.03074285774230957,
      "pass": true,
      "relative_difference": 0.0007706865344215391,
      "second_mean_s": 0.030766559982299803,
      "tp": 2
    },
    {
      "batch_spec": "q8ks80k",
      "first_mean_s": 0.038945382690429686,
      "pass": true,
      "relative_difference": 0.001695543775275234,
      "second_mean_s": 0.03901147232055664,
      "tp": 2
    },
    {
      "batch_spec": "q8ks96k",
      "first_mean_s": 0.04714819869995117,
      "pass": true,
      "relative_difference": 0.0014946162280953233,
      "second_mean_s": 0.0472187198638916,
      "tp": 2
    },
    {
      "batch_spec": "q1ks8k",
      "first_mean_s": 0.0003469599992036819,
      "pass": true,
      "relative_difference": 0.008642208702722183,
      "second_mean_s": 0.00034397439956665037,
      "tp": 4
    },
    {
      "batch_spec": "q2ks66k",
      "first_mean_s": 0.004817302370071412,
      "pass": true,
      "relative_difference": 0.020290162871872575,
      "second_mean_s": 0.004916048002243043,
      "tp": 4
    },
    {
      "batch_spec": "q4ks100k",
      "first_mean_s": 0.014412275123596191,
      "pass": true,
      "relative_difference": 0.007127958343441791,
      "second_mean_s": 0.01451537265777588,
      "tp": 4
    },
    {
      "batch_spec": "q512s128k",
      "first_mean_s": 0.003182867193222046,
      "pass": true,
      "relative_difference": 0.018068901625077752,
      "second_mean_s": 0.003240902423858643,
      "tp": 4
    },
    {
      "batch_spec": "q512s4k",
      "first_mean_s": 0.00015103680044412614,
      "pass": true,
      "relative_difference": 0.011099630528025208,
      "second_mean_s": 0.0001493696004152298,
      "tp": 4
    },
    {
      "batch_spec": "q6ks134k",
      "first_mean_s": 0.02411243553161621,
      "pass": true,
      "relative_difference": 0.007733466059061839,
      "second_mean_s": 0.024299632072448733,
      "tp": 4
    },
    {
      "batch_spec": "q8ks112k",
      "first_mean_s": 0.027747158622741696,
      "pass": true,
      "relative_difference": 0.009591652827304903,
      "second_mean_s": 0.028014582252502435,
      "tp": 4
    },
    {
      "batch_spec": "q8ks128k",
      "first_mean_s": 0.03186994876861572,
      "pass": true,
      "relative_difference": 0.008816561960069082,
      "second_mean_s": 0.03215217628479004,
      "tp": 4
    },
    {
      "batch_spec": "q8ks136k",
      "first_mean_s": 0.033897081375122075,
      "pass": true,
      "relative_difference": 0.009532159165047292,
      "second_mean_s": 0.03422174110412597,
      "tp": 4
    },
    {
      "batch_spec": "q8ks48k",
      "first_mean_s": 0.011354345512390137,
      "pass": true,
      "relative_difference": 0.010122318271387163,
      "second_mean_s": 0.011469862461090087,
      "tp": 4
    },
    {
      "batch_spec": "q8ks64k",
      "first_mean_s": 0.015451993656158448,
      "pass": true,
      "relative_difference": 0.00978342795663941,
      "second_mean_s": 0.015603910255432129,
      "tp": 4
    },
    {
      "batch_spec": "q8ks80k",
      "first_mean_s": 0.01954985942840576,
      "pass": true,
      "relative_difference": 0.012041804397861736,
      "second_mean_s": 0.019786701011657717,
      "tp": 4
    },
    {
      "batch_spec": "q8ks96k",
      "first_mean_s": 0.02367282257080078,
      "pass": true,
      "relative_difference": 0.008957957353947301,
      "second_mean_s": 0.023885836791992184,
      "tp": 4
    }
  ],
  "maximum_relative_difference": 0.04648308449369336,
  "schema": "frontier-code-longctx-repeat-check-v1",
  "status": "PASS",
  "threshold": 0.05
}