{
 "run": "007c947738f44da5",
 "model": "openai/gpt-4o-2024-08-06",
 "protocol": "coser-gca-1.1 · 2026-11-01",
 "criteria": {
  "reproduced": {
   "avg": 1.0,
   "dim": 2.5
  },
  "same_range": {
   "avg": 3.0,
   "dim": 5
  }
 },
 "paper_main_table": {
  "sc": {
   "mean": 61.59,
   "pm": 0.66
  },
  "an": {
   "mean": 48.93,
   "pm": 0.48
  },
  "cf": {
   "mean": 48.95,
   "pm": 1.73
  },
  "sq": {
   "mean": 80.33,
   "pm": 0.59
  },
  "avg": {
   "mean": 59.95,
   "pm": 0.5
  },
  "bleu": {
   "mean": 5.9,
   "pm": 0.16
  }
 },
 "platform": {
  "sc": 62.26833333333333,
  "an": 42.05083333333334,
  "cf": 32.82833333333333,
  "sq": 81.5675,
  "avg": 54.678749999999994
 },
 "delta": {
  "sc": 0.6783333333333275,
  "an": -6.879166666666663,
  "cf": -16.12166666666667,
  "sq": 1.2374999999999972,
  "avg": -5.271250000000009
 },
 "verdict": "not reproduced",
 "platform_summary_rounded": {
  "overall": 54.68,
  "ci": {
   "overall": [
    53.52,
    55.8
   ],
   "groups": {
    "sc": [
     60.55,
     64.01
    ],
    "an": [
     39.84,
     44.21
    ],
    "cf": [
     30.29,
     35.43
    ],
    "sq": [
     80.39,
     82.68
    ]
   }
  },
  "dims": {
   "sc": 62.27,
   "an": 42.05,
   "cf": 32.83,
   "sq": 81.57,
   "bleu": 4.95
  }
 },
 "bleu": {
  "platform": 4.949248804120384,
  "paper": 5.9
 },
 "paper_splits": {
  "id": {
   "sc": 59.88,
   "an": 48.11,
   "cf": 47.1,
   "sq": 79.06,
   "avg": 58.54
  },
  "ood": {
   "sc": 62.29,
   "an": 49.48,
   "cf": 49.9,
   "sq": 80.18,
   "avg": 60.46
  }
 },
 "platform_splits": {
  "id": {
   "sessions": 300,
   "means": {
    "sc": 62.045,
    "an": 42.36666666666667,
    "cf": 30.513333333333332,
    "sq": 81.985,
    "bleu": 4.578574173357653,
    "avg": 54.22749999999999
   }
  },
  "ood": {
   "sessions": 300,
   "means": {
    "sc": 62.49166666666667,
    "an": 41.735,
    "cf": 35.14333333333333,
    "sq": 81.15,
    "bleu": 5.319923434883119,
    "avg": 55.13
   }
  }
 },
 "runs": [
  {
   "sessions": 200,
   "means": {
    "sc": 62.3075,
    "an": 42.9325,
    "cf": 35.14,
    "sq": 82.0325,
    "bleu": 4.862389677806833,
    "avg": 55.603125
   }
  },
  {
   "sessions": 200,
   "means": {
    "sc": 62.91,
    "an": 41.36,
    "cf": 31.53,
    "sq": 81.3325,
    "bleu": 4.968714970966928,
    "avg": 54.283125
   }
  },
  {
   "sessions": 200,
   "means": {
    "sc": 61.5875,
    "an": 41.86,
    "cf": 31.815,
    "sq": 81.3375,
    "bleu": 5.016641763587398,
    "avg": 54.15
   }
  }
 ],
 "across_runs": {
  "sc": {
   "mean": 62.26833333333334,
   "sd": 0.6621193875226219
  },
  "an": {
   "mean": 42.05083333333333,
   "sd": 0.803431442833383
  },
  "cf": {
   "mean": 32.82833333333333,
   "sd": 2.007027237815504
  },
  "sq": {
   "mean": 81.56750000000001,
   "sd": 0.402709572769259
  },
  "bleu": {
   "mean": 4.949248804120386,
   "sd": 0.07894697733724007
  },
  "avg": {
   "mean": 54.67875,
   "sd": 0.8032947291467809
  }
 },
 "with_ceiling_100": {
  "means": {
   "sc": 62.26833333333333,
   "an": 42.05083333333334,
   "cf": 32.79333333333334,
   "sq": 81.54833333333333,
   "avg": 54.66520833333334
  },
  "sessions_dims_above_100": 3
 },
 "floor": {
  "session_dims_at_0": {
   "sc": 1,
   "an": 75,
   "cf": 161,
   "sq": 1
  }
 },
 "R": {
  "mean": 12.036666666666667,
  "median": 13.0,
  "min": 3,
  "max": 20,
  "histogram": {
   "3": 7,
   "4": 9,
   "5": 13,
   "6": 29,
   "7": 27,
   "8": 35,
   "9": 47,
   "10": 50,
   "11": 36,
   "12": 34,
   "13": 34,
   "14": 90,
   "15": 79,
   "16": 59,
   "17": 32,
   "18": 12,
   "19": 1,
   "20": 6
  }
 },
 "lines": {
  "ai": {
   "lines": 7222,
   "empty": 0,
   "truncated": 0,
   "refused": 0,
   "retried": 0,
   "fallback": null,
   "ended": null
  },
  "env": {
   "lines": 2080,
   "empty": 0,
   "truncated": 0,
   "refused": 0,
   "retried": 0,
   "fallback": null,
   "ended": null
  },
  "nsp": {
   "lines": 9302,
   "empty": 0,
   "truncated": 0,
   "refused": 0,
   "retried": 0,
   "fallback": 342,
   "ended": 366
  }
 },
 "nsp": {
  "fallback_rate": 0.0367662868200387,
  "ended_rate_per_session": 0.61
 },
 "flaws_per_session": {
  "sc": 3.545,
  "an": 5.956666666666667,
  "cf": 7.248333333333333,
  "sq": 2.75
 },
 "severity_sum_per_session": {
  "sc": 11.158333333333333,
  "an": 16.218333333333334,
  "cf": 19.638333333333332,
  "sq": 7.305
 },
 "hosts": [
  {
   "provider": "OpenAI",
   "n": 7222
  }
 ],
 "usage": {
  "calls": 21004,
  "promptTokens": 31617116,
  "completionTokens": 1529578,
  "cost": 60.0120066
 },
 "eligibility": {
  "eligible": false,
  "fullProtocol": true,
  "complete": true,
  "currentVersion": true,
  "pinned": true,
  "singleHost": true,
  "reason": "CoSER 协议的锚定实验（矩阵 V09）尚未判定，平台评测暂时只生成报告，不计入排行榜"
 },
 "flaws_ge_10_share": {
  "sc": 0.0,
  "an": 0.12666666666666668,
  "cf": 0.25333333333333335,
  "sq": 0.005
 },
 "by_length": [
  {
   "R": [
    3,
    6
   ],
   "sessions": 58,
   "sc": 56.189655172413794,
   "an": 47.13793103448276,
   "cf": 47.327586206896555,
   "sq": 75.06896551724138,
   "avg": 56.43103448275862
  },
  {
   "R": [
    7,
    9
   ],
   "sessions": 109,
   "sc": 56.11009174311926,
   "an": 41.330275229357795,
   "cf": 37.50917431192661,
   "sq": 79.11009174311927,
   "avg": 53.51490825688073
  },
  {
   "R": [
    10,
    12
   ],
   "sessions": 120,
   "sc": 59.09166666666667,
   "an": 42.19583333333333,
   "cf": 32.55833333333333,
   "sq": 80.96666666666667,
   "avg": 53.703125
  },
  {
   "R": [
    13,
    15
   ],
   "sessions": 203,
   "sc": 67.31773399014779,
   "an": 42.00738916256157,
   "cf": 29.839901477832512,
   "sq": 83.4384236453202,
   "avg": 55.650862068965516
  },
  {
   "R": [
    16,
    20
   ],
   "sessions": 110,
   "sc": 65.72272727272727,
   "an": 40.00454545454546,
   "cf": 26.354545454545455,
   "sq": 84.63181818181818,
   "avg": 54.178409090909085
  }
 ]
}