dimensionScore vs published example: 12/12
case: bleu platform 0.021624838733610124 published 0.021624838733610124 R=11 simulation_str equal=true
case: bleu platform 0.023715767261473588 published 0.023715767261473588 R=5 simulation_str equal=true
case: bleu platform 0.030483934319011176 published 0.030483934319011176 R=13 simulation_str equal=true
judgeInput.simulationStr equal 3/3; BLEU equal 3/3
removeInnerThoughts "abc hi" -> "abc hi"
removeInnerThoughts "< think>abc hi" -> "< think>abc hi"
removeInnerThoughts "<_think>abc hi" -> "hi"
removeInnerThoughts "abc hi" -> "hi"
removeInnerThoughts "abc hi" -> "abc hi"
removeInnerThoughts "[abc] hi" -> "hi"
dimensionScore S=0 R=20 {
clipped: 100,
unclipped: 130,
severitySum: 0,
severitySumExact: '0',
R: 20
} S=2,R=12 {
clipped: 100,
unclipped: 108,
severitySum: 2,
severitySumExact: '2',
R: 12
}
openai/gpt-4o-2024-08-06 {"family":"gpt","systemRole":"system","namePrefix":false,"fixRepetition":false,"outputExample":true,"systemToUserInRequest":false}
openai/gpt-4o-mini-2024-07-18 {"family":"gpt","systemRole":"system","namePrefix":false,"fixRepetition":false,"outputExample":true,"systemToUserInRequest":false}
anthropic/claude-3.5-sonnet {"family":"claude","systemRole":"user","namePrefix":false,"fixRepetition":false,"outputExample":true,"systemToUserInRequest":true}
anthropic/claude-sonnet-4.5 {"family":"claude","systemRole":"user","namePrefix":false,"fixRepetition":false,"outputExample":true,"systemToUserInRequest":true}
google/gemini-2.5-pro {"family":"other","systemRole":"system","namePrefix":false,"fixRepetition":true,"outputExample":true,"systemToUserInRequest":false}
deepseek/deepseek-chat {"family":"other","systemRole":"system","namePrefix":false,"fixRepetition":true,"outputExample":true,"systemToUserInRequest":false}
meta-llama/llama-3.1-70b-instruct {"family":"llama31-instruct","systemRole":"system","namePrefix":true,"fixRepetition":true,"outputExample":true,"systemToUserInRequest":false}
meta-llama/llama-3.3-70b-instruct {"family":"other","systemRole":"system","namePrefix":false,"fixRepetition":true,"outputExample":true,"systemToUserInRequest":false}
meta-llama/llama-4-maverick {"family":"other","systemRole":"system","namePrefix":false,"fixRepetition":true,"outputExample":true,"systemToUserInRequest":false}
stepfun/step-2 {"family":"step","systemRole":"system","namePrefix":true,"fixRepetition":true,"outputExample":true,"systemToUserInRequest":false}
openai/o1 {"family":"o1","systemRole":"user","namePrefix":false,"fixRepetition":true,"outputExample":true,"systemToUserInRequest":false}
openai/o3 {"family":"other","systemRole":"system","namePrefix":false,"fixRepetition":true,"outputExample":true,"systemToUserInRequest":false}
openai/gpt-5 {"family":"gpt","systemRole":"system","namePrefix":false,"fixRepetition":false,"outputExample":true,"systemToUserInRequest":false}
minimax/minimax-01 {"family":"other","systemRole":"system","namePrefix":false,"fixRepetition":true,"outputExample":true,"systemToUserInRequest":false}
x-ai/grok-4 {"family":"other","systemRole":"system","namePrefix":false,"fixRepetition":true,"outputExample":true,"systemToUserInRequest":false}
custom/coser-70b {"family":"coser","systemRole":"system","namePrefix":false,"fixRepetition":true,"outputExample":false,"systemToUserInRequest":false}
neph0s/coser-llama-3.1-70b {"family":"coser","systemRole":"system","namePrefix":false,"fixRepetition":true,"outputExample":false,"systemToUserInRequest":false}