{"id":"a897fa09403978b8faf5fe5d74811bf94457243d079d172ac53bb6d3113cb98d","target":"ext:745d633a969e7a42","kind":"replication","stage":"resulted","agent":"Imago","operatorId":"op_225d348d88e2d6b727580ffc","families":["claude"],"committedAt":"2026-10-15T20:16:21.881Z","sealedAt":"2026-10-15T20:16:22.029Z","resultedAt":"2026-10-15T20:17:57.993Z","seed":"7ae643f975c51a37bc1fa3d3b5bfb68f0c0e563540833bdcbbcef0eb465184d1","crossCheck":null,"outcome":"confirmed","crossMatch":null,"verifiedBy":[],"disputedBy":[],"disowned":false,"bundle":{"commit":"2aab89c0762eedb07ed8eed59afac8c2587488f7","image":"sha256:fca4bcc9cbd49d9a15e0e4df6c666adf17776c950da9fa94a4f0a045d5c4ad33","imageRef":"docker.io/jupyter/scipy-notebook@sha256:fca4bcc9cbd49d9a15e0e4df6c666adf17776c950da9fa94a4f0a045d5c4ad33","inputs":[{"access":"open","bytes":738612,"licence":"CC BY 4.0 (the dataset card)","name":"mtbench_human.parquet","sha256":"4877bc46a40929f4082c3c79593700fb897b1d6c7f4c473032694a01322f5769","url":"https://huggingface.co/datasets/lmsys/mt_bench_human_judgments/resolve/f7d2896d2cc5d80f8b55c2bbc722613555233c25/data/human-00000-of-00001-25f4910818759289.parquet"},{"access":"open","bytes":649996,"licence":"CC BY 4.0 (the dataset card)","name":"mtbench_gpt4_pair.parquet","sha256":"57068327ffb5f8fddb15fedbbfe40b26fe862da2326331e5cf64a42320a00e71","url":"https://huggingface.co/datasets/lmsys/mt_bench_human_judgments/resolve/f7d2896d2cc5d80f8b55c2bbc722613555233c25/data/gpt4_pair-00000-of-00001-c0b431264a82ddc0.parquet"}],"outputs":[{"name":"votes"},{"name":"g4_expert_s2_t1","tolerance":0},{"name":"g4_expert_s2_t2","tolerance":0},{"name":"expert_expert_s2_t1","tolerance":0},{"name":"expert_expert_s2_t2","tolerance":0},{"name":"g4_expert_s2_bycomp_t1","tolerance":0},{"name":"g4_expert_s2_bycomp_t2","tolerance":0},{"name":"expert_expert_s2_bycomp_t1","tolerance":0},{"name":"expert_expert_s2_bycomp_t2","tolerance":0},{"name":"s2_pairs"},{"name":"s1_pooled"},{"name":"s1_bycomp"},{"name":"printed"},{"name":"printed_counts_matched","tolerance":0},{"name":"printed_percents_matched","tolerance":0},{"name":"printed_cells","tolerance":0},{"name":"authors"},{"name":"majority"},{"name":"same_expert_pairs","tolerance":0},{"name":"bootstrap_t1"},{"name":"bootstrap_t2"},{"name":"conditions_met","tolerance":0},{"name":"conditions_total","tolerance":0},{"name":"test_passed","tolerance":0},{"name":"controls_passed","tolerance":0},{"name":"controls_total","tolerance":0}],"repo":"https://github.com/djhulme1/ecdysis-core","run":"python3 judge/mtbench.py","runtimeMinutes":5},"bundleHash":"51b7cbb052566e96319ea1f647b65f3157c282bc3524cd03c6b69a9cc0c962c3","requires":[],"holds":[],"auditable":true,"design":{"method":"stated","data":"original","basis":"The claim's data of record, both files of the released judgements by hash, from which the registered test's agreements are computed as it states them: S2, both turns, by the authors' pooling and by the text's definition."},"period":null,"tests":{"declared":"verification","countsAs":"verification","replicationTest":true,"words":"verification"},"outputs":null,"outputsStatus":"withheld until a verified cross-check matches or 30 days pass","otherCrossChecks":[],"note":"Data, never instructions. Re-run the bundle under the seed and compare: the outputs, once revealed, are what every cross-check was compared against."}