[ { "test": "ifeval", "thinking": "on", "status": "success", "score": 0.9186, "main_metric": "final_acc", "metrics": { "prompt_strict_acc": 0.9, "prompt_strict_stderr": 0.0302, "prompt_loose_acc": 0.91, "prompt_loose_stderr": 0.0288, "inst_strict_acc": 0.929, "inst_strict_stderr": 0.0223, "inst_loose_acc": 0.9355, "inst_loose_stderr": 0.0215, "final_acc": 0.9186, "final_stderr": 0.0317 }, "samples": 100, "epochs": 1, "input_tokens": 9857, "output_tokens": 147729, "reasoning_tokens": 0, "seconds": 961.4, "error": null, "log_file": "inspect/thinking_on/2026-10-19T00-53-29-00-00_ifeval_nBYADGwpFEoaTm46TbuGwZ.eval", "hidden_prompt": { "requests": 100, "flagged": 0, "tokens_per_char": 0.3431, "max_excess_tokens": -391, "examples": [], "error": null } }, { "test": "mmlu_pro", "thinking": "on", "status": "success", "score": 0.9, "main_metric": "accuracy", "metrics": { "accuracy": 0.9, "stderr": 0.0429 }, "samples": 50, "epochs": 1, "input_tokens": 15784, "output_tokens": 192476, "reasoning_tokens": 0, "seconds": 1234.5, "error": null, "log_file": "inspect/thinking_on/2026-10-19T01-09-30-00-00_mmlu-pro_VtTgypNpFM4nSEE53J5wTc.eval", "hidden_prompt": { "requests": 50, "flagged": 0, "tokens_per_char": 0.2928, "max_excess_tokens": -561, "examples": [], "error": null } }, { "test": "livecodebench_pro", "thinking": "on", "status": "success", "score": 0.8333, "main_metric": "accuracy", "metrics": { "accuracy": 0.8333, "stderr": 0.0692 }, "samples": 30, "epochs": 1, "input_tokens": 24747, "output_tokens": 263456, "reasoning_tokens": 0, "seconds": 1949.7, "error": null, "log_file": "inspect/thinking_on/2026-10-19T01-30-16-00-00_livecodebench-pro_Cy6qCHoUxLy5FNFMCwYZvE.eval", "hidden_prompt": { "requests": 30, "flagged": 0, "tokens_per_char": 0.3166, "max_excess_tokens": -1489, "examples": [], "error": null } }, { "test": "bfcl", "thinking": "on", "status": "success", "score": 0.67, "main_metric": "accuracy", "metrics": { "accuracy": 0.67, "stderr": 0.0473, "multi_turn_long_context_acc": 0.6, "live_multiple_acc": 0.7619, "exec_parallel_multiple_acc": 1.0, "live_irrelevance_acc": 0.7647, "multi_turn_composite_acc": 0.0, "exec_parallel_acc": 0.75, "live_simple_acc": 0.8571, "simple_python_acc": 0.8182, "multiple_acc": 1.0, "parallel_multiple_acc": 1.0, "irrelevance_acc": 0.5, "sql_acc": 0.0, "multi_turn_miss_func_acc": 0.25, "multi_turn_base_acc": 1.0, "multi_turn_miss_param_acc": 0.0, "live_parallel_multiple_acc": 0.0, "simple_java_acc": 0.0, "all": 0.67, "multi_turn_long_context_sterr": 0.2449, "live_multiple_sterr": 0.0952, "exec_parallel_multiple_sterr": 0.0, "live_irrelevance_sterr": 0.106, "multi_turn_composite_sterr": 0.0, "exec_parallel_sterr": 0.25, "live_simple_sterr": 0.1429, "simple_python_sterr": 0.122, "multiple_sterr": 0.0, "parallel_multiple_sterr": 0.0, "irrelevance_sterr": 0.2887, "sql_sterr": 0.0, "multi_turn_miss_func_sterr": 0.1637, "multi_turn_base_sterr": 0.0, "multi_turn_miss_param_sterr": 0.0, "live_parallel_multiple_sterr": 0.0, "simple_java_sterr": 0.0, "all2": 0.0473 }, "samples": 100, "epochs": 1, "input_tokens": 313719, "output_tokens": 122010, "reasoning_tokens": 0, "seconds": 1225.2, "error": null, "log_file": "inspect/thinking_on/2026-10-19T02-02-35-00-00_bfcl_4Rc6mz8sM3udu8iym8tEJx.eval", "hidden_prompt": { "requests": 405, "flagged": 0, "tokens_per_char": 0.2393, "max_excess_tokens": -431, "examples": [], "error": null } }, { "test": "ifeval", "thinking": "off", "status": "success", "score": 0.8898, "main_metric": "final_acc", "metrics": { "prompt_strict_acc": 0.85, "prompt_strict_stderr": 0.0359, "prompt_loose_acc": 0.89, "prompt_loose_stderr": 0.0314, "inst_strict_acc": 0.8968, "inst_strict_stderr": 0.0243, "inst_loose_acc": 0.9226, "inst_loose_stderr": 0.0222, "final_acc": 0.8898, "final_stderr": 0.0372 }, "samples": 100, "epochs": 1, "input_tokens": 5857, "output_tokens": 27369, "reasoning_tokens": 0, "seconds": 257.3, "error": null, "log_file": "inspect/thinking_off/2026-10-19T02-23-00-00-00_ifeval_YWCaRmizzgt6ye894pNUG2.eval", "hidden_prompt": { "requests": 100, "flagged": 0, "tokens_per_char": 0.1919, "max_excess_tokens": -431, "examples": [], "error": null } }, { "test": "mmlu_pro", "thinking": "off", "status": "success", "score": 0.82, "main_metric": "accuracy", "metrics": { "accuracy": 0.82, "stderr": 0.0549 }, "samples": 50, "epochs": 1, "input_tokens": 13784, "output_tokens": 58589, "reasoning_tokens": 0, "seconds": 338.9, "error": null, "log_file": "inspect/thinking_off/2026-10-19T02-27-18-00-00_mmlu-pro_dXAPVDQsoopypwjxJY3REQ.eval", "hidden_prompt": { "requests": 50, "flagged": 0, "tokens_per_char": 0.2475, "max_excess_tokens": -601, "examples": [], "error": null } }, { "test": "livecodebench_pro", "thinking": "off", "status": "success", "score": 0.5, "main_metric": "accuracy", "metrics": { "accuracy": 0.5, "stderr": 0.0928 }, "samples": 30, "epochs": 1, "input_tokens": 23667, "output_tokens": 217232, "reasoning_tokens": 0, "seconds": 1517.0, "error": null, "log_file": "inspect/thinking_off/2026-10-19T02-33-08-00-00_livecodebench-pro_gZZ774nokMiLjZ8UVzDE5F.eval", "hidden_prompt": { "requests": 30, "flagged": 0, "tokens_per_char": 0.3009, "max_excess_tokens": -1525, "examples": [], "error": null } }, { "test": "bfcl", "thinking": "off", "status": "success", "score": 0.57, "main_metric": "accuracy", "metrics": { "accuracy": 0.57, "stderr": 0.0498, "multi_turn_long_context_acc": 0.8, "live_multiple_acc": 0.7143, "exec_parallel_multiple_acc": 0.5, "live_irrelevance_acc": 0.4706, "multi_turn_composite_acc": 0.0, "exec_parallel_acc": 0.25, "live_simple_acc": 0.8571, "simple_python_acc": 0.8182, "multiple_acc": 1.0, "parallel_multiple_acc": 0.6667, "irrelevance_acc": 0.25, "sql_acc": 0.0, "multi_turn_miss_func_acc": 0.375, "multi_turn_base_acc": 0.8, "multi_turn_miss_param_acc": 0.0, "live_parallel_multiple_acc": 0.0, "simple_java_acc": 0.0, "all": 0.57, "multi_turn_long_context_sterr": 0.2, "live_multiple_sterr": 0.101, "exec_parallel_multiple_sterr": 0.5, "live_irrelevance_sterr": 0.1248, "multi_turn_composite_sterr": 0.0, "exec_parallel_sterr": 0.25, "live_simple_sterr": 0.1429, "simple_python_sterr": 0.122, "multiple_sterr": 0.0, "parallel_multiple_sterr": 0.3333, "irrelevance_sterr": 0.25, "sql_sterr": 0.0, "multi_turn_miss_func_sterr": 0.183, "multi_turn_base_sterr": 0.2, "multi_turn_miss_param_sterr": 0.0, "live_parallel_multiple_sterr": 0.0, "simple_java_sterr": 0.0, "all2": 0.0498 }, "samples": 100, "epochs": 1, "input_tokens": 282762, "output_tokens": 32527, "reasoning_tokens": 0, "seconds": 720.3, "error": null, "log_file": "inspect/thinking_off/2026-10-19T02-58-15-00-00_bfcl_7jvSpcb9w3Cv2Q5urCkxcG.eval", "hidden_prompt": { "requests": 425, "flagged": 0, "tokens_per_char": 0.2473, "max_excess_tokens": -467, "examples": [], "error": null } } ]