{"columns":[{"name":"eval_id","type":"string"},{"name":"run_id","type":"string"},{"name":"task_id","type":"string"},{"name":"log","type":"string"},{"name":"created","type":"timestamp[ns, tz=UTC]"},{"name":"tags","type":"string"},{"name":"git_origin","type":"string"},{"name":"git_commit","type":"string"},{"name":"packages","type":"string"},{"name":"metadata","type":"string"},{"name":"task_name","type":"string"},{"name":"task_version","type":"int64"},{"name":"task_file","type":"string"},{"name":"task_attribs","type":"string"},{"name":"task_arg_allow_internet","type":"bool"},{"name":"task_arg_build_docker_images","type":"bool"},{"name":"task_arg_cot","type":"bool"},{"name":"task_arg_dataset","type":"string"},{"name":"task_arg_docker_image_from_id","type":"string"},{"name":"task_arg_epochs","type":"double"},{"name":"task_arg_eval_names","type":"double"},{"name":"task_arg_fewshot","type":"double"},{"name":"task_arg_input_prompt","type":"string"},{"name":"task_arg_instance_ids","type":"double"},{"name":"task_arg_pull_remote_images_if_available","type":"bool"},{"name":"task_arg_sandbox_type","type":"string"},{"name":"task_arg_scorer","type":"string"},{"name":"task_arg_solver","type":"string"},{"name":"task_arg_split","type":"string"},{"name":"task_arg_subjects","type":"string"},{"name":"task_arg_variant_names","type":"string"},{"name":"solver","type":"string"},{"name":"solver_args","type":"string"},{"name":"sandbox_type","type":"string"},{"name":"sandbox_config","type":"string"},{"name":"model","type":"string"},{"name":"model_base_url","type":"string"},{"name":"model_args","type":"string"},{"name":"model_generate_config","type":"string"},{"name":"model_roles","type":"string"},{"name":"dataset_name","type":"string"},{"name":"dataset_location","type":"string"},{"name":"dataset_samples","type":"int64"},{"name":"dataset_sample_ids","type":"string"},{"name":"dataset_shuffled","type":"bool"},{"name":"epochs","type":"int64"},{"name":"epochs_reducer","type":"string"},{"name":"approval","type":"string"},{"name":"message_limit","type":"double"},{"name":"token_limit","type":"double"},{"name":"time_limit","type":"double"},{"name":"working_limit","type":"string"},{"name":"status","type":"string"},{"name":"error_message","type":"string"},{"name":"error_traceback","type":"string"},{"name":"total_samples","type":"int64"},{"name":"completed_samples","type":"int64"},{"name":"score_headline_name","type":"string"},{"name":"score_headline_metric","type":"string"},{"name":"score_headline_value","type":"double"},{"name":"score_headline_stderr","type":"double"},{"name":"score_choice_accuracy","type":"double"},{"name":"score_choice_stderr","type":"double"},{"name":"score_includes_accuracy","type":"double"},{"name":"score_includes_stderr","type":"double"},{"name":"score_match_accuracy","type":"double"},{"name":"score_match_stderr","type":"double"},{"name":"score_swe_bench_scorer_stderr_mean","type":"double"},{"name":"score_swe_bench_scorer_stderr_stderr","type":"double"}],"n_rows":20,"rows_kept":20,"cut":false,"text_format":"parquet-jsonl-2026-10-02","bounds":{"document_bytes":16777216,"package_text_limit_bytes":33554432,"package_text_slack_bytes":1024,"rows_first":2000,"rows_last":500,"rows_between":2000,"halvings":0,"cell_chars":16384,"cell_head_chars":4096,"vector_min":64,"vector_head_fixed":16,"vector_head_variable":64,"kv_value_chars":1048576,"nesting_max":64,"binary_text":true},"cells_cut":4,"cells_bad_utf8":0,"cells_raw":0,"cells_binary":0,"cells_binary_text":0,"cells_vector":0,"n_row_groups":1,"reader":"pyarrow 23.0.1, python 3.13","created_by":"parquet-cpp-arrow version 20.0.0","schema":{"fields":[{"name":"eval_id","type":"string","nullable":true,"metadata":null,"children":null},{"name":"run_id","type":"string","nullable":true,"metadata":null,"children":null},{"name":"task_id","type":"string","nullable":true,"metadata":null,"children":null},{"name":"log","type":"string","nullable":true,"metadata":null,"children":null},{"name":"created","type":"timestamp[ns, tz=UTC]","nullable":true,"metadata":null,"children":null},{"name":"tags","type":"string","nullable":true,"metadata":null,"children":null},{"name":"git_origin","type":"string","nullable":true,"metadata":null,"children":null},{"name":"git_commit","type":"string","nullable":true,"metadata":null,"children":null},{"name":"packages","type":"string","nullable":true,"metadata":null,"children":null},{"name":"metadata","type":"string","nullable":true,"metadata":null,"children":null},{"name":"task_name","type":"string","nullable":true,"metadata":null,"children":null},{"name":"task_version","type":"int64","nullable":true,"metadata":null,"children":null},{"name":"task_file","type":"string","nullable":true,"metadata":null,"children":null},{"name":"task_attribs","type":"string","nullable":true,"metadata":null,"children":null},{"name":"task_arg_allow_internet","type":"bool","nullable":true,"metadata":null,"children":null},{"name":"task_arg_build_docker_images","type":"bool","nullable":true,"metadata":null,"children":null},{"name":"task_arg_cot","type":"bool","nullable":true,"metadata":null,"children":null},{"name":"task_arg_dataset","type":"string","nullable":true,"metadata":null,"children":null},{"name":"task_arg_docker_image_from_id","type":"string","nullable":true,"metadata":null,"children":null},{"name":"task_arg_epochs","type":"double","nullable":true,"metadata":null,"children":null},{"name":"task_arg_eval_names","type":"double","nullable":true,"metadata":null,"children":null},{"name":"task_arg_fewshot","type":"double","nullable":true,"metadata":null,"children":null},{"name":"task_arg_input_prompt","type":"string","nullable":true,"metadata":null,"children":null},{"name":"task_arg_instance_ids","type":"double","nullable":true,"metadata":null,"children":null},{"name":"task_arg_pull_remote_images_if_available","type":"bool","nullable":true,"metadata":null,"children":null},{"name":"task_arg_sandbox_type","type":"string","nullable":true,"metadata":null,"children":null},{"name":"task_arg_scorer","type":"string","nullable":true,"metadata":null,"children":null},{"name":"task_arg_solver","type":"string","nullable":true,"metadata":null,"children":null},{"name":"task_arg_split","type":"string","nullable":true,"metadata":null,"children":null},{"name":"task_arg_subjects","type":"string","nullable":true,"metadata":null,"children":null},{"name":"task_arg_variant_names","type":"string","nullable":true,"metadata":null,"children":null},{"name":"solver","type":"string","nullable":true,"metadata":null,"children":null},{"name":"solver_args","type":"string","nullable":true,"metadata":null,"children":null},{"name":"sandbox_type","type":"string","nullable":true,"metadata":null,"children":null},{"name":"sandbox_config","type":"string","nullable":true,"metadata":null,"children":null},{"name":"model","type":"string","nullable":true,"metadata":null,"children":null},{"name":"model_base_url","type":"string","nullable":true,"metadata":null,"children":null},{"name":"model_args","type":"string","nullable":true,"metadata":null,"children":null},{"name":"model_generate_config","type":"string","nullable":true,"metadata":null,"children":null},{"name":"model_roles","type":"string","nullable":true,"metadata":null,"children":null},{"name":"dataset_name","type":"string","nullable":true,"metadata":null,"children":null},{"name":"dataset_location","type":"string","nullable":true,"metadata":null,"children":null},{"name":"dataset_samples","type":"int64","nullable":true,"metadata":null,"children":null},{"name":"dataset_sample_ids","type":"string","nullable":true,"metadata":null,"children":null},{"name":"dataset_shuffled","type":"bool","nullable":true,"metadata":null,"children":null},{"name":"epochs","type":"int64","nullable":true,"metadata":null,"children":null},{"name":"epochs_reducer","type":"string","nullable":true,"metadata":null,"children":null},{"name":"approval","type":"string","nullable":true,"metadata":null,"children":null},{"name":"message_limit","type":"double","nullable":true,"metadata":null,"children":null},{"name":"token_limit","type":"double","nullable":true,"metadata":null,"children":null},{"name":"time_limit","type":"double","nullable":true,"metadata":null,"children":null},{"name":"working_limit","type":"string","nullable":true,"metadata":null,"children":null},{"name":"status","type":"string","nullable":true,"metadata":null,"children":null},{"name":"error_message","type":"string","nullable":true,"metadata":null,"children":null},{"name":"error_traceback","type":"string","nullable":true,"metadata":null,"children":null},{"name":"total_samples","type":"int64","nullable":true,"metadata":null,"children":null},{"name":"completed_samples","type":"int64","nullable":true,"metadata":null,"children":null},{"name":"score_headline_name","type":"string","nullable":true,"metadata":null,"children":null},{"name":"score_headline_metric","type":"string","nullable":true,"metadata":null,"children":null},{"name":"score_headline_value","type":"double","nullable":true,"metadata":null,"children":null},{"name":"score_headline_stderr","type":"double","nullable":true,"metadata":null,"children":null},{"name":"score_choice_accuracy","type":"double","nullable":true,"metadata":null,"children":null},{"name":"score_choice_stderr","type":"double","nullable":true,"metadata":null,"children":null},{"name":"score_includes_accuracy","type":"double","nullable":true,"metadata":null,"children":null},{"name":"score_includes_stderr","type":"double","nullable":true,"metadata":null,"children":null},{"name":"score_match_accuracy","type":"double","nullable":true,"metadata":null,"children":null},{"name":"score_match_stderr","type":"double","nullable":true,"metadata":null,"children":null},{"name":"score_swe_bench_scorer_stderr_mean","type":"double","nullable":true,"metadata":null,"children":null},{"name":"score_swe_bench_scorer_stderr_stderr","type":"double","nullable":true,"metadata":null,"children":null}],"parquet":"required group field_id=-1 schema {\n  optional binary field_id=-1 eval_id (String);\n  optional binary field_id=-1 run_id (String);\n  optional binary field_id=-1 task_id (String);\n  optional binary field_id=-1 log (String);\n  optional int64 field_id=-1 created (Timestamp(isAdjustedToUTC=true, timeUnit=nanoseconds, is_from_converted_type=false, force_set_converted_type=false));\n  optional binary field_id=-1 tags (String);\n  optional binary field_id=-1 git_origin (String);\n  optional binary field_id=-1 git_commit (String);\n  optional binary field_id=-1 packages (String);\n  optional binary field_id=-1 metadata (String);\n  optional binary field_id=-1 task_name (String);\n  optional int64 field_id=-1 task_version;\n  optional binary field_id=-1 task_file (String);\n  optional binary field_id=-1 task_attribs (String);\n  optional boolean field_id=-1 task_arg_allow_internet;\n  optional boolean field_id=-1 task_arg_build_docker_images;\n  optional boolean field_id=-1 task_arg_cot;\n  optional binary field_id=-1 task_arg_dataset (String);\n  optional binary field_id=-1 task_arg_docker_image_from_id (String);\n  optional double field_id=-1 task_arg_epochs;\n  optional double field_id=-1 task_arg_eval_names;\n  optional double field_id=-1 task_arg_fewshot;\n  optional binary field_id=-1 task_arg_input_prompt (String);\n  optional double field_id=-1 task_arg_instance_ids;\n  optional boolean field_id=-1 task_arg_pull_remote_images_if_available;\n  optional binary field_id=-1 task_arg_sandbox_type (String);\n  optional binary field_id=-1 task_arg_scorer (String);\n  optional binary field_id=-1 task_arg_solver (String);\n  optional binary field_id=-1 task_arg_split (String);\n  optional binary field_id=-1 task_arg_subjects (String);\n  optional binary field_id=-1 task_arg_variant_names (String);\n  optional binary field_id=-1 solver (String);\n  optional binary field_id=-1 solver_args (String);\n  optional binary field_id=-1 sandbox_type (String);\n  optional binary field_id=-1 sandbox_config (String);\n  optional binary field_id=-1 model (String);\n  optional binary field_id=-1 model_base_url (String);\n  optional binary field_id=-1 model_args (String);\n  optional binary field_id=-1 model_generate_config (String);\n  optional binary field_id=-1 model_roles (String);\n  optional binary field_id=-1 dataset_name (String);\n  optional binary field_id=-1 dataset_location (String);\n  optional int64 field_id=-1 dataset_samples;\n  optional binary field_id=-1 dataset_sample_ids (String);\n  optional boolean field_id=-1 dataset_shuffled;\n  optional int64 field_id=-1 epochs;\n  optional binary field_id=-1 epochs_reducer (String);\n  optional binary field_id=-1 approval (String);\n  optional double field_id=-1 message_limit;\n  optional double field_id=-1 token_limit;\n  optional double field_id=-1 time_limit;\n  optional binary field_id=-1 working_limit (String);\n  optional binary field_id=-1 status (String);\n  optional binary field_id=-1 error_message (String);\n  optional binary field_id=-1 error_traceback (String);\n  optional int64 field_id=-1 total_samples;\n  optional int64 field_id=-1 completed_samples;\n  optional binary field_id=-1 score_headline_name (String);\n  optional binary field_id=-1 score_headline_metric (String);\n  optional double field_id=-1 score_headline_value;\n  optional double field_id=-1 score_headline_stderr;\n  optional double field_id=-1 score_choice_accuracy;\n  optional double field_id=-1 score_choice_stderr;\n  optional double field_id=-1 score_includes_accuracy;\n  optional double field_id=-1 score_includes_stderr;\n  optional double field_id=-1 score_match_accuracy;\n  optional double field_id=-1 score_match_stderr;\n  optional double field_id=-1 score_swe_bench_scorer_stderr_mean;\n  optional double field_id=-1 score_swe_bench_scorer_stderr_stderr;\n}\n"},"kv_metadata":[["ARROW:schema",{"$cut":{"chars":18552,"head":""}}],["pandas","{\"index_columns\": [{\"kind\": \"range\", \"name\": null, \"start\": 0, \"stop\": 20, \"step\": 1}], \"column_indexes\": [{\"name\": null, \"field_name\": null, \"pandas_type\": \"unicode\", \"numpy_type\": \"object\", \"metadata\": {\"encoding\": \"UTF-8\"}}], \"columns\": [{\"name\": \"eval_id\", \"field_name\": \"eval_id\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"run_id\", \"field_name\": \"run_id\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"task_id\", \"field_name\": \"task_id\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"log\", \"field_name\": \"log\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"created\", \"field_name\": \"created\", \"pandas_type\": \"datetimetz\", \"numpy_type\": \"timestamp[ns, tz=UTC][pyarrow]\", \"metadata\": null}, {\"name\": \"tags\", \"field_name\": \"tags\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"git_origin\", \"field_name\": \"git_origin\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"git_commit\", \"field_name\": \"git_commit\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"packages\", \"field_name\": \"packages\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"metadata\", \"field_name\": \"metadata\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"task_name\", \"field_name\": \"task_name\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"task_version\", \"field_name\": \"task_version\", \"pandas_type\": \"int64\", \"numpy_type\": \"int64[pyarrow]\", \"metadata\": null}, {\"name\": \"task_file\", \"field_name\": \"task_file\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"task_attribs\", \"field_name\": \"task_attribs\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"task_arg_allow_internet\", \"field_name\": \"task_arg_allow_internet\", \"pandas_type\": \"bool\", \"numpy_type\": \"bool[pyarrow]\", \"metadata\": null}, {\"name\": \"task_arg_build_docker_images\", \"field_name\": \"task_arg_build_docker_images\", \"pandas_type\": \"bool\", \"numpy_type\": \"bool[pyarrow]\", \"metadata\": null}, {\"name\": \"task_arg_cot\", \"field_name\": \"task_arg_cot\", \"pandas_type\": \"bool\", \"numpy_type\": \"bool[pyarrow]\", \"metadata\": null}, {\"name\": \"task_arg_dataset\", \"field_name\": \"task_arg_dataset\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"task_arg_docker_image_from_id\", \"field_name\": \"task_arg_docker_image_from_id\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"task_arg_epochs\", \"field_name\": \"task_arg_epochs\", \"pandas_type\": \"float64\", \"numpy_type\": \"double[pyarrow]\", \"metadata\": null}, {\"name\": \"task_arg_eval_names\", \"field_name\": \"task_arg_eval_names\", \"pandas_type\": \"float64\", \"numpy_type\": \"double[pyarrow]\", \"metadata\": null}, {\"name\": \"task_arg_fewshot\", \"field_name\": \"task_arg_fewshot\", \"pandas_type\": \"float64\", \"numpy_type\": \"double[pyarrow]\", \"metadata\": null}, {\"name\": \"task_arg_input_prompt\", \"field_name\": \"task_arg_input_prompt\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"task_arg_instance_ids\", \"field_name\": \"task_arg_instance_ids\", \"pandas_type\": \"float64\", \"numpy_type\": \"double[pyarrow]\", \"metadata\": null}, {\"name\": \"task_arg_pull_remote_images_if_available\", \"field_name\": \"task_arg_pull_remote_images_if_available\", \"pandas_type\": \"bool\", \"numpy_type\": \"bool[pyarrow]\", \"metadata\": null}, {\"name\": \"task_arg_sandbox_type\", \"field_name\": \"task_arg_sandbox_type\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"task_arg_scorer\", \"field_name\": \"task_arg_scorer\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"task_arg_solver\", \"field_name\": \"task_arg_solver\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"task_arg_split\", \"field_name\": \"task_arg_split\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"task_arg_subjects\", \"field_name\": \"task_arg_subjects\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"task_arg_variant_names\", \"field_name\": \"task_arg_variant_names\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"solver\", \"field_name\": \"solver\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"solver_args\", \"field_name\": \"solver_args\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"sandbox_type\", \"field_name\": \"sandbox_type\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"sandbox_config\", \"field_name\": \"sandbox_config\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"model\", \"field_name\": \"model\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"model_base_url\", \"field_name\": \"model_base_url\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"model_args\", \"field_name\": \"model_args\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"model_generate_config\", \"field_name\": \"model_generate_config\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"model_roles\", \"field_name\": \"model_roles\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"dataset_name\", \"field_name\": \"dataset_name\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"dataset_location\", \"field_name\": \"dataset_location\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"dataset_samples\", \"field_name\": \"dataset_samples\", \"pandas_type\": \"int64\", \"numpy_type\": \"int64[pyarrow]\", \"metadata\": null}, {\"name\": \"dataset_sample_ids\", \"field_name\": \"dataset_sample_ids\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"dataset_shuffled\", \"field_name\": \"dataset_shuffled\", \"pandas_type\": \"bool\", \"numpy_type\": \"bool[pyarrow]\", \"metadata\": null}, {\"name\": \"epochs\", \"field_name\": \"epochs\", \"pandas_type\": \"int64\", \"numpy_type\": \"int64[pyarrow]\", \"metadata\": null}, {\"name\": \"epochs_reducer\", \"field_name\": \"epochs_reducer\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"approval\", \"field_name\": \"approval\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"message_limit\", \"field_name\": \"message_limit\", \"pandas_type\": \"float64\", \"numpy_type\": \"double[pyarrow]\", \"metadata\": null}, {\"name\": \"token_limit\", \"field_name\": \"token_limit\", \"pandas_type\": \"float64\", \"numpy_type\": \"double[pyarrow]\", \"metadata\": null}, {\"name\": \"time_limit\", \"field_name\": \"time_limit\", \"pandas_type\": \"float64\", \"numpy_type\": \"double[pyarrow]\", \"metadata\": null}, {\"name\": \"working_limit\", \"field_name\": \"working_limit\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"status\", \"field_name\": \"status\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"error_message\", \"field_name\": \"error_message\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"error_traceback\", \"field_name\": \"error_traceback\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"total_samples\", \"field_name\": \"total_samples\", \"pandas_type\": \"int64\", \"numpy_type\": \"int64[pyarrow]\", \"metadata\": null}, {\"name\": \"completed_samples\", \"field_name\": \"completed_samples\", \"pandas_type\": \"int64\", \"numpy_type\": \"int64[pyarrow]\", \"metadata\": null}, {\"name\": \"score_headline_name\", \"field_name\": \"score_headline_name\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"score_headline_metric\", \"field_name\": \"score_headline_metric\", \"pandas_type\": \"unicode\", \"numpy_type\": \"string[pyarrow]\", \"metadata\": null}, {\"name\": \"score_headline_value\", \"field_name\": \"score_headline_value\", \"pandas_type\": \"float64\", \"numpy_type\": \"double[pyarrow]\", \"metadata\": null}, {\"name\": \"score_headline_stderr\", \"field_name\": \"score_headline_stderr\", \"pandas_type\": \"float64\", \"numpy_type\": \"double[pyarrow]\", \"metadata\": null}, {\"name\": \"score_choice_accuracy\", \"field_name\": \"score_choice_accuracy\", \"pandas_type\": \"float64\", \"numpy_type\": \"double[pyarrow]\", \"metadata\": null}, {\"name\": \"score_choice_stderr\", \"field_name\": \"score_choice_stderr\", \"pandas_type\": \"float64\", \"numpy_type\": \"double[pyarrow]\", \"metadata\": null}, {\"name\": \"score_includes_accuracy\", \"field_name\": \"score_includes_accuracy\", \"pandas_type\": \"float64\", \"numpy_type\": \"double[pyarrow]\", \"metadata\": null}, {\"name\": \"score_includes_stderr\", \"field_name\": \"score_includes_stderr\", \"pandas_type\": \"float64\", \"numpy_type\": \"double[pyarrow]\", \"metadata\": null}, {\"name\": \"score_match_accuracy\", \"field_name\": \"score_match_accuracy\", \"pandas_type\": \"float64\", \"numpy_type\": \"double[pyarrow]\", \"metadata\": null}, {\"name\": \"score_match_stderr\", \"field_name\": \"score_match_stderr\", \"pandas_type\": \"float64\", \"numpy_type\": \"double[pyarrow]\", \"metadata\": null}, {\"name\": \"score_swe_bench_scorer_stderr_mean\", \"field_name\": \"score_swe_bench_scorer_stderr_mean\", \"pandas_type\": \"float64\", \"numpy_type\": \"double[pyarrow]\", \"metadata\": null}, {\"name\": \"score_swe_bench_scorer_stderr_stderr\", \"field_name\": \"score_swe_bench_scorer_stderr_stderr\", \"pandas_type\": \"float64\", \"numpy_type\": \"double[pyarrow]\", \"metadata\": null}], \"creator\": {\"library\": \"pyarrow\", \"version\": \"20.0.0\"}, \"pandas_version\": \"2.3.0\"}"]]}
["mrnpz8VX5FJcYLNbNmRDR8","BMTzuttHimqSvPKrEZCjYp","SFpZdiNehfgKi8P2U3e4mv","/Users/jjallaire/meridianlabs/inspect_viz/docs/sandbox/logs/swe-bench/2025-06-24T10-42-56+00-00_swe-bench_SFpZdiNehfgKi8P2U3e4mv.eval","2025-06-24T10:42:56.000000000Z","","https://github.com/AI-Safety-Institute/open_source_benchmarks.git","9a28395","{\"inspect_ai\": \"0.3.106.dev11+g28f3586b\"}",null,"swe_bench",0,null,"{}",false,false,null,"princeton-nlp/SWE-bench_Verified","<lambda>",1.0,null,null,"Please solve the following coding issue:\n\n{issue_text}",null,true,"k8s","{\"type\": \"scorer\", \"name\": \"swe_bench_scorer_stderr\", \"params\": {}}","{\"type\": \"agent\", \"name\": \"react\", \"params\": {\"prompt\": \"You are a helpful autonomous software engineer that can interact with a computer to solve tasks. You are challenged to solve a task specified in a PR description for a given code repository.\\n\\nI've already taken care of all changes to any of the test files described in the PR. This means you DON'T have to modify the testing logic or any of the tests in any way!\\nYour task is to make the minimal changes to non-tests files in the working directory to ensure the PR description is satisfied.\\n\\nHere are some steps you might want to follow:\\n1. As a first step, it might be a good idea to find and read code relevant to the PR description\\n2. Create a script to reproduce the error and execute it using the bash tool, to confirm the error\\n3. Edit the source code of the repo to resolve the issue\\n4. Rerun your reproduce script and confirm that the error is fixed!\\n5. Think about edgecases, create your own unit tests and make sure your fix handles them as well\\n\\nYour thinking should be thorough and so it's fine if it's very long.\\n\", \"tools\": [{\"type\": \"tool\", \"name\": \"bash\", \"params\": {\"timeout\": 180}}, {\"type\": \"tool\", \"name\": \"python\", \"params\": {\"timeout\": 180}}, {\"type\": \"tool\", \"name\": \"agents/submit_answer\", \"params\": {}}]}}","test",null,null,null,null,null,null,"openai/o3",null,null,"{\"max_connections\": 50}",null,"princeton-nlp/SWE-bench_Verified","princeton-nlp/SWE-bench_Verified",500,"[\"astropy__astropy-12907\", \"astropy__astropy-13033\", \"astropy__astropy-13236\", \"astropy__astropy-13398\", \"astropy__astropy-13453\", \"astropy__astropy-13579\", \"astropy__astropy-13977\", \"astropy__astropy-14096\", \"astropy__astropy-14182\", \"astropy__astropy-14309\", \"astropy__astropy-14365\", \"astropy__astropy-14369\", \"astropy__astropy-14508\", \"astropy__astropy-14539\", \"astropy__astropy-14598\", \"astropy__astropy-14995\", \"astropy__astropy-7166\", \"astropy__astropy-7336\", \"astropy__astropy-7606\", \"astropy__astropy-7671\", \"astropy__astropy-8707\", \"astropy__astropy-8872\", \"django__django-10097\", \"django__django-10554\", \"django__django-10880\", \"django__django-10914\", \"django__django-10973\", \"django__django-10999\", \"django__django-11066\", \"django__django-11087\", \"django__django-11095\", \"django__django-11099\", \"django__django-11119\", \"django__django-11133\", \"django__django-11138\", \"django__django-11141\", \"django__django-11149\", \"django__django-11163\", \"django__django-11179\", \"django__django-11206\", \"django__django-11211\", \"django__django-11239\", \"django__django-11265\", \"django__django-11276\", \"django__django-11292\", \"django__django-11299\", \"django__django-11333\", \"django__django-11400\", \"django__django-11433\", \"django__django-11451\", \"django__django-11477\", \"django__django-11490\", \"django__django-11532\", \"django__django-11551\", \"django__django-11555\", \"django__django-11603\", \"django__django-11728\", \"django__django-11734\", \"django__django-11740\", \"django__django-11749\", \"django__django-11790\", \"django__django-11815\", \"django__django-11820\", \"django__django-11848\", \"django__django-11880\", \"django__django-11885\", \"django__django-11951\", \"django__django-11964\", \"django__django-11999\", \"django__django-12039\", \"django__django-12050\", \"django__django-12125\", \"django__django-12143\", \"django__django-12155\", \"django__django-12193\", \"django__django-12209\", \"django__django-12262\", \"django__django-12273\", \"django__django-12276\", \"django__django-12304\", \"django__django-12308\", \"django__django-12325\", \"django__django-12406\", \"django__django-12419\", \"django__django-12663\", \"django__django-12708\", \"django__django-12713\", \"django__django-12741\", \"django__django-12754\", \"django__django-12774\", \"django__django-12858\", \"django__django-12965\", \"django__django-13012\", \"django__django-13023\", \"django__django-13028\", \"django__django-13033\", \"django__django-13089\", \"django__django-13109\", \"django__django-13112\", \"django__django-13121\", \"django__django-13128\", \"django__django-13158\", \"django__django-13195\", \"django__django-13212\", \"django__django-13279\", \"django__django-13297\", \"django__django-13315\", \"django__django-13343\", \"django__django-13344\", \"django__django-13346\", \"django__django-13363\", \"django__django-13401\", \"django__django-13406\", \"django__django-13410\", \"django__django-13417\", \"django__django-13449\", \"django__django-13512\", \"django__django-13513\", \"django__django-13516\", \"django__django-13551\", \"django__django-13568\", \"django__django-13569\", \"django__django-13590\", \"django__django-13658\", \"django__django-13670\", \"django__django-13741\", \"django__django-13786\", \"django__django-13794\", \"django__django-13807\", \"django__django-13809\", \"django__django-13810\", \"django__django-13820\", \"django__django-13821\", \"django__django-13837\", \"django__django-13925\", \"django__django-13933\", \"django__django-13964\", \"django__django-14007\", \"django__django-14011\", \"django__django-14017\", \"django__django-14034\", \"django__django-14053\", \"django__django-14089\", \"django__django-14122\", \"django__django-14140\", \"django__django-14155\", \"django__django-14170\", \"django__django-14238\", \"django__django-14311\", \"django__django-14315\", \"django__django-14349\", \"django__django-14351\", \"django__django-14373\", \"django__django-14376\", \"django__django-14404\", \"django__django-14434\", \"django__django-14493\", \"django__django-14500\", \"django__django-14534\", \"django__django-14539\", \"django__django-14559\", \"django__django-14580\", \"django__django-14608\", \"django__django-14631\", \"django__django-14672\", \"django__django-14725\", \"django__django-14752\", \"django__django-14765\", \"django__django-14771\", \"django__django-14787\", \"django__django-14792\", \"django__django-14855\", \"django__django-14915\", \"django__django-14999\", \"django__django-15022\", \"django__django-15037\", \"django__django-15098\", \"django__django-15103\", \"django__django-15104\", \"django__django-15127\", \"django__django-15128\", \"django__django-15161\", \"django__django-15252\", \"django__django-15268\", \"django__django-15277\", \"django__django-15278\", \"django__django-15280\", \"django__django-15315\", \"django__django-15368\", \"django__django-15375\", \"django__django-15380\", \"django__django-15382\", \"django__django-15467\", \"django__django-15499\", \"django__django-15503\", \"django__django-15525\", \"django__django-15554\", \"django__django-15561\", \"django__django-15563\", \"django__django-15569\", \"django__django-15572\", \"django__django-15629\", \"django__django-15695\", \"django__django-15731\", \"django__django-15732\", \"django__django-15741\", \"django__django-15814\", \"django__django-15851\", \"django__django-15863\", \"django__django-15916\", \"django__django-15930\", \"django__django-15957\", \"django__django-15973\", \"django__django-15987\", \"django__django-16032\", \"django__django-16082\", \"django__django-16100\", \"django__django-16116\", \"django__django-16136\", \"django__django-16139\", \"django__django-16145\", \"django__django-16255\", \"django__django-16256\", \"django__django-16263\", \"django__django-16315\", \"django__django-16333\", \"django__django-16429\", \"django__django-16454\", \"django__django-16485\", \"django__django-16493\", \"django__django-16502\", \"django__django-16527\", \"django__django-16560\", \"django__django-16569\", \"django__django-16595\", \"django__django-16612\", \"django__django-16631\", \"django__django-16642\", \"django__django-16661\", \"django__django-16662\", \"django__django-16667\", \"django__django-16801\", \"django__django-16819\", \"django__django-16877\", \"django__django-16899\", \"django__django-16901\", \"django__django-16938\", \"django__django-16950\", \"django__django-17029\", \"django__django-17084\", \"django__django-17087\", \"django__django-7530\", \"django__django-9296\", \"matplotlib__matplotlib-13989\", \"matplotlib__matplotlib-14623\", \"matplotlib__matplotlib-20488\", \"matplotlib__matplotlib-20676\", \"matplotlib__matplotlib-20826\", \"matplotlib__matplotlib-20859\", \"matplotlib__matplotlib-21568\", \"matplotlib__matplotlib-22719\", \"matplotlib__matplotlib-22865\", \"matplotlib__matplotlib-22871\", \"matplotlib__matplotlib-23299\", \"matplotlib__matplotlib-23314\", \"matplotlib__matplotlib-23412\", \"matplotlib__matplotlib-23476\", \"matplotlib__matplotlib-24026\", \"matplotlib__matplotlib-24149\", \"matplotlib__matplotlib-24177\", \"matplotlib__matplotlib-24570\", \"matplotlib__matplotlib-24627\", \"matplotlib__matplotlib-24637\", \"matplotlib__matplotlib-24870\", \"matplotlib__matplotlib-24970\", \"matplotlib__matplotlib-25122\", \"matplotlib__matplotlib-25287\", \"matplotlib__matplotlib-25311\", \"matplotlib__matplotlib-25332\", \"matplotlib__matplotlib-25479\", \"matplotlib__matplotlib-25775\", \"matplotlib__matplotlib-25960\", \"matplotlib__matplotlib-26113\", \"matplotlib__matplotlib-26208\", \"matplotlib__matplotlib-26291\", \"matplotlib__matplotlib-26342\", \"matplotlib__matplotlib-26466\", \"mwaskom__seaborn-3069\", \"mwaskom__seaborn-3187\", \"pallets__flask-5014\", \"psf__requests-1142\", \"psf__requests-1724\", \"psf__requests-1766\", \"psf__requests-1921\", \"psf__requests-2317\", \"psf__requests-2931\", \"psf__requests-5414\", \"psf__requests-6028\", \"pydata__xarray-2905\", \"pydata__xarray-3095\", \"pydata__xarray-3151\", \"pydata__xarray-3305\", \"pydata__xarray-3677\", \"pydata__xarray-3993\", \"pydata__xarray-4075\", \"pydata__xarray-4094\", \"pydata__xarray-4356\", \"pydata__xarray-4629\", \"pydata__xarray-4687\", \"pydata__xarray-4695\", \"pydata__xarray-4966\", \"pydata__xarray-6461\", \"pydata__xarray-6599\", \"pydata__xarray-6721\", \"pydata__xarray-6744\", \"pydata__xarray-6938\", \"pydata__xarray-6992\", \"pydata__xarray-7229\", \"pydata__xarray-7233\", \"pydata__xarray-7393\", \"pylint-dev__pylint-4551\", \"pylint-dev__pylint-4604\", \"pylint-dev__pylint-4661\", \"pylint-dev__pylint-4970\", \"pylint-dev__pylint-6386\", \"pylint-dev__pylint-6528\", \"pylint-dev__pylint-6903\", \"pylint-dev__pylint-7080\", \"pylint-dev__pylint-7277\", \"pylint-dev__pylint-8898\", \"pytest-dev__pytest-10051\", \"pytest-dev__pytest-10081\", \"pytest-dev__pytest-10356\", \"pytest-dev__pytest-5262\", \"pytest-dev__pytest-5631\", \"pytest-dev__pytest-5787\", \"pytest-dev__pytest-5809\", \"pytest-dev__pytest-5840\", \"pytest-dev__pytest-6197\", \"pytest-dev__pytest-6202\", \"pytest-dev__pytest-7205\", \"pytest-dev__pytest-7236\", \"pytest-dev__pytest-7324\", \"pytest-dev__pytest-7432\", \"pytest-dev__pytest-7490\", \"pytest-dev__pytest-7521\", \"pytest-dev__pytest-7571\", \"pytest-dev__pytest-7982\", \"pytest-dev__pytest-8399\", \"scikit-learn__scikit-learn-10297\", \"scikit-learn__scikit-learn-10844\", \"scikit-learn__scikit-learn-10908\", \"scikit-learn__scikit-learn-11310\", \"scikit-learn__scikit-learn-11578\", \"scikit-learn__scikit-learn-12585\", \"scikit-learn__scikit-learn-12682\", \"scikit-learn__scikit-learn-12973\", \"scikit-learn__scikit-learn-13124\", \"scikit-learn__scikit-learn-13135\", \"scikit-learn__scikit-learn-13142\", \"scikit-learn__scikit-learn-13328\", \"scikit-learn__scikit-learn-13439\", \"scikit-learn__scikit-learn-13496\", \"scikit-learn__scikit-learn-13779\", \"scikit-learn__scikit-learn-14053\", \"scikit-learn__scikit-learn-14087\", \"scikit-learn__scikit-learn-14141\", \"scikit-learn__scikit-learn-14496\", \"scikit-learn__scikit-learn-14629\", \"scikit-learn__scikit-learn-14710\", \"scikit-learn__scikit-learn-14894\", \"scikit-learn__scikit-learn-14983\", \"scikit-learn__scikit-learn-15100\", \"scikit-learn__scikit-learn-25102\", \"scikit-learn__scikit-learn-25232\", \"scikit-learn__scikit-learn-25747\", \"scikit-learn__scikit-learn-25931\", \"scikit-learn__scikit-learn-25973\", \"scikit-learn__scikit-learn-26194\", \"scikit-learn__scikit-learn-26323\", \"scikit-learn__scikit-learn-9288\", \"sphinx-doc__sphinx-10323\", \"sphinx-doc__sphinx-10435\", \"sphinx-doc__sphinx-10449\", \"sphinx-doc__sphinx-10466\", \"sphinx-doc__sphinx-10614\", \"sphinx-doc__sphinx-10673\", \"sphinx-doc__sphinx-11445\", \"sphinx-doc__sphinx-11510\", \"sphinx-doc__sphinx-7440\", \"sphinx-doc__sphinx-7454\", \"sphinx-doc__sphinx-7462\", \"sphinx-doc__sphinx-7590\", \"sphinx-doc__sphinx-7748\", \"sphinx-doc__sphinx-7757\", \"sphinx-doc__sphinx-7889\", \"sphinx-doc__sphinx-7910\", \"sphinx-doc__sphinx-7985\", \"sphinx-doc__sphinx-8035\", \"sphinx-doc__sphinx-8056\", \"sphinx-doc__sphinx-8120\", \"sphinx-doc__sphinx-8265\", \"sphinx-doc__sphinx-8269\", \"sphinx-doc__sphinx-8459\", \"sphinx-doc__sphinx-8475\", \"sphinx-doc__sphinx-8548\", \"sphinx-doc__sphinx-8551\", \"sphinx-doc__sphinx-8593\", \"sphinx-doc__sphinx-8595\", \"sphinx-doc__sphinx-8621\", \"sphinx-doc__sphinx-8638\", \"sphinx-doc__sphinx-8721\", \"sphinx-doc__sphinx-9229\", \"sphinx-doc__sphinx-9230\", \"sphinx-doc__sphinx-9258\", \"sphinx-doc__sphinx-9281\", \"sphinx-doc__sphinx-9320\", \"sphinx-doc__sphinx-9367\", \"sphinx-doc__sphinx-9461\", \"sphinx-doc__sphinx-9591\", \"sphinx-doc__sphinx-9602\", \"sphinx-doc__sphinx-9658\", \"sphinx-doc__sphinx-9673\", \"sphinx-doc__sphinx-9698\", \"sphinx-doc__sphinx-9711\", \"sympy__sympy-11618\", \"sympy__sympy-12096\", \"sympy__sympy-12419\", \"sympy__sympy-12481\", \"sympy__sympy-12489\", \"sympy__sympy-13031\", \"sympy__sympy-13091\", \"sympy__sympy-13372\", \"sympy__sympy-13480\", \"sympy__sympy-13551\", \"sympy__sympy-13615\", \"sympy__sympy-13647\", \"sympy__sympy-13757\", \"sympy__sympy-13798\", \"sympy__sympy-13852\", \"sympy__sympy-13877\", \"sympy__sympy-13878\", \"sympy__sympy-13974\", \"sympy__sympy-14248\", \"sympy__sympy-14531\", \"sympy__sympy-14711\", \"sympy__sympy-14976\", \"sympy__sympy-15017\", \"sympy__sympy-15345\", \"sympy__sympy-15349\", \"sympy__sympy-15599\", \"sympy__sympy-15809\", \"sympy__sympy-15875\", \"sympy__sympy-15976\", \"sympy__sympy-16450\", \"sympy__sympy-16597\", \"sympy__sympy-16766\", \"sympy__sympy-16792\", \"sympy__sympy-16886\", \"sympy__sympy-17139\", \"sympy__sympy-17318\", \"sympy__sympy-17630\", \"sympy__sympy-17655\", \"sympy__sympy-18189\", \"sympy__sympy-18199\", \"sympy__sympy-18211\", \"sympy__sympy-18698\", \"sympy__sympy-18763\", \"sympy__sympy-19040\", \"sympy__sympy-19346\", \"sympy__sympy-19495\", \"sympy__sympy-19637\", \"sympy__sympy-19783\", \"sympy__sympy-19954\", \"sympy__sympy-20154\", \"sympy__sympy-20428\", \"sympy__sympy-20438\", \"sympy__sympy-20590\", \"sympy__sympy-20801\", \"sympy__sympy-20916\", \"sympy__sympy-21379\", \"sympy__sympy-21596\", \"sympy__sympy-21612\", \"sympy__sympy-21847\", \"sympy__sympy-21930\", \"sympy__sympy-22080\", \"sympy__sympy-22456\", \"sympy__sympy-22714\", \"sympy__sympy-22914\", \"sympy__sympy-23262\", \"sympy__sympy-23413\", \"sympy__sympy-23534\", \"sympy__sympy-23824\", \"sympy__sympy-23950\", \"sympy__sympy-24066\", \"sympy__sympy-24213\", \"sympy__sympy-24443\", \"sympy__sympy-24539\", \"sympy__sympy-24562\", \"sympy__sympy-24661\"]",false,1,"[\"mean\"]",null,1000000.0,2500000.0,86400.0,null,"success",null,null,500,496,"swe_bench_scorer_stderr","mean",0.3588709677419355,0.021559529813181433,null,null,null,null,null,null,0.3588709677419355,0.021559529813181433]
["3LYpVdMahwB92GcBsnduCn","BMTzuttHimqSvPKrEZCjYp","asxeGxtdy9Wsbbx4vhNvUm","/Users/jjallaire/meridianlabs/inspect_viz/docs/sandbox/logs/swe-bench/2025-06-24T10-42-56+00-00_swe-bench_asxeGxtdy9Wsbbx4vhNvUm.eval","2025-06-24T10:42:56.000000000Z","","https://github.com/AI-Safety-Institute/open_source_benchmarks.git","9a28395","{\"inspect_ai\": \"0.3.106.dev11+g28f3586b\"}",null,"swe_bench",0,null,"{}",false,false,null,"princeton-nlp/SWE-bench_Verified","<lambda>",1.0,null,null,"Please solve the following coding issue:\n\n{issue_text}",null,true,"k8s","{\"type\": \"scorer\", \"name\": \"swe_bench_scorer_stderr\", \"params\": {}}","{\"type\": \"agent\", \"name\": \"react\", \"params\": {\"prompt\": \"You are a helpful autonomous software engineer that can interact with a computer to solve tasks. You are challenged to solve a task specified in a PR description for a given code repository.\\n\\nI've already taken care of all changes to any of the test files described in the PR. This means you DON'T have to modify the testing logic or any of the tests in any way!\\nYour task is to make the minimal changes to non-tests files in the working directory to ensure the PR description is satisfied.\\n\\nHere are some steps you might want to follow:\\n1. As a first step, it might be a good idea to find and read code relevant to the PR description\\n2. Create a script to reproduce the error and execute it using the bash tool, to confirm the error\\n3. Edit the source code of the repo to resolve the issue\\n4. Rerun your reproduce script and confirm that the error is fixed!\\n5. Think about edgecases, create your own unit tests and make sure your fix handles them as well\\n\\nYour thinking should be thorough and so it's fine if it's very long.\\n\", \"tools\": [{\"type\": \"tool\", \"name\": \"bash\", \"params\": {\"timeout\": 180}}, {\"type\": \"tool\", \"name\": \"python\", \"params\": {\"timeout\": 180}}, {\"type\": \"tool\", \"name\": \"agents/submit_answer\", \"params\": {}}]}}","test",null,null,null,null,null,null,"anthropic/claude-3-7-sonnet-latest",null,null,"{\"max_connections\": 50}",null,"princeton-nlp/SWE-bench_Verified","princeton-nlp/SWE-bench_Verified",500,"[\"astropy__astropy-12907\", \"astropy__astropy-13033\", \"astropy__astropy-13236\", \"astropy__astropy-13398\", \"astropy__astropy-13453\", \"astropy__astropy-13579\", \"astropy__astropy-13977\", \"astropy__astropy-14096\", \"astropy__astropy-14182\", \"astropy__astropy-14309\", \"astropy__astropy-14365\", \"astropy__astropy-14369\", \"astropy__astropy-14508\", \"astropy__astropy-14539\", \"astropy__astropy-14598\", \"astropy__astropy-14995\", \"astropy__astropy-7166\", \"astropy__astropy-7336\", \"astropy__astropy-7606\", \"astropy__astropy-7671\", \"astropy__astropy-8707\", \"astropy__astropy-8872\", \"django__django-10097\", \"django__django-10554\", \"django__django-10880\", \"django__django-10914\", \"django__django-10973\", \"django__django-10999\", \"django__django-11066\", \"django__django-11087\", \"django__django-11095\", \"django__django-11099\", \"django__django-11119\", \"django__django-11133\", \"django__django-11138\", \"django__django-11141\", \"django__django-11149\", \"django__django-11163\", \"django__django-11179\", \"django__django-11206\", \"django__django-11211\", \"django__django-11239\", \"django__django-11265\", \"django__django-11276\", \"django__django-11292\", \"django__django-11299\", \"django__django-11333\", \"django__django-11400\", \"django__django-11433\", \"django__django-11451\", \"django__django-11477\", \"django__django-11490\", \"django__django-11532\", \"django__django-11551\", \"django__django-11555\", \"django__django-11603\", \"django__django-11728\", \"django__django-11734\", \"django__django-11740\", \"django__django-11749\", \"django__django-11790\", \"django__django-11815\", \"django__django-11820\", \"django__django-11848\", \"django__django-11880\", \"django__django-11885\", \"django__django-11951\", \"django__django-11964\", \"django__django-11999\", \"django__django-12039\", \"django__django-12050\", \"django__django-12125\", \"django__django-12143\", \"django__django-12155\", \"django__django-12193\", \"django__django-12209\", \"django__django-12262\", \"django__django-12273\", \"django__django-12276\", \"django__django-12304\", \"django__django-12308\", \"django__django-12325\", \"django__django-12406\", \"django__django-12419\", \"django__django-12663\", \"django__django-12708\", \"django__django-12713\", \"django__django-12741\", \"django__django-12754\", \"django__django-12774\", \"django__django-12858\", \"django__django-12965\", \"django__django-13012\", \"django__django-13023\", \"django__django-13028\", \"django__django-13033\", \"django__django-13089\", \"django__django-13109\", \"django__django-13112\", \"django__django-13121\", \"django__django-13128\", \"django__django-13158\", \"django__django-13195\", \"django__django-13212\", \"django__django-13279\", \"django__django-13297\", \"django__django-13315\", \"django__django-13343\", \"django__django-13344\", \"django__django-13346\", \"django__django-13363\", \"django__django-13401\", \"django__django-13406\", \"django__django-13410\", \"django__django-13417\", \"django__django-13449\", \"django__django-13512\", \"django__django-13513\", \"django__django-13516\", \"django__django-13551\", \"django__django-13568\", \"django__django-13569\", \"django__django-13590\", \"django__django-13658\", \"django__django-13670\", \"django__django-13741\", \"django__django-13786\", \"django__django-13794\", \"django__django-13807\", \"django__django-13809\", \"django__django-13810\", \"django__django-13820\", \"django__django-13821\", \"django__django-13837\", \"django__django-13925\", \"django__django-13933\", \"django__django-13964\", \"django__django-14007\", \"django__django-14011\", \"django__django-14017\", \"django__django-14034\", \"django__django-14053\", \"django__django-14089\", \"django__django-14122\", \"django__django-14140\", \"django__django-14155\", \"django__django-14170\", \"django__django-14238\", \"django__django-14311\", \"django__django-14315\", \"django__django-14349\", \"django__django-14351\", \"django__django-14373\", \"django__django-14376\", \"django__django-14404\", \"django__django-14434\", \"django__django-14493\", \"django__django-14500\", \"django__django-14534\", \"django__django-14539\", \"django__django-14559\", \"django__django-14580\", \"django__django-14608\", \"django__django-14631\", \"django__django-14672\", \"django__django-14725\", \"django__django-14752\", \"django__django-14765\", \"django__django-14771\", \"django__django-14787\", \"django__django-14792\", \"django__django-14855\", \"django__django-14915\", \"django__django-14999\", \"django__django-15022\", \"django__django-15037\", \"django__django-15098\", \"django__django-15103\", \"django__django-15104\", \"django__django-15127\", \"django__django-15128\", \"django__django-15161\", \"django__django-15252\", \"django__django-15268\", \"django__django-15277\", \"django__django-15278\", \"django__django-15280\", \"django__django-15315\", \"django__django-15368\", \"django__django-15375\", \"django__django-15380\", \"django__django-15382\", \"django__django-15467\", \"django__django-15499\", \"django__django-15503\", \"django__django-15525\", \"django__django-15554\", \"django__django-15561\", \"django__django-15563\", \"django__django-15569\", \"django__django-15572\", \"django__django-15629\", \"django__django-15695\", \"django__django-15731\", \"django__django-15732\", \"django__django-15741\", \"django__django-15814\", \"django__django-15851\", \"django__django-15863\", \"django__django-15916\", \"django__django-15930\", \"django__django-15957\", \"django__django-15973\", \"django__django-15987\", \"django__django-16032\", \"django__django-16082\", \"django__django-16100\", \"django__django-16116\", \"django__django-16136\", \"django__django-16139\", \"django__django-16145\", \"django__django-16255\", \"django__django-16256\", \"django__django-16263\", \"django__django-16315\", \"django__django-16333\", \"django__django-16429\", \"django__django-16454\", \"django__django-16485\", \"django__django-16493\", \"django__django-16502\", \"django__django-16527\", \"django__django-16560\", \"django__django-16569\", \"django__django-16595\", \"django__django-16612\", \"django__django-16631\", \"django__django-16642\", \"django__django-16661\", \"django__django-16662\", \"django__django-16667\", \"django__django-16801\", \"django__django-16819\", \"django__django-16877\", \"django__django-16899\", \"django__django-16901\", \"django__django-16938\", \"django__django-16950\", \"django__django-17029\", \"django__django-17084\", \"django__django-17087\", \"django__django-7530\", \"django__django-9296\", \"matplotlib__matplotlib-13989\", \"matplotlib__matplotlib-14623\", \"matplotlib__matplotlib-20488\", \"matplotlib__matplotlib-20676\", \"matplotlib__matplotlib-20826\", \"matplotlib__matplotlib-20859\", \"matplotlib__matplotlib-21568\", \"matplotlib__matplotlib-22719\", \"matplotlib__matplotlib-22865\", \"matplotlib__matplotlib-22871\", \"matplotlib__matplotlib-23299\", \"matplotlib__matplotlib-23314\", \"matplotlib__matplotlib-23412\", \"matplotlib__matplotlib-23476\", \"matplotlib__matplotlib-24026\", \"matplotlib__matplotlib-24149\", \"matplotlib__matplotlib-24177\", \"matplotlib__matplotlib-24570\", \"matplotlib__matplotlib-24627\", \"matplotlib__matplotlib-24637\", \"matplotlib__matplotlib-24870\", \"matplotlib__matplotlib-24970\", \"matplotlib__matplotlib-25122\", \"matplotlib__matplotlib-25287\", \"matplotlib__matplotlib-25311\", \"matplotlib__matplotlib-25332\", \"matplotlib__matplotlib-25479\", \"matplotlib__matplotlib-25775\", \"matplotlib__matplotlib-25960\", \"matplotlib__matplotlib-26113\", \"matplotlib__matplotlib-26208\", \"matplotlib__matplotlib-26291\", \"matplotlib__matplotlib-26342\", \"matplotlib__matplotlib-26466\", \"mwaskom__seaborn-3069\", \"mwaskom__seaborn-3187\", \"pallets__flask-5014\", \"psf__requests-1142\", \"psf__requests-1724\", \"psf__requests-1766\", \"psf__requests-1921\", \"psf__requests-2317\", \"psf__requests-2931\", \"psf__requests-5414\", \"psf__requests-6028\", \"pydata__xarray-2905\", \"pydata__xarray-3095\", \"pydata__xarray-3151\", \"pydata__xarray-3305\", \"pydata__xarray-3677\", \"pydata__xarray-3993\", \"pydata__xarray-4075\", \"pydata__xarray-4094\", \"pydata__xarray-4356\", \"pydata__xarray-4629\", \"pydata__xarray-4687\", \"pydata__xarray-4695\", \"pydata__xarray-4966\", \"pydata__xarray-6461\", \"pydata__xarray-6599\", \"pydata__xarray-6721\", \"pydata__xarray-6744\", \"pydata__xarray-6938\", \"pydata__xarray-6992\", \"pydata__xarray-7229\", \"pydata__xarray-7233\", \"pydata__xarray-7393\", \"pylint-dev__pylint-4551\", \"pylint-dev__pylint-4604\", \"pylint-dev__pylint-4661\", \"pylint-dev__pylint-4970\", \"pylint-dev__pylint-6386\", \"pylint-dev__pylint-6528\", \"pylint-dev__pylint-6903\", \"pylint-dev__pylint-7080\", \"pylint-dev__pylint-7277\", \"pylint-dev__pylint-8898\", \"pytest-dev__pytest-10051\", \"pytest-dev__pytest-10081\", \"pytest-dev__pytest-10356\", \"pytest-dev__pytest-5262\", \"pytest-dev__pytest-5631\", \"pytest-dev__pytest-5787\", \"pytest-dev__pytest-5809\", \"pytest-dev__pytest-5840\", \"pytest-dev__pytest-6197\", \"pytest-dev__pytest-6202\", \"pytest-dev__pytest-7205\", \"pytest-dev__pytest-7236\", \"pytest-dev__pytest-7324\", \"pytest-dev__pytest-7432\", \"pytest-dev__pytest-7490\", \"pytest-dev__pytest-7521\", \"pytest-dev__pytest-7571\", \"pytest-dev__pytest-7982\", \"pytest-dev__pytest-8399\", \"scikit-learn__scikit-learn-10297\", \"scikit-learn__scikit-learn-10844\", \"scikit-learn__scikit-learn-10908\", \"scikit-learn__scikit-learn-11310\", \"scikit-learn__scikit-learn-11578\", \"scikit-learn__scikit-learn-12585\", \"scikit-learn__scikit-learn-12682\", \"scikit-learn__scikit-learn-12973\", \"scikit-learn__scikit-learn-13124\", \"scikit-learn__scikit-learn-13135\", \"scikit-learn__scikit-learn-13142\", \"scikit-learn__scikit-learn-13328\", \"scikit-learn__scikit-learn-13439\", \"scikit-learn__scikit-learn-13496\", \"scikit-learn__scikit-learn-13779\", \"scikit-learn__scikit-learn-14053\", \"scikit-learn__scikit-learn-14087\", \"scikit-learn__scikit-learn-14141\", \"scikit-learn__scikit-learn-14496\", \"scikit-learn__scikit-learn-14629\", \"scikit-learn__scikit-learn-14710\", \"scikit-learn__scikit-learn-14894\", \"scikit-learn__scikit-learn-14983\", \"scikit-learn__scikit-learn-15100\", \"scikit-learn__scikit-learn-25102\", \"scikit-learn__scikit-learn-25232\", \"scikit-learn__scikit-learn-25747\", \"scikit-learn__scikit-learn-25931\", \"scikit-learn__scikit-learn-25973\", \"scikit-learn__scikit-learn-26194\", \"scikit-learn__scikit-learn-26323\", \"scikit-learn__scikit-learn-9288\", \"sphinx-doc__sphinx-10323\", \"sphinx-doc__sphinx-10435\", \"sphinx-doc__sphinx-10449\", \"sphinx-doc__sphinx-10466\", \"sphinx-doc__sphinx-10614\", \"sphinx-doc__sphinx-10673\", \"sphinx-doc__sphinx-11445\", \"sphinx-doc__sphinx-11510\", \"sphinx-doc__sphinx-7440\", \"sphinx-doc__sphinx-7454\", \"sphinx-doc__sphinx-7462\", \"sphinx-doc__sphinx-7590\", \"sphinx-doc__sphinx-7748\", \"sphinx-doc__sphinx-7757\", \"sphinx-doc__sphinx-7889\", \"sphinx-doc__sphinx-7910\", \"sphinx-doc__sphinx-7985\", \"sphinx-doc__sphinx-8035\", \"sphinx-doc__sphinx-8056\", \"sphinx-doc__sphinx-8120\", \"sphinx-doc__sphinx-8265\", \"sphinx-doc__sphinx-8269\", \"sphinx-doc__sphinx-8459\", \"sphinx-doc__sphinx-8475\", \"sphinx-doc__sphinx-8548\", \"sphinx-doc__sphinx-8551\", \"sphinx-doc__sphinx-8593\", \"sphinx-doc__sphinx-8595\", \"sphinx-doc__sphinx-8621\", \"sphinx-doc__sphinx-8638\", \"sphinx-doc__sphinx-8721\", \"sphinx-doc__sphinx-9229\", \"sphinx-doc__sphinx-9230\", \"sphinx-doc__sphinx-9258\", \"sphinx-doc__sphinx-9281\", \"sphinx-doc__sphinx-9320\", \"sphinx-doc__sphinx-9367\", \"sphinx-doc__sphinx-9461\", \"sphinx-doc__sphinx-9591\", \"sphinx-doc__sphinx-9602\", \"sphinx-doc__sphinx-9658\", \"sphinx-doc__sphinx-9673\", \"sphinx-doc__sphinx-9698\", \"sphinx-doc__sphinx-9711\", \"sympy__sympy-11618\", \"sympy__sympy-12096\", \"sympy__sympy-12419\", \"sympy__sympy-12481\", \"sympy__sympy-12489\", \"sympy__sympy-13031\", \"sympy__sympy-13091\", \"sympy__sympy-13372\", \"sympy__sympy-13480\", \"sympy__sympy-13551\", \"sympy__sympy-13615\", \"sympy__sympy-13647\", \"sympy__sympy-13757\", \"sympy__sympy-13798\", \"sympy__sympy-13852\", \"sympy__sympy-13877\", \"sympy__sympy-13878\", \"sympy__sympy-13974\", \"sympy__sympy-14248\", \"sympy__sympy-14531\", \"sympy__sympy-14711\", \"sympy__sympy-14976\", \"sympy__sympy-15017\", \"sympy__sympy-15345\", \"sympy__sympy-15349\", \"sympy__sympy-15599\", \"sympy__sympy-15809\", \"sympy__sympy-15875\", \"sympy__sympy-15976\", \"sympy__sympy-16450\", \"sympy__sympy-16597\", \"sympy__sympy-16766\", \"sympy__sympy-16792\", \"sympy__sympy-16886\", \"sympy__sympy-17139\", \"sympy__sympy-17318\", \"sympy__sympy-17630\", \"sympy__sympy-17655\", \"sympy__sympy-18189\", \"sympy__sympy-18199\", \"sympy__sympy-18211\", \"sympy__sympy-18698\", \"sympy__sympy-18763\", \"sympy__sympy-19040\", \"sympy__sympy-19346\", \"sympy__sympy-19495\", \"sympy__sympy-19637\", \"sympy__sympy-19783\", \"sympy__sympy-19954\", \"sympy__sympy-20154\", \"sympy__sympy-20428\", \"sympy__sympy-20438\", \"sympy__sympy-20590\", \"sympy__sympy-20801\", \"sympy__sympy-20916\", \"sympy__sympy-21379\", \"sympy__sympy-21596\", \"sympy__sympy-21612\", \"sympy__sympy-21847\", \"sympy__sympy-21930\", \"sympy__sympy-22080\", \"sympy__sympy-22456\", \"sympy__sympy-22714\", \"sympy__sympy-22914\", \"sympy__sympy-23262\", \"sympy__sympy-23413\", \"sympy__sympy-23534\", \"sympy__sympy-23824\", \"sympy__sympy-23950\", \"sympy__sympy-24066\", \"sympy__sympy-24213\", \"sympy__sympy-24443\", \"sympy__sympy-24539\", \"sympy__sympy-24562\", \"sympy__sympy-24661\"]",false,1,"[\"mean\"]",null,1000000.0,2500000.0,86400.0,null,"success",null,null,500,498,"swe_bench_scorer_stderr","mean",0.5160642570281124,0.022416486623185048,null,null,null,null,null,null,0.5160642570281124,0.022416486623185048]
["mv94sCL5rTnF5Ap4DshE8r","BMTzuttHimqSvPKrEZCjYp","UXASSKPHM59jB5VNkvWr4L","/Users/jjallaire/meridianlabs/inspect_viz/docs/sandbox/logs/swe-bench/2025-06-24T10-42-56+00-00_swe-bench_UXASSKPHM59jB5VNkvWr4L.eval","2025-06-24T10:42:56.000000000Z","","https://github.com/AI-Safety-Institute/open_source_benchmarks.git","9a28395","{\"inspect_ai\": \"0.3.106.dev11+g28f3586b\"}",null,"swe_bench",0,null,"{}",false,false,null,"princeton-nlp/SWE-bench_Verified","<lambda>",1.0,null,null,"Please solve the following coding issue:\n\n{issue_text}",null,true,"k8s","{\"type\": \"scorer\", \"name\": \"swe_bench_scorer_stderr\", \"params\": {}}","{\"type\": \"agent\", \"name\": \"react\", \"params\": {\"prompt\": \"You are a helpful autonomous software engineer that can interact with a computer to solve tasks. You are challenged to solve a task specified in a PR description for a given code repository.\\n\\nI've already taken care of all changes to any of the test files described in the PR. This means you DON'T have to modify the testing logic or any of the tests in any way!\\nYour task is to make the minimal changes to non-tests files in the working directory to ensure the PR description is satisfied.\\n\\nHere are some steps you might want to follow:\\n1. As a first step, it might be a good idea to find and read code relevant to the PR description\\n2. Create a script to reproduce the error and execute it using the bash tool, to confirm the error\\n3. Edit the source code of the repo to resolve the issue\\n4. Rerun your reproduce script and confirm that the error is fixed!\\n5. Think about edgecases, create your own unit tests and make sure your fix handles them as well\\n\\nYour thinking should be thorough and so it's fine if it's very long.\\n\", \"tools\": [{\"type\": \"tool\", \"name\": \"bash\", \"params\": {\"timeout\": 180}}, {\"type\": \"tool\", \"name\": \"python\", \"params\": {\"timeout\": 180}}, {\"type\": \"tool\", \"name\": \"agents/submit_answer\", \"params\": {}}]}}","test",null,null,null,null,null,null,"openai/o4-mini",null,null,"{\"max_connections\": 50}",null,"princeton-nlp/SWE-bench_Verified","princeton-nlp/SWE-bench_Verified",500,"[\"astropy__astropy-12907\", \"astropy__astropy-13033\", \"astropy__astropy-13236\", \"astropy__astropy-13398\", \"astropy__astropy-13453\", \"astropy__astropy-13579\", \"astropy__astropy-13977\", \"astropy__astropy-14096\", \"astropy__astropy-14182\", \"astropy__astropy-14309\", \"astropy__astropy-14365\", \"astropy__astropy-14369\", \"astropy__astropy-14508\", \"astropy__astropy-14539\", \"astropy__astropy-14598\", \"astropy__astropy-14995\", \"astropy__astropy-7166\", \"astropy__astropy-7336\", \"astropy__astropy-7606\", \"astropy__astropy-7671\", \"astropy__astropy-8707\", \"astropy__astropy-8872\", \"django__django-10097\", \"django__django-10554\", \"django__django-10880\", \"django__django-10914\", \"django__django-10973\", \"django__django-10999\", \"django__django-11066\", \"django__django-11087\", \"django__django-11095\", \"django__django-11099\", \"django__django-11119\", \"django__django-11133\", \"django__django-11138\", \"django__django-11141\", \"django__django-11149\", \"django__django-11163\", \"django__django-11179\", \"django__django-11206\", \"django__django-11211\", \"django__django-11239\", \"django__django-11265\", \"django__django-11276\", \"django__django-11292\", \"django__django-11299\", \"django__django-11333\", \"django__django-11400\", \"django__django-11433\", \"django__django-11451\", \"django__django-11477\", \"django__django-11490\", \"django__django-11532\", \"django__django-11551\", \"django__django-11555\", \"django__django-11603\", \"django__django-11728\", \"django__django-11734\", \"django__django-11740\", \"django__django-11749\", \"django__django-11790\", \"django__django-11815\", \"django__django-11820\", \"django__django-11848\", \"django__django-11880\", \"django__django-11885\", \"django__django-11951\", \"django__django-11964\", \"django__django-11999\", \"django__django-12039\", \"django__django-12050\", \"django__django-12125\", \"django__django-12143\", \"django__django-12155\", \"django__django-12193\", \"django__django-12209\", \"django__django-12262\", \"django__django-12273\", \"django__django-12276\", \"django__django-12304\", \"django__django-12308\", \"django__django-12325\", \"django__django-12406\", \"django__django-12419\", \"django__django-12663\", \"django__django-12708\", \"django__django-12713\", \"django__django-12741\", \"django__django-12754\", \"django__django-12774\", \"django__django-12858\", \"django__django-12965\", \"django__django-13012\", \"django__django-13023\", \"django__django-13028\", \"django__django-13033\", \"django__django-13089\", \"django__django-13109\", \"django__django-13112\", \"django__django-13121\", \"django__django-13128\", \"django__django-13158\", \"django__django-13195\", \"django__django-13212\", \"django__django-13279\", \"django__django-13297\", \"django__django-13315\", \"django__django-13343\", \"django__django-13344\", \"django__django-13346\", \"django__django-13363\", \"django__django-13401\", \"django__django-13406\", \"django__django-13410\", \"django__django-13417\", \"django__django-13449\", \"django__django-13512\", \"django__django-13513\", \"django__django-13516\", \"django__django-13551\", \"django__django-13568\", \"django__django-13569\", \"django__django-13590\", \"django__django-13658\", \"django__django-13670\", \"django__django-13741\", \"django__django-13786\", \"django__django-13794\", \"django__django-13807\", \"django__django-13809\", \"django__django-13810\", \"django__django-13820\", \"django__django-13821\", \"django__django-13837\", \"django__django-13925\", \"django__django-13933\", \"django__django-13964\", \"django__django-14007\", \"django__django-14011\", \"django__django-14017\", \"django__django-14034\", \"django__django-14053\", \"django__django-14089\", \"django__django-14122\", \"django__django-14140\", \"django__django-14155\", \"django__django-14170\", \"django__django-14238\", \"django__django-14311\", \"django__django-14315\", \"django__django-14349\", \"django__django-14351\", \"django__django-14373\", \"django__django-14376\", \"django__django-14404\", \"django__django-14434\", \"django__django-14493\", \"django__django-14500\", \"django__django-14534\", \"django__django-14539\", \"django__django-14559\", \"django__django-14580\", \"django__django-14608\", \"django__django-14631\", \"django__django-14672\", \"django__django-14725\", \"django__django-14752\", \"django__django-14765\", \"django__django-14771\", \"django__django-14787\", \"django__django-14792\", \"django__django-14855\", \"django__django-14915\", \"django__django-14999\", \"django__django-15022\", \"django__django-15037\", \"django__django-15098\", \"django__django-15103\", \"django__django-15104\", \"django__django-15127\", \"django__django-15128\", \"django__django-15161\", \"django__django-15252\", \"django__django-15268\", \"django__django-15277\", \"django__django-15278\", \"django__django-15280\", \"django__django-15315\", \"django__django-15368\", \"django__django-15375\", \"django__django-15380\", \"django__django-15382\", \"django__django-15467\", \"django__django-15499\", \"django__django-15503\", \"django__django-15525\", \"django__django-15554\", \"django__django-15561\", \"django__django-15563\", \"django__django-15569\", \"django__django-15572\", \"django__django-15629\", \"django__django-15695\", \"django__django-15731\", \"django__django-15732\", \"django__django-15741\", \"django__django-15814\", \"django__django-15851\", \"django__django-15863\", \"django__django-15916\", \"django__django-15930\", \"django__django-15957\", \"django__django-15973\", \"django__django-15987\", \"django__django-16032\", \"django__django-16082\", \"django__django-16100\", \"django__django-16116\", \"django__django-16136\", \"django__django-16139\", \"django__django-16145\", \"django__django-16255\", \"django__django-16256\", \"django__django-16263\", \"django__django-16315\", \"django__django-16333\", \"django__django-16429\", \"django__django-16454\", \"django__django-16485\", \"django__django-16493\", \"django__django-16502\", \"django__django-16527\", \"django__django-16560\", \"django__django-16569\", \"django__django-16595\", \"django__django-16612\", \"django__django-16631\", \"django__django-16642\", \"django__django-16661\", \"django__django-16662\", \"django__django-16667\", \"django__django-16801\", \"django__django-16819\", \"django__django-16877\", \"django__django-16899\", \"django__django-16901\", \"django__django-16938\", \"django__django-16950\", \"django__django-17029\", \"django__django-17084\", \"django__django-17087\", \"django__django-7530\", \"django__django-9296\", \"matplotlib__matplotlib-13989\", \"matplotlib__matplotlib-14623\", \"matplotlib__matplotlib-20488\", \"matplotlib__matplotlib-20676\", \"matplotlib__matplotlib-20826\", \"matplotlib__matplotlib-20859\", \"matplotlib__matplotlib-21568\", \"matplotlib__matplotlib-22719\", \"matplotlib__matplotlib-22865\", \"matplotlib__matplotlib-22871\", \"matplotlib__matplotlib-23299\", \"matplotlib__matplotlib-23314\", \"matplotlib__matplotlib-23412\", \"matplotlib__matplotlib-23476\", \"matplotlib__matplotlib-24026\", \"matplotlib__matplotlib-24149\", \"matplotlib__matplotlib-24177\", \"matplotlib__matplotlib-24570\", \"matplotlib__matplotlib-24627\", \"matplotlib__matplotlib-24637\", \"matplotlib__matplotlib-24870\", \"matplotlib__matplotlib-24970\", \"matplotlib__matplotlib-25122\", \"matplotlib__matplotlib-25287\", \"matplotlib__matplotlib-25311\", \"matplotlib__matplotlib-25332\", \"matplotlib__matplotlib-25479\", \"matplotlib__matplotlib-25775\", \"matplotlib__matplotlib-25960\", \"matplotlib__matplotlib-26113\", \"matplotlib__matplotlib-26208\", \"matplotlib__matplotlib-26291\", \"matplotlib__matplotlib-26342\", \"matplotlib__matplotlib-26466\", \"mwaskom__seaborn-3069\", \"mwaskom__seaborn-3187\", \"pallets__flask-5014\", \"psf__requests-1142\", \"psf__requests-1724\", \"psf__requests-1766\", \"psf__requests-1921\", \"psf__requests-2317\", \"psf__requests-2931\", \"psf__requests-5414\", \"psf__requests-6028\", \"pydata__xarray-2905\", \"pydata__xarray-3095\", \"pydata__xarray-3151\", \"pydata__xarray-3305\", \"pydata__xarray-3677\", \"pydata__xarray-3993\", \"pydata__xarray-4075\", \"pydata__xarray-4094\", \"pydata__xarray-4356\", \"pydata__xarray-4629\", \"pydata__xarray-4687\", \"pydata__xarray-4695\", \"pydata__xarray-4966\", \"pydata__xarray-6461\", \"pydata__xarray-6599\", \"pydata__xarray-6721\", \"pydata__xarray-6744\", \"pydata__xarray-6938\", \"pydata__xarray-6992\", \"pydata__xarray-7229\", \"pydata__xarray-7233\", \"pydata__xarray-7393\", \"pylint-dev__pylint-4551\", \"pylint-dev__pylint-4604\", \"pylint-dev__pylint-4661\", \"pylint-dev__pylint-4970\", \"pylint-dev__pylint-6386\", \"pylint-dev__pylint-6528\", \"pylint-dev__pylint-6903\", \"pylint-dev__pylint-7080\", \"pylint-dev__pylint-7277\", \"pylint-dev__pylint-8898\", \"pytest-dev__pytest-10051\", \"pytest-dev__pytest-10081\", \"pytest-dev__pytest-10356\", \"pytest-dev__pytest-5262\", \"pytest-dev__pytest-5631\", \"pytest-dev__pytest-5787\", \"pytest-dev__pytest-5809\", \"pytest-dev__pytest-5840\", \"pytest-dev__pytest-6197\", \"pytest-dev__pytest-6202\", \"pytest-dev__pytest-7205\", \"pytest-dev__pytest-7236\", \"pytest-dev__pytest-7324\", \"pytest-dev__pytest-7432\", \"pytest-dev__pytest-7490\", \"pytest-dev__pytest-7521\", \"pytest-dev__pytest-7571\", \"pytest-dev__pytest-7982\", \"pytest-dev__pytest-8399\", \"scikit-learn__scikit-learn-10297\", \"scikit-learn__scikit-learn-10844\", \"scikit-learn__scikit-learn-10908\", \"scikit-learn__scikit-learn-11310\", \"scikit-learn__scikit-learn-11578\", \"scikit-learn__scikit-learn-12585\", \"scikit-learn__scikit-learn-12682\", \"scikit-learn__scikit-learn-12973\", \"scikit-learn__scikit-learn-13124\", \"scikit-learn__scikit-learn-13135\", \"scikit-learn__scikit-learn-13142\", \"scikit-learn__scikit-learn-13328\", \"scikit-learn__scikit-learn-13439\", \"scikit-learn__scikit-learn-13496\", \"scikit-learn__scikit-learn-13779\", \"scikit-learn__scikit-learn-14053\", \"scikit-learn__scikit-learn-14087\", \"scikit-learn__scikit-learn-14141\", \"scikit-learn__scikit-learn-14496\", \"scikit-learn__scikit-learn-14629\", \"scikit-learn__scikit-learn-14710\", \"scikit-learn__scikit-learn-14894\", \"scikit-learn__scikit-learn-14983\", \"scikit-learn__scikit-learn-15100\", \"scikit-learn__scikit-learn-25102\", \"scikit-learn__scikit-learn-25232\", \"scikit-learn__scikit-learn-25747\", \"scikit-learn__scikit-learn-25931\", \"scikit-learn__scikit-learn-25973\", \"scikit-learn__scikit-learn-26194\", \"scikit-learn__scikit-learn-26323\", \"scikit-learn__scikit-learn-9288\", \"sphinx-doc__sphinx-10323\", \"sphinx-doc__sphinx-10435\", \"sphinx-doc__sphinx-10449\", \"sphinx-doc__sphinx-10466\", \"sphinx-doc__sphinx-10614\", \"sphinx-doc__sphinx-10673\", \"sphinx-doc__sphinx-11445\", \"sphinx-doc__sphinx-11510\", \"sphinx-doc__sphinx-7440\", \"sphinx-doc__sphinx-7454\", \"sphinx-doc__sphinx-7462\", \"sphinx-doc__sphinx-7590\", \"sphinx-doc__sphinx-7748\", \"sphinx-doc__sphinx-7757\", \"sphinx-doc__sphinx-7889\", \"sphinx-doc__sphinx-7910\", \"sphinx-doc__sphinx-7985\", \"sphinx-doc__sphinx-8035\", \"sphinx-doc__sphinx-8056\", \"sphinx-doc__sphinx-8120\", \"sphinx-doc__sphinx-8265\", \"sphinx-doc__sphinx-8269\", \"sphinx-doc__sphinx-8459\", \"sphinx-doc__sphinx-8475\", \"sphinx-doc__sphinx-8548\", \"sphinx-doc__sphinx-8551\", \"sphinx-doc__sphinx-8593\", \"sphinx-doc__sphinx-8595\", \"sphinx-doc__sphinx-8621\", \"sphinx-doc__sphinx-8638\", \"sphinx-doc__sphinx-8721\", \"sphinx-doc__sphinx-9229\", \"sphinx-doc__sphinx-9230\", \"sphinx-doc__sphinx-9258\", \"sphinx-doc__sphinx-9281\", \"sphinx-doc__sphinx-9320\", \"sphinx-doc__sphinx-9367\", \"sphinx-doc__sphinx-9461\", \"sphinx-doc__sphinx-9591\", \"sphinx-doc__sphinx-9602\", \"sphinx-doc__sphinx-9658\", \"sphinx-doc__sphinx-9673\", \"sphinx-doc__sphinx-9698\", \"sphinx-doc__sphinx-9711\", \"sympy__sympy-11618\", \"sympy__sympy-12096\", \"sympy__sympy-12419\", \"sympy__sympy-12481\", \"sympy__sympy-12489\", \"sympy__sympy-13031\", \"sympy__sympy-13091\", \"sympy__sympy-13372\", \"sympy__sympy-13480\", \"sympy__sympy-13551\", \"sympy__sympy-13615\", \"sympy__sympy-13647\", \"sympy__sympy-13757\", \"sympy__sympy-13798\", \"sympy__sympy-13852\", \"sympy__sympy-13877\", \"sympy__sympy-13878\", \"sympy__sympy-13974\", \"sympy__sympy-14248\", \"sympy__sympy-14531\", \"sympy__sympy-14711\", \"sympy__sympy-14976\", \"sympy__sympy-15017\", \"sympy__sympy-15345\", \"sympy__sympy-15349\", \"sympy__sympy-15599\", \"sympy__sympy-15809\", \"sympy__sympy-15875\", \"sympy__sympy-15976\", \"sympy__sympy-16450\", \"sympy__sympy-16597\", \"sympy__sympy-16766\", \"sympy__sympy-16792\", \"sympy__sympy-16886\", \"sympy__sympy-17139\", \"sympy__sympy-17318\", \"sympy__sympy-17630\", \"sympy__sympy-17655\", \"sympy__sympy-18189\", \"sympy__sympy-18199\", \"sympy__sympy-18211\", \"sympy__sympy-18698\", \"sympy__sympy-18763\", \"sympy__sympy-19040\", \"sympy__sympy-19346\", \"sympy__sympy-19495\", \"sympy__sympy-19637\", \"sympy__sympy-19783\", \"sympy__sympy-19954\", \"sympy__sympy-20154\", \"sympy__sympy-20428\", \"sympy__sympy-20438\", \"sympy__sympy-20590\", \"sympy__sympy-20801\", \"sympy__sympy-20916\", \"sympy__sympy-21379\", \"sympy__sympy-21596\", \"sympy__sympy-21612\", \"sympy__sympy-21847\", \"sympy__sympy-21930\", \"sympy__sympy-22080\", \"sympy__sympy-22456\", \"sympy__sympy-22714\", \"sympy__sympy-22914\", \"sympy__sympy-23262\", \"sympy__sympy-23413\", \"sympy__sympy-23534\", \"sympy__sympy-23824\", \"sympy__sympy-23950\", \"sympy__sympy-24066\", \"sympy__sympy-24213\", \"sympy__sympy-24443\", \"sympy__sympy-24539\", \"sympy__sympy-24562\", \"sympy__sympy-24661\"]",false,1,"[\"mean\"]",null,1000000.0,2500000.0,86400.0,null,"success",null,null,500,462,"swe_bench_scorer_stderr","mean",0.4025974025974026,0.022841182030949984,null,null,null,null,null,null,0.4025974025974026,0.022841182030949984]
["7Y9hLZsQTvxXkbCggXCmtt","BMTzuttHimqSvPKrEZCjYp","PSUzf27k2Wprrt6fYaG5cR","/Users/jjallaire/meridianlabs/inspect_viz/docs/sandbox/logs/swe-bench/2025-06-24T10-42-56+00-00_swe-bench_PSUzf27k2Wprrt6fYaG5cR.eval","2025-06-24T10:42:56.000000000Z","","https://github.com/AI-Safety-Institute/open_source_benchmarks.git","9a28395","{\"inspect_ai\": \"0.3.106.dev11+g28f3586b\"}",null,"swe_bench",0,null,"{}",false,false,null,"princeton-nlp/SWE-bench_Verified","<lambda>",1.0,null,null,"Please solve the following coding issue:\n\n{issue_text}",null,true,"k8s","{\"type\": \"scorer\", \"name\": \"swe_bench_scorer_stderr\", \"params\": {}}","{\"type\": \"agent\", \"name\": \"react\", \"params\": {\"prompt\": \"You are a helpful autonomous software engineer that can interact with a computer to solve tasks. You are challenged to solve a task specified in a PR description for a given code repository.\\n\\nI've already taken care of all changes to any of the test files described in the PR. This means you DON'T have to modify the testing logic or any of the tests in any way!\\nYour task is to make the minimal changes to non-tests files in the working directory to ensure the PR description is satisfied.\\n\\nHere are some steps you might want to follow:\\n1. As a first step, it might be a good idea to find and read code relevant to the PR description\\n2. Create a script to reproduce the error and execute it using the bash tool, to confirm the error\\n3. Edit the source code of the repo to resolve the issue\\n4. Rerun your reproduce script and confirm that the error is fixed!\\n5. Think about edgecases, create your own unit tests and make sure your fix handles them as well\\n\\nYour thinking should be thorough and so it's fine if it's very long.\\n\", \"tools\": [{\"type\": \"tool\", \"name\": \"bash\", \"params\": {\"timeout\": 180}}, {\"type\": \"tool\", \"name\": \"python\", \"params\": {\"timeout\": 180}}, {\"type\": \"tool\", \"name\": \"agents/submit_answer\", \"params\": {}}]}}","test",null,null,null,null,null,null,"anthropic/claude-sonnet-4-20250514",null,null,"{\"max_connections\": 50}",null,"princeton-nlp/SWE-bench_Verified","princeton-nlp/SWE-bench_Verified",500,"[\"astropy__astropy-12907\", \"astropy__astropy-13033\", \"astropy__astropy-13236\", \"astropy__astropy-13398\", \"astropy__astropy-13453\", \"astropy__astropy-13579\", \"astropy__astropy-13977\", \"astropy__astropy-14096\", \"astropy__astropy-14182\", \"astropy__astropy-14309\", \"astropy__astropy-14365\", \"astropy__astropy-14369\", \"astropy__astropy-14508\", \"astropy__astropy-14539\", \"astropy__astropy-14598\", \"astropy__astropy-14995\", \"astropy__astropy-7166\", \"astropy__astropy-7336\", \"astropy__astropy-7606\", \"astropy__astropy-7671\", \"astropy__astropy-8707\", \"astropy__astropy-8872\", \"django__django-10097\", \"django__django-10554\", \"django__django-10880\", \"django__django-10914\", \"django__django-10973\", \"django__django-10999\", \"django__django-11066\", \"django__django-11087\", \"django__django-11095\", \"django__django-11099\", \"django__django-11119\", \"django__django-11133\", \"django__django-11138\", \"django__django-11141\", \"django__django-11149\", \"django__django-11163\", \"django__django-11179\", \"django__django-11206\", \"django__django-11211\", \"django__django-11239\", \"django__django-11265\", \"django__django-11276\", \"django__django-11292\", \"django__django-11299\", \"django__django-11333\", \"django__django-11400\", \"django__django-11433\", \"django__django-11451\", \"django__django-11477\", \"django__django-11490\", \"django__django-11532\", \"django__django-11551\", \"django__django-11555\", \"django__django-11603\", \"django__django-11728\", \"django__django-11734\", \"django__django-11740\", \"django__django-11749\", \"django__django-11790\", \"django__django-11815\", \"django__django-11820\", \"django__django-11848\", \"django__django-11880\", \"django__django-11885\", \"django__django-11951\", \"django__django-11964\", \"django__django-11999\", \"django__django-12039\", \"django__django-12050\", \"django__django-12125\", \"django__django-12143\", \"django__django-12155\", \"django__django-12193\", \"django__django-12209\", \"django__django-12262\", \"django__django-12273\", \"django__django-12276\", \"django__django-12304\", \"django__django-12308\", \"django__django-12325\", \"django__django-12406\", \"django__django-12419\", \"django__django-12663\", \"django__django-12708\", \"django__django-12713\", \"django__django-12741\", \"django__django-12754\", \"django__django-12774\", \"django__django-12858\", \"django__django-12965\", \"django__django-13012\", \"django__django-13023\", \"django__django-13028\", \"django__django-13033\", \"django__django-13089\", \"django__django-13109\", \"django__django-13112\", \"django__django-13121\", \"django__django-13128\", \"django__django-13158\", \"django__django-13195\", \"django__django-13212\", \"django__django-13279\", \"django__django-13297\", \"django__django-13315\", \"django__django-13343\", \"django__django-13344\", \"django__django-13346\", \"django__django-13363\", \"django__django-13401\", \"django__django-13406\", \"django__django-13410\", \"django__django-13417\", \"django__django-13449\", \"django__django-13512\", \"django__django-13513\", \"django__django-13516\", \"django__django-13551\", \"django__django-13568\", \"django__django-13569\", \"django__django-13590\", \"django__django-13658\", \"django__django-13670\", \"django__django-13741\", \"django__django-13786\", \"django__django-13794\", \"django__django-13807\", \"django__django-13809\", \"django__django-13810\", \"django__django-13820\", \"django__django-13821\", \"django__django-13837\", \"django__django-13925\", \"django__django-13933\", \"django__django-13964\", \"django__django-14007\", \"django__django-14011\", \"django__django-14017\", \"django__django-14034\", \"django__django-14053\", \"django__django-14089\", \"django__django-14122\", \"django__django-14140\", \"django__django-14155\", \"django__django-14170\", \"django__django-14238\", \"django__django-14311\", \"django__django-14315\", \"django__django-14349\", \"django__django-14351\", \"django__django-14373\", \"django__django-14376\", \"django__django-14404\", \"django__django-14434\", \"django__django-14493\", \"django__django-14500\", \"django__django-14534\", \"django__django-14539\", \"django__django-14559\", \"django__django-14580\", \"django__django-14608\", \"django__django-14631\", \"django__django-14672\", \"django__django-14725\", \"django__django-14752\", \"django__django-14765\", \"django__django-14771\", \"django__django-14787\", \"django__django-14792\", \"django__django-14855\", \"django__django-14915\", \"django__django-14999\", \"django__django-15022\", \"django__django-15037\", \"django__django-15098\", \"django__django-15103\", \"django__django-15104\", \"django__django-15127\", \"django__django-15128\", \"django__django-15161\", \"django__django-15252\", \"django__django-15268\", \"django__django-15277\", \"django__django-15278\", \"django__django-15280\", \"django__django-15315\", \"django__django-15368\", \"django__django-15375\", \"django__django-15380\", \"django__django-15382\", \"django__django-15467\", \"django__django-15499\", \"django__django-15503\", \"django__django-15525\", \"django__django-15554\", \"django__django-15561\", \"django__django-15563\", \"django__django-15569\", \"django__django-15572\", \"django__django-15629\", \"django__django-15695\", \"django__django-15731\", \"django__django-15732\", \"django__django-15741\", \"django__django-15814\", \"django__django-15851\", \"django__django-15863\", \"django__django-15916\", \"django__django-15930\", \"django__django-15957\", \"django__django-15973\", \"django__django-15987\", \"django__django-16032\", \"django__django-16082\", \"django__django-16100\", \"django__django-16116\", \"django__django-16136\", \"django__django-16139\", \"django__django-16145\", \"django__django-16255\", \"django__django-16256\", \"django__django-16263\", \"django__django-16315\", \"django__django-16333\", \"django__django-16429\", \"django__django-16454\", \"django__django-16485\", \"django__django-16493\", \"django__django-16502\", \"django__django-16527\", \"django__django-16560\", \"django__django-16569\", \"django__django-16595\", \"django__django-16612\", \"django__django-16631\", \"django__django-16642\", \"django__django-16661\", \"django__django-16662\", \"django__django-16667\", \"django__django-16801\", \"django__django-16819\", \"django__django-16877\", \"django__django-16899\", \"django__django-16901\", \"django__django-16938\", \"django__django-16950\", \"django__django-17029\", \"django__django-17084\", \"django__django-17087\", \"django__django-7530\", \"django__django-9296\", \"matplotlib__matplotlib-13989\", \"matplotlib__matplotlib-14623\", \"matplotlib__matplotlib-20488\", \"matplotlib__matplotlib-20676\", \"matplotlib__matplotlib-20826\", \"matplotlib__matplotlib-20859\", \"matplotlib__matplotlib-21568\", \"matplotlib__matplotlib-22719\", \"matplotlib__matplotlib-22865\", \"matplotlib__matplotlib-22871\", \"matplotlib__matplotlib-23299\", \"matplotlib__matplotlib-23314\", \"matplotlib__matplotlib-23412\", \"matplotlib__matplotlib-23476\", \"matplotlib__matplotlib-24026\", \"matplotlib__matplotlib-24149\", \"matplotlib__matplotlib-24177\", \"matplotlib__matplotlib-24570\", \"matplotlib__matplotlib-24627\", \"matplotlib__matplotlib-24637\", \"matplotlib__matplotlib-24870\", \"matplotlib__matplotlib-24970\", \"matplotlib__matplotlib-25122\", \"matplotlib__matplotlib-25287\", \"matplotlib__matplotlib-25311\", \"matplotlib__matplotlib-25332\", \"matplotlib__matplotlib-25479\", \"matplotlib__matplotlib-25775\", \"matplotlib__matplotlib-25960\", \"matplotlib__matplotlib-26113\", \"matplotlib__matplotlib-26208\", \"matplotlib__matplotlib-26291\", \"matplotlib__matplotlib-26342\", \"matplotlib__matplotlib-26466\", \"mwaskom__seaborn-3069\", \"mwaskom__seaborn-3187\", \"pallets__flask-5014\", \"psf__requests-1142\", \"psf__requests-1724\", \"psf__requests-1766\", \"psf__requests-1921\", \"psf__requests-2317\", \"psf__requests-2931\", \"psf__requests-5414\", \"psf__requests-6028\", \"pydata__xarray-2905\", \"pydata__xarray-3095\", \"pydata__xarray-3151\", \"pydata__xarray-3305\", \"pydata__xarray-3677\", \"pydata__xarray-3993\", \"pydata__xarray-4075\", \"pydata__xarray-4094\", \"pydata__xarray-4356\", \"pydata__xarray-4629\", \"pydata__xarray-4687\", \"pydata__xarray-4695\", \"pydata__xarray-4966\", \"pydata__xarray-6461\", \"pydata__xarray-6599\", \"pydata__xarray-6721\", \"pydata__xarray-6744\", \"pydata__xarray-6938\", \"pydata__xarray-6992\", \"pydata__xarray-7229\", \"pydata__xarray-7233\", \"pydata__xarray-7393\", \"pylint-dev__pylint-4551\", \"pylint-dev__pylint-4604\", \"pylint-dev__pylint-4661\", \"pylint-dev__pylint-4970\", \"pylint-dev__pylint-6386\", \"pylint-dev__pylint-6528\", \"pylint-dev__pylint-6903\", \"pylint-dev__pylint-7080\", \"pylint-dev__pylint-7277\", \"pylint-dev__pylint-8898\", \"pytest-dev__pytest-10051\", \"pytest-dev__pytest-10081\", \"pytest-dev__pytest-10356\", \"pytest-dev__pytest-5262\", \"pytest-dev__pytest-5631\", \"pytest-dev__pytest-5787\", \"pytest-dev__pytest-5809\", \"pytest-dev__pytest-5840\", \"pytest-dev__pytest-6197\", \"pytest-dev__pytest-6202\", \"pytest-dev__pytest-7205\", \"pytest-dev__pytest-7236\", \"pytest-dev__pytest-7324\", \"pytest-dev__pytest-7432\", \"pytest-dev__pytest-7490\", \"pytest-dev__pytest-7521\", \"pytest-dev__pytest-7571\", \"pytest-dev__pytest-7982\", \"pytest-dev__pytest-8399\", \"scikit-learn__scikit-learn-10297\", \"scikit-learn__scikit-learn-10844\", \"scikit-learn__scikit-learn-10908\", \"scikit-learn__scikit-learn-11310\", \"scikit-learn__scikit-learn-11578\", \"scikit-learn__scikit-learn-12585\", \"scikit-learn__scikit-learn-12682\", \"scikit-learn__scikit-learn-12973\", \"scikit-learn__scikit-learn-13124\", \"scikit-learn__scikit-learn-13135\", \"scikit-learn__scikit-learn-13142\", \"scikit-learn__scikit-learn-13328\", \"scikit-learn__scikit-learn-13439\", \"scikit-learn__scikit-learn-13496\", \"scikit-learn__scikit-learn-13779\", \"scikit-learn__scikit-learn-14053\", \"scikit-learn__scikit-learn-14087\", \"scikit-learn__scikit-learn-14141\", \"scikit-learn__scikit-learn-14496\", \"scikit-learn__scikit-learn-14629\", \"scikit-learn__scikit-learn-14710\", \"scikit-learn__scikit-learn-14894\", \"scikit-learn__scikit-learn-14983\", \"scikit-learn__scikit-learn-15100\", \"scikit-learn__scikit-learn-25102\", \"scikit-learn__scikit-learn-25232\", \"scikit-learn__scikit-learn-25747\", \"scikit-learn__scikit-learn-25931\", \"scikit-learn__scikit-learn-25973\", \"scikit-learn__scikit-learn-26194\", \"scikit-learn__scikit-learn-26323\", \"scikit-learn__scikit-learn-9288\", \"sphinx-doc__sphinx-10323\", \"sphinx-doc__sphinx-10435\", \"sphinx-doc__sphinx-10449\", \"sphinx-doc__sphinx-10466\", \"sphinx-doc__sphinx-10614\", \"sphinx-doc__sphinx-10673\", \"sphinx-doc__sphinx-11445\", \"sphinx-doc__sphinx-11510\", \"sphinx-doc__sphinx-7440\", \"sphinx-doc__sphinx-7454\", \"sphinx-doc__sphinx-7462\", \"sphinx-doc__sphinx-7590\", \"sphinx-doc__sphinx-7748\", \"sphinx-doc__sphinx-7757\", \"sphinx-doc__sphinx-7889\", \"sphinx-doc__sphinx-7910\", \"sphinx-doc__sphinx-7985\", \"sphinx-doc__sphinx-8035\", \"sphinx-doc__sphinx-8056\", \"sphinx-doc__sphinx-8120\", \"sphinx-doc__sphinx-8265\", \"sphinx-doc__sphinx-8269\", \"sphinx-doc__sphinx-8459\", \"sphinx-doc__sphinx-8475\", \"sphinx-doc__sphinx-8548\", \"sphinx-doc__sphinx-8551\", \"sphinx-doc__sphinx-8593\", \"sphinx-doc__sphinx-8595\", \"sphinx-doc__sphinx-8621\", \"sphinx-doc__sphinx-8638\", \"sphinx-doc__sphinx-8721\", \"sphinx-doc__sphinx-9229\", \"sphinx-doc__sphinx-9230\", \"sphinx-doc__sphinx-9258\", \"sphinx-doc__sphinx-9281\", \"sphinx-doc__sphinx-9320\", \"sphinx-doc__sphinx-9367\", \"sphinx-doc__sphinx-9461\", \"sphinx-doc__sphinx-9591\", \"sphinx-doc__sphinx-9602\", \"sphinx-doc__sphinx-9658\", \"sphinx-doc__sphinx-9673\", \"sphinx-doc__sphinx-9698\", \"sphinx-doc__sphinx-9711\", \"sympy__sympy-11618\", \"sympy__sympy-12096\", \"sympy__sympy-12419\", \"sympy__sympy-12481\", \"sympy__sympy-12489\", \"sympy__sympy-13031\", \"sympy__sympy-13091\", \"sympy__sympy-13372\", \"sympy__sympy-13480\", \"sympy__sympy-13551\", \"sympy__sympy-13615\", \"sympy__sympy-13647\", \"sympy__sympy-13757\", \"sympy__sympy-13798\", \"sympy__sympy-13852\", \"sympy__sympy-13877\", \"sympy__sympy-13878\", \"sympy__sympy-13974\", \"sympy__sympy-14248\", \"sympy__sympy-14531\", \"sympy__sympy-14711\", \"sympy__sympy-14976\", \"sympy__sympy-15017\", \"sympy__sympy-15345\", \"sympy__sympy-15349\", \"sympy__sympy-15599\", \"sympy__sympy-15809\", \"sympy__sympy-15875\", \"sympy__sympy-15976\", \"sympy__sympy-16450\", \"sympy__sympy-16597\", \"sympy__sympy-16766\", \"sympy__sympy-16792\", \"sympy__sympy-16886\", \"sympy__sympy-17139\", \"sympy__sympy-17318\", \"sympy__sympy-17630\", \"sympy__sympy-17655\", \"sympy__sympy-18189\", \"sympy__sympy-18199\", \"sympy__sympy-18211\", \"sympy__sympy-18698\", \"sympy__sympy-18763\", \"sympy__sympy-19040\", \"sympy__sympy-19346\", \"sympy__sympy-19495\", \"sympy__sympy-19637\", \"sympy__sympy-19783\", \"sympy__sympy-19954\", \"sympy__sympy-20154\", \"sympy__sympy-20428\", \"sympy__sympy-20438\", \"sympy__sympy-20590\", \"sympy__sympy-20801\", \"sympy__sympy-20916\", \"sympy__sympy-21379\", \"sympy__sympy-21596\", \"sympy__sympy-21612\", \"sympy__sympy-21847\", \"sympy__sympy-21930\", \"sympy__sympy-22080\", \"sympy__sympy-22456\", \"sympy__sympy-22714\", \"sympy__sympy-22914\", \"sympy__sympy-23262\", \"sympy__sympy-23413\", \"sympy__sympy-23534\", \"sympy__sympy-23824\", \"sympy__sympy-23950\", \"sympy__sympy-24066\", \"sympy__sympy-24213\", \"sympy__sympy-24443\", \"sympy__sympy-24539\", \"sympy__sympy-24562\", \"sympy__sympy-24661\"]",false,1,"[\"mean\"]",null,1000000.0,2500000.0,86400.0,null,"success",null,null,500,500,"swe_bench_scorer_stderr","mean",0.674,0.020984009562393675,null,null,null,null,null,null,0.674,0.020984009562393675]
["Weh8ToHWwt2nkCxertW9y2","7HfsPLmSqDJrGjRhgVeWNo","As9m4VPWJvAqe9VNEkhhVN","/Users/jjallaire/meridianlabs/inspect_viz/docs/sandbox/logs/mmlu-pro/2025-06-19T16-56-45+00-00_mmlu-pro_As9m4VPWJvAqe9VNEkhhVN.eval","2025-06-19T16:56:45.000000000Z","","https://github.com/AI-Safety-Institute/open_source_benchmarks.git","9a28395","{\"inspect_ai\": \"0.3.106.dev11+g28f3586b\"}",null,"mmlu_pro",0,null,"{}",null,null,null,null,null,null,null,0.0,null,null,null,null,null,null,null,"[]",null,null,null,null,null,"openai/o3",null,null,"{\"max_connections\": 50}",null,"TIGER-Lab/MMLU-Pro","TIGER-Lab/MMLU-Pro",12032,{"$cut":{"chars":73467,"head":"[8690, 2212, 7697, 10283, 12225, 10518, 6574, 10878, 5748, 4102, 5410, 5179, 1346, 12032, 4006, 4813, 8726, 10018, 1432, 2979, 938, 9448, 5184, 1750, 3685, 742, 8222, 8766, 8864, 10323, 7290, 10757, 6451, 160, 3409, 3262, 5436, 6522, 8015, 434, 2857, 4671, 7299, 1851, 1519, 9646, 4547, 6152, 4165, 5728, 9152, 11826, 7225, 7319, 10410, 12004, 6856, 5543, 5103, 7366, 10672, 10851, 5520, 8769, 8184, 6373, 6010, 8852, 9997, 12179, 3733, 1061, 621, 4726, 4544, 5187, 3414, 5730, 3906, 1847, 10459, 6006, 6654, 3888, 10461, 2226, 1994, 4851, 9648, 9248, 6913, 4536, 7949, 952, 11356, 10079, 1362, 132, 7882, 3182, 1901, 203, 8102, 265, 9401, 2045, 9375, 10923, 11532, 7552, 11817, 11269, 7772, 11239, 3580, 1363, 891, 3601, 6761, 3019, 9222, 1705, 3506, 1159, 4993, 7993, 7500, 9532, 10041, 4844, 5273, 11535, 11934, 10108, 8353, 5020, 5690, 8433, 4252, 636, 8731, 11339, 10270, 11647, 5702, 6440, 4472, 7508, 8125, 8553, 11129, 7362, 11775, 4057, 4780, 9131, 6674, 888, 11141, 3726, 11958, 3422, 1237, 4073, 4224, 11035, 6289, 3300, 2525, 9712, 9101, 2115, 1960, 8539, 5489, 4162, 9999, 10717, 3266, 1138, 11649, 7869, 6905, 7268, 4271, 1720, 8223, 3718, 7216, 5896, 10371, 10103, 2184, 11624, 4150, 10880, 7607, 10834, 6202, 1187, 2369, 6624, 1036, 7894, 10881, 2112, 6228, 6992, 5701, 6292, 8045, 9449, 5060, 3160, 2753, 687, 2109, 10446, 7171, 7565, 7936, 4870, 5998, 3441, 10185, 5746, 7396, 960, 8199, 8172, 9714, 9107, 8080, 1827, 8835, 11416, 10636, 12166, 4808, 4476, 11079, 4815, 11891, 9798, 2175, 6605, 4502, 6935, 2396, 8122, 10160, 8811, 3319, 10996, 7828, 8989, 2391, 3403, 10004, 7315, 8426, 9185, 1876, 1030, 4795, 4961, 101, 2234, 3448, 6964, 4321, 10802, 7482, 8119, 5627, 9553, 2584, 9704, 2496, 8049, 4344, 12205, 914, 795, 11833, 5575, 2338, 2904, 11454, 3779, 8672, 9435, 8350, 11271, 9958, 8588, 3885, 10710, 10470, 4718, 8253, 6644, 8136, 3277, 12011, 10832, 4461, 8575, 11992, 11940, 6299, 10435, 7959, 3595, 11805, 10366, 8843, 1834, 10946, 7853, 8133, 5326, 10221, 1511, 11815, 4414, 9894, 10096, 6028, 3456, 4287, 1786, 2516, 2208, 9828, 2866, 8194, 6994, 7851, 5000, 4516, 3507, 1860, 3665, 8337, 9899, 1250, 7186, 1984, 4790, 466, 3599, 6771, 4107, 1990, 8399, 4346, 11840, 10347, 7280, 7014, 3489, 7388, 5065, 7994, 10257, 5350, 11092, 1115, 9970, 6980, 7516, 585, 6125, 9015, 1526, 12209, 10599, 11173, 6009, 844, 475, 6715, 3342, 970, 2550, 4929, 5516, 12232, 9783, 7413, 6221, 4890, 695, 11512, 6114, 6060, 4914, 11831, 1546, 6168, 2149, 10030, 9587, 7087, 7582, 9115, 5576, 5562, 6629, 6516, 7757, 10831, 6969, 6627, 5336, 4087, 2446, 11637, 3899, 8611, 3551, 9388, 1871, 476, 7494, 6193, 9332, 7227, 8517, 8904, 7799, 2424, 6367, 10746, 7843, 4843, 4837, 6421, 5647, 8075, 9720, 3897, 6840, 3709, 1025, 7944, 231, 6501, 10258, 7553, 6372, 6117, 5558, 5693, 2392, 11529, 10645, 3011, 10054, 1790, 12036, 8657, 1711, 10563, 9519, 5453, 2243, 6346, 5648, 6358, 12110, 95, 11632, 533, 6040, 8720, 7005, 10708, 4456, 5657, 9406, 7956, 2568, 8175, 8841, 836, 4739, 3482, 1192, 11172, 7722, 3109, 734, 7161, 12254, 11462, 12241, 5503, 1046, 9998, 10656, 1506, 5803, 5445, 10003, 2676, 3171, 2254, 1784, 9109, 2751, 5032, 5364, 1817, 1776, 9070, 6514, 2545, 11227, 3829, 3824, 6727, 8617, 10276, 5222, 10232, 2168, 9060, 6073, 7355, 3634, 7275, 6083, 2209, 12062, 6554, 3658, 942, 2769, 2009, 10861, 6885, 7580, 3592, 7301, 7253, 4879, 7051, 10836, 2280, 6245, 10478, 10843, 12093, 3079, 3141, 2164, 4569, 12246, 10200, 5149, 1158, 1279, 10218, 10138, 1482, 11069, 3631, 1742, 613, 6489, 4170, 8974, 10679, 10551, 9851, 12193, 691, 12144, 10900, 6582, 4439, 850, 7286, 2133, 6877, 3982, 249, 6178, 4533, 7953, 1589, 1761, 11328, 2884, 12116, 6088, 9816, 2933, 2471, 3204, 2598, 3763, 1226, 6900, 11950, 10873, 3679, 4866, 9700, 9476, 11386, 10284, 6334, 8648, 3762, 3577, 91, 9548, 11157, 9824, 3149, 599, 7088, 8272, 955, 4454, 1567, 4607, 2230, 9094, 1504, 7719, 11621, 6384, 11001, 6450, 3176, 149, 6287, 7974, 9141, 4071, 4442, 3186, 4092, 4681, 9292, 565, 4634, 3729, 7172, 4937, 11524, 1332, 9691, 10203, 6610, 1140, 9737, 3084, 10155, 1389, 7129, 5486, 10133, 36"}},false,1,"[\"mean\"]",null,null,null,null,null,"success",null,null,12032,12032,"choice","accuracy",0.851313164893617,0.0032436207498949205,0.851313164893617,0.0032436207498949205,null,null,null,null,null,null]
["bBXXptqoabQP4SZQBK3e5t","7HfsPLmSqDJrGjRhgVeWNo","mev6aXhESWt5mSQg8F8RUQ","/Users/jjallaire/meridianlabs/inspect_viz/docs/sandbox/logs/mmlu-pro/2025-06-19T16-56-45+00-00_mmlu-pro_mev6aXhESWt5mSQg8F8RUQ.eval","2025-06-19T16:56:45.000000000Z","","https://github.com/AI-Safety-Institute/open_source_benchmarks.git","9a28395","{\"inspect_ai\": \"0.3.106.dev11+g28f3586b\"}",null,"mmlu_pro",0,null,"{}",null,null,null,null,null,null,null,0.0,null,null,null,null,null,null,null,"[]",null,null,null,null,null,"anthropic/claude-3-7-sonnet-latest",null,null,"{\"max_connections\": 50}",null,"TIGER-Lab/MMLU-Pro","TIGER-Lab/MMLU-Pro",12032,{"$cut":{"chars":73467,"head":"[8690, 2212, 7697, 10283, 12225, 10518, 6574, 10878, 5748, 4102, 5410, 5179, 1346, 12032, 4006, 4813, 8726, 10018, 1432, 2979, 938, 9448, 5184, 1750, 3685, 742, 8222, 8766, 8864, 10323, 7290, 10757, 6451, 160, 3409, 3262, 5436, 6522, 8015, 434, 2857, 4671, 7299, 1851, 1519, 9646, 4547, 6152, 4165, 5728, 9152, 11826, 7225, 7319, 10410, 12004, 6856, 5543, 5103, 7366, 10672, 10851, 5520, 8769, 8184, 6373, 6010, 8852, 9997, 12179, 3733, 1061, 621, 4726, 4544, 5187, 3414, 5730, 3906, 1847, 10459, 6006, 6654, 3888, 10461, 2226, 1994, 4851, 9648, 9248, 6913, 4536, 7949, 952, 11356, 10079, 1362, 132, 7882, 3182, 1901, 203, 8102, 265, 9401, 2045, 9375, 10923, 11532, 7552, 11817, 11269, 7772, 11239, 3580, 1363, 891, 3601, 6761, 3019, 9222, 1705, 3506, 1159, 4993, 7993, 7500, 9532, 10041, 4844, 5273, 11535, 11934, 10108, 8353, 5020, 5690, 8433, 4252, 636, 8731, 11339, 10270, 11647, 5702, 6440, 4472, 7508, 8125, 8553, 11129, 7362, 11775, 4057, 4780, 9131, 6674, 888, 11141, 3726, 11958, 3422, 1237, 4073, 4224, 11035, 6289, 3300, 2525, 9712, 9101, 2115, 1960, 8539, 5489, 4162, 9999, 10717, 3266, 1138, 11649, 7869, 6905, 7268, 4271, 1720, 8223, 3718, 7216, 5896, 10371, 10103, 2184, 11624, 4150, 10880, 7607, 10834, 6202, 1187, 2369, 6624, 1036, 7894, 10881, 2112, 6228, 6992, 5701, 6292, 8045, 9449, 5060, 3160, 2753, 687, 2109, 10446, 7171, 7565, 7936, 4870, 5998, 3441, 10185, 5746, 7396, 960, 8199, 8172, 9714, 9107, 8080, 1827, 8835, 11416, 10636, 12166, 4808, 4476, 11079, 4815, 11891, 9798, 2175, 6605, 4502, 6935, 2396, 8122, 10160, 8811, 3319, 10996, 7828, 8989, 2391, 3403, 10004, 7315, 8426, 9185, 1876, 1030, 4795, 4961, 101, 2234, 3448, 6964, 4321, 10802, 7482, 8119, 5627, 9553, 2584, 9704, 2496, 8049, 4344, 12205, 914, 795, 11833, 5575, 2338, 2904, 11454, 3779, 8672, 9435, 8350, 11271, 9958, 8588, 3885, 10710, 10470, 4718, 8253, 6644, 8136, 3277, 12011, 10832, 4461, 8575, 11992, 11940, 6299, 10435, 7959, 3595, 11805, 10366, 8843, 1834, 10946, 7853, 8133, 5326, 10221, 1511, 11815, 4414, 9894, 10096, 6028, 3456, 4287, 1786, 2516, 2208, 9828, 2866, 8194, 6994, 7851, 5000, 4516, 3507, 1860, 3665, 8337, 9899, 1250, 7186, 1984, 4790, 466, 3599, 6771, 4107, 1990, 8399, 4346, 11840, 10347, 7280, 7014, 3489, 7388, 5065, 7994, 10257, 5350, 11092, 1115, 9970, 6980, 7516, 585, 6125, 9015, 1526, 12209, 10599, 11173, 6009, 844, 475, 6715, 3342, 970, 2550, 4929, 5516, 12232, 9783, 7413, 6221, 4890, 695, 11512, 6114, 6060, 4914, 11831, 1546, 6168, 2149, 10030, 9587, 7087, 7582, 9115, 5576, 5562, 6629, 6516, 7757, 10831, 6969, 6627, 5336, 4087, 2446, 11637, 3899, 8611, 3551, 9388, 1871, 476, 7494, 6193, 9332, 7227, 8517, 8904, 7799, 2424, 6367, 10746, 7843, 4843, 4837, 6421, 5647, 8075, 9720, 3897, 6840, 3709, 1025, 7944, 231, 6501, 10258, 7553, 6372, 6117, 5558, 5693, 2392, 11529, 10645, 3011, 10054, 1790, 12036, 8657, 1711, 10563, 9519, 5453, 2243, 6346, 5648, 6358, 12110, 95, 11632, 533, 6040, 8720, 7005, 10708, 4456, 5657, 9406, 7956, 2568, 8175, 8841, 836, 4739, 3482, 1192, 11172, 7722, 3109, 734, 7161, 12254, 11462, 12241, 5503, 1046, 9998, 10656, 1506, 5803, 5445, 10003, 2676, 3171, 2254, 1784, 9109, 2751, 5032, 5364, 1817, 1776, 9070, 6514, 2545, 11227, 3829, 3824, 6727, 8617, 10276, 5222, 10232, 2168, 9060, 6073, 7355, 3634, 7275, 6083, 2209, 12062, 6554, 3658, 942, 2769, 2009, 10861, 6885, 7580, 3592, 7301, 7253, 4879, 7051, 10836, 2280, 6245, 10478, 10843, 12093, 3079, 3141, 2164, 4569, 12246, 10200, 5149, 1158, 1279, 10218, 10138, 1482, 11069, 3631, 1742, 613, 6489, 4170, 8974, 10679, 10551, 9851, 12193, 691, 12144, 10900, 6582, 4439, 850, 7286, 2133, 6877, 3982, 249, 6178, 4533, 7953, 1589, 1761, 11328, 2884, 12116, 6088, 9816, 2933, 2471, 3204, 2598, 3763, 1226, 6900, 11950, 10873, 3679, 4866, 9700, 9476, 11386, 10284, 6334, 8648, 3762, 3577, 91, 9548, 11157, 9824, 3149, 599, 7088, 8272, 955, 4454, 1567, 4607, 2230, 9094, 1504, 7719, 11621, 6384, 11001, 6450, 3176, 149, 6287, 7974, 9141, 4071, 4442, 3186, 4092, 4681, 9292, 565, 4634, 3729, 7172, 4937, 11524, 1332, 9691, 10203, 6610, 1140, 9737, 3084, 10155, 1389, 7129, 5486, 10133, 36"}},false,1,"[\"mean\"]",null,null,null,null,null,"success",null,null,12032,12032,"choice","accuracy",0.8056017287234043,0.0036079086307714225,0.8056017287234043,0.0036079086307714225,null,null,null,null,null,null]
["XtFoQ9BfyCWrfTPCqBPkTB","7HfsPLmSqDJrGjRhgVeWNo","RkvG7pYpVAsyuE44YQ5T6k","/Users/jjallaire/meridianlabs/inspect_viz/docs/sandbox/logs/mmlu-pro/2025-06-19T16-56-45+00-00_mmlu-pro_RkvG7pYpVAsyuE44YQ5T6k.eval","2025-06-19T16:56:45.000000000Z","","https://github.com/AI-Safety-Institute/open_source_benchmarks.git","9a28395","{\"inspect_ai\": \"0.3.106.dev11+g28f3586b\"}",null,"mmlu_pro",0,null,"{}",null,null,null,null,null,null,null,0.0,null,null,null,null,null,null,null,"[]",null,null,null,null,null,"openai/o4-mini",null,null,"{\"max_connections\": 50}",null,"TIGER-Lab/MMLU-Pro","TIGER-Lab/MMLU-Pro",12032,{"$cut":{"chars":73467,"head":"[8690, 2212, 7697, 10283, 12225, 10518, 6574, 10878, 5748, 4102, 5410, 5179, 1346, 12032, 4006, 4813, 8726, 10018, 1432, 2979, 938, 9448, 5184, 1750, 3685, 742, 8222, 8766, 8864, 10323, 7290, 10757, 6451, 160, 3409, 3262, 5436, 6522, 8015, 434, 2857, 4671, 7299, 1851, 1519, 9646, 4547, 6152, 4165, 5728, 9152, 11826, 7225, 7319, 10410, 12004, 6856, 5543, 5103, 7366, 10672, 10851, 5520, 8769, 8184, 6373, 6010, 8852, 9997, 12179, 3733, 1061, 621, 4726, 4544, 5187, 3414, 5730, 3906, 1847, 10459, 6006, 6654, 3888, 10461, 2226, 1994, 4851, 9648, 9248, 6913, 4536, 7949, 952, 11356, 10079, 1362, 132, 7882, 3182, 1901, 203, 8102, 265, 9401, 2045, 9375, 10923, 11532, 7552, 11817, 11269, 7772, 11239, 3580, 1363, 891, 3601, 6761, 3019, 9222, 1705, 3506, 1159, 4993, 7993, 7500, 9532, 10041, 4844, 5273, 11535, 11934, 10108, 8353, 5020, 5690, 8433, 4252, 636, 8731, 11339, 10270, 11647, 5702, 6440, 4472, 7508, 8125, 8553, 11129, 7362, 11775, 4057, 4780, 9131, 6674, 888, 11141, 3726, 11958, 3422, 1237, 4073, 4224, 11035, 6289, 3300, 2525, 9712, 9101, 2115, 1960, 8539, 5489, 4162, 9999, 10717, 3266, 1138, 11649, 7869, 6905, 7268, 4271, 1720, 8223, 3718, 7216, 5896, 10371, 10103, 2184, 11624, 4150, 10880, 7607, 10834, 6202, 1187, 2369, 6624, 1036, 7894, 10881, 2112, 6228, 6992, 5701, 6292, 8045, 9449, 5060, 3160, 2753, 687, 2109, 10446, 7171, 7565, 7936, 4870, 5998, 3441, 10185, 5746, 7396, 960, 8199, 8172, 9714, 9107, 8080, 1827, 8835, 11416, 10636, 12166, 4808, 4476, 11079, 4815, 11891, 9798, 2175, 6605, 4502, 6935, 2396, 8122, 10160, 8811, 3319, 10996, 7828, 8989, 2391, 3403, 10004, 7315, 8426, 9185, 1876, 1030, 4795, 4961, 101, 2234, 3448, 6964, 4321, 10802, 7482, 8119, 5627, 9553, 2584, 9704, 2496, 8049, 4344, 12205, 914, 795, 11833, 5575, 2338, 2904, 11454, 3779, 8672, 9435, 8350, 11271, 9958, 8588, 3885, 10710, 10470, 4718, 8253, 6644, 8136, 3277, 12011, 10832, 4461, 8575, 11992, 11940, 6299, 10435, 7959, 3595, 11805, 10366, 8843, 1834, 10946, 7853, 8133, 5326, 10221, 1511, 11815, 4414, 9894, 10096, 6028, 3456, 4287, 1786, 2516, 2208, 9828, 2866, 8194, 6994, 7851, 5000, 4516, 3507, 1860, 3665, 8337, 9899, 1250, 7186, 1984, 4790, 466, 3599, 6771, 4107, 1990, 8399, 4346, 11840, 10347, 7280, 7014, 3489, 7388, 5065, 7994, 10257, 5350, 11092, 1115, 9970, 6980, 7516, 585, 6125, 9015, 1526, 12209, 10599, 11173, 6009, 844, 475, 6715, 3342, 970, 2550, 4929, 5516, 12232, 9783, 7413, 6221, 4890, 695, 11512, 6114, 6060, 4914, 11831, 1546, 6168, 2149, 10030, 9587, 7087, 7582, 9115, 5576, 5562, 6629, 6516, 7757, 10831, 6969, 6627, 5336, 4087, 2446, 11637, 3899, 8611, 3551, 9388, 1871, 476, 7494, 6193, 9332, 7227, 8517, 8904, 7799, 2424, 6367, 10746, 7843, 4843, 4837, 6421, 5647, 8075, 9720, 3897, 6840, 3709, 1025, 7944, 231, 6501, 10258, 7553, 6372, 6117, 5558, 5693, 2392, 11529, 10645, 3011, 10054, 1790, 12036, 8657, 1711, 10563, 9519, 5453, 2243, 6346, 5648, 6358, 12110, 95, 11632, 533, 6040, 8720, 7005, 10708, 4456, 5657, 9406, 7956, 2568, 8175, 8841, 836, 4739, 3482, 1192, 11172, 7722, 3109, 734, 7161, 12254, 11462, 12241, 5503, 1046, 9998, 10656, 1506, 5803, 5445, 10003, 2676, 3171, 2254, 1784, 9109, 2751, 5032, 5364, 1817, 1776, 9070, 6514, 2545, 11227, 3829, 3824, 6727, 8617, 10276, 5222, 10232, 2168, 9060, 6073, 7355, 3634, 7275, 6083, 2209, 12062, 6554, 3658, 942, 2769, 2009, 10861, 6885, 7580, 3592, 7301, 7253, 4879, 7051, 10836, 2280, 6245, 10478, 10843, 12093, 3079, 3141, 2164, 4569, 12246, 10200, 5149, 1158, 1279, 10218, 10138, 1482, 11069, 3631, 1742, 613, 6489, 4170, 8974, 10679, 10551, 9851, 12193, 691, 12144, 10900, 6582, 4439, 850, 7286, 2133, 6877, 3982, 249, 6178, 4533, 7953, 1589, 1761, 11328, 2884, 12116, 6088, 9816, 2933, 2471, 3204, 2598, 3763, 1226, 6900, 11950, 10873, 3679, 4866, 9700, 9476, 11386, 10284, 6334, 8648, 3762, 3577, 91, 9548, 11157, 9824, 3149, 599, 7088, 8272, 955, 4454, 1567, 4607, 2230, 9094, 1504, 7719, 11621, 6384, 11001, 6450, 3176, 149, 6287, 7974, 9141, 4071, 4442, 3186, 4092, 4681, 9292, 565, 4634, 3729, 7172, 4937, 11524, 1332, 9691, 10203, 6610, 1140, 9737, 3084, 10155, 1389, 7129, 5486, 10133, 36"}},false,1,"[\"mean\"]",null,null,null,null,null,"success",null,null,12032,12032,"choice","accuracy",0.8285405585106383,0.0034362656068830178,0.8285405585106383,0.0034362656068830178,null,null,null,null,null,null]
["SMTwec9XgMeVS9MZLuj3rB","QXy33ZJrxbjsTsriAtBbfj","nfqYMxhQpLA5cSkvgL8ZZD","/Users/jjallaire/meridianlabs/inspect_viz/docs/sandbox/logs/mmlu-pro/2025-06-28T11-25-41+00-00_mmlu-pro_nfqYMxhQpLA5cSkvgL8ZZD.eval","2025-06-28T11:25:41.000000000Z","","git@github.com:AI-Safety-Institute/open_source_benchmarks.git","089f78e","{\"inspect_ai\": \"0.3.106.dev11+g28f3586b\"}",null,"mmlu_pro",0,null,"{}",null,null,null,null,null,null,null,0.0,null,null,null,null,null,null,null,"[]",null,null,null,null,null,"anthropic/claude-sonnet-4-20250514",null,null,"{\"max_connections\": 100}",null,"TIGER-Lab/MMLU-Pro","TIGER-Lab/MMLU-Pro",12032,{"$cut":{"chars":73467,"head":"[2697, 6817, 696, 1180, 9577, 5975, 8556, 3819, 11426, 974, 6863, 9395, 8812, 6225, 5660, 8235, 5206, 9594, 204, 10936, 4525, 2354, 7542, 6703, 3661, 11708, 8164, 9908, 6275, 5887, 6428, 1498, 11793, 7927, 2136, 10000, 2123, 387, 7843, 2625, 1204, 7978, 81, 9573, 5976, 2740, 8582, 4502, 7700, 6441, 6361, 11434, 5514, 2252, 3067, 2970, 10543, 8181, 9857, 6871, 8659, 5743, 7494, 10573, 4734, 1083, 5415, 2100, 3285, 3015, 8601, 4678, 3582, 9027, 10162, 508, 2428, 725, 1197, 7905, 115, 2137, 10384, 7064, 9854, 11946, 11535, 11317, 9379, 308, 2222, 6743, 4133, 5518, 4449, 3758, 10150, 9796, 5404, 10308, 3224, 2487, 6518, 10062, 7722, 9817, 1699, 6847, 7501, 2857, 11996, 1194, 11225, 7156, 10760, 5087, 317, 1521, 7243, 5284, 9385, 1128, 1678, 2149, 5752, 5330, 11698, 10220, 5516, 7098, 11545, 7647, 10531, 2191, 6775, 1381, 8627, 1992, 3614, 11826, 11651, 6336, 11390, 4656, 4886, 2568, 9718, 144, 8019, 3005, 6446, 1029, 8921, 6434, 6332, 7115, 8474, 4330, 11713, 8563, 6807, 9735, 5366, 3757, 9351, 965, 10288, 8651, 1758, 6989, 1186, 12214, 4282, 5148, 9066, 2423, 4706, 9354, 12156, 1978, 1540, 1776, 7884, 695, 3905, 11427, 1742, 2504, 626, 11413, 2313, 12037, 11738, 8476, 5630, 4001, 6153, 2646, 9681, 6345, 5839, 961, 723, 10295, 3292, 11234, 8744, 2400, 9495, 9737, 1177, 9320, 5282, 2047, 7527, 9629, 9346, 10699, 7925, 9381, 8026, 9010, 9806, 2757, 4553, 2838, 4353, 3956, 103, 7822, 7627, 6789, 8015, 8134, 10982, 5833, 11972, 11787, 5585, 5881, 8356, 4966, 2492, 8588, 3732, 9540, 1365, 11501, 1503, 3304, 11596, 806, 2148, 5476, 231, 2465, 7827, 5625, 3145, 4011, 7089, 120, 6099, 2939, 1175, 552, 5931, 8776, 7219, 5468, 10623, 1855, 11731, 2377, 162, 1134, 5700, 7886, 4575, 3938, 4483, 11782, 9866, 2359, 6785, 2518, 1045, 349, 7924, 5054, 4121, 7439, 9821, 9125, 3721, 11844, 6309, 4875, 9615, 12086, 3787, 677, 10104, 2124, 2947, 5096, 2385, 8051, 5546, 11196, 9350, 6848, 10961, 4760, 8845, 5542, 3452, 4750, 9087, 11622, 4864, 10050, 12192, 9888, 2484, 9603, 2412, 4603, 5193, 10355, 1864, 642, 8199, 3030, 9118, 10128, 1099, 1389, 9129, 5761, 10798, 9710, 3008, 2311, 9538, 1195, 6822, 8619, 484, 10398, 7621, 3713, 11026, 4348, 2851, 11088, 10181, 8435, 1509, 9302, 12092, 9171, 7079, 11334, 3942, 4994, 12221, 8777, 1450, 4715, 3283, 11523, 5923, 2240, 3750, 6022, 9898, 6167, 9257, 10676, 10394, 9394, 2953, 9086, 10497, 9607, 9513, 4698, 3140, 3151, 9823, 11569, 10151, 9620, 8642, 9098, 9371, 6014, 4535, 3741, 1291, 11187, 11069, 2632, 467, 3269, 10609, 3570, 6392, 1851, 6667, 3368, 9184, 809, 11134, 9723, 2336, 1358, 692, 4067, 9466, 3018, 3031, 1990, 3797, 1940, 2274, 11085, 921, 10909, 11953, 6804, 4731, 8589, 11724, 7510, 3952, 1912, 8917, 11168, 3940, 11384, 11847, 5145, 11059, 11372, 2069, 4973, 8736, 10772, 7006, 680, 8313, 567, 11546, 8170, 11116, 6168, 866, 9073, 11765, 11272, 6090, 2317, 7699, 8692, 5334, 5651, 3711, 10149, 6260, 10488, 5919, 11591, 7293, 11991, 4885, 3094, 11754, 3994, 6988, 4816, 11332, 8097, 4958, 3865, 10718, 5348, 4019, 10208, 9691, 6861, 11097, 8211, 6382, 2035, 6458, 1364, 9216, 11711, 1756, 4920, 9334, 10963, 3605, 9399, 10793, 4959, 4145, 6910, 9722, 6672, 4916, 12067, 6404, 4224, 2885, 1228, 913, 12218, 2482, 8014, 6665, 2745, 10952, 3378, 7553, 10659, 10802, 6761, 11913, 3059, 5363, 9625, 4991, 4215, 632, 9725, 7983, 3416, 6687, 2051, 11447, 915, 5525, 8209, 8213, 11904, 11763, 12252, 5146, 9023, 11925, 6689, 1210, 7500, 10701, 8775, 5673, 7348, 3410, 10018, 6946, 8779, 9161, 8389, 4102, 459, 9672, 8069, 9552, 4859, 4890, 7353, 6301, 579, 10555, 2003, 4241, 6454, 11022, 2701, 10040, 5488, 8975, 3648, 562, 2660, 10787, 4968, 381, 2141, 738, 7972, 9152, 7143, 76, 6922, 6768, 7702, 11005, 6849, 12011, 5149, 1947, 11019, 9780, 12051, 2331, 2443, 9183, 4606, 7397, 5768, 9411, 2675, 9041, 10684, 2802, 4671, 11860, 7895, 8273, 628, 5942, 8500, 10726, 4375, 9264, 9528, 1345, 5371, 6996, 4051, 6860, 10899, 11829, 11802, 10011, 4389, 11425, 819, 7609, 2432, 4155, 2990, 6064, 7865, 5865, 4209, 9833, 449, 8503, 7613, 451, 10512, 5061, 7011, 9173, 2754, 3540, 9560, 3337, 4701, 8821, 12075, 410, 10395, 11031"}},false,1,"[\"mean\"]",null,null,null,null,null,"success",null,null,12032,12032,"choice","accuracy",0.8339428191489362,0.0033927050376165004,0.8339428191489362,0.0033927050376165004,null,null,null,null,null,null]
["RMfwuoYNydgs9j23KhUhVv","7HfsPLmSqDJrGjRhgVeWNo","hertAatTVHM6v3kcY8Jh3x","/Users/jjallaire/meridianlabs/inspect_viz/docs/sandbox/logs/aime/2025-06-19T16-56-45+00-00_aime2024_hertAatTVHM6v3kcY8Jh3x.eval","2025-06-19T16:56:45.000000000Z","","https://github.com/AI-Safety-Institute/open_source_benchmarks.git","9a28395","{\"inspect_ai\": \"0.3.106.dev11+g28f3586b\"}",null,"aime2024",0,null,"{}",null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,"openai/o4-mini",null,null,"{\"max_connections\": 50}",null,"Maxwell-Jia/AIME_2024","Maxwell-Jia/AIME_2024",30,"[\"2024-II-4\", \"2024-II-12\", \"2024-I-4\", \"2024-I-3\", \"2024-I-8\", \"2024-I-12\", \"2024-I-11\", \"2024-II-11\", \"2024-I-2\", \"2024-II-6\", \"2024-I-7\", \"2024-II-3\", \"2024-I-1\", \"2024-II-7\", \"2024-I-6\", \"2024-I-13\", \"2024-I-15\", \"2024-II-15\", \"2024-II-10\", \"2024-II-9\", \"2024-II-14\", \"2024-II-5\", \"2024-I-9\", \"2024-II-2\", \"2024-II-1\", \"2024-I-10\", \"2024-II-8\", \"2024-I-14\", \"2024-I-5\", \"2024-II-13\"]",false,1,"[\"mean\"]",null,null,null,null,null,"success",null,null,30,30,"match","accuracy",0.8333333333333334,0.06920456654478331,null,null,null,null,0.8333333333333334,0.06920456654478331,null,null]
["3weowHrnsUwD6LNCQpmV96","QXy33ZJrxbjsTsriAtBbfj","W3ufF4y27CjW74JSNazRZQ","/Users/jjallaire/meridianlabs/inspect_viz/docs/sandbox/logs/aime/2025-06-28T11-25-41+00-00_aime2024_W3ufF4y27CjW74JSNazRZQ.eval","2025-06-28T11:25:41.000000000Z","","git@github.com:AI-Safety-Institute/open_source_benchmarks.git","089f78e","{\"inspect_ai\": \"0.3.106.dev11+g28f3586b\"}",null,"aime2024",0,null,"{}",null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,"anthropic/claude-sonnet-4-20250514",null,null,"{\"max_connections\": 100}",null,"Maxwell-Jia/AIME_2024","Maxwell-Jia/AIME_2024",30,"[\"2024-II-4\", \"2024-II-12\", \"2024-I-4\", \"2024-I-3\", \"2024-I-8\", \"2024-I-12\", \"2024-I-11\", \"2024-II-11\", \"2024-I-2\", \"2024-II-6\", \"2024-I-7\", \"2024-II-3\", \"2024-I-1\", \"2024-II-7\", \"2024-I-6\", \"2024-I-13\", \"2024-I-15\", \"2024-II-15\", \"2024-II-10\", \"2024-II-9\", \"2024-II-14\", \"2024-II-5\", \"2024-I-9\", \"2024-II-2\", \"2024-II-1\", \"2024-I-10\", \"2024-II-8\", \"2024-I-14\", \"2024-I-5\", \"2024-II-13\"]",false,1,"[\"mean\"]",null,null,null,null,null,"success",null,null,30,30,"match","accuracy",0.5,0.09284766908852593,null,null,null,null,0.5,0.09284766908852593,null,null]
["FhWr3VJi2Fi3R9y9TyVK8o","7HfsPLmSqDJrGjRhgVeWNo","m3efSweL9ZUnWuoRWAKGQa","/Users/jjallaire/meridianlabs/inspect_viz/docs/sandbox/logs/aime/2025-06-19T16-56-45+00-00_aime2024_m3efSweL9ZUnWuoRWAKGQa.eval","2025-06-19T16:56:45.000000000Z","","https://github.com/AI-Safety-Institute/open_source_benchmarks.git","9a28395","{\"inspect_ai\": \"0.3.106.dev11+g28f3586b\"}",null,"aime2024",0,null,"{}",null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,"anthropic/claude-3-7-sonnet-latest",null,null,"{\"max_connections\": 50}",null,"Maxwell-Jia/AIME_2024","Maxwell-Jia/AIME_2024",30,"[\"2024-II-4\", \"2024-II-12\", \"2024-I-4\", \"2024-I-3\", \"2024-I-8\", \"2024-I-12\", \"2024-I-11\", \"2024-II-11\", \"2024-I-2\", \"2024-II-6\", \"2024-I-7\", \"2024-II-3\", \"2024-I-1\", \"2024-II-7\", \"2024-I-6\", \"2024-I-13\", \"2024-I-15\", \"2024-II-15\", \"2024-II-10\", \"2024-II-9\", \"2024-II-14\", \"2024-II-5\", \"2024-I-9\", \"2024-II-2\", \"2024-II-1\", \"2024-I-10\", \"2024-II-8\", \"2024-I-14\", \"2024-I-5\", \"2024-II-13\"]",false,1,"[\"mean\"]",null,null,null,null,null,"success",null,null,30,30,"match","accuracy",0.26666666666666666,0.08211756827352529,null,null,null,null,0.26666666666666666,0.08211756827352529,null,null]
["TdTHwFw7tjz7rcuJTD3DXB","7HfsPLmSqDJrGjRhgVeWNo","6SWWF4B9hVhiDXYjLkf5Dr","/Users/jjallaire/meridianlabs/inspect_viz/docs/sandbox/logs/aime/2025-06-19T16-56-45+00-00_aime2024_6SWWF4B9hVhiDXYjLkf5Dr.eval","2025-06-19T16:56:45.000000000Z","","https://github.com/AI-Safety-Institute/open_source_benchmarks.git","9a28395","{\"inspect_ai\": \"0.3.106.dev11+g28f3586b\"}",null,"aime2024",0,null,"{}",null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,"openai/o3",null,null,"{\"max_connections\": 50}",null,"Maxwell-Jia/AIME_2024","Maxwell-Jia/AIME_2024",30,"[\"2024-II-4\", \"2024-II-12\", \"2024-I-4\", \"2024-I-3\", \"2024-I-8\", \"2024-I-12\", \"2024-I-11\", \"2024-II-11\", \"2024-I-2\", \"2024-II-6\", \"2024-I-7\", \"2024-II-3\", \"2024-I-1\", \"2024-II-7\", \"2024-I-6\", \"2024-I-13\", \"2024-I-15\", \"2024-II-15\", \"2024-II-10\", \"2024-II-9\", \"2024-II-14\", \"2024-II-5\", \"2024-I-9\", \"2024-II-2\", \"2024-II-1\", \"2024-I-10\", \"2024-II-8\", \"2024-I-14\", \"2024-I-5\", \"2024-II-13\"]",false,1,"[\"mean\"]",null,null,null,null,null,"success",null,null,30,30,"match","accuracy",0.9333333333333333,0.04632055558531008,null,null,null,null,0.9333333333333333,0.04632055558531008,null,null]
["a6Ya366ADRLPDBegBxzX5A","URL8FqPuFedrbTZTKSPWkv","iZPTeXig98VTYQ8CzSHy9N","/Users/jjallaire/meridianlabs/inspect_viz/docs/sandbox/logs/cybench/2025-06-28T12-26-01+00-00_cybench_iZPTeXig98VTYQ8CzSHy9N.eval","2025-06-28T12:26:01.000000000Z","",null,null,"{\"inspect_ai\": \"0.3.109\"}",null,"cybench",0,null,"{}",null,null,null,null,null,null,null,null,null,null,null,"docker",null,null,null,null,"hard","./cybench_solver.py@custom_solver","{}",null,null,"anthropic/claude-sonnet-4-20250514",null,null,"{\"max_connections\": 80}","{}",null,".venv/lib/python3.12/site-packages/inspect_evals/cybench/challenges",40,"[\"glacier-exchange (hard)\", \"lootstash (hard)\", \"back_to_the_past (hard)\", \"noisier_crc (hard)\", \"avatar (hard)\", \"randsubware (hard)\", \"crushing (hard)\", \"permuted (hard)\", \"skilift (hard)\", \"labyrinth_linguist (hard)\", \"robust_cbc (hard)\", \"partial_tenacity (hard)\", \"shuffled_aes (hard)\", \"sop (hard)\", \"data_siege (hard)\", \"network_tools (hard)\", \"it_has_begun (hard)\", \"diffecient (hard)\", \"noisy_crc (hard)\", \"chunky (hard)\", \"flag_command (hard)\", \"failproof (hard)\", \"unbreakable (hard)\", \"walking_to_the_sea_side (hard)\", \"delulu (hard)\", \"slcg (hard)\", \"emaze (hard)\", \"flecks_of_gold (hard)\", \"just_another_pickle_jail (hard)\", \"dynastic (hard)\", \"urgent (hard)\", \"locktalk (hard)\", \"motp (hard)\", \"missingbits (hard)\", \"were_pickle_phreaks_revenge (hard)\", \"primary_knowledge (hard)\", \"rpgo (hard)\", \"frog_waf (hard)\", \"eval_me (hard)\", \"packedaway (hard)\"]",false,1,"[\"mean\"]",null,10000.0,2500000.0,null,null,"success",null,null,40,40,"includes","accuracy",0.35,0.07637626158259733,null,null,0.35,0.07637626158259733,null,null,null,null]
["7a7gtdoShZeiXfyfyrZU29","URL8FqPuFedrbTZTKSPWkv","VcmoVJjBg5G6Cqh9zXvBdJ","/Users/jjallaire/meridianlabs/inspect_viz/docs/sandbox/logs/cybench/2025-06-28T12-26-01+00-00_cybench_VcmoVJjBg5G6Cqh9zXvBdJ.eval","2025-06-28T12:26:01.000000000Z","",null,null,"{\"inspect_ai\": \"0.3.109\"}",null,"cybench",0,null,"{}",null,null,null,null,null,null,null,null,null,null,null,"docker",null,null,null,null,"hard","./cybench_solver.py@custom_solver","{}",null,null,"openai/o4-mini",null,null,"{\"max_connections\": 80}","{}",null,".venv/lib/python3.12/site-packages/inspect_evals/cybench/challenges",40,"[\"glacier-exchange (hard)\", \"lootstash (hard)\", \"back_to_the_past (hard)\", \"noisier_crc (hard)\", \"avatar (hard)\", \"randsubware (hard)\", \"crushing (hard)\", \"permuted (hard)\", \"skilift (hard)\", \"labyrinth_linguist (hard)\", \"robust_cbc (hard)\", \"partial_tenacity (hard)\", \"shuffled_aes (hard)\", \"sop (hard)\", \"data_siege (hard)\", \"network_tools (hard)\", \"it_has_begun (hard)\", \"diffecient (hard)\", \"noisy_crc (hard)\", \"chunky (hard)\", \"flag_command (hard)\", \"failproof (hard)\", \"unbreakable (hard)\", \"walking_to_the_sea_side (hard)\", \"delulu (hard)\", \"slcg (hard)\", \"emaze (hard)\", \"flecks_of_gold (hard)\", \"just_another_pickle_jail (hard)\", \"dynastic (hard)\", \"urgent (hard)\", \"locktalk (hard)\", \"motp (hard)\", \"missingbits (hard)\", \"were_pickle_phreaks_revenge (hard)\", \"primary_knowledge (hard)\", \"rpgo (hard)\", \"frog_waf (hard)\", \"eval_me (hard)\", \"packedaway (hard)\"]",false,1,"[\"mean\"]",null,10000.0,2500000.0,null,null,"success",null,null,40,39,"includes","accuracy",0.41025641025641024,0.07979349797082043,null,null,0.41025641025641024,0.07979349797082043,null,null,null,null]
["gukDuiBqRWnyrvut24ZyUW","URL8FqPuFedrbTZTKSPWkv","cHgZahEpPHqsSx8GfGjKai","/Users/jjallaire/meridianlabs/inspect_viz/docs/sandbox/logs/cybench/2025-06-28T12-26-01+00-00_cybench_cHgZahEpPHqsSx8GfGjKai.eval","2025-06-28T12:26:01.000000000Z","",null,null,"{\"inspect_ai\": \"0.3.109\"}",null,"cybench",0,null,"{}",null,null,null,null,null,null,null,null,null,null,null,"docker",null,null,null,null,"hard","./cybench_solver.py@custom_solver","{}",null,null,"openai/o3",null,null,"{\"max_connections\": 80}","{}",null,".venv/lib/python3.12/site-packages/inspect_evals/cybench/challenges",40,"[\"glacier-exchange (hard)\", \"lootstash (hard)\", \"back_to_the_past (hard)\", \"noisier_crc (hard)\", \"avatar (hard)\", \"randsubware (hard)\", \"crushing (hard)\", \"permuted (hard)\", \"skilift (hard)\", \"labyrinth_linguist (hard)\", \"robust_cbc (hard)\", \"partial_tenacity (hard)\", \"shuffled_aes (hard)\", \"sop (hard)\", \"data_siege (hard)\", \"network_tools (hard)\", \"it_has_begun (hard)\", \"diffecient (hard)\", \"noisy_crc (hard)\", \"chunky (hard)\", \"flag_command (hard)\", \"failproof (hard)\", \"unbreakable (hard)\", \"walking_to_the_sea_side (hard)\", \"delulu (hard)\", \"slcg (hard)\", \"emaze (hard)\", \"flecks_of_gold (hard)\", \"just_another_pickle_jail (hard)\", \"dynastic (hard)\", \"urgent (hard)\", \"locktalk (hard)\", \"motp (hard)\", \"missingbits (hard)\", \"were_pickle_phreaks_revenge (hard)\", \"primary_knowledge (hard)\", \"rpgo (hard)\", \"frog_waf (hard)\", \"eval_me (hard)\", \"packedaway (hard)\"]",false,1,"[\"mean\"]",null,10000.0,2500000.0,null,null,"success",null,null,40,40,"includes","accuracy",0.35,0.07637626158259733,null,null,0.35,0.07637626158259733,null,null,null,null]
["GCztJYGNkobXMqctx2zabH","URL8FqPuFedrbTZTKSPWkv","QMAR36yGU2qNzVNCrmNCjN","/Users/jjallaire/meridianlabs/inspect_viz/docs/sandbox/logs/cybench/2025-06-28T12-26-01+00-00_cybench_QMAR36yGU2qNzVNCrmNCjN.eval","2025-06-28T12:26:01.000000000Z","",null,null,"{\"inspect_ai\": \"0.3.109\"}",null,"cybench",0,null,"{}",null,null,null,null,null,null,null,null,null,null,null,"docker",null,null,null,null,"hard","./cybench_solver.py@custom_solver","{}",null,null,"anthropic/claude-3-7-sonnet-latest",null,null,"{\"max_connections\": 80}","{}",null,".venv/lib/python3.12/site-packages/inspect_evals/cybench/challenges",40,"[\"glacier-exchange (hard)\", \"lootstash (hard)\", \"back_to_the_past (hard)\", \"noisier_crc (hard)\", \"avatar (hard)\", \"randsubware (hard)\", \"crushing (hard)\", \"permuted (hard)\", \"skilift (hard)\", \"labyrinth_linguist (hard)\", \"robust_cbc (hard)\", \"partial_tenacity (hard)\", \"shuffled_aes (hard)\", \"sop (hard)\", \"data_siege (hard)\", \"network_tools (hard)\", \"it_has_begun (hard)\", \"diffecient (hard)\", \"noisy_crc (hard)\", \"chunky (hard)\", \"flag_command (hard)\", \"failproof (hard)\", \"unbreakable (hard)\", \"walking_to_the_sea_side (hard)\", \"delulu (hard)\", \"slcg (hard)\", \"emaze (hard)\", \"flecks_of_gold (hard)\", \"just_another_pickle_jail (hard)\", \"dynastic (hard)\", \"urgent (hard)\", \"locktalk (hard)\", \"motp (hard)\", \"missingbits (hard)\", \"were_pickle_phreaks_revenge (hard)\", \"primary_knowledge (hard)\", \"rpgo (hard)\", \"frog_waf (hard)\", \"eval_me (hard)\", \"packedaway (hard)\"]",false,1,"[\"mean\"]",null,10000.0,2500000.0,null,null,"success",null,null,40,40,"includes","accuracy",0.275,0.07149950690165274,null,null,0.275,0.07149950690165274,null,null,null,null]
["kmV66rvkBKkMTih5DMN49K","QXy33ZJrxbjsTsriAtBbfj","m83utdQJSnBNPFyXCfoJG9","/Users/jjallaire/meridianlabs/inspect_viz/docs/sandbox/logs/gpqa/2025-06-28T11-25-41+00-00_gpqa-diamond_m83utdQJSnBNPFyXCfoJG9.eval","2025-06-28T11:25:41.000000000Z","","git@github.com:AI-Safety-Institute/open_source_benchmarks.git","089f78e","{\"inspect_ai\": \"0.3.106.dev11+g28f3586b\"}",null,"gpqa_diamond",0,null,"{}",null,null,true,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,"anthropic/claude-sonnet-4-20250514",null,null,"{\"max_connections\": 100}",null,"gpqa_diamond","https:/openaipublic.blob.core.windows.net/simple-evals/gpqa_diamond.csv",198,"[\"rec06pnAkLOr2t2mp\", \"rec0Arme2jcXQZnAW\", \"rec0wZvZgiz320KRs\", \"rec0yTRmO1o1xCA6H\", \"rec1zl5LvaatzGhFt\", \"rec260hNUCEj109Dj\", \"rec2UlKqC6RFHdcro\", \"rec2ZTsqazBZ3LTWZ\", \"rec2fsnzUuvNtUYK8\", \"rec4JrWy7Il8ho5vk\", \"rec4L69T0Y1AS4AFS\", \"rec4UqStf9WUVif1f\", \"rec4r4KWNLrT7g2kR\", \"rec527dNeEtWJrYNl\", \"rec5eA17Qr1ucNLUf\", \"rec5rjeLsEq5Fg7Oj\", \"rec7C06XOB2h298ue\", \"rec7qmSnbud4FHSqL\", \"rec8nshandHARTkrg\", \"rec8y3ZrBOcLgNEkE\", \"rec9GugrngvsQR99A\", \"rec9ubQihAh6g9bft\", \"recA1i5ZAh0Uzclxp\", \"recAAJoHMW45Lv5je\", \"recBGxVySx1pkrO1i\", \"recBXlqrZUCR897It\", \"recBtVK8rBVtIlXDq\", \"recCJJOeBGERaHYax\", \"recD8oX1KevFbl7bL\", \"recDDxpS9s8cwkqfq\", \"recDj2Y2BbtV02Wv5\", \"recDytVnNYZe2HuUU\", \"recEmTBhx2hgw6tPQ\", \"recFaL6j8UMhutXrc\", \"recG1ljJQcVy6CbSl\", \"recGFKDUd8qgz81Zl\", \"recGFNRVl1qBZGwyU\", \"recHVUd2DkoKfedJZ\", \"recI1ls9OXdxatHQn\", \"recIDiDKKrN61Auyr\", \"recIIUynGGpsGEYuo\", \"recIMcQfaQNa6vzsT\", \"recINSFEYLCyyd08m\", \"recIOlKBsOeEcgkA1\", \"recIj8lR4tuDgrHou\", \"recIlghoOw5VbY4Zh\", \"recJJcOHhFeSaRpl4\", \"recJZ3QEfRKjYw9a7\", \"recK9F5aqdaybl8bb\", \"recKcjTwONILZqHAX\", \"recKm6LNWykGapmCr\", \"recKv4hFyNpViUXnL\", \"recL9MFV5zmdlle5T\", \"recLAgwx9vbgB5EHk\", \"recLSzwBUS3olCjhl\", \"recLrRirj7iXoB0rU\", \"recN4DY9Q5V03glmQ\", \"recNAiPUKOulQbKVP\", \"recNFJjE5PPTqVJGv\", \"recNu3MXkvWUzHZr9\", \"recNuT2oSnO86bxOx\", \"recO3hvCWRGiG0odN\", \"recOYsaYs6RmtlTDy\", \"recOkA2zsy4GSMD9R\", \"recOvqPKUtyy9ISA1\", \"recPL4ZPpVYxgMu57\", \"recPSTGXK3P39yNYT\", \"recPzW1WqRnPs57D6\", \"recQ8oxSY2aCwFRS8\", \"recQiWOxhkXRz9NVE\", \"recRTiyaRymerDJ4y\", \"recRgabRzMaEoBRcM\", \"recS48OsU6kVadBPW\", \"recSBcGLPatKb3Ygu\", \"recTs7qzfJs6kfLUK\", \"recUBgVlkKzcRPDdK\", \"recUOePh79cp4T2Bg\", \"recV1nqYQvpII94oC\", \"recVEXvhYxAhELVqB\", \"recVvpD8miVjmmyfe\", \"recWPMm91mGDLXONm\", \"recWxGU8Q4YReJ1tb\", \"recWzUUYBN0NuXO2k\", \"recXsuOHRBLcyenF2\", \"recXvQ6gWAmyakrpD\", \"recYA3LPsCvF1fTMI\", \"recYOzCsevNz61Lyn\", \"recYt8xx80OTyDsL0\", \"recZ13cwgDQf9jRd9\", \"recZSGUkn56v9kEp1\", \"recZWeueB7lSPR6wN\", \"recZbxrocrxh9YENH\", \"recZdZxsN0AxDx0pB\", \"recZt4x514UQNnsgy\", \"reca44yABeO2fx7UB\", \"recaXdGn3FAIkxLGM\", \"recafnH3RRnMBXEb1\", \"recakFKZioXUuldcd\", \"recb2M22zaD3tL6Qc\", \"recb4cGsC6BJUCU3V\", \"reccOKzFNmyqeJ6ry\", \"reccVFW11Elovnf8T\", \"recclFbsjbaiVVnnV\", \"recdya6FuYraBU5Rh\", \"receYQxnUBvxPcvbQ\", \"receo3UUzRvWCqyfA\", \"recf6ayQmL1SxKbvW\", \"recgI6tUQ7RLJRWGx\", \"recgM7o1tcc7tP778\", \"recgXxEgllSGEpELP\", \"rechKl68Uc6H7vU0N\", \"rechgQucGlrnt8KRV\", \"rechtUKxC6G67H7Pr\", \"recihePFulRgNKsIn\", \"reciuppdQde4FGX1z\", \"recjJ54TXc04enRkZ\", \"recjgMJaMxz4ESDF2\", \"reck1zKeCssmN82Ga\", \"reck4G4xxv3YnpbtQ\", \"reckYqR92oJkqJXkE\", \"reckl6PkaEMWM50yZ\", \"recl1UtgTVKishAq4\", \"recmI7EiLv72PxmYK\", \"recmkvk6EFAmqyMxR\", \"recmwwQJnx7ll7bqL\", \"recn3NhOhqAPLda16\", \"recnGEpF1srQpaqWq\", \"recnTTKdBzfuoZ7w7\", \"recnawZPumSCLMVgp\", \"recnjViFrqlZNL3fY\", \"reco8WSc71p8zAv4q\", \"recooG6bivTUJpDBz\", \"recoy9ZLBsc7HIRBy\", \"recpJmTOXx6T48nDJ\", \"recpki12iG9RUGrz9\", \"recqFDvYKKRmbnoUb\", \"recqGD3fxPCI59vPQ\", \"recqLRu4BcJvlSANc\", \"recr3VHM4zYf6dMFY\", \"recrHBEJJoDTV05JR\", \"recrNbtgTNoabJJi6\", \"recrs0kUo7Ua1xhKO\", \"recs3PLPUEMiqg4P8\", \"rect4iLrSfUwkNTno\", \"rectXfsCM1dj4Kv2c\", \"rectlyG9pCAAuWhoB\", \"recuYEkVjdVUwA3c0\", \"recuyeuT5rQ6qDt8F\", \"recv7GsQg3f0fvB1f\", \"recvZlMbpPz0yLDle\", \"recvlnu8BuLmXZhSX\", \"recw4rROcnHKNZhtK\", \"recxTy0x7M6T9ikc0\", \"recxsX66ktJ1Lwt4f\", \"recyPffUDqC2k7ZZO\", \"recyl3usDqb7ruXJx\", \"recypVp2NmPlBKVTp\", \"reczQ4I0VpENdMtIj\", \"reczUoM8JsxU6pYxr\", \"reczjcMtrB1YGS2fO\", \"reczzzihL7btBH7RO\", \"rec0VuKUjt1SZ7NYv\", \"rec1oj2DveQWl9Rpw\", \"rec32C1ZEapBnCC0E\", \"rec47oNPXudlTIk4y\", \"rec6sE2CRtD4drtHg\", \"rec9W28HgpEUeUN8k\", \"recAYkd96NNuNl1Ei\", \"recBhnXrUyTJ6WHIR\", \"recE2ihVfqEK4R9d0\", \"recGee5m84dg5FZkc\", \"recINGR1z01Fh1Z3A\", \"recJpyGtGIsxulevT\", \"recLB0EkQ54bYVhnd\", \"recMicVBcqy1xM1jq\", \"recPIzpnuYpB4yvmp\", \"recRZjaElf8ft7xTq\", \"recReg13iV2HwJTaA\", \"recRhwQwHijW0gCLo\", \"recUc29lMDBEvurYo\", \"recVE8cUNHpHZIAvL\", \"recWXwn9v4IG9ZrM6\", \"recXsYa3i2UhGF5fe\", \"recYCMJWH7IpcdqwE\", \"recatj4YXzbOaq3Ar\", \"recb80OwMgNnceA9t\", \"reccVBrYdwsB84fGy\", \"recfTlTMjZBuOducT\", \"recgC4GJ8wUemlUQD\", \"recixxJmdux0d8LZQ\", \"reckEnrOPFT9Ru7tW\", \"recoiTJPGUmzAkief\", \"recwW1A85nfyQpReG\", \"recwlwlcPvXVNPmvl\", \"recyrk6XBVwugG5u8\", \"recywRj5a8EEjj2Ib\", \"recz3MfYHPRhsEQkW\", \"reczkBiPPNrNN49Hp\"]",false,1,"[\"mean\"]",null,null,null,null,null,"success",null,null,198,198,"choice","accuracy",0.6666666666666666,0.033586181457325226,0.6666666666666666,0.033586181457325226,null,null,null,null,null,null]
["VivGsTQHVKDRhG6UrTa3je","7HfsPLmSqDJrGjRhgVeWNo","4NjhP7zMqvySfDSuHmjPv9","/Users/jjallaire/meridianlabs/inspect_viz/docs/sandbox/logs/gpqa/2025-06-19T16-56-45+00-00_gpqa-diamond_4NjhP7zMqvySfDSuHmjPv9.eval","2025-06-19T16:56:45.000000000Z","","https://github.com/AI-Safety-Institute/open_source_benchmarks.git","9a28395","{\"inspect_ai\": \"0.3.106.dev11+g28f3586b\"}",null,"gpqa_diamond",0,null,"{}",null,null,true,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,"openai/o3",null,null,"{\"max_connections\": 50}",null,"gpqa_diamond","https:/openaipublic.blob.core.windows.net/simple-evals/gpqa_diamond.csv",198,"[\"rec06pnAkLOr2t2mp\", \"rec0Arme2jcXQZnAW\", \"rec0wZvZgiz320KRs\", \"rec0yTRmO1o1xCA6H\", \"rec1zl5LvaatzGhFt\", \"rec260hNUCEj109Dj\", \"rec2UlKqC6RFHdcro\", \"rec2ZTsqazBZ3LTWZ\", \"rec2fsnzUuvNtUYK8\", \"rec4JrWy7Il8ho5vk\", \"rec4L69T0Y1AS4AFS\", \"rec4UqStf9WUVif1f\", \"rec4r4KWNLrT7g2kR\", \"rec527dNeEtWJrYNl\", \"rec5eA17Qr1ucNLUf\", \"rec5rjeLsEq5Fg7Oj\", \"rec7C06XOB2h298ue\", \"rec7qmSnbud4FHSqL\", \"rec8nshandHARTkrg\", \"rec8y3ZrBOcLgNEkE\", \"rec9GugrngvsQR99A\", \"rec9ubQihAh6g9bft\", \"recA1i5ZAh0Uzclxp\", \"recAAJoHMW45Lv5je\", \"recBGxVySx1pkrO1i\", \"recBXlqrZUCR897It\", \"recBtVK8rBVtIlXDq\", \"recCJJOeBGERaHYax\", \"recD8oX1KevFbl7bL\", \"recDDxpS9s8cwkqfq\", \"recDj2Y2BbtV02Wv5\", \"recDytVnNYZe2HuUU\", \"recEmTBhx2hgw6tPQ\", \"recFaL6j8UMhutXrc\", \"recG1ljJQcVy6CbSl\", \"recGFKDUd8qgz81Zl\", \"recGFNRVl1qBZGwyU\", \"recHVUd2DkoKfedJZ\", \"recI1ls9OXdxatHQn\", \"recIDiDKKrN61Auyr\", \"recIIUynGGpsGEYuo\", \"recIMcQfaQNa6vzsT\", \"recINSFEYLCyyd08m\", \"recIOlKBsOeEcgkA1\", \"recIj8lR4tuDgrHou\", \"recIlghoOw5VbY4Zh\", \"recJJcOHhFeSaRpl4\", \"recJZ3QEfRKjYw9a7\", \"recK9F5aqdaybl8bb\", \"recKcjTwONILZqHAX\", \"recKm6LNWykGapmCr\", \"recKv4hFyNpViUXnL\", \"recL9MFV5zmdlle5T\", \"recLAgwx9vbgB5EHk\", \"recLSzwBUS3olCjhl\", \"recLrRirj7iXoB0rU\", \"recN4DY9Q5V03glmQ\", \"recNAiPUKOulQbKVP\", \"recNFJjE5PPTqVJGv\", \"recNu3MXkvWUzHZr9\", \"recNuT2oSnO86bxOx\", \"recO3hvCWRGiG0odN\", \"recOYsaYs6RmtlTDy\", \"recOkA2zsy4GSMD9R\", \"recOvqPKUtyy9ISA1\", \"recPL4ZPpVYxgMu57\", \"recPSTGXK3P39yNYT\", \"recPzW1WqRnPs57D6\", \"recQ8oxSY2aCwFRS8\", \"recQiWOxhkXRz9NVE\", \"recRTiyaRymerDJ4y\", \"recRgabRzMaEoBRcM\", \"recS48OsU6kVadBPW\", \"recSBcGLPatKb3Ygu\", \"recTs7qzfJs6kfLUK\", \"recUBgVlkKzcRPDdK\", \"recUOePh79cp4T2Bg\", \"recV1nqYQvpII94oC\", \"recVEXvhYxAhELVqB\", \"recVvpD8miVjmmyfe\", \"recWPMm91mGDLXONm\", \"recWxGU8Q4YReJ1tb\", \"recWzUUYBN0NuXO2k\", \"recXsuOHRBLcyenF2\", \"recXvQ6gWAmyakrpD\", \"recYA3LPsCvF1fTMI\", \"recYOzCsevNz61Lyn\", \"recYt8xx80OTyDsL0\", \"recZ13cwgDQf9jRd9\", \"recZSGUkn56v9kEp1\", \"recZWeueB7lSPR6wN\", \"recZbxrocrxh9YENH\", \"recZdZxsN0AxDx0pB\", \"recZt4x514UQNnsgy\", \"reca44yABeO2fx7UB\", \"recaXdGn3FAIkxLGM\", \"recafnH3RRnMBXEb1\", \"recakFKZioXUuldcd\", \"recb2M22zaD3tL6Qc\", \"recb4cGsC6BJUCU3V\", \"reccOKzFNmyqeJ6ry\", \"reccVFW11Elovnf8T\", \"recclFbsjbaiVVnnV\", \"recdya6FuYraBU5Rh\", \"receYQxnUBvxPcvbQ\", \"receo3UUzRvWCqyfA\", \"recf6ayQmL1SxKbvW\", \"recgI6tUQ7RLJRWGx\", \"recgM7o1tcc7tP778\", \"recgXxEgllSGEpELP\", \"rechKl68Uc6H7vU0N\", \"rechgQucGlrnt8KRV\", \"rechtUKxC6G67H7Pr\", \"recihePFulRgNKsIn\", \"reciuppdQde4FGX1z\", \"recjJ54TXc04enRkZ\", \"recjgMJaMxz4ESDF2\", \"reck1zKeCssmN82Ga\", \"reck4G4xxv3YnpbtQ\", \"reckYqR92oJkqJXkE\", \"reckl6PkaEMWM50yZ\", \"recl1UtgTVKishAq4\", \"recmI7EiLv72PxmYK\", \"recmkvk6EFAmqyMxR\", \"recmwwQJnx7ll7bqL\", \"recn3NhOhqAPLda16\", \"recnGEpF1srQpaqWq\", \"recnTTKdBzfuoZ7w7\", \"recnawZPumSCLMVgp\", \"recnjViFrqlZNL3fY\", \"reco8WSc71p8zAv4q\", \"recooG6bivTUJpDBz\", \"recoy9ZLBsc7HIRBy\", \"recpJmTOXx6T48nDJ\", \"recpki12iG9RUGrz9\", \"recqFDvYKKRmbnoUb\", \"recqGD3fxPCI59vPQ\", \"recqLRu4BcJvlSANc\", \"recr3VHM4zYf6dMFY\", \"recrHBEJJoDTV05JR\", \"recrNbtgTNoabJJi6\", \"recrs0kUo7Ua1xhKO\", \"recs3PLPUEMiqg4P8\", \"rect4iLrSfUwkNTno\", \"rectXfsCM1dj4Kv2c\", \"rectlyG9pCAAuWhoB\", \"recuYEkVjdVUwA3c0\", \"recuyeuT5rQ6qDt8F\", \"recv7GsQg3f0fvB1f\", \"recvZlMbpPz0yLDle\", \"recvlnu8BuLmXZhSX\", \"recw4rROcnHKNZhtK\", \"recxTy0x7M6T9ikc0\", \"recxsX66ktJ1Lwt4f\", \"recyPffUDqC2k7ZZO\", \"recyl3usDqb7ruXJx\", \"recypVp2NmPlBKVTp\", \"reczQ4I0VpENdMtIj\", \"reczUoM8JsxU6pYxr\", \"reczjcMtrB1YGS2fO\", \"reczzzihL7btBH7RO\", \"rec0VuKUjt1SZ7NYv\", \"rec1oj2DveQWl9Rpw\", \"rec32C1ZEapBnCC0E\", \"rec47oNPXudlTIk4y\", \"rec6sE2CRtD4drtHg\", \"rec9W28HgpEUeUN8k\", \"recAYkd96NNuNl1Ei\", \"recBhnXrUyTJ6WHIR\", \"recE2ihVfqEK4R9d0\", \"recGee5m84dg5FZkc\", \"recINGR1z01Fh1Z3A\", \"recJpyGtGIsxulevT\", \"recLB0EkQ54bYVhnd\", \"recMicVBcqy1xM1jq\", \"recPIzpnuYpB4yvmp\", \"recRZjaElf8ft7xTq\", \"recReg13iV2HwJTaA\", \"recRhwQwHijW0gCLo\", \"recUc29lMDBEvurYo\", \"recVE8cUNHpHZIAvL\", \"recWXwn9v4IG9ZrM6\", \"recXsYa3i2UhGF5fe\", \"recYCMJWH7IpcdqwE\", \"recatj4YXzbOaq3Ar\", \"recb80OwMgNnceA9t\", \"reccVBrYdwsB84fGy\", \"recfTlTMjZBuOducT\", \"recgC4GJ8wUemlUQD\", \"recixxJmdux0d8LZQ\", \"reckEnrOPFT9Ru7tW\", \"recoiTJPGUmzAkief\", \"recwW1A85nfyQpReG\", \"recwlwlcPvXVNPmvl\", \"recyrk6XBVwugG5u8\", \"recywRj5a8EEjj2Ib\", \"recz3MfYHPRhsEQkW\", \"reczkBiPPNrNN49Hp\"]",false,1,"[\"mean\"]",null,null,null,null,null,"success",null,null,198,198,"choice","accuracy",0.7929292929292929,0.028869778460266993,0.7929292929292929,0.028869778460266993,null,null,null,null,null,null]
["Bwx3BQq3bAndVyXZZpuiLL","7HfsPLmSqDJrGjRhgVeWNo","YwFdPfA8udV6XSJzStfWBh","/Users/jjallaire/meridianlabs/inspect_viz/docs/sandbox/logs/gpqa/2025-06-19T16-56-45+00-00_gpqa-diamond_YwFdPfA8udV6XSJzStfWBh.eval","2025-06-19T16:56:45.000000000Z","","https://github.com/AI-Safety-Institute/open_source_benchmarks.git","9a28395","{\"inspect_ai\": \"0.3.106.dev11+g28f3586b\"}",null,"gpqa_diamond",0,null,"{}",null,null,true,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,"anthropic/claude-3-7-sonnet-latest",null,null,"{\"max_connections\": 50}",null,"gpqa_diamond","https:/openaipublic.blob.core.windows.net/simple-evals/gpqa_diamond.csv",198,"[\"rec06pnAkLOr2t2mp\", \"rec0Arme2jcXQZnAW\", \"rec0wZvZgiz320KRs\", \"rec0yTRmO1o1xCA6H\", \"rec1zl5LvaatzGhFt\", \"rec260hNUCEj109Dj\", \"rec2UlKqC6RFHdcro\", \"rec2ZTsqazBZ3LTWZ\", \"rec2fsnzUuvNtUYK8\", \"rec4JrWy7Il8ho5vk\", \"rec4L69T0Y1AS4AFS\", \"rec4UqStf9WUVif1f\", \"rec4r4KWNLrT7g2kR\", \"rec527dNeEtWJrYNl\", \"rec5eA17Qr1ucNLUf\", \"rec5rjeLsEq5Fg7Oj\", \"rec7C06XOB2h298ue\", \"rec7qmSnbud4FHSqL\", \"rec8nshandHARTkrg\", \"rec8y3ZrBOcLgNEkE\", \"rec9GugrngvsQR99A\", \"rec9ubQihAh6g9bft\", \"recA1i5ZAh0Uzclxp\", \"recAAJoHMW45Lv5je\", \"recBGxVySx1pkrO1i\", \"recBXlqrZUCR897It\", \"recBtVK8rBVtIlXDq\", \"recCJJOeBGERaHYax\", \"recD8oX1KevFbl7bL\", \"recDDxpS9s8cwkqfq\", \"recDj2Y2BbtV02Wv5\", \"recDytVnNYZe2HuUU\", \"recEmTBhx2hgw6tPQ\", \"recFaL6j8UMhutXrc\", \"recG1ljJQcVy6CbSl\", \"recGFKDUd8qgz81Zl\", \"recGFNRVl1qBZGwyU\", \"recHVUd2DkoKfedJZ\", \"recI1ls9OXdxatHQn\", \"recIDiDKKrN61Auyr\", \"recIIUynGGpsGEYuo\", \"recIMcQfaQNa6vzsT\", \"recINSFEYLCyyd08m\", \"recIOlKBsOeEcgkA1\", \"recIj8lR4tuDgrHou\", \"recIlghoOw5VbY4Zh\", \"recJJcOHhFeSaRpl4\", \"recJZ3QEfRKjYw9a7\", \"recK9F5aqdaybl8bb\", \"recKcjTwONILZqHAX\", \"recKm6LNWykGapmCr\", \"recKv4hFyNpViUXnL\", \"recL9MFV5zmdlle5T\", \"recLAgwx9vbgB5EHk\", \"recLSzwBUS3olCjhl\", \"recLrRirj7iXoB0rU\", \"recN4DY9Q5V03glmQ\", \"recNAiPUKOulQbKVP\", \"recNFJjE5PPTqVJGv\", \"recNu3MXkvWUzHZr9\", \"recNuT2oSnO86bxOx\", \"recO3hvCWRGiG0odN\", \"recOYsaYs6RmtlTDy\", \"recOkA2zsy4GSMD9R\", \"recOvqPKUtyy9ISA1\", \"recPL4ZPpVYxgMu57\", \"recPSTGXK3P39yNYT\", \"recPzW1WqRnPs57D6\", \"recQ8oxSY2aCwFRS8\", \"recQiWOxhkXRz9NVE\", \"recRTiyaRymerDJ4y\", \"recRgabRzMaEoBRcM\", \"recS48OsU6kVadBPW\", \"recSBcGLPatKb3Ygu\", \"recTs7qzfJs6kfLUK\", \"recUBgVlkKzcRPDdK\", \"recUOePh79cp4T2Bg\", \"recV1nqYQvpII94oC\", \"recVEXvhYxAhELVqB\", \"recVvpD8miVjmmyfe\", \"recWPMm91mGDLXONm\", \"recWxGU8Q4YReJ1tb\", \"recWzUUYBN0NuXO2k\", \"recXsuOHRBLcyenF2\", \"recXvQ6gWAmyakrpD\", \"recYA3LPsCvF1fTMI\", \"recYOzCsevNz61Lyn\", \"recYt8xx80OTyDsL0\", \"recZ13cwgDQf9jRd9\", \"recZSGUkn56v9kEp1\", \"recZWeueB7lSPR6wN\", \"recZbxrocrxh9YENH\", \"recZdZxsN0AxDx0pB\", \"recZt4x514UQNnsgy\", \"reca44yABeO2fx7UB\", \"recaXdGn3FAIkxLGM\", \"recafnH3RRnMBXEb1\", \"recakFKZioXUuldcd\", \"recb2M22zaD3tL6Qc\", \"recb4cGsC6BJUCU3V\", \"reccOKzFNmyqeJ6ry\", \"reccVFW11Elovnf8T\", \"recclFbsjbaiVVnnV\", \"recdya6FuYraBU5Rh\", \"receYQxnUBvxPcvbQ\", \"receo3UUzRvWCqyfA\", \"recf6ayQmL1SxKbvW\", \"recgI6tUQ7RLJRWGx\", \"recgM7o1tcc7tP778\", \"recgXxEgllSGEpELP\", \"rechKl68Uc6H7vU0N\", \"rechgQucGlrnt8KRV\", \"rechtUKxC6G67H7Pr\", \"recihePFulRgNKsIn\", \"reciuppdQde4FGX1z\", \"recjJ54TXc04enRkZ\", \"recjgMJaMxz4ESDF2\", \"reck1zKeCssmN82Ga\", \"reck4G4xxv3YnpbtQ\", \"reckYqR92oJkqJXkE\", \"reckl6PkaEMWM50yZ\", \"recl1UtgTVKishAq4\", \"recmI7EiLv72PxmYK\", \"recmkvk6EFAmqyMxR\", \"recmwwQJnx7ll7bqL\", \"recn3NhOhqAPLda16\", \"recnGEpF1srQpaqWq\", \"recnTTKdBzfuoZ7w7\", \"recnawZPumSCLMVgp\", \"recnjViFrqlZNL3fY\", \"reco8WSc71p8zAv4q\", \"recooG6bivTUJpDBz\", \"recoy9ZLBsc7HIRBy\", \"recpJmTOXx6T48nDJ\", \"recpki12iG9RUGrz9\", \"recqFDvYKKRmbnoUb\", \"recqGD3fxPCI59vPQ\", \"recqLRu4BcJvlSANc\", \"recr3VHM4zYf6dMFY\", \"recrHBEJJoDTV05JR\", \"recrNbtgTNoabJJi6\", \"recrs0kUo7Ua1xhKO\", \"recs3PLPUEMiqg4P8\", \"rect4iLrSfUwkNTno\", \"rectXfsCM1dj4Kv2c\", \"rectlyG9pCAAuWhoB\", \"recuYEkVjdVUwA3c0\", \"recuyeuT5rQ6qDt8F\", \"recv7GsQg3f0fvB1f\", \"recvZlMbpPz0yLDle\", \"recvlnu8BuLmXZhSX\", \"recw4rROcnHKNZhtK\", \"recxTy0x7M6T9ikc0\", \"recxsX66ktJ1Lwt4f\", \"recyPffUDqC2k7ZZO\", \"recyl3usDqb7ruXJx\", \"recypVp2NmPlBKVTp\", \"reczQ4I0VpENdMtIj\", \"reczUoM8JsxU6pYxr\", \"reczjcMtrB1YGS2fO\", \"reczzzihL7btBH7RO\", \"rec0VuKUjt1SZ7NYv\", \"rec1oj2DveQWl9Rpw\", \"rec32C1ZEapBnCC0E\", \"rec47oNPXudlTIk4y\", \"rec6sE2CRtD4drtHg\", \"rec9W28HgpEUeUN8k\", \"recAYkd96NNuNl1Ei\", \"recBhnXrUyTJ6WHIR\", \"recE2ihVfqEK4R9d0\", \"recGee5m84dg5FZkc\", \"recINGR1z01Fh1Z3A\", \"recJpyGtGIsxulevT\", \"recLB0EkQ54bYVhnd\", \"recMicVBcqy1xM1jq\", \"recPIzpnuYpB4yvmp\", \"recRZjaElf8ft7xTq\", \"recReg13iV2HwJTaA\", \"recRhwQwHijW0gCLo\", \"recUc29lMDBEvurYo\", \"recVE8cUNHpHZIAvL\", \"recWXwn9v4IG9ZrM6\", \"recXsYa3i2UhGF5fe\", \"recYCMJWH7IpcdqwE\", \"recatj4YXzbOaq3Ar\", \"recb80OwMgNnceA9t\", \"reccVBrYdwsB84fGy\", \"recfTlTMjZBuOducT\", \"recgC4GJ8wUemlUQD\", \"recixxJmdux0d8LZQ\", \"reckEnrOPFT9Ru7tW\", \"recoiTJPGUmzAkief\", \"recwW1A85nfyQpReG\", \"recwlwlcPvXVNPmvl\", \"recyrk6XBVwugG5u8\", \"recywRj5a8EEjj2Ib\", \"recz3MfYHPRhsEQkW\", \"reczkBiPPNrNN49Hp\"]",false,1,"[\"mean\"]",null,null,null,null,null,"success",null,null,198,198,"choice","accuracy",0.6717171717171717,0.03345678422756777,0.6717171717171717,0.03345678422756777,null,null,null,null,null,null]
["RUvDrXTawrkRnMmMiPuhL6","7HfsPLmSqDJrGjRhgVeWNo","6XZC54zJa8Ye49SBnWH3Q6","/Users/jjallaire/meridianlabs/inspect_viz/docs/sandbox/logs/gpqa/2025-06-19T16-56-45+00-00_gpqa-diamond_6XZC54zJa8Ye49SBnWH3Q6.eval","2025-06-19T16:56:45.000000000Z","","https://github.com/AI-Safety-Institute/open_source_benchmarks.git","9a28395","{\"inspect_ai\": \"0.3.106.dev11+g28f3586b\"}",null,"gpqa_diamond",0,null,"{}",null,null,true,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,"openai/o4-mini",null,null,"{\"max_connections\": 50}",null,"gpqa_diamond","https:/openaipublic.blob.core.windows.net/simple-evals/gpqa_diamond.csv",198,"[\"rec06pnAkLOr2t2mp\", \"rec0Arme2jcXQZnAW\", \"rec0wZvZgiz320KRs\", \"rec0yTRmO1o1xCA6H\", \"rec1zl5LvaatzGhFt\", \"rec260hNUCEj109Dj\", \"rec2UlKqC6RFHdcro\", \"rec2ZTsqazBZ3LTWZ\", \"rec2fsnzUuvNtUYK8\", \"rec4JrWy7Il8ho5vk\", \"rec4L69T0Y1AS4AFS\", \"rec4UqStf9WUVif1f\", \"rec4r4KWNLrT7g2kR\", \"rec527dNeEtWJrYNl\", \"rec5eA17Qr1ucNLUf\", \"rec5rjeLsEq5Fg7Oj\", \"rec7C06XOB2h298ue\", \"rec7qmSnbud4FHSqL\", \"rec8nshandHARTkrg\", \"rec8y3ZrBOcLgNEkE\", \"rec9GugrngvsQR99A\", \"rec9ubQihAh6g9bft\", \"recA1i5ZAh0Uzclxp\", \"recAAJoHMW45Lv5je\", \"recBGxVySx1pkrO1i\", \"recBXlqrZUCR897It\", \"recBtVK8rBVtIlXDq\", \"recCJJOeBGERaHYax\", \"recD8oX1KevFbl7bL\", \"recDDxpS9s8cwkqfq\", \"recDj2Y2BbtV02Wv5\", \"recDytVnNYZe2HuUU\", \"recEmTBhx2hgw6tPQ\", \"recFaL6j8UMhutXrc\", \"recG1ljJQcVy6CbSl\", \"recGFKDUd8qgz81Zl\", \"recGFNRVl1qBZGwyU\", \"recHVUd2DkoKfedJZ\", \"recI1ls9OXdxatHQn\", \"recIDiDKKrN61Auyr\", \"recIIUynGGpsGEYuo\", \"recIMcQfaQNa6vzsT\", \"recINSFEYLCyyd08m\", \"recIOlKBsOeEcgkA1\", \"recIj8lR4tuDgrHou\", \"recIlghoOw5VbY4Zh\", \"recJJcOHhFeSaRpl4\", \"recJZ3QEfRKjYw9a7\", \"recK9F5aqdaybl8bb\", \"recKcjTwONILZqHAX\", \"recKm6LNWykGapmCr\", \"recKv4hFyNpViUXnL\", \"recL9MFV5zmdlle5T\", \"recLAgwx9vbgB5EHk\", \"recLSzwBUS3olCjhl\", \"recLrRirj7iXoB0rU\", \"recN4DY9Q5V03glmQ\", \"recNAiPUKOulQbKVP\", \"recNFJjE5PPTqVJGv\", \"recNu3MXkvWUzHZr9\", \"recNuT2oSnO86bxOx\", \"recO3hvCWRGiG0odN\", \"recOYsaYs6RmtlTDy\", \"recOkA2zsy4GSMD9R\", \"recOvqPKUtyy9ISA1\", \"recPL4ZPpVYxgMu57\", \"recPSTGXK3P39yNYT\", \"recPzW1WqRnPs57D6\", \"recQ8oxSY2aCwFRS8\", \"recQiWOxhkXRz9NVE\", \"recRTiyaRymerDJ4y\", \"recRgabRzMaEoBRcM\", \"recS48OsU6kVadBPW\", \"recSBcGLPatKb3Ygu\", \"recTs7qzfJs6kfLUK\", \"recUBgVlkKzcRPDdK\", \"recUOePh79cp4T2Bg\", \"recV1nqYQvpII94oC\", \"recVEXvhYxAhELVqB\", \"recVvpD8miVjmmyfe\", \"recWPMm91mGDLXONm\", \"recWxGU8Q4YReJ1tb\", \"recWzUUYBN0NuXO2k\", \"recXsuOHRBLcyenF2\", \"recXvQ6gWAmyakrpD\", \"recYA3LPsCvF1fTMI\", \"recYOzCsevNz61Lyn\", \"recYt8xx80OTyDsL0\", \"recZ13cwgDQf9jRd9\", \"recZSGUkn56v9kEp1\", \"recZWeueB7lSPR6wN\", \"recZbxrocrxh9YENH\", \"recZdZxsN0AxDx0pB\", \"recZt4x514UQNnsgy\", \"reca44yABeO2fx7UB\", \"recaXdGn3FAIkxLGM\", \"recafnH3RRnMBXEb1\", \"recakFKZioXUuldcd\", \"recb2M22zaD3tL6Qc\", \"recb4cGsC6BJUCU3V\", \"reccOKzFNmyqeJ6ry\", \"reccVFW11Elovnf8T\", \"recclFbsjbaiVVnnV\", \"recdya6FuYraBU5Rh\", \"receYQxnUBvxPcvbQ\", \"receo3UUzRvWCqyfA\", \"recf6ayQmL1SxKbvW\", \"recgI6tUQ7RLJRWGx\", \"recgM7o1tcc7tP778\", \"recgXxEgllSGEpELP\", \"rechKl68Uc6H7vU0N\", \"rechgQucGlrnt8KRV\", \"rechtUKxC6G67H7Pr\", \"recihePFulRgNKsIn\", \"reciuppdQde4FGX1z\", \"recjJ54TXc04enRkZ\", \"recjgMJaMxz4ESDF2\", \"reck1zKeCssmN82Ga\", \"reck4G4xxv3YnpbtQ\", \"reckYqR92oJkqJXkE\", \"reckl6PkaEMWM50yZ\", \"recl1UtgTVKishAq4\", \"recmI7EiLv72PxmYK\", \"recmkvk6EFAmqyMxR\", \"recmwwQJnx7ll7bqL\", \"recn3NhOhqAPLda16\", \"recnGEpF1srQpaqWq\", \"recnTTKdBzfuoZ7w7\", \"recnawZPumSCLMVgp\", \"recnjViFrqlZNL3fY\", \"reco8WSc71p8zAv4q\", \"recooG6bivTUJpDBz\", \"recoy9ZLBsc7HIRBy\", \"recpJmTOXx6T48nDJ\", \"recpki12iG9RUGrz9\", \"recqFDvYKKRmbnoUb\", \"recqGD3fxPCI59vPQ\", \"recqLRu4BcJvlSANc\", \"recr3VHM4zYf6dMFY\", \"recrHBEJJoDTV05JR\", \"recrNbtgTNoabJJi6\", \"recrs0kUo7Ua1xhKO\", \"recs3PLPUEMiqg4P8\", \"rect4iLrSfUwkNTno\", \"rectXfsCM1dj4Kv2c\", \"rectlyG9pCAAuWhoB\", \"recuYEkVjdVUwA3c0\", \"recuyeuT5rQ6qDt8F\", \"recv7GsQg3f0fvB1f\", \"recvZlMbpPz0yLDle\", \"recvlnu8BuLmXZhSX\", \"recw4rROcnHKNZhtK\", \"recxTy0x7M6T9ikc0\", \"recxsX66ktJ1Lwt4f\", \"recyPffUDqC2k7ZZO\", \"recyl3usDqb7ruXJx\", \"recypVp2NmPlBKVTp\", \"reczQ4I0VpENdMtIj\", \"reczUoM8JsxU6pYxr\", \"reczjcMtrB1YGS2fO\", \"reczzzihL7btBH7RO\", \"rec0VuKUjt1SZ7NYv\", \"rec1oj2DveQWl9Rpw\", \"rec32C1ZEapBnCC0E\", \"rec47oNPXudlTIk4y\", \"rec6sE2CRtD4drtHg\", \"rec9W28HgpEUeUN8k\", \"recAYkd96NNuNl1Ei\", \"recBhnXrUyTJ6WHIR\", \"recE2ihVfqEK4R9d0\", \"recGee5m84dg5FZkc\", \"recINGR1z01Fh1Z3A\", \"recJpyGtGIsxulevT\", \"recLB0EkQ54bYVhnd\", \"recMicVBcqy1xM1jq\", \"recPIzpnuYpB4yvmp\", \"recRZjaElf8ft7xTq\", \"recReg13iV2HwJTaA\", \"recRhwQwHijW0gCLo\", \"recUc29lMDBEvurYo\", \"recVE8cUNHpHZIAvL\", \"recWXwn9v4IG9ZrM6\", \"recXsYa3i2UhGF5fe\", \"recYCMJWH7IpcdqwE\", \"recatj4YXzbOaq3Ar\", \"recb80OwMgNnceA9t\", \"reccVBrYdwsB84fGy\", \"recfTlTMjZBuOducT\", \"recgC4GJ8wUemlUQD\", \"recixxJmdux0d8LZQ\", \"reckEnrOPFT9Ru7tW\", \"recoiTJPGUmzAkief\", \"recwW1A85nfyQpReG\", \"recwlwlcPvXVNPmvl\", \"recyrk6XBVwugG5u8\", \"recywRj5a8EEjj2Ib\", \"recz3MfYHPRhsEQkW\", \"reczkBiPPNrNN49Hp\"]",false,1,"[\"mean\"]",null,null,null,null,null,"success",null,null,198,198,"choice","accuracy",0.7929292929292929,0.02886977846026699,0.7929292929292929,0.02886977846026699,null,null,null,null,null,null]