{
  "created_at": "2026-10-04T07:05:23.337889+00:00",
  "finished_at": "2026-10-04T07:09:26.933039+00:00",
  "framework_version": "0.1.0",
  "harnesses": [
    {
      "attempts": 5,
      "id": "opencode",
      "mean_duration_seconds": 13.15048036617227,
      "mean_score": 100,
      "name": "OpenCode",
      "successes": 5,
      "total_duration_seconds": 65.75240183086134,
      "version": "1.18.34"
    },
    {
      "attempts": 5,
      "id": "pi",
      "mean_duration_seconds": 15.595742934057489,
      "mean_score": 97.5,
      "name": "pi",
      "successes": 4,
      "total_duration_seconds": 77.97871467028745,
      "version": "0.99.2"
    },
    {
      "attempts": 5,
      "id": "oh-my-pi",
      "mean_duration_seconds": 19.44321526638232,
      "mean_score": 97.5,
      "name": "oh-my-pi",
      "successes": 4,
      "total_duration_seconds": 97.21607633191161,
      "version": "18.4.8"
    }
  ],
  "methodology": {
    "durations": "Host wall time includes agent startup, grading and cleanup; separate agent/grader times recorded",
    "execution_errors": "Timeouts and harness/infrastructure errors count as 0, with partial grading retained",
    "limits": "Docker bounds client CPU/RAM/time, not remote server GPU/RAM or total tokens",
    "repeats": "Fresh independent workspaces, serial execution, seeded shuffled attempt order; not pass@k",
    "scope": "Synthetic Python tasks, not a production-repository or contamination-resistant benchmark",
    "score": "100 × hidden unittest methods passed / total; each task equally weighted",
    "submission": "Only regular UTF-8 solution/**/*.py files are graded; tests/configs are never imported on host",
    "task_sets": "Selected sets expand in order, followed by explicit tasks; overlapping tasks run once per harness/repeat and appear in each applicable set breakdown, without double-counting overall metrics",
    "termination": "Final emitted response only; length is a diagnostic warning, not an execution error; grading and status are unchanged",
    "usage": "Primary harness event stream only; unknown counts are null; background calls may be omitted"
  },
  "model": {
    "api_profile": "llama-cpp",
    "context_window": 196608,
    "id": "qwen3.8-27b",
    "max_tokens": 4096,
    "reasoning": false,
    "thinking": "off"
  },
  "parameters": {
    "cpus": 2,
    "grade_timeout": 60,
    "memory": "4g",
    "repeats": 1,
    "seed": 42,
    "timeout": 600,
    "tool_profile": "native"
  },
  "results": [
    {
      "duration_seconds": 18.21833316795528,
      "failed_checks": [],
      "harness": "opencode",
      "repeat": 1,
      "score": 100,
      "status": "success",
      "task_id": "system-refactor",
      "tests_passed": 5,
      "tests_total": 5
    },
    {
      "duration_seconds": 16.62936175102368,
      "failed_checks": [],
      "harness": "oh-my-pi",
      "repeat": 1,
      "score": 100,
      "status": "success",
      "task_id": "performance",
      "tests_passed": 3,
      "tests_total": 3
    },
    {
      "duration_seconds": 12.974061766872182,
      "failed_checks": [],
      "harness": "oh-my-pi",
      "repeat": 1,
      "score": 100,
      "status": "success",
      "task_id": "system-refactor",
      "tests_passed": 5,
      "tests_total": 5
    },
    {
      "duration_seconds": 14.524599356111139,
      "failed_checks": [],
      "harness": "pi",
      "repeat": 1,
      "score": 100,
      "status": "success",
      "task_id": "system-refactor",
      "tests_passed": 5,
      "tests_total": 5
    },
    {
      "duration_seconds": 7.202246976783499,
      "failed_checks": [],
      "harness": "opencode",
      "repeat": 1,
      "score": 100,
      "status": "success",
      "task_id": "performance",
      "tests_passed": 3,
      "tests_total": 3
    },
    {
      "duration_seconds": 18.179182254942134,
      "failed_checks": [],
      "harness": "pi",
      "repeat": 1,
      "score": 100,
      "status": "success",
      "task_id": "performance",
      "tests_passed": 3,
      "tests_total": 3
    },
    {
      "duration_seconds": 12.40892429696396,
      "failed_checks": [],
      "harness": "opencode",
      "repeat": 1,
      "score": 100,
      "status": "success",
      "task_id": "small-feature",
      "tests_passed": 4,
      "tests_total": 4
    },
    {
      "duration_seconds": 10.306982821086422,
      "failed_checks": [],
      "harness": "opencode",
      "repeat": 1,
      "score": 100,
      "status": "success",
      "task_id": "bug-fix",
      "tests_passed": 5,
      "tests_total": 5
    },
    {
      "duration_seconds": 15.726988853188232,
      "failed_checks": [
        "_agent_bench_hidden_tests.NoteContract.test_descriptors_closed_on_success_and_errors"
      ],
      "harness": "pi",
      "repeat": 1,
      "score": 87.5,
      "status": "failed",
      "task_id": "security-hardening",
      "tests_passed": 7,
      "tests_total": 8
    },
    {
      "duration_seconds": 22.834601955022663,
      "failed_checks": [],
      "harness": "pi",
      "repeat": 1,
      "score": 100,
      "status": "success",
      "task_id": "small-feature",
      "tests_passed": 4,
      "tests_total": 4
    },
    {
      "duration_seconds": 37.45767129282467,
      "failed_checks": [],
      "harness": "oh-my-pi",
      "repeat": 1,
      "score": 100,
      "status": "success",
      "task_id": "small-feature",
      "tests_passed": 4,
      "tests_total": 4
    },
    {
      "duration_seconds": 17.615914568072185,
      "failed_checks": [],
      "harness": "opencode",
      "repeat": 1,
      "score": 100,
      "status": "success",
      "task_id": "security-hardening",
      "tests_passed": 8,
      "tests_total": 8
    },
    {
      "duration_seconds": 6.713342251023278,
      "failed_checks": [],
      "harness": "pi",
      "repeat": 1,
      "score": 100,
      "status": "success",
      "task_id": "bug-fix",
      "tests_passed": 5,
      "tests_total": 5
    },
    {
      "duration_seconds": 12.374352645128965,
      "failed_checks": [],
      "harness": "oh-my-pi",
      "repeat": 1,
      "score": 100,
      "status": "success",
      "task_id": "bug-fix",
      "tests_passed": 5,
      "tests_total": 5
    },
    {
      "duration_seconds": 17.78062887606211,
      "failed_checks": [
        "_agent_bench_hidden_tests.NoteContract.test_descriptors_closed_on_success_and_errors"
      ],
      "harness": "oh-my-pi",
      "repeat": 1,
      "score": 87.5,
      "status": "failed",
      "task_id": "security-hardening",
      "tests_passed": 7,
      "tests_total": 8
    }
  ],
  "run_id": "20261004T070523Z-3662bf45",
  "source_sha256": "cf6004cd5715e003953cb8abeedb71ea5ef389a440f8dfd87afea1b09c64f25a",
  "tasks": [
    {
      "fixture_sha256": "b9ed6431df126798b8576c4b5ca11d7175bfe4305b88192110e4909296406546",
      "hidden_tests_sha256": "cf5758b0318ee3c60700daaf32883b67dbf26eea6e66389207673bf5c8433a75",
      "id": "bug-fix",
      "title": "Fix invoice reconciliation precision"
    },
    {
      "fixture_sha256": "1dec5b9e9f2ac9f9ed2bd32ad5c9053fe3de236fbf854ef4a4119886963495ba",
      "hidden_tests_sha256": "08a4a000fae1cd136b5be92273726333f3d67036973b8c0f111b139f831261bf",
      "id": "small-feature",
      "title": "Add nested configuration overlays and deletion"
    },
    {
      "fixture_sha256": "cb0f527c95fdaba91a537c043f1313b609dfd472c59c88aa2e4740406d5b1496",
      "hidden_tests_sha256": "ad726d0c4f26369d044cf9662fe043877ed909388f2189ef47fd3bb9d239d9c3",
      "id": "system-refactor",
      "title": "Centralize pricing across checkout and reporting"
    },
    {
      "fixture_sha256": "ab9ca138fd759e77e590b319a6bbc4aca2abd6b77eb9d3c4d52c0d77b3f50207",
      "hidden_tests_sha256": "90308887d1c8ccb6cef61ce2f730ee13a1e970966c3c83aeacd6413ac9a689fb",
      "id": "security-hardening",
      "title": "Harden filesystem-backed note reads"
    },
    {
      "fixture_sha256": "8b171bfe7caa4efb25946610a3391fc8e9932b375b513778e3ec36b9eef935de",
      "hidden_tests_sha256": "1ba90a815ccdd29f90d12abfcf37691860028ab9104598b3ade99d8cdf99fb76",
      "id": "performance",
      "title": "Make rolling event counts linear"
    }
  ]
}