{
  "schema": 2,
  "model": "Qwen/Qwen2.5-Coder-7B-Instruct",
  "endpoint_family": "openai-compatible",
  "lang": "python",
  "spec_profile": null,
  "n": 20,
  "solved": 20,
  "pass_at_1": 1.0,
  "final_pass_rate": 1.0,
  "spec_tokens_o200k": 14,
  "task_prompt_tokens_o200k": 2023,
  "generated_tokens": 927,
  "repair_prompt_tokens": 0,
  "tokens_per_correct_output_only": 46.35,
  "total_tokens_session_amortized": 2964,
  "tokens_per_correct_session_amortized": 148.2,
  "total_tokens_cold_requests": 3230,
  "tokens_per_correct_cold_requests": 161.5,
  "frozen_language_manifest": {
    "schema": 1,
    "version": "0.0.7",
    "files": {
      "src/compiler.mjs": "cfab9f58fcffe59c33ac8f7791258c456743cfd689e30f7773cda7b2cff100d1",
      "docs/LLM_CORE.md": "8908f84dea12e98ccd8c820de011f1ca576f2fea79bc020482b316cc468b0b85",
      "docs/packs/sequence.md": "1cd6ea28d0283fa88aaa27555ed99727403c4c34a81cd1e52fe43786ce585cbd",
      "docs/LLM_SPEC.md": "7cbafad983debbb42ea1fa0fde3057a6e9e437716a07c6a79e1089c86c4df62f",
      "package.json": "d90dd19c68141cf90dee0f5a9637cb95e68e33cfe85fbb39c74e9d7b492b6053"
    },
    "language_spec_sha256": "e2a9592b895f9d4c16f3ba639d09c51d12ce0df1bf0558657be5fef79d875f31"
  },
  "rows": [
    {
      "id": "HumanEval/40",
      "pass": true,
      "attempts": [
        {
          "code": "def triples_sum_to_zero(l: list) -> bool:\n    n = len(l)\n    for i in range(n-2):\n        seen = set()\n        target = -l[i]\n        for j in range(i+1, n):\n            if (target - l[j]) in seen:\n                return True\n            seen.add(l[j])\n    return False",
          "tokens": 74,
          "pass": true,
          "error": "",
          "seconds": 11.396
        }
      ],
      "output_tokens": 74,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/41",
      "pass": true,
      "attempts": [
        {
          "code": "def car_race_collision(n: int) -> int:\n    return n * n",
          "tokens": 17,
          "pass": true,
          "error": "",
          "seconds": 7.677
        }
      ],
      "output_tokens": 17,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/42",
      "pass": true,
      "attempts": [
        {
          "code": "def incr_list(l: list) -> list:\n    return [x + 1 for x in l]",
          "tokens": 22,
          "pass": true,
          "error": "",
          "seconds": 8.049
        }
      ],
      "output_tokens": 22,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/43",
      "pass": true,
      "attempts": [
        {
          "code": "def pairs_sum_to_zero(l):\n    seen = set()\n    for num in l:\n        if -num in seen:\n            return True\n        seen.add(num)\n    return False",
          "tokens": 37,
          "pass": true,
          "error": "",
          "seconds": 9.061
        }
      ],
      "output_tokens": 37,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/44",
      "pass": true,
      "attempts": [
        {
          "code": "def change_base(x: int, base: int) -> str:\n    if x == 0:\n        return \"0\"\n    digits = []\n    while x:\n        digits.append(int(x % base))\n        x //= base\n    return ''.join(str(x) for x in digits[::-1])",
          "tokens": 62,
          "pass": true,
          "error": "",
          "seconds": 9.573
        }
      ],
      "output_tokens": 62,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/45",
      "pass": true,
      "attempts": [
        {
          "code": "def triangle_area(a, h):\n    return 0.5 * a * h",
          "tokens": 17,
          "pass": true,
          "error": "",
          "seconds": 7.939
        }
      ],
      "output_tokens": 17,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/46",
      "pass": true,
      "attempts": [
        {
          "code": "def fib4(n: int) -> int:\n    if n == 0 or n == 1 or n == 3:\n        return 0\n    elif n == 2:\n        return 2\n    \n    a, b, c, d = 0, 0, 2, 0\n    for _ in range(4, n + 1):\n        a, b, c, d = b, c, d, a + b + c + d\n    return d",
          "tokens": 104,
          "pass": true,
          "error": "",
          "seconds": 11.657
        }
      ],
      "output_tokens": 104,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/47",
      "pass": true,
      "attempts": [
        {
          "code": "def median(l: list):\n    sorted_l = sorted(l)\n    n = len(sorted_l)\n    mid = n // 2\n    if n % 2 == 0:\n        return (sorted_l[mid - 1] + sorted_l[mid]) / 2\n    else:\n        return sorted_l[mid]",
          "tokens": 66,
          "pass": true,
          "error": "",
          "seconds": 10.155
        }
      ],
      "output_tokens": 66,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/48",
      "pass": true,
      "attempts": [
        {
          "code": "def is_palindrome(text: str) -> bool:\n    return text == text[::-1]",
          "tokens": 19,
          "pass": true,
          "error": "",
          "seconds": 8.564
        }
      ],
      "output_tokens": 19,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/49",
      "pass": true,
      "attempts": [
        {
          "code": "def modp(n: int, p: int) -> int:\n    return pow(2, n, p)",
          "tokens": 24,
          "pass": true,
          "error": "",
          "seconds": 8.791
        }
      ],
      "output_tokens": 24,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/50",
      "pass": true,
      "attempts": [
        {
          "code": "def decode_shift(s: str):\n    \"\"\"\n    takes as input string encoded with encode_shift function. Returns decoded string.\n    \"\"\"\n    return \"\".join([chr(((ord(ch) - 5 - ord(\"a\")) % 26) + ord(\"a\")) for ch in s])",
          "tokens": 58,
          "pass": true,
          "error": "",
          "seconds": 10.848
        }
      ],
      "output_tokens": 58,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/51",
      "pass": true,
      "attempts": [
        {
          "code": "def remove_vowels(text):\n    return ''.join([char for char in text if char.lower() not in 'aeiou'])",
          "tokens": 26,
          "pass": true,
          "error": "",
          "seconds": 8.616
        }
      ],
      "output_tokens": 26,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/52",
      "pass": true,
      "attempts": [
        {
          "code": "def below_threshold(l: list, t: int) -> bool:\n    return all(x < t for x in l)",
          "tokens": 25,
          "pass": true,
          "error": "",
          "seconds": 8.426
        }
      ],
      "output_tokens": 25,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/53",
      "pass": true,
      "attempts": [
        {
          "code": "def add(x: int, y: int) -> int:\n    return x + y",
          "tokens": 18,
          "pass": true,
          "error": "",
          "seconds": 8.361
        }
      ],
      "output_tokens": 18,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/54",
      "pass": true,
      "attempts": [
        {
          "code": "def same_chars(s0: str, s1: str) -> bool:\n    return set(s0) == set(s1)",
          "tokens": 27,
          "pass": true,
          "error": "",
          "seconds": 29.36
        }
      ],
      "output_tokens": 27,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/55",
      "pass": true,
      "attempts": [
        {
          "code": "def fib(n: int) -> int:\n    if n <= 0:\n        return 0\n    elif n == 1:\n        return 1\n    else:\n        a, b = 0, 1\n        for _ in range(2, n + 1):\n            a, b = b, a + b\n        return b",
          "tokens": 74,
          "pass": true,
          "error": "",
          "seconds": 10.709
        }
      ],
      "output_tokens": 74,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/56",
      "pass": true,
      "attempts": [
        {
          "code": "def correct_bracketing(brackets: str) -> bool:\n    balance = 0\n    for bracket in brackets:\n        if bracket == '<':\n            balance += 1\n        elif bracket == '>':\n            balance -= 1\n        if balance < 0:\n            return False\n    return balance == 0",
          "tokens": 66,
          "pass": true,
          "error": "",
          "seconds": 9.906
        }
      ],
      "output_tokens": 66,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/57",
      "pass": true,
      "attempts": [
        {
          "code": "def monotonic(l: list) -> bool:\n    return all(x <= y for x, y in zip(l, l[1:])) or all(x >= y for x, y in zip(l, l[1:]))",
          "tokens": 47,
          "pass": true,
          "error": "",
          "seconds": 9.044
        }
      ],
      "output_tokens": 47,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/58",
      "pass": true,
      "attempts": [
        {
          "code": "def common(l1: list, l2: list) -> list:\n    return sorted(set(l1) & set(l2))",
          "tokens": 27,
          "pass": true,
          "error": "",
          "seconds": 7.443
        }
      ],
      "output_tokens": 27,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/59",
      "pass": true,
      "attempts": [
        {
          "code": "def largest_prime_factor(n: int) -> int:\n    def is_prime(num: int) -> bool:\n        if num <= 1:\n            return False\n        for i in range(2, int(num**0.5) + 1):\n            if num % i == 0:\n                return False\n        return True\n\n    largest_factor = None\n    for i in range(2, n + 1):\n        while n % i == 0 and is_prime(i):\n            largest_factor = i\n            n //= i\n    return largest_factor",
          "tokens": 117,
          "pass": true,
          "error": "",
          "seconds": 11.623
        }
      ],
      "output_tokens": 117,
      "repair_prompt_tokens": 0
    }
  ]
}