{
  "schema": 2,
  "model": "Qwen/Qwen2.5-Coder-7B-Instruct",
  "endpoint_family": "openai-compatible",
  "lang": "python",
  "spec_profile": null,
  "n": 20,
  "solved": 18,
  "pass_at_1": 0.85,
  "final_pass_rate": 0.9,
  "spec_tokens_o200k": 14,
  "task_prompt_tokens_o200k": 3043,
  "generated_tokens": 1165,
  "repair_prompt_tokens": 71,
  "tokens_per_correct_output_only": 68.66666666666667,
  "total_tokens_session_amortized": 4293,
  "tokens_per_correct_session_amortized": 238.5,
  "total_tokens_cold_requests": 4559,
  "tokens_per_correct_cold_requests": 253.27777777777777,
  "frozen_language_manifest": {
    "schema": 1,
    "version": "0.0.8",
    "files": {
      "src/compiler.mjs": "f6e06599ca090fa4fc57d50a3f142723e0b207e7e437ff423faafc2d71e9461d",
      "docs/LLM_CORE.md": "7ef989b1d37b119a6a57a2cba5aa7a5e1353d837c4e5ac48fea671d7e985b3bb",
      "docs/packs/sequence.md": "1cd6ea28d0283fa88aaa27555ed99727403c4c34a81cd1e52fe43786ce585cbd",
      "docs/LLM_SPEC.md": "fc460794045b3722ca9dee0d8221d7920f367086cb66c68f8fd337d0a2ffa5c4",
      "package.json": "fbdfe0f4806ef5ec2eaab2eb06eb3a5b09d1b85d5faef27ff61cde956133768b"
    },
    "language_spec_sha256": "2d7f1e21e02d7873b594ce35407ac6e880f27a78599683b67ac0b0641b98f1ba"
  },
  "rows": [
    {
      "id": "HumanEval/60",
      "pass": true,
      "attempts": [
        {
          "code": "def sum_to_n(n: int) -> int:\n    return n * (n + 1) // 2",
          "tokens": 24,
          "pass": true,
          "error": "",
          "seconds": 11.878
        }
      ],
      "output_tokens": 24,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/61",
      "pass": true,
      "attempts": [
        {
          "code": "def correct_bracketing(brackets: str) -> bool:\n    balance = 0\n    for bracket in brackets:\n        if bracket == '(':\n            balance += 1\n        elif bracket == ')':\n            balance -= 1\n        if balance < 0:\n            return False\n    return balance == 0",
          "tokens": 66,
          "pass": true,
          "error": "",
          "seconds": 10.838
        }
      ],
      "output_tokens": 66,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/62",
      "pass": true,
      "attempts": [
        {
          "code": "def derivative(xs: list) -> list:\n    return [i * x for i, x in enumerate(xs)][1:]",
          "tokens": 25,
          "pass": true,
          "error": "",
          "seconds": 8.699
        }
      ],
      "output_tokens": 25,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/63",
      "pass": true,
      "attempts": [
        {
          "code": "def fibfib(n: int) -> int:\n    if n == 0 or n == 1:\n        return 0\n    elif n == 2:\n        return 1\n    else:\n        a, b, c = 0, 0, 1\n        for _ in range(3, n + 1):\n            a, b, c = b, c, a + b + c\n        return c",
          "tokens": 91,
          "pass": true,
          "error": "",
          "seconds": 12.857
        }
      ],
      "output_tokens": 91,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/64",
      "pass": true,
      "attempts": [
        {
          "code": "def vowels_count(s):\n    vowels = \"aeiou\"\n    count = sum(1 for char in s.lower() if char in vowels)\n    if s.endswith('y') or s.endswith('Y'):\n        count += 1\n    return count",
          "tokens": 51,
          "pass": true,
          "error": "",
          "seconds": 9.475
        }
      ],
      "output_tokens": 51,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/65",
      "pass": false,
      "attempts": [
        {
          "code": "def circular_shift(x, shift):\n    s = str(x)\n    n = len(s)\n    if shift >= n:\n        return s[::-1]\n    else:\n        shift %= n\n        return s[-shift:] + s[:-shift]",
          "tokens": 50,
          "pass": false,
          "error": "AssertionError()",
          "seconds": 9.965
        },
        {
          "code": "def circular_shift(x, shift):\n    s = str(x)\n    n = len(s)\n    if shift >= n:\n        return s[::-1]\n    else:\n        shift %= n\n        return s[-shift:] + s[:-shift]",
          "tokens": 50,
          "pass": false,
          "error": "AssertionError()",
          "seconds": 10.52
        }
      ],
      "output_tokens": 100,
      "repair_prompt_tokens": 16
    },
    {
      "id": "HumanEval/66",
      "pass": true,
      "attempts": [
        {
          "code": "def digitSum(s):\n    return sum(ord(c) for c in s if c.isupper())",
          "tokens": 20,
          "pass": true,
          "error": "",
          "seconds": 8.105
        }
      ],
      "output_tokens": 20,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/67",
      "pass": true,
      "attempts": [
        {
          "code": "def fruit_distribution(s, n):\n    parts = s.split()\n    apples = int(parts[0])\n    oranges = int(parts[3])\n    return n - apples - oranges",
          "tokens": 36,
          "pass": true,
          "error": "",
          "seconds": 10.065
        }
      ],
      "output_tokens": 36,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/68",
      "pass": true,
      "attempts": [
        {
          "code": "def pluck(arr):\n    min_val = float('inf')\n    min_index = -1\n    for i, val in enumerate(arr):\n        if val % 2 == 0 and val < min_val:\n            min_val = val\n            min_index = i\n    return [min_val, min_index] if min_index != -1 else []",
          "tokens": 73,
          "pass": true,
          "error": "",
          "seconds": 10.807
        }
      ],
      "output_tokens": 73,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/69",
      "pass": true,
      "attempts": [
        {
          "code": "from collections import Counter\n\ndef search(lst):\n    freq = Counter(lst)\n    for num in sorted(freq.keys(), reverse=True):\n        if freq[num] >= num:\n            return num\n    return -1",
          "tokens": 42,
          "pass": true,
          "error": "",
          "seconds": 11.204
        }
      ],
      "output_tokens": 42,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/70",
      "pass": true,
      "attempts": [
        {
          "code": "def strange_sort_list(lst):\n    sorted_lst = sorted(lst)\n    result = []\n    while sorted_lst:\n        if sorted_lst:\n            result.append(sorted_lst.pop(0))\n        if sorted_lst:\n            result.append(sorted_lst.pop())\n    return result",
          "tokens": 51,
          "pass": true,
          "error": "",
          "seconds": 10.629
        }
      ],
      "output_tokens": 51,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/71",
      "pass": true,
      "attempts": [
        {
          "code": "import math\n\ndef triangle_area(a, b, c):\n    if a + b > c and a + c > b and b + c > a:\n        s = (a + b + c) / 2\n        area = math.sqrt(s * (s - a) * (s - b) * (s - c))\n        return round(area, 2)\n    else:\n        return -1",
          "tokens": 85,
          "pass": true,
          "error": "",
          "seconds": 12.503
        }
      ],
      "output_tokens": 85,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/72",
      "pass": true,
      "attempts": [
        {
          "code": "def will_it_fly(q, w):\n    return q == q[::-1] and sum(q) <= w",
          "tokens": 23,
          "pass": true,
          "error": "",
          "seconds": 8.835
        }
      ],
      "output_tokens": 23,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/73",
      "pass": true,
      "attempts": [
        {
          "code": "def smallest_change(arr):\n    left = 0\n    right = len(arr) - 1\n    changes = 0\n    \n    while left < right:\n        if arr[left] != arr[right]:\n            changes += 1\n        left += 1\n        right -= 1\n    \n    return changes",
          "tokens": 63,
          "pass": true,
          "error": "",
          "seconds": 12.056
        }
      ],
      "output_tokens": 63,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/74",
      "pass": true,
      "attempts": [
        {
          "code": "def total_match(lst1, lst2):\n    total_chars_lst1 = sum(len(s) for s in lst1)\n    total_chars_lst2 = sum(len(s) for s in lst2)\n    \n    if total_chars_lst1 <= total_chars_lst2:\n        return lst1\n    else:\n        return lst2",
          "tokens": 66,
          "pass": true,
          "error": "",
          "seconds": 13.698
        }
      ],
      "output_tokens": 66,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/75",
      "pass": true,
      "attempts": [
        {
          "code": "def is_multiply_prime(a):\n    def is_prime(n):\n        if n <= 1:\n            return False\n        for i in range(2, int(n**0.5) + 1):\n            if n % i == 0:\n                return False\n        return True\n    \n    primes = [i for i in range(2, 100) if is_prime(i)]\n    \n    for i in range(len(primes)):\n        for j in range(i, len(primes)):\n            for k in range(j, len(primes)):\n                if primes[i] * primes[j] * primes[k] == a:\n                    return True\n    return False",
          "tokens": 135,
          "pass": true,
          "error": "",
          "seconds": 14.011
        }
      ],
      "output_tokens": 135,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/76",
      "pass": false,
      "attempts": [
        {
          "code": "def is_simple_power(x, n):\n    return n ** int(n ** (1 / float(n))) == x",
          "tokens": 23,
          "pass": false,
          "error": "AssertionError('This prints if this assert fails 1 (good for debugging!)')",
          "seconds": 10.649
        },
        {
          "code": "def is_simple_power(x, n):\n    return n ** round(n ** (1 / float(n))) == x",
          "tokens": 23,
          "pass": false,
          "error": "AssertionError('This prints if this assert fails 1 (good for debugging!)')",
          "seconds": 9.185
        }
      ],
      "output_tokens": 46,
      "repair_prompt_tokens": 30
    },
    {
      "id": "HumanEval/77",
      "pass": true,
      "attempts": [
        {
          "code": "def iscube(a):\n    return round(a ** (1/3)) ** 3 == a",
          "tokens": 20,
          "pass": false,
          "error": "TypeError(\"type complex doesn't define __round__ method\")",
          "seconds": 8.996
        },
        {
          "code": "def iscube(a):\n    root = round(abs(a) ** (1/3))\n    return root ** 3 == abs(a) and (a >= 0 or root % 2 == 1)",
          "tokens": 43,
          "pass": true,
          "error": "",
          "seconds": 11.304
        }
      ],
      "output_tokens": 63,
      "repair_prompt_tokens": 25
    },
    {
      "id": "HumanEval/78",
      "pass": true,
      "attempts": [
        {
          "code": "def hex_key(num):\n    prime_hex_digits = {'2', '3', '5', '7', 'B', 'D'}\n    return sum(1 for digit in num if digit in prime_hex_digits)",
          "tokens": 44,
          "pass": true,
          "error": "",
          "seconds": 9.281
        }
      ],
      "output_tokens": 44,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/79",
      "pass": true,
      "attempts": [
        {
          "code": "def decimal_to_binary(decimal):\n    if decimal == 0:\n        return \"db0db\"\n    binary = \"\"\n    while decimal > 0:\n        binary = str(decimal % 2) + binary\n        decimal //= 2\n    return \"db\" + binary + \"db\"",
          "tokens": 61,
          "pass": true,
          "error": "",
          "seconds": 12.086
        }
      ],
      "output_tokens": 61,
      "repair_prompt_tokens": 0
    }
  ]
}