{
  "schema": 2,
  "model": "Qwen/Qwen2.5-Coder-7B-Instruct",
  "endpoint_family": "openai-compatible",
  "lang": "lir",
  "spec_profile": "compact",
  "n": 20,
  "solved": 5,
  "pass_at_1": 0.2,
  "final_pass_rate": 0.25,
  "spec_tokens_o200k": 288,
  "task_prompt_tokens_o200k": 3043,
  "generated_tokens": 1961,
  "repair_prompt_tokens": 693,
  "tokens_per_correct_output_only": 530.8,
  "total_tokens_session_amortized": 5985,
  "tokens_per_correct_session_amortized": 1197.0,
  "total_tokens_cold_requests": 11457,
  "tokens_per_correct_cold_requests": 2291.4,
  "frozen_language_manifest": {
    "schema": 1,
    "version": "0.0.8",
    "files": {
      "src/compiler.mjs": "f6e06599ca090fa4fc57d50a3f142723e0b207e7e437ff423faafc2d71e9461d",
      "docs/LLM_CORE.md": "7ef989b1d37b119a6a57a2cba5aa7a5e1353d837c4e5ac48fea671d7e985b3bb",
      "docs/packs/sequence.md": "1cd6ea28d0283fa88aaa27555ed99727403c4c34a81cd1e52fe43786ce585cbd",
      "docs/LLM_SPEC.md": "fc460794045b3722ca9dee0d8221d7920f367086cb66c68f8fd337d0a2ffa5c4",
      "package.json": "fbdfe0f4806ef5ec2eaab2eb06eb3a5b09d1b85d5faef27ff61cde956133768b"
    },
    "language_spec_sha256": "2d7f1e21e02d7873b594ce35407ac6e880f27a78599683b67ac0b0641b98f1ba"
  },
  "rows": [
    {
      "id": "HumanEval/60",
      "pass": true,
      "attempts": [
        {
          "code": "sum_to_n=n=>n*(n+1)/2",
          "tokens": 12,
          "pass": true,
          "error": "",
          "seconds": 10.576
        }
      ],
      "output_tokens": 12,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/61",
      "pass": false,
      "attempts": [
        {
          "code": "correct_bracketing(brackets) = {\n  stack = [];\n  for char in brackets {\n    if char == '(' {\n      stack.push(char);\n    } else if char == ')' {\n      if stack.len() == 0 {\n        return false;\n      }\n      stack.pop();\n    }\n  }\n  return stack.len() == 0;\n}",
          "tokens": 71,
          "pass": false,
          "error": "RuntimeError('compile: lse;\\n      }\\n      stack.pop();\\n    }\\n  }\\n  return stack.len() == 0;\\n}\\n    at compile (file:///workspace/scratch/b8233941f598/recovered/compact-llm-ir-v0.0.8/src/compiler.mjs:190:11)\\n    at file:///workspace/scratch/b8233941f598/recovered/compact-llm-ir-v0.0.8/src/cli.mjs:5:13\\n    at ModuleJob.run (node:internal/modules/esm/module_job:439:25)\\n    at async node:internal/modules/esm/loader:643:26\\n    at async asyncRunEntryPointWithESMLoader (node:internal/modules/run_main:101:5)\\n\\nNode.js v24.19.0\\n')",
          "seconds": 11.504
        },
        {
          "code": "correct_bracketing(brackets) = {\n  stack = [];\n  for char in brackets {\n    if char == '(' {\n      stack.push(char);\n    } else if char == ')' {\n      if stack.len() == 0 {\n        return false;\n      }\n      stack.pop();\n    }\n  }\n  return stack.len() == 0;\n}",
          "tokens": 71,
          "pass": false,
          "error": "RuntimeError('compile: lse;\\n      }\\n      stack.pop();\\n    }\\n  }\\n  return stack.len() == 0;\\n}\\n    at compile (file:///workspace/scratch/b8233941f598/recovered/compact-llm-ir-v0.0.8/src/compiler.mjs:190:11)\\n    at file:///workspace/scratch/b8233941f598/recovered/compact-llm-ir-v0.0.8/src/cli.mjs:5:13\\n    at ModuleJob.run (node:internal/modules/esm/module_job:439:25)\\n    at async node:internal/modules/esm/loader:643:26\\n    at async asyncRunEntryPointWithESMLoader (node:internal/modules/run_main:101:5)\\n\\nNode.js v24.19.0\\n')",
          "seconds": 13.361
        }
      ],
      "output_tokens": 142,
      "repair_prompt_tokens": 196
    },
    {
      "id": "HumanEval/62",
      "pass": true,
      "attempts": [
        {
          "code": "derivative(xs) = xs.slice(1).map(i => i * (xs.len() - i))",
          "tokens": 22,
          "pass": false,
          "error": "AssertionError()",
          "seconds": 11.461
        },
        {
          "code": "derivative(xs) = xs.slice(1).map((i, idx) => i * (idx + 1))",
          "tokens": 25,
          "pass": true,
          "error": "",
          "seconds": 9.365
        }
      ],
      "output_tokens": 47,
      "repair_prompt_tokens": 16
    },
    {
      "id": "HumanEval/63",
      "pass": true,
      "attempts": [
        {
          "code": "fibfib(n){\n    if n==0 return 0;\n    if n==1 return 0;\n    if n==2 return 1;\n    let a=0, b=0, c=1, d;\n    for i from 3 to n{\n        d=a+b+c;\n        a=b;\n        b=c;\n        c=d;\n    }\n    return c;\n}",
          "tokens": 81,
          "pass": true,
          "error": "",
          "seconds": 12.038
        }
      ],
      "output_tokens": 81,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/64",
      "pass": false,
      "attempts": [
        {
          "code": "vowels_count(s) {\n    let count = 0;\n    let vowels = \"aeiou\";\n    for char in s.lower() {\n        if vowels.includes(char) || (char == 'y' && s.len() - 1 == s.index(char)) {\n            count++;\n        }\n    }\n    return count;\n}",
          "tokens": 67,
          "pass": false,
          "error": "RuntimeError('s.toLowerCase is not a function or its return value is not iterable')",
          "seconds": 14.597
        },
        {
          "code": "vowels_count(s) {\n    let count = 0;\n    let vowels = \"aeiou\";\n    for char in s.lower() {\n        if vowels.includes(char) || (char == 'y' && s.len() - 1 == s.index(char)) {\n            count++;\n        }\n    }\n    return count;\n}",
          "tokens": 67,
          "pass": false,
          "error": "RuntimeError('s.toLowerCase is not a function or its return value is not iterable')",
          "seconds": 12.322
        }
      ],
      "output_tokens": 134,
      "repair_prompt_tokens": 32
    },
    {
      "id": "HumanEval/65",
      "pass": true,
      "attempts": [
        {
          "code": "circular_shift(x,shift){\n    let s=str(x);\n    let n=len(s);\n    if shift>n{\n        return rev(s);\n    }\n    let r=s.slice(-shift)+s.slice(0,n-shift);\n    return r;\n}",
          "tokens": 53,
          "pass": true,
          "error": "",
          "seconds": 10.489
        }
      ],
      "output_tokens": 53,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/66",
      "pass": false,
      "attempts": [
        {
          "code": "digitSum=s=>s.filter(c=>c>=65&&c<=90).map(c=>c.charCodeAt(0)).sum()",
          "tokens": 28,
          "pass": false,
          "error": "RuntimeError('s.filter is not a function')",
          "seconds": 9.308
        },
        {
          "code": "digitSum=s=>s.split('').filter(c=>c>=65&&c<=90).reduce((acc,c)=>acc+c.charCodeAt(0),0)",
          "tokens": 33,
          "pass": false,
          "error": "AssertionError('Error')",
          "seconds": 11.715
        }
      ],
      "output_tokens": 61,
      "repair_prompt_tokens": 23
    },
    {
      "id": "HumanEval/67",
      "pass": false,
      "attempts": [
        {
          "code": "fruit_distribution(s,n)=n-(int(rev(split(s,\" \")[0]))+int(rev(split(split(s,\"and\")[1],\" \")[0])))",
          "tokens": 31,
          "pass": false,
          "error": "RuntimeError('split is not defined')",
          "seconds": 10.027
        },
        {
          "code": "fruit_distribution(s,n)=n-(int(str.split(s,\" \")[0])+int(str.split(str.split(s,\"and\")[1],\" \")[0]))",
          "tokens": 29,
          "pass": false,
          "error": "RuntimeError('str is not defined')",
          "seconds": 10.434
        }
      ],
      "output_tokens": 60,
      "repair_prompt_tokens": 21
    },
    {
      "id": "HumanEval/68",
      "pass": false,
      "attempts": [
        {
          "code": "pluck(arr){\n    let minEven = -1;\n    let minIndex = -1;\n    each i in arr{\n        if arr[i] % 2 == 0{\n            if minEven == -1 || arr[i] < minEven || (arr[i] == minEven && i < minIndex){\n                minEven = arr[i];\n                minIndex = i;\n            }\n        }\n    }\n    if minEven != -1{\n        return [minEven, minIndex];\n    }else{\n        return [];\n    }\n}",
          "tokens": 111,
          "pass": false,
          "error": "AssertionError('Error')",
          "seconds": 12.099
        },
        {
          "code": "pluck(arr){\n    let minEven = -1;\n    let minIndex = -1;\n    each i in arr{\n        if arr[i] % 2 == 0{\n            if minEven == -1 || arr[i] < minEven || (arr[i] == minEven && i < minIndex){\n                minEven = arr[i];\n                minIndex = i;\n            }\n        }\n    }\n    if minEven != -1{\n        return [minEven, minIndex];\n    }else{\n        return [];\n    }\n}",
          "tokens": 111,
          "pass": false,
          "error": "AssertionError('Error')",
          "seconds": 12.494
        }
      ],
      "output_tokens": 222,
      "repair_prompt_tokens": 18
    },
    {
      "id": "HumanEval/69",
      "pass": false,
      "attempts": [
        {
          "code": "search(lst){\n    freq = lst.map(x => [x, lst.filter(y => y == x).len()]);\n    sortedFreq = freq.sort((a, b) => b[0] - a[0]);\n    for item in sortedFreq{\n        if item[0] <= item[1]{return item[0]}\n    }\n    return -1\n}",
          "tokens": 75,
          "pass": false,
          "error": "RuntimeError('lst.filter(...).length is not a function')",
          "seconds": 10.619
        },
        {
          "code": "search(lst){\n    freq = lst.map(x => [x, lst.filter(y => y == x).len()]);\n    sortedFreq = freq.sort((a, b) => b[0] - a[0]);\n    for item in sortedFreq{\n        if item[0] <= item[1]{return item[0]}\n    }\n    return -1\n}",
          "tokens": 75,
          "pass": false,
          "error": "RuntimeError('lst.filter(...).length is not a function')",
          "seconds": 10.717
        }
      ],
      "output_tokens": 150,
      "repair_prompt_tokens": 26
    },
    {
      "id": "HumanEval/70",
      "pass": false,
      "attempts": [
        {
          "code": "strange_sort_list(lst){\n    sorted_lst = sort(lst);\n    result = [];\n    while len(sorted_lst) > 0{\n        if len(result) % 2 == 0{\n            append(result, min(sorted_lst));\n        } else {\n            append(result, max(sorted_lst));\n        }\n        sorted_lst = filter(sorted_lst, x != last(result));\n    }\n    return result;\n}",
          "tokens": 81,
          "pass": false,
          "error": "RuntimeError('append is not defined')",
          "seconds": 12.293
        },
        {
          "code": "strange_sort_list(lst){\n    sorted_lst = sort(lst);\n    result = [];\n    while len(sorted_lst) > 0{\n        if len(result) % 2 == 0{\n            append(result, min(sorted_lst));\n        } else {\n            append(result, max(sorted_lst));\n        }\n        sorted_lst = filter(sorted_lst, x != last(result));\n    }\n    return result;\n}",
          "tokens": 81,
          "pass": false,
          "error": "RuntimeError('append is not defined')",
          "seconds": 10.689
        }
      ],
      "output_tokens": 162,
      "repair_prompt_tokens": 21
    },
    {
      "id": "HumanEval/71",
      "pass": false,
      "attempts": [
        {
          "code": "triangle_area(a,b,c){\n    if (a+b<=c || a+c<=b || b+c<=a){return -1}\n    s=(a+b+c)/2\n    area=sqrt(s*(s-a)*(s-b)*(s-c))\n    return round(area,2)\n}",
          "tokens": 60,
          "pass": false,
          "error": "RuntimeError('sqrt is not defined')",
          "seconds": 11.412
        },
        {
          "code": "triangle_area(a,b,c){\n    if (a+b<=c || a+c<=b || b+c<=a){return -1}\n    s=(a+b+c)/2\n    area=sqrt(s*(s-a)*(s-b)*(s-c))\n    return round(area,2)\n}",
          "tokens": 60,
          "pass": false,
          "error": "RuntimeError('sqrt is not defined')",
          "seconds": 10.447
        }
      ],
      "output_tokens": 120,
      "repair_prompt_tokens": 21
    },
    {
      "id": "HumanEval/72",
      "pass": false,
      "attempts": [
        {
          "code": "will_it_fly(q,w){\n    if q.rev==q and sum(q)<=w{\n        return true\n    }\n    return false\n}",
          "tokens": 30,
          "pass": false,
          "error": "AssertionError()",
          "seconds": 10.798
        },
        {
          "code": "will_it_fly(q,w){\n    if q.rev()==q and sum(q)<=w{\n        return true\n    }\n    return false\n}",
          "tokens": 30,
          "pass": false,
          "error": "RuntimeError('__rev(...) is not a function')",
          "seconds": 8.635
        }
      ],
      "output_tokens": 60,
      "repair_prompt_tokens": 16
    },
    {
      "id": "HumanEval/73",
      "pass": false,
      "attempts": [
        {
          "code": "smallest_change(arr) = {\n    let n = len(arr);\n    let count = 0;\n    for i from 0 to n-1 {\n        if arr[i] != arr[n-i-1] {\n            count++;\n        }\n    }\n    return count / 2;\n}",
          "tokens": 60,
          "pass": false,
          "error": "RuntimeError('compile: [n-i-1] {\\n            count++;\\n        }\\n    }\\n    return count / 2;\\n}\\n    at compile (file:///workspace/scratch/b8233941f598/recovered/compact-llm-ir-v0.0.8/src/compiler.mjs:190:11)\\n    at file:///workspace/scratch/b8233941f598/recovered/compact-llm-ir-v0.0.8/src/cli.mjs:5:13\\n    at ModuleJob.run (node:internal/modules/esm/module_job:439:25)\\n    at async node:internal/modules/esm/loader:643:26\\n    at async asyncRunEntryPointWithESMLoader (node:internal/modules/run_main:101:5)\\n\\nNode.js v24.19.0\\n')",
          "seconds": 9.971
        },
        {
          "code": "smallest_change(arr) = {\n    let n = len(arr);\n    let count = 0;\n    for i from 0 to n/2 - 1 {\n        if arr[i] != arr[n-i-1] {\n            count++;\n        }\n    }\n    return count;\n}",
          "tokens": 60,
          "pass": false,
          "error": "RuntimeError('compile:  arr[n-i-1] {\\n            count++;\\n        }\\n    }\\n    return count;\\n}\\n    at compile (file:///workspace/scratch/b8233941f598/recovered/compact-llm-ir-v0.0.8/src/compiler.mjs:190:11)\\n    at file:///workspace/scratch/b8233941f598/recovered/compact-llm-ir-v0.0.8/src/cli.mjs:5:13\\n    at ModuleJob.run (node:internal/modules/esm/module_job:439:25)\\n    at async node:internal/modules/esm/loader:643:26\\n    at async asyncRunEntryPointWithESMLoader (node:internal/modules/run_main:101:5)\\n\\nNode.js v24.19.0\\n')",
          "seconds": 9.642
        }
      ],
      "output_tokens": 120,
      "repair_prompt_tokens": 194
    },
    {
      "id": "HumanEval/74",
      "pass": false,
      "attempts": [
        {
          "code": "total_match(lst1,lst2){\n  len1=lst1.len;\n  len2=lst2.len;\n  sum1=lst1.sum(x=>x.len);\n  sum2=lst2.sum(x=>x.len);\n  if sum1<len2{return lst1}\n  if sum2<len1{return lst2}\n  return lst1\n}",
          "tokens": 74,
          "pass": false,
          "error": "RuntimeError('lst1.reduce(...) is not a function')",
          "seconds": 13.105
        },
        {
          "code": "total_match(lst1,lst2){\n  len1=lst1.len;\n  len2=lst2.len;\n  sum1=lst1.sum(x=>x.len);\n  sum2=lst2.sum(x=>x.len);\n  if(sum1<len2)return lst1;\n  if(sum2<len1)return lst2;\n  return lst1;\n}",
          "tokens": 74,
          "pass": false,
          "error": "RuntimeError('compile:        ^\\n\\nSyntaxError: LIR: unsupported line: if(sum1<len2)return lst1\\n    at compile (file:///workspace/scratch/b8233941f598/recovered/compact-llm-ir-v0.0.8/src/compiler.mjs:190:11)\\n    at file:///workspace/scratch/b8233941f598/recovered/compact-llm-ir-v0.0.8/src/cli.mjs:5:13\\n    at ModuleJob.run (node:internal/modules/esm/module_job:439:25)\\n    at async node:internal/modules/esm/loader:643:26\\n    at async asyncRunEntryPointWithESMLoader (node:internal/modules/run_main:101:5)\\n\\nNode.js v24.19.0\\n')",
          "seconds": 10.586
        }
      ],
      "output_tokens": 148,
      "repair_prompt_tokens": 25
    },
    {
      "id": "HumanEval/75",
      "pass": false,
      "attempts": [
        {
          "code": "is_multiply_prime(a){\n    if a <= 1 { return false }\n    let count = 0\n    for i from 2 to a {\n        if a % i == 0 {\n            let j = i\n            while j * j <= a {\n                if a % j == 0 {\n                    count++\n                    if count > 3 { return false }\n                }\n                j++\n            }\n        }\n    }\n    return count == 3\n}",
          "tokens": 96,
          "pass": false,
          "error": "AssertionError()",
          "seconds": 16.507
        },
        {
          "code": "is_multiply_prime(a){\n    if a <= 1 { return false }\n    let count = 0\n    for i from 2 to a {\n        if a % i == 0 {\n            let j = i\n            while j * j <= a {\n                if a % j == 0 {\n                    count++\n                    if count > 3 { return false }\n                }\n                j++\n            }\n        }\n    }\n    return count == 3\n}",
          "tokens": 96,
          "pass": false,
          "error": "AssertionError()",
          "seconds": 11.288
        }
      ],
      "output_tokens": 192,
      "repair_prompt_tokens": 16
    },
    {
      "id": "HumanEval/76",
      "pass": false,
      "attempts": [
        {
          "code": "is_simple_power(x,n){\n    return pow(n,int(x))==x\n}",
          "tokens": 17,
          "pass": false,
          "error": "AssertionError('This prints if this assert fails 1 (good for debugging!)')",
          "seconds": 10.516
        },
        {
          "code": "is_simple_power(x,n){\n    return pow(n,int(x))==x\n}",
          "tokens": 17,
          "pass": false,
          "error": "AssertionError('This prints if this assert fails 1 (good for debugging!)')",
          "seconds": 7.969
        }
      ],
      "output_tokens": 34,
      "repair_prompt_tokens": 30
    },
    {
      "id": "HumanEval/77",
      "pass": false,
      "attempts": [
        {
          "code": "iscube(a){\n  b=int(pow(abs(a),1/3))\n  return b*b*b==a\n}",
          "tokens": 24,
          "pass": false,
          "error": "AssertionError('Third test error: False')",
          "seconds": 11.057
        },
        {
          "code": "iscube(a){\n  b=int(pow(abs(a),1/3))\n  return b*b*b==a\n}",
          "tokens": 24,
          "pass": false,
          "error": "AssertionError('Third test error: False')",
          "seconds": 7.919
        }
      ],
      "output_tokens": 48,
      "repair_prompt_tokens": 22
    },
    {
      "id": "HumanEval/78",
      "pass": true,
      "attempts": [
        {
          "code": "hex_key(s) {\n    let count = 0;\n    for char in s {\n        if char == '2' or char == '3' or char == '5' or char == '7' or char == 'B' or char == 'D' {\n            count++;\n        }\n    }\n    return count;\n}",
          "tokens": 68,
          "pass": true,
          "error": "",
          "seconds": 12.745
        }
      ],
      "output_tokens": 68,
      "repair_prompt_tokens": 0
    },
    {
      "id": "HumanEval/79",
      "pass": false,
      "attempts": [
        {
          "code": "decimal_to_binary(d)=d.toString(2).padStart(8,'0').slice(2,-2)",
          "tokens": 24,
          "pass": false,
          "error": "AssertionError()",
          "seconds": 8.268
        },
        {
          "code": "decimal_to_binary(d)=\"db\"+d.toString(2).padStart(8,'0')+\"db\"",
          "tokens": 23,
          "pass": false,
          "error": "AssertionError()",
          "seconds": 10.85
        }
      ],
      "output_tokens": 47,
      "repair_prompt_tokens": 16
    }
  ]
}