Add support for the Fennel programming language

Author: rudolf-adamkovic

dataset_builder/humaneval_to_fnl.py

@@ -0,0 +1,66 @@
+"""
+This script translates problems from the OpenAI HumanEval dataset into Fennel.
+
+- Home: https://fennel-lang.org/
+- Compiler: https://git.sr.ht/~technomancy/fennel
+- Test library: https://git.sr.ht/~technomancy/faith
+"""
+
+import ast
+from typing import List
+
+
+class Translator:
+
+    USub = "-"
+
+    stop = ["\n(fn", "\n;", "\n("]
+
+    def file_ext(self):
+        return "fnl"
+
+    def translate_prompt(
+        self, name: str, args: List[ast.arg], _returns, description: str
+    ) -> str:
+        fnl_args = " ".join([arg.arg for arg in args])
+        fnl_description = description.replace('"', '\\"')
+        self.entry_point = name
+        return f'(fn {name} [{fnl_args}]\n"{fnl_description}"\n'
+
+    def test_suite_prefix_lines(self, entry_point) -> List[str]:
+        return [
+            "(local faith (require :faith))",
+            f"(local candidate {entry_point})",
+            "(fn test-human-eval []",
+        ]
+
+    def test_suite_suffix_lines(self) -> List[str]:
+        return [")", "{: test-human-eval}"]
+
+    def deep_equality(self, left: str, right: str) -> str:
+        return f"  (faith.= {right} {left})"  # Expected on the left.
+
+    def gen_literal(self, c: bool | str | int | float):
+        if type(c) is bool:
+            return "true" if c else "false"
+        elif type(c) is str:
+            return f'"{c}"'
+        elif c is None:
+            return "nil"
+        return repr(c)
+
+    def gen_var(self, variable: str) -> str:
+        return variable
+
+    def gen_list(self, list: List[str]) -> str:
+        return "[" + " ".join(list) + "]"
+
+    def gen_tuple(self, tuple: List[str]) -> str:
+        return "[" + " ".join(tuple) + "]"
+
+    def gen_dict(self, keys: List[str], values: List[str]) -> str:
+        pairs = " ".join(f"{k} {v}" for k, v in zip(keys, values))
+        return "{" + pairs + "}"
+
+    def gen_call(self, func: str, args: List[str]) -> str:
+        return "(" + func + " " + " ".join(args) + ")"

dataset_builder/libexperiments.py

@@ -42,6 +42,7 @@ def path(self) -> Path:
     "elixir",
     "clj",
     "ada",
+    "fnl",
 ]
 MODELS = ["davinci", "incoder", "codegen"]
 

dataset_builder/terms.csv

@@ -26,3 +26,4 @@ Matlab,m,array,array,array,dictionary,<missing>,true,false
 Haskell,hs,list,list,tuple,association list,Nothing,True,False
 Clojure,clj,vector,list,vector,map,nil,true,false
 Dart,dart,list,list,record,map,null,true,false
+Fennel,fnl,table,table,table,table,nil,true,false

evaluation/Dockerfile

@@ -106,6 +106,14 @@ RUN apt-get update -yqq && apt-get install -yqq dart
 # Lean
 # RUN wget https://github.com/leanprover/lean4/releases/download/v4.6.0-rc1/lean-4.6.0-rc1-linux.zip -O /tmp/lean.zip && unzip /tmp/lean.zip -d /root/lean/ && ln -s /root/lean/bin/lean /bin/lean
 
+# Fennel
+# TODO Once on Ubuntu 25.10+, replace with `apt install -y fennel'.
+RUN curl -L -O https://fennel-lang.org/downloads/fennel-1.6.1
+RUN curl -L -O https://git.sr.ht/~technomancy/faith/blob/main/faith.fnl
+RUN chmod +x fennel-1.6.1
+RUN ln -s /fennel-1.6.1 /bin/fennel
+ENV FENNEL_PATH=/?.fnl
+
 # install numpy for humanevalplus
 RUN python3 -m pip install numpy
 

evaluation/src/containerized_eval.py

@@ -29,6 +29,7 @@
 import eval_v
 import eval_lean
 import eval_dart
+import eval_fnl
 import tempfile
 
 
@@ -65,6 +66,7 @@
     "coq": (eval_v.eval_script, ".v"),
     "lean": (eval_lean.eval_script, ".lean"),
     "dart": (eval_dart.eval_script, ".dart"),
+    "fnl": (eval_fnl.eval_script, ".fnl")
 }
 
 def eval_string_script(language, program):

evaluation/src/eval_fnl.py

@@ -0,0 +1,39 @@
+"""
+Evaluates a generated Fennel program (.fnl).
+"""
+import os
+from pathlib import Path
+from safe_subprocess import run
+from libeval import run_without_exn
+
+def eval_script(path: Path):
+
+    module_dir = path.parent
+    module = path.stem
+
+    old_path = os.environ["FENNEL_PATH"]
+    new_path = f'{old_path};{module_dir}/?.fnl'
+
+    args = ["fennel", "/faith.fnl", "--tests", module]
+    env = {'FENNEL_PATH': new_path}
+
+    result = run(args, env = env)
+
+    if result.timeout:
+        status = "Timeout"
+    elif result.exit_code != 0:
+        status = "Exception"
+    elif "\n1 passed, 0 failed, 0 error(s), 0 skipped\n" in result.stdout:
+        status = "OK"
+    else: # test failure
+        status = "Exception"
+
+    return {
+        "status": status,
+        "exit_code": result.exit_code,
+        "stdout": result.stdout,
+        "stderr": result.stderr,
+    }
+
+if __name__ == "__main__":
+    main()