adding tests for class_eval

Author: zhenningdavidliu

src/inspect_evals/class_eval/class_eval.py

@@ -31,10 +31,6 @@
 )
 from inspect_ai.util import ExecResult, sandbox
 
-
-import importlib
-import unittest
-
 from utils import InferenceUtil, ModelName, GenerationStrategy, construct_prompt
 
 
@@ -89,12 +85,8 @@ def class_eval_scorer() -> Scorer:
     async def score(state:TaskState, target: Target) -> Score:
 
         result = {}
-        test_code = state.metadata["class_name"] + str(state.metadata["task_id"])
-
+        
         generated_code = find_code(state.output.completion)
-
-        #test_classes = state.metadata["test_classes"]
-
         code = generated_code + "\n" + state.metadata["test"]
 
         explanation = ""
@@ -161,10 +153,3 @@ def record_to_sample(
             "fields": record["fields"],
         }
     )
-
-
-def sample_to_fewshot(sample: Sample) -> str:
-    metadata = sample.metadata
-    return (
-        f"{metadata['class_description']}\n"
-    )

src/inspect_evals/class_eval/test.py

@@ -1,9 +1,20 @@
 import unittest
+from inspect_ai.solver import (
+    generate, 
+    system_message
+)
 
 from utils import construct_prompt
-from class_eval import record_to_sample, find_code
+from class_eval import record_to_sample, find_code, class_eval
 
-from test_data import record, sample, prompt, sample_code, raw_code
+from test_data import (
+    record, 
+    sample, 
+    prompt, 
+    sample_code, 
+    raw_code, 
+    INSTRUCTION
+)
 
 class TestClassEval(unittest.TestCase):
 
@@ -13,14 +24,14 @@ def test_record_to_sample(self):
     def test_find_code(self):
         self.assertEqual(find_code(raw_code), sample_code)
 
+    def test_task(self):
+        task = class_eval(k_shot = 5)
+        self.assertEqual(task.dataset.name, 'FudanSELab/ClassEval')
+        self.assertEqual(task.epochs, 5)
+        self.assertEqual(task.sandbox.type, "docker")
+
+
 class TestUtils(unittest.TestCase):
 
     def test_construct_prompt(self):
         self.assertEqual(construct_prompt(record), prompt)
-
-
-# if __name__ == '__main__':
-
-#     #Save record to file
-#     with open('test_data2.py', 'w') as f:
-#         f.write(f'{record_to_sample(record)}')

src/inspect_evals/class_eval/test_data.py

@@ -76,4 +76,11 @@ def add(self, x, y):
 Test text 
 """
 
-sample_code = r"""print("Hello, World!")"""
+sample_code = r"""print("Hello, World!")"""
+
+
+INSTRUCTION = """
+
+You are an expert Python programmer. You will be given a task, and the tests that your code must pass.
+
+"""