Improve error handling, return hardhat output

api/engine.py

@@ -4,20 +4,46 @@
 
 import os
 import subprocess
+import shutil
+import tempfile
+from typing import Tuple
 
 
-def evaluate_solution(task_id: str, solution: str):
+def evaluate_solution(task_id: str, solution: str) -> Tuple[bool, str]:
     initial_dir = os.getcwd()
     task_dir = task_id.replace('/', '_')
-    task_path = 'tasks/' + task_dir
-    os.chdir(task_path)
+    original_task_path = 'tasks/{}'.format(task_dir)
+    
+    if not os.path.exists(original_task_path):
+        raise FileNotFoundError('Task not found: {}'.format(task_id))
+    
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_task_path = os.path.join(temp_dir, task_dir)
 
-    with open('contracts/Task.sol', 'w') as f:
-        f.write(solution)
-    try:
-        subprocess.run(['npx', 'hardhat', 'test'], check=True)
-        return True
-    except subprocess.CalledProcessError:
-        return False
-    finally:
-        os.chdir(initial_dir)
+        # Replicate the directory structure with symbolic links
+        shutil.copytree(
+            original_task_path,
+            temp_task_path,
+            symlinks=True,
+            dirs_exist_ok=True,
+        )
+        
+        os.chdir(temp_task_path)
+        
+        # Write solution with UTF-8 encoding
+        with open('contracts/Task.sol', 'w', encoding='utf-8') as f:
+            f.write(solution)
+
+        try:
+            result = subprocess.run(
+                ['npx', 'hardhat', 'test'],
+                capture_output=True,
+                text=True,
+                check=True,
+                encoding='utf-8',  # Specify UTF-8 encoding for subprocess
+            )
+            return True, result.stdout
+        except subprocess.CalledProcessError as e:
+            return False, e.stderr
+        finally:
+            os.chdir(initial_dir)

api/resources/root.py

@@ -3,17 +3,10 @@
 
 import falcon
 from engine import evaluate_solution
-from threading import Lock
-from typing import Dict
 import logging
 
 
 class RootResource:
-    def __init__(self):
-        self.main_lock = Lock()
-        self.task_locks: Dict[str, Lock] = {}
-        self.max_tasks = 5
-
     def on_get(self, request, response):
         response.text = 'Human Eval for Solidity Server v1.2410.0'
 
@@ -31,19 +24,11 @@ class RootResource:
         solution = payload['solution']
 
         try:
-            with self.main_lock:
-                if len(self.task_locks) >= self.max_tasks:
-                    # Remove the oldest task lock if we've reached the limit
-                    oldest_task = next(iter(self.task_locks))
-                    del self.task_locks[oldest_task]
-
-                if task_id not in self.task_locks:
-                    self.task_locks[task_id] = Lock()
-
-            with self.task_locks[task_id]:
-                passed = evaluate_solution(task_id, solution)
-
-            response.media = {'passed': passed}
+            passed, output = evaluate_solution(task_id, solution)
+            response.media = {
+                'passed': passed,
+                'output': output,
+            }
         except FileNotFoundError as e:
             logging.error('Task not found: {}'.format(str(e)))
             response.status = falcon.HTTP_404