update

backend-cli.py

@@ -96,7 +96,7 @@ def process_evaluation(task: Task, eval_request: EvalRequest) -> dict:
     return results
 
 
-def process_finished_requests() -> bool:
+def process_finished_requests(thr: int) -> bool:
     sanity_checks()
 
     current_finished_status = [FINISHED_STATUS, FAILED_STATUS]
@@ -118,33 +118,34 @@ def process_finished_requests() -> bool:
     result_name_to_result = {r.eval_name: r for r in eval_results}
 
     for eval_request in eval_requests:
-        result_name: str = request_to_result_name(eval_request)
+        if eval_request.likes >= thr:
+            result_name: str = request_to_result_name(eval_request)
 
-        # Check the corresponding result
-        from typing import Optional
-        eval_result: Optional[EvalResult] = result_name_to_result[result_name] if result_name in result_name_to_result else None
+            # Check the corresponding result
+            from typing import Optional
+            eval_result: Optional[EvalResult] = result_name_to_result[result_name] if result_name in result_name_to_result else None
 
-        task_lst = TASKS_HARNESS.copy()
-        random.shuffle(task_lst)
+            task_lst = TASKS_HARNESS.copy()
+            random.shuffle(task_lst)
 
-        # Iterate over tasks and, if we do not have results for a task, run the relevant evaluations
-        for task in task_lst:
-            task_name = task.benchmark
+            # Iterate over tasks and, if we do not have results for a task, run the relevant evaluations
+            for task in task_lst:
+                task_name = task.benchmark
 
-            if eval_result is None or task_name not in eval_result.results:
-                eval_request: EvalRequest = result_name_to_request[result_name]
+                if eval_result is None or task_name not in eval_result.results:
+                    eval_request: EvalRequest = result_name_to_request[result_name]
 
-                my_snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
-                set_eval_request(api=API, eval_request=eval_request, set_to_status=RUNNING_STATUS, hf_repo=QUEUE_REPO,
-                                 local_dir=EVAL_REQUESTS_PATH_BACKEND)
+                    my_snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
+                    set_eval_request(api=API, eval_request=eval_request, set_to_status=RUNNING_STATUS, hf_repo=QUEUE_REPO,
+                                     local_dir=EVAL_REQUESTS_PATH_BACKEND)
 
-                results = process_evaluation(task, eval_request)
+                    results = process_evaluation(task, eval_request)
 
-                my_snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
-                set_eval_request(api=API, eval_request=eval_request, set_to_status=FINISHED_STATUS, hf_repo=QUEUE_REPO,
-                                 local_dir=EVAL_REQUESTS_PATH_BACKEND)
+                    my_snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)
+                    set_eval_request(api=API, eval_request=eval_request, set_to_status=FINISHED_STATUS, hf_repo=QUEUE_REPO,
+                                     local_dir=EVAL_REQUESTS_PATH_BACKEND)
 
-                return True
+                    return True
 
     return False
 
@@ -190,10 +191,13 @@ if __name__ == "__main__":
     import socket
     if socket.gethostname() not in {'hamburg'}:
         import time
-        time.sleep(60 * random.randint(3, 10))
+        time.sleep(60 * random.randint(2, 5))
 
     # res = False
     res = process_pending_requests()
 
     if res is False:
-        res = process_finished_requests()
+        res = process_finished_requests(100)
+
+    if res is False:
+        res = process_finished_requests(0)