*.py

globals.py

@@ -11,6 +11,7 @@ job_results: Dict[str, JobResult] = {}  # {model_provider_key: JobResult}
 results_lock: threading.Lock = threading.Lock()
 
 # Configuration
+NUM_MODELS_RUN: int = 10
 RESULTS_DATASET_NAME: str = "IPTesting/inference-provider-test-results"
 LOCAL_CONFIG_FILE: str = "model_providers.txt"
 TASKS: str = "extended|ifeval|0,lighteval|gsm_plus|0,lighteval|gpqa:diamond|0"

utils/io.py

@@ -13,7 +13,7 @@ def get_models_providers() -> List[Tuple[str, List[str]]]:
         filter="text-generation",
         sort="likes",
         direction=-1,
-        limit=1,
+        limit=globals.NUM_MODELS_RUN,
         expand="inferenceProviderMapping"
     )
 

utils/jobs.py

@@ -91,7 +91,7 @@ def run_single_job(model: str, provider: str, tasks: str) -> Optional[str]:
             "--results-org", "IPTesting",
             "--max-samples", "10"
         ],
-        namespace="clefourrier",
+        namespace=globals.NAMESPACE,
         secrets={"HF_TOKEN": os.getenv("HF_TOKEN")},
         token=os.getenv("HF_TOKEN")
     )