slight reorg

app.py

@@ -3,8 +3,8 @@ import time
 from apscheduler.schedulers.background import BackgroundScheduler
 import threading
 import globals
-from utils.io import initialize_models_providers_file, save_results, load_results, load_models_providers, get_results_table
-from utils.jobs import run_single_job, launch_jobs, update_job_statuses
+from utils.io import initialize_models_providers_file, save_results, load_results, load_models_providers, get_results_table, load_models_providers_str
+from utils.jobs import run_single_job, launch_jobs, update_job_statuses, relaunch_failed_jobs
 from typing import List, Optional
 
 
@@ -31,12 +31,19 @@ def create_app() -> gr.Blocks:
         with gr.Row():
             init_btn = gr.Button("Fetch and Initialize Models/Providers", variant="secondary")
             launch_btn = gr.Button("Launch All Jobs", variant="primary")
-            relaunch_all_btn = gr.Button("Relaunch All", variant="primary")
             relaunch_failed_btn = gr.Button("Relaunch Failed", variant="stop")
             refresh_btn = gr.Button("Refresh Results", variant="secondary")
 
         output = gr.Textbox(label="Status", interactive=False)
 
+        # Accordion for viewing models/providers list
+        with gr.Accordion("Models/Providers Configuration", open=False):
+            models_providers_display = gr.Code(
+                label="Current Models and Providers",
+                value=load_models_providers_str(),
+                interactive=False,
+            )
+
         with gr.Row():
             with gr.Column():
                 gr.Markdown("## Job Results")
@@ -52,77 +59,22 @@ def create_app() -> gr.Blocks:
                     elem_id="results_table"
                 )
 
-        def update_model_choices() -> gr.update:
-            models_providers = load_models_providers(globals.LOCAL_CONFIG_FILE)
-            model_choices = sorted(list(set([mp[0] for mp in models_providers])))
-            return gr.update(choices=model_choices, value=model_choices[0] if model_choices else None)
-
-
-        def update_provider_choices(model: Optional[str]) -> gr.update:
-            """Update provider dropdown based on selected model."""
-            if not model:
-                return gr.update(choices=[])
-
-            # Get providers for the selected model from the config file
-            models_providers = load_models_providers(globals.LOCAL_CONFIG_FILE)
-            providers = [mp[1] for mp in models_providers if mp[0] == model]
-
-            return gr.update(choices=providers, value=providers[0] if providers else None)
 
         # Event handlers
         init_btn.click(
             fn=initialize_models_providers_file,
             outputs=output
         )
+        init_btn.click(
+            fn=load_models_providers_str,
+            outputs=models_providers_display
+        )
 
         launch_btn.click(
             fn=launch_jobs,
             outputs=output
         )
 
-        def relaunch_all_jobs():
-            """Relaunch all existing model-provider combinations from job results."""
-            if not globals.job_results:
-                return "No existing jobs to relaunch"
-
-            relaunched_count = 0
-            for key, info in globals.job_results.items():
-                model = info["model"]
-                provider = info["provider"]
-                job_id = run_single_job(model, provider, globals.TASKS)
-                if job_id != -1:
-                    relaunched_count += 1
-                time.sleep(2)  # Small delay between launches to avoid rate limiting
-
-            return f"Relaunched {relaunched_count}/{len(globals.job_results)} jobs"
-
-        relaunch_all_btn.click(
-            fn=relaunch_all_jobs,
-            outputs=output
-        )
-
-        def relaunch_failed_jobs():
-            """Relaunch only failed model-provider combinations from job results."""
-            if not globals.job_results:
-                return "No existing jobs to relaunch"
-
-            failed_jobs = [(key, info) for key, info in globals.job_results.items()
-                          if info.get("status") in ["ERROR", "FAILED"]]
-
-            if not failed_jobs:
-                return "No failed jobs to relaunch"
-
-            relaunched_count = 0
-            for key, info in failed_jobs:
-                model = info["model"]
-                provider = info["provider"]
-                job_id = run_single_job(model, provider, globals.TASKS)
-                if job_id != -1:
-                    relaunched_count += 1
-                time.sleep(2)  # Small delay between launches to avoid rate limiting
-
-            return f"Relaunched {relaunched_count}/{len(failed_jobs)} failed jobs"
-
         relaunch_failed_btn.click(
             fn=relaunch_failed_jobs,
             outputs=output

globals.py

@@ -11,7 +11,7 @@ job_results: Dict[str, JobResult] = {}  # {model_provider_key: JobResult}
 results_lock: threading.Lock = threading.Lock()
 
 # Configuration
-NUM_MODELS_RUN: int = 10
+NUM_MODELS_RUN: int = 100
 RESULTS_DATASET_NAME: str = "IPTesting/inference-provider-test-results"
 LOCAL_CONFIG_FILE: str = "model_providers.txt"
 TASKS: str = "extended|ifeval|0,lighteval|gsm_plus|0,lighteval|gpqa:diamond|0"

utils/io.py

@@ -48,6 +48,12 @@ def initialize_models_providers_file(file_path: str = globals.LOCAL_CONFIG_FILE)
         return f"Initialized {count} model-provider combinations"
 
 
+def load_models_providers_str(file_path: str = globals.LOCAL_CONFIG_FILE) -> str:
+    mp_list = load_models_providers(file_path)
+
+    return "\n".join([f"{model} : {provider}" for (model, provider) in mp_list])
+
+
 def load_models_providers(file_path: str = globals.LOCAL_CONFIG_FILE) -> List[Tuple[str, str]]:
     """Load models and providers from text file."""
     models_providers = []

utils/jobs.py

@@ -118,7 +118,7 @@ def run_single_job(model: str, provider: str, tasks: str = globals.TASKS) -> Opt
     print(f"Job launched: ID={job_id}, model={model}, provider={provider}")
     return job_id
 
-
+# Todo: factorize both following functions
 def launch_jobs(tasks: str = globals.TASKS, config_file: str = globals.LOCAL_CONFIG_FILE):
     """Launch jobs for all models and providers."""
     models_providers = load_models_providers(config_file)
@@ -140,6 +140,29 @@ def launch_jobs(tasks: str = globals.TASKS, config_file: str = globals.LOCAL_CON
     print(f"Launched {launched_count}/{len(models_providers)} jobs successfully")
     return f"Launched {launched_count} jobs"
 
+def relaunch_failed_jobs():
+    """Relaunch only failed model-provider combinations from job results."""
+    if not globals.job_results:
+        return "No existing jobs to relaunch"
+
+    failed_jobs = [(key, info) for key, info in globals.job_results.items()
+                    if info.get("status") in ["ERROR", "FAILED"]]
+
+    if not failed_jobs:
+        return "No failed jobs to relaunch"
+
+    relaunched_count = 0
+    for key, info in failed_jobs:
+        model = info["model"]
+        provider = info["provider"]
+        job_id = run_single_job(model, provider, globals.TASKS)
+        if job_id != -1:
+            relaunched_count += 1
+        time.sleep(2)  # Small delay between launches to avoid rate limiting
+
+    return f"Relaunched {relaunched_count}/{len(failed_jobs)} failed jobs"
+
+
 
 def update_job_statuses() -> None:
     """Check and update the status of active jobs."""