File size: 822 Bytes
7f5506e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 |
"""Global variables and configuration for the Inference Provider Testing Dashboard."""
import threading
from typing import Dict, Any, Optional
# Type definition for job result entries
JobResult = Dict[str, Any] # {model, provider, last_run, status, current_score, previous_score, job_id}
# Global variables to track jobs
job_results: Dict[str, JobResult] = {} # {model_provider_key: JobResult}
results_lock: threading.Lock = threading.Lock()
# Configuration
RESULTS_DATASET_NAME: str = "IPTesting/inference-provider-test-results"
LOCAL_CONFIG_FILE: str = "model_providers.txt"
TASKS: str = "extended|ifeval|0,lighteval|gsm_plus|0,lighteval|gpqa:diamond|0"
def get_model_provider_key(model: str, provider: str) -> str:
"""Create a unique key for model-provider combination."""
return f"{model}||{provider}"
|