Spaces:
Sleeping
Sleeping
File size: 1,062 Bytes
7f5506e 6e44082 7f5506e 80d548a 6e44082 7f5506e f5cbbfe 0b4b222 5906706 7f5506e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 |
"""Global variables and configuration for the Inference Provider Testing Dashboard."""
import threading
from typing import Dict, Any, Optional
# Type definition for job result entries
JobResult = Dict[str, Any] # {model, provider, last_run, status, current_score, previous_score, job_id, start_time, duration, completed_at, runs: [{job_id, score, status, start_time, duration, completed_at}]}
# Global variables to track jobs
job_results: Dict[str, JobResult] = {} # {model_provider_key: JobResult}
results_lock: threading.Lock = threading.Lock()
# Configuration
NUM_MODELS_RUN: int = 100
NUM_RUNS_PER_JOB: int = 4 # Number of times to run each job for variance reduction
RESULTS_DATASET_NAME: str = "IPTesting/inference-provider-test-results"
LOCAL_CONFIG_FILE: str = "/home/user/app/model_providers.txt"
TASKS: str = "extended|ifeval|0,lighteval|gpqa:diamond|0"
NAMESPACE: str = "huggingface"
def get_model_provider_key(model: str, provider: str) -> str:
"""Create a unique key for model-provider combination."""
return f"{model}||{provider}"
|