Spaces:
Running
Running
David Pomerenke
commited on
Commit
·
c29b8da
1
Parent(s):
2eeba23
Fix: don't cache model metadata forever
Browse files- evals/models.py +6 -13
evals/models.py
CHANGED
|
@@ -121,12 +121,13 @@ async def transcribe(path, model="elevenlabs/scribe_v1"):
|
|
| 121 |
|
| 122 |
models = pd.DataFrame(models, columns=["id"])
|
| 123 |
|
| 124 |
-
|
| 125 |
@cache
|
|
|
|
|
|
|
|
|
|
| 126 |
def get_or_metadata(id):
|
| 127 |
# get metadata from OpenRouter
|
| 128 |
-
|
| 129 |
-
models = response.json()["data"]
|
| 130 |
metadata = next((m for m in models if m["slug"] == id), None)
|
| 131 |
return metadata
|
| 132 |
|
|
@@ -163,21 +164,13 @@ def get_hf_metadata(row):
|
|
| 163 |
return empty
|
| 164 |
|
| 165 |
|
| 166 |
-
or_metadata = models["id"].apply(get_or_metadata)
|
| 167 |
-
hf_metadata = or_metadata.apply(get_hf_metadata)
|
| 168 |
-
|
| 169 |
-
|
| 170 |
def get_cost(row):
|
| 171 |
cost = float(row["endpoint"]["pricing"]["completion"])
|
| 172 |
return round(cost * 1_000_000, 2)
|
| 173 |
|
| 174 |
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
models[exists],
|
| 178 |
-
or_metadata[exists],
|
| 179 |
-
hf_metadata[exists],
|
| 180 |
-
)
|
| 181 |
creation_date_hf = pd.to_datetime(hf_metadata.str["creation_date"]).dt.date
|
| 182 |
creation_date_or = pd.to_datetime(
|
| 183 |
or_metadata.str["created_at"].str.split("T").str[0]
|
|
|
|
| 121 |
|
| 122 |
models = pd.DataFrame(models, columns=["id"])
|
| 123 |
|
|
|
|
| 124 |
@cache
|
| 125 |
+
def get_models(date):
|
| 126 |
+
return get("https://openrouter.ai/api/frontend/models/").json()["data"]
|
| 127 |
+
|
| 128 |
def get_or_metadata(id):
|
| 129 |
# get metadata from OpenRouter
|
| 130 |
+
models = get_models(date.today())
|
|
|
|
| 131 |
metadata = next((m for m in models if m["slug"] == id), None)
|
| 132 |
return metadata
|
| 133 |
|
|
|
|
| 164 |
return empty
|
| 165 |
|
| 166 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 167 |
def get_cost(row):
|
| 168 |
cost = float(row["endpoint"]["pricing"]["completion"])
|
| 169 |
return round(cost * 1_000_000, 2)
|
| 170 |
|
| 171 |
|
| 172 |
+
or_metadata = models["id"].apply(get_or_metadata)
|
| 173 |
+
hf_metadata = or_metadata.apply(get_hf_metadata)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 174 |
creation_date_hf = pd.to_datetime(hf_metadata.str["creation_date"]).dt.date
|
| 175 |
creation_date_or = pd.to_datetime(
|
| 176 |
or_metadata.str["created_at"].str.split("T").str[0]
|