Spaces:
Running
Running
Verify trust_remote_code in tokenizer
Browse files
app.py
CHANGED
@@ -15,7 +15,7 @@ from gradio_huggingfacehub_search import HuggingfaceHubSearch
|
|
15 |
from apscheduler.schedulers.background import BackgroundScheduler
|
16 |
|
17 |
from textwrap import dedent
|
18 |
-
from transformers import
|
19 |
|
20 |
import mlx_lm
|
21 |
from mlx_lm import convert
|
@@ -109,8 +109,8 @@ def process_model(model_id, q_method, oauth_token: gr.OAuthToken | None):
|
|
109 |
model_name = model_id.split('/')[-1]
|
110 |
username = whoami(oauth_token.token)["name"]
|
111 |
try:
|
112 |
-
# Fail early if the
|
113 |
-
_ =
|
114 |
|
115 |
if q_method == "FP16":
|
116 |
upload_repo = f"{username}/{model_name}-mlx-fp16"
|
|
|
15 |
from apscheduler.schedulers.background import BackgroundScheduler
|
16 |
|
17 |
from textwrap import dedent
|
18 |
+
from transformers import AutoTokenizer
|
19 |
|
20 |
import mlx_lm
|
21 |
from mlx_lm import convert
|
|
|
109 |
model_name = model_id.split('/')[-1]
|
110 |
username = whoami(oauth_token.token)["name"]
|
111 |
try:
|
112 |
+
# Fail early if the tokenizer requires custom code
|
113 |
+
_ = AutoTokenizer.from_pretrained(model_id, trust_remote_code=False)
|
114 |
|
115 |
if q_method == "FP16":
|
116 |
upload_repo = f"{username}/{model_name}-mlx-fp16"
|