We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
PEFT support does not work with local directories
### System Info

text-generation-inference version: main branch

The rest of the system info is probably not relevant.

### Information

- [X] Docker
- [X] The CLI directly

### Tasks

- [X] An officially supported command
- [ ] My own modifications

### Reproduction

Steps to reproduce:

1. Create a directory that contains PEFT weights, for example: `mypeft` which contains two files: `adapter_config.json` and `adapter_model.bin`
2. Try to download the base model using the following command: `text-generation-server download-weights mypeft`

### Expected behavior

It should be able to download the base model weights and merge the PEFT weights into the base model.

However, as mentioned in [this comment](https://github.com/huggingface/text-generation-inference/pull/762#issuecomment-1728042436):

> it seems the checks for PEFT model is only in the block checking if the model is not a local model

There is a workaround as documented in [this comment](https://github.com/huggingface/text-generation-inference/pull/762#issuecomment-1729689087)



--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `server/text_generation_server/cli.py`
Content:
```
1 import os
2 import sys
3 import typer
4 
5 from pathlib import Path
6 from loguru import logger
7 from typing import Optional
8 from enum import Enum
9 from huggingface_hub import hf_hub_download
10 
11 
12 app = typer.Typer()
13 
14 
15 class Quantization(str, Enum):
16     bitsandbytes = "bitsandbytes"
17     bitsandbytes_nf4 = "bitsandbytes-nf4"
18     bitsandbytes_fp4 = "bitsandbytes-fp4"
19     gptq = "gptq"
20     awq = "awq"
21     eetq = "eetq"
22 
23 
24 class Dtype(str, Enum):
25     float16 = "float16"
26     bloat16 = "bfloat16"
27 
28 
29 @app.command()
30 def serve(
31     model_id: str,
32     revision: Optional[str] = None,
33     sharded: bool = False,
34     quantize: Optional[Quantization] = None,
35     dtype: Optional[Dtype] = None,
36     trust_remote_code: bool = False,
37     uds_path: Path = "/tmp/text-generation-server",
38     logger_level: str = "INFO",
39     json_output: bool = False,
40     otlp_endpoint: Optional[str] = None,
41 ):
42     if sharded:
43         assert (
44             os.getenv("RANK", None) is not None
45         ), "RANK must be set when sharded is True"
46         assert (
47             os.getenv("WORLD_SIZE", None) is not None
48         ), "WORLD_SIZE must be set when sharded is True"
49         assert (
50             os.getenv("MASTER_ADDR", None) is not None
51         ), "MASTER_ADDR must be set when sharded is True"
52         assert (
53             os.getenv("MASTER_PORT", None) is not None
54         ), "MASTER_PORT must be set when sharded is True"
55 
56     # Remove default handler
57     logger.remove()
58     logger.add(
59         sys.stdout,
60         format="{message}",
61         filter="text_generation_server",
62         level=logger_level,
63         serialize=json_output,
64         backtrace=True,
65         diagnose=False,
66     )
67 
68     # Import here after the logger is added to log potential import exceptions
69     from text_generation_server import server
70     from text_generation_server.tracing import setup_tracing
71 
72     # Setup OpenTelemetry distributed tracing
73     if otlp_endpoint is not None:
74         setup_tracing(shard=os.getenv("RANK", 0), otlp_endpoint=otlp_endpoint)
75 
76     # Downgrade enum into str for easier management later on
77     quantize = None if quantize is None else quantize.value
78     dtype = None if dtype is None else dtype.value
79     if dtype is not None and quantize is not None:
80         raise RuntimeError(
81             "Only 1 can be set between `dtype` and `quantize`, as they both decide how goes the final model."
82         )
83     server.serve(
84         model_id, revision, sharded, quantize, dtype, trust_remote_code, uds_path
85     )
86 
87 
88 @app.command()
89 def download_weights(
90     model_id: str,
91     revision: Optional[str] = None,
92     extension: str = ".safetensors",
93     auto_convert: bool = True,
94     logger_level: str = "INFO",
95     json_output: bool = False,
96     trust_remote_code: bool = False,
97 ):
98     # Remove default handler
99     logger.remove()
100     logger.add(
101         sys.stdout,
102         format="{message}",
103         filter="text_generation_server",
104         level=logger_level,
105         serialize=json_output,
106         backtrace=True,
107         diagnose=False,
108     )
109 
110     # Import here after the logger is added to log potential import exceptions
111     from text_generation_server import utils
112 
113     # Test if files were already download
114     try:
115         utils.weight_files(model_id, revision, extension)
116         logger.info("Files are already present on the host. " "Skipping download.")
117         return
118     # Local files not found
119     except (utils.LocalEntryNotFoundError, FileNotFoundError):
120         pass
121 
122     is_local_model = (Path(model_id).exists() and Path(model_id).is_dir()) or os.getenv(
123         "WEIGHTS_CACHE_OVERRIDE", None
124     ) is not None
125 
126     if not is_local_model:
127         try:
128             adapter_config_filename = hf_hub_download(
129                 model_id, revision=revision, filename="adapter_config.json"
130             )
131             utils.download_and_unload_peft(
132                 model_id, revision, trust_remote_code=trust_remote_code
133             )
134             is_local_model = True
135             utils.weight_files(model_id, revision, extension)
136             return
137         except (utils.LocalEntryNotFoundError, utils.EntryNotFoundError):
138             pass
139 
140         # Try to download weights from the hub
141         try:
142             filenames = utils.weight_hub_files(model_id, revision, extension)
143             utils.download_weights(filenames, model_id, revision)
144             # Successfully downloaded weights
145             return
146 
147         # No weights found on the hub with this extension
148         except utils.EntryNotFoundError as e:
149             # Check if we want to automatically convert to safetensors or if we can use .bin weights instead
150             if not extension == ".safetensors" or not auto_convert:
151                 raise e
152 
153     # Try to see if there are local pytorch weights
154     try:
155         # Get weights for a local model, a hub cached model and inside the WEIGHTS_CACHE_OVERRIDE
156         local_pt_files = utils.weight_files(model_id, revision, ".bin")
157 
158     # No local pytorch weights
159     except utils.LocalEntryNotFoundError:
160         if extension == ".safetensors":
161             logger.warning(
162                 f"No safetensors weights found for model {model_id} at revision {revision}. "
163                 f"Downloading PyTorch weights."
164             )
165 
166         # Try to see if there are pytorch weights on the hub
167         pt_filenames = utils.weight_hub_files(model_id, revision, ".bin")
168         # Download pytorch weights
169         local_pt_files = utils.download_weights(pt_filenames, model_id, revision)
170 
171     if auto_convert:
172         logger.warning(
173             f"No safetensors weights found for model {model_id} at revision {revision}. "
174             f"Converting PyTorch weights to safetensors."
175         )
176 
177         # Safetensors final filenames
178         local_st_files = [
179             p.parent / f"{p.stem.lstrip('pytorch_')}.safetensors"
180             for p in local_pt_files
181         ]
182         try:
183             import transformers
184             import json
185 
186             if is_local_model:
187                 config_filename = os.path.join(model_id, "config.json")
188             else:
189                 config_filename = hf_hub_download(
190                     model_id, revision=revision, filename="config.json"
191                 )
192             with open(config_filename, "r") as f:
193                 config = json.load(f)
194             architecture = config["architectures"][0]
195 
196             class_ = getattr(transformers, architecture)
197 
198             # Name for this varible depends on transformers version.
199             discard_names = getattr(class_, "_tied_weights_keys", [])
200 
201         except Exception as e:
202             discard_names = []
203         # Convert pytorch weights to safetensors
204         utils.convert_files(local_pt_files, local_st_files, discard_names)
205 
206 
207 @app.command()
208 def quantize(
209     model_id: str,
210     output_dir: str,
211     revision: Optional[str] = None,
212     logger_level: str = "INFO",
213     json_output: bool = False,
214     trust_remote_code: bool = False,
215     upload_to_model_id: Optional[str] = None,
216     percdamp: float = 0.01,
217     act_order: bool = False,
218 ):
219     if revision is None:
220         revision = "main"
221     download_weights(
222         model_id=model_id,
223         revision=revision,
224         logger_level=logger_level,
225         json_output=json_output,
226     )
227     from text_generation_server.utils.gptq.quantize import quantize
228 
229     quantize(
230         model_id=model_id,
231         bits=4,
232         groupsize=128,
233         output_dir=output_dir,
234         revision=revision,
235         trust_remote_code=trust_remote_code,
236         upload_to_model_id=upload_to_model_id,
237         percdamp=percdamp,
238         act_order=act_order,
239     )
240 
241 
242 if __name__ == "__main__":
243     app()
244
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py
--- a/server/text_generation_server/cli.py
+++ b/server/text_generation_server/cli.py
@@ -150,6 +150,17 @@
             if not extension == ".safetensors" or not auto_convert:
                 raise e
 
+    else:
+        # Try to load as a local PEFT model
+        try:
+            utils.download_and_unload_peft(
+                model_id, revision, trust_remote_code=trust_remote_code
+            )
+            utils.weight_files(model_id, revision, extension)
+            return
+        except (utils.LocalEntryNotFoundError, utils.EntryNotFoundError):
+            pass
+
     # Try to see if there are local pytorch weights
     try:
         # Get weights for a local model, a hub cached model and inside the WEIGHTS_CACHE_OVERRIDE

{"golden_diff": "diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py\n--- a/server/text_generation_server/cli.py\n+++ b/server/text_generation_server/cli.py\n@@ -150,6 +150,17 @@\n             if not extension == \".safetensors\" or not auto_convert:\n                 raise e\n \n+    else:\n+        # Try to load as a local PEFT model\n+        try:\n+            utils.download_and_unload_peft(\n+                model_id, revision, trust_remote_code=trust_remote_code\n+            )\n+            utils.weight_files(model_id, revision, extension)\n+            return\n+        except (utils.LocalEntryNotFoundError, utils.EntryNotFoundError):\n+            pass\n+\n     # Try to see if there are local pytorch weights\n     try:\n         # Get weights for a local model, a hub cached model and inside the WEIGHTS_CACHE_OVERRIDE\n", "issue": "PEFT support does not work with local directories\n### System Info\n\ntext-generation-inference version: main branch\r\n\r\nThe rest of the system info is probably not relevant.\n\n### Information\n\n- [X] Docker\n- [X] The CLI directly\n\n### Tasks\n\n- [X] An officially supported command\n- [ ] My own modifications\n\n### Reproduction\n\nSteps to reproduce:\r\n\r\n1. Create a directory that contains PEFT weights, for example: `mypeft` which contains two files: `adapter_config.json` and `adapter_model.bin`\r\n2. Try to download the base model using the following command: `text-generation-server download-weights mypeft`\n\n### Expected behavior\n\nIt should be able to download the base model weights and merge the PEFT weights into the base model.\r\n\r\nHowever, as mentioned in [this comment](https://github.com/huggingface/text-generation-inference/pull/762#issuecomment-1728042436):\r\n\r\n> it seems the checks for PEFT model is only in the block checking if the model is not a local model\r\n\r\nThere is a workaround as documented in [this comment](https://github.com/huggingface/text-generation-inference/pull/762#issuecomment-1729689087)\r\n\r\n\n", "before_files": [{"content": "import os\nimport sys\nimport typer\n\nfrom pathlib import Path\nfrom loguru import logger\nfrom typing import Optional\nfrom enum import Enum\nfrom huggingface_hub import hf_hub_download\n\n\napp = typer.Typer()\n\n\nclass Quantization(str, Enum):\n    bitsandbytes = \"bitsandbytes\"\n    bitsandbytes_nf4 = \"bitsandbytes-nf4\"\n    bitsandbytes_fp4 = \"bitsandbytes-fp4\"\n    gptq = \"gptq\"\n    awq = \"awq\"\n    eetq = \"eetq\"\n\n\nclass Dtype(str, Enum):\n    float16 = \"float16\"\n    bloat16 = \"bfloat16\"\n\n\n@app.command()\ndef serve(\n    model_id: str,\n    revision: Optional[str] = None,\n    sharded: bool = False,\n    quantize: Optional[Quantization] = None,\n    dtype: Optional[Dtype] = None,\n    trust_remote_code: bool = False,\n    uds_path: Path = \"/tmp/text-generation-server\",\n    logger_level: str = \"INFO\",\n    json_output: bool = False,\n    otlp_endpoint: Optional[str] = None,\n):\n    if sharded:\n        assert (\n            os.getenv(\"RANK\", None) is not None\n        ), \"RANK must be set when sharded is True\"\n        assert (\n            os.getenv(\"WORLD_SIZE\", None) is not None\n        ), \"WORLD_SIZE must be set when sharded is True\"\n        assert (\n            os.getenv(\"MASTER_ADDR\", None) is not None\n        ), \"MASTER_ADDR must be set when sharded is True\"\n        assert (\n            os.getenv(\"MASTER_PORT\", None) is not None\n        ), \"MASTER_PORT must be set when sharded is True\"\n\n    # Remove default handler\n    logger.remove()\n    logger.add(\n        sys.stdout,\n        format=\"{message}\",\n        filter=\"text_generation_server\",\n        level=logger_level,\n        serialize=json_output,\n        backtrace=True,\n        diagnose=False,\n    )\n\n    # Import here after the logger is added to log potential import exceptions\n    from text_generation_server import server\n    from text_generation_server.tracing import setup_tracing\n\n    # Setup OpenTelemetry distributed tracing\n    if otlp_endpoint is not None:\n        setup_tracing(shard=os.getenv(\"RANK\", 0), otlp_endpoint=otlp_endpoint)\n\n    # Downgrade enum into str for easier management later on\n    quantize = None if quantize is None else quantize.value\n    dtype = None if dtype is None else dtype.value\n    if dtype is not None and quantize is not None:\n        raise RuntimeError(\n            \"Only 1 can be set between `dtype` and `quantize`, as they both decide how goes the final model.\"\n        )\n    server.serve(\n        model_id, revision, sharded, quantize, dtype, trust_remote_code, uds_path\n    )\n\n\n@app.command()\ndef download_weights(\n    model_id: str,\n    revision: Optional[str] = None,\n    extension: str = \".safetensors\",\n    auto_convert: bool = True,\n    logger_level: str = \"INFO\",\n    json_output: bool = False,\n    trust_remote_code: bool = False,\n):\n    # Remove default handler\n    logger.remove()\n    logger.add(\n        sys.stdout,\n        format=\"{message}\",\n        filter=\"text_generation_server\",\n        level=logger_level,\n        serialize=json_output,\n        backtrace=True,\n        diagnose=False,\n    )\n\n    # Import here after the logger is added to log potential import exceptions\n    from text_generation_server import utils\n\n    # Test if files were already download\n    try:\n        utils.weight_files(model_id, revision, extension)\n        logger.info(\"Files are already present on the host. \" \"Skipping download.\")\n        return\n    # Local files not found\n    except (utils.LocalEntryNotFoundError, FileNotFoundError):\n        pass\n\n    is_local_model = (Path(model_id).exists() and Path(model_id).is_dir()) or os.getenv(\n        \"WEIGHTS_CACHE_OVERRIDE\", None\n    ) is not None\n\n    if not is_local_model:\n        try:\n            adapter_config_filename = hf_hub_download(\n                model_id, revision=revision, filename=\"adapter_config.json\"\n            )\n            utils.download_and_unload_peft(\n                model_id, revision, trust_remote_code=trust_remote_code\n            )\n            is_local_model = True\n            utils.weight_files(model_id, revision, extension)\n            return\n        except (utils.LocalEntryNotFoundError, utils.EntryNotFoundError):\n            pass\n\n        # Try to download weights from the hub\n        try:\n            filenames = utils.weight_hub_files(model_id, revision, extension)\n            utils.download_weights(filenames, model_id, revision)\n            # Successfully downloaded weights\n            return\n\n        # No weights found on the hub with this extension\n        except utils.EntryNotFoundError as e:\n            # Check if we want to automatically convert to safetensors or if we can use .bin weights instead\n            if not extension == \".safetensors\" or not auto_convert:\n                raise e\n\n    # Try to see if there are local pytorch weights\n    try:\n        # Get weights for a local model, a hub cached model and inside the WEIGHTS_CACHE_OVERRIDE\n        local_pt_files = utils.weight_files(model_id, revision, \".bin\")\n\n    # No local pytorch weights\n    except utils.LocalEntryNotFoundError:\n        if extension == \".safetensors\":\n            logger.warning(\n                f\"No safetensors weights found for model {model_id} at revision {revision}. \"\n                f\"Downloading PyTorch weights.\"\n            )\n\n        # Try to see if there are pytorch weights on the hub\n        pt_filenames = utils.weight_hub_files(model_id, revision, \".bin\")\n        # Download pytorch weights\n        local_pt_files = utils.download_weights(pt_filenames, model_id, revision)\n\n    if auto_convert:\n        logger.warning(\n            f\"No safetensors weights found for model {model_id} at revision {revision}. \"\n            f\"Converting PyTorch weights to safetensors.\"\n        )\n\n        # Safetensors final filenames\n        local_st_files = [\n            p.parent / f\"{p.stem.lstrip('pytorch_')}.safetensors\"\n            for p in local_pt_files\n        ]\n        try:\n            import transformers\n            import json\n\n            if is_local_model:\n                config_filename = os.path.join(model_id, \"config.json\")\n            else:\n                config_filename = hf_hub_download(\n                    model_id, revision=revision, filename=\"config.json\"\n                )\n            with open(config_filename, \"r\") as f:\n                config = json.load(f)\n            architecture = config[\"architectures\"][0]\n\n            class_ = getattr(transformers, architecture)\n\n            # Name for this varible depends on transformers version.\n            discard_names = getattr(class_, \"_tied_weights_keys\", [])\n\n        except Exception as e:\n            discard_names = []\n        # Convert pytorch weights to safetensors\n        utils.convert_files(local_pt_files, local_st_files, discard_names)\n\n\n@app.command()\ndef quantize(\n    model_id: str,\n    output_dir: str,\n    revision: Optional[str] = None,\n    logger_level: str = \"INFO\",\n    json_output: bool = False,\n    trust_remote_code: bool = False,\n    upload_to_model_id: Optional[str] = None,\n    percdamp: float = 0.01,\n    act_order: bool = False,\n):\n    if revision is None:\n        revision = \"main\"\n    download_weights(\n        model_id=model_id,\n        revision=revision,\n        logger_level=logger_level,\n        json_output=json_output,\n    )\n    from text_generation_server.utils.gptq.quantize import quantize\n\n    quantize(\n        model_id=model_id,\n        bits=4,\n        groupsize=128,\n        output_dir=output_dir,\n        revision=revision,\n        trust_remote_code=trust_remote_code,\n        upload_to_model_id=upload_to_model_id,\n        percdamp=percdamp,\n        act_order=act_order,\n    )\n\n\nif __name__ == \"__main__\":\n    app()\n", "path": "server/text_generation_server/cli.py"}], "after_files": [{"content": "import os\nimport sys\nimport typer\n\nfrom pathlib import Path\nfrom loguru import logger\nfrom typing import Optional\nfrom enum import Enum\nfrom huggingface_hub import hf_hub_download\n\n\napp = typer.Typer()\n\n\nclass Quantization(str, Enum):\n    bitsandbytes = \"bitsandbytes\"\n    bitsandbytes_nf4 = \"bitsandbytes-nf4\"\n    bitsandbytes_fp4 = \"bitsandbytes-fp4\"\n    gptq = \"gptq\"\n    awq = \"awq\"\n    eetq = \"eetq\"\n\n\nclass Dtype(str, Enum):\n    float16 = \"float16\"\n    bloat16 = \"bfloat16\"\n\n\n@app.command()\ndef serve(\n    model_id: str,\n    revision: Optional[str] = None,\n    sharded: bool = False,\n    quantize: Optional[Quantization] = None,\n    dtype: Optional[Dtype] = None,\n    trust_remote_code: bool = False,\n    uds_path: Path = \"/tmp/text-generation-server\",\n    logger_level: str = \"INFO\",\n    json_output: bool = False,\n    otlp_endpoint: Optional[str] = None,\n):\n    if sharded:\n        assert (\n            os.getenv(\"RANK\", None) is not None\n        ), \"RANK must be set when sharded is True\"\n        assert (\n            os.getenv(\"WORLD_SIZE\", None) is not None\n        ), \"WORLD_SIZE must be set when sharded is True\"\n        assert (\n            os.getenv(\"MASTER_ADDR\", None) is not None\n        ), \"MASTER_ADDR must be set when sharded is True\"\n        assert (\n            os.getenv(\"MASTER_PORT\", None) is not None\n        ), \"MASTER_PORT must be set when sharded is True\"\n\n    # Remove default handler\n    logger.remove()\n    logger.add(\n        sys.stdout,\n        format=\"{message}\",\n        filter=\"text_generation_server\",\n        level=logger_level,\n        serialize=json_output,\n        backtrace=True,\n        diagnose=False,\n    )\n\n    # Import here after the logger is added to log potential import exceptions\n    from text_generation_server import server\n    from text_generation_server.tracing import setup_tracing\n\n    # Setup OpenTelemetry distributed tracing\n    if otlp_endpoint is not None:\n        setup_tracing(shard=os.getenv(\"RANK\", 0), otlp_endpoint=otlp_endpoint)\n\n    # Downgrade enum into str for easier management later on\n    quantize = None if quantize is None else quantize.value\n    dtype = None if dtype is None else dtype.value\n    if dtype is not None and quantize is not None:\n        raise RuntimeError(\n            \"Only 1 can be set between `dtype` and `quantize`, as they both decide how goes the final model.\"\n        )\n    server.serve(\n        model_id, revision, sharded, quantize, dtype, trust_remote_code, uds_path\n    )\n\n\n@app.command()\ndef download_weights(\n    model_id: str,\n    revision: Optional[str] = None,\n    extension: str = \".safetensors\",\n    auto_convert: bool = True,\n    logger_level: str = \"INFO\",\n    json_output: bool = False,\n    trust_remote_code: bool = False,\n):\n    # Remove default handler\n    logger.remove()\n    logger.add(\n        sys.stdout,\n        format=\"{message}\",\n        filter=\"text_generation_server\",\n        level=logger_level,\n        serialize=json_output,\n        backtrace=True,\n        diagnose=False,\n    )\n\n    # Import here after the logger is added to log potential import exceptions\n    from text_generation_server import utils\n\n    # Test if files were already download\n    try:\n        utils.weight_files(model_id, revision, extension)\n        logger.info(\"Files are already present on the host. \" \"Skipping download.\")\n        return\n    # Local files not found\n    except (utils.LocalEntryNotFoundError, FileNotFoundError):\n        pass\n\n    is_local_model = (Path(model_id).exists() and Path(model_id).is_dir()) or os.getenv(\n        \"WEIGHTS_CACHE_OVERRIDE\", None\n    ) is not None\n\n    if not is_local_model:\n        try:\n            adapter_config_filename = hf_hub_download(\n                model_id, revision=revision, filename=\"adapter_config.json\"\n            )\n            utils.download_and_unload_peft(\n                model_id, revision, trust_remote_code=trust_remote_code\n            )\n            is_local_model = True\n            utils.weight_files(model_id, revision, extension)\n            return\n        except (utils.LocalEntryNotFoundError, utils.EntryNotFoundError):\n            pass\n\n        # Try to download weights from the hub\n        try:\n            filenames = utils.weight_hub_files(model_id, revision, extension)\n            utils.download_weights(filenames, model_id, revision)\n            # Successfully downloaded weights\n            return\n\n        # No weights found on the hub with this extension\n        except utils.EntryNotFoundError as e:\n            # Check if we want to automatically convert to safetensors or if we can use .bin weights instead\n            if not extension == \".safetensors\" or not auto_convert:\n                raise e\n\n    else:\n        # Try to load as a local PEFT model\n        try:\n            utils.download_and_unload_peft(\n                model_id, revision, trust_remote_code=trust_remote_code\n            )\n            utils.weight_files(model_id, revision, extension)\n            return\n        except (utils.LocalEntryNotFoundError, utils.EntryNotFoundError):\n            pass\n\n    # Try to see if there are local pytorch weights\n    try:\n        # Get weights for a local model, a hub cached model and inside the WEIGHTS_CACHE_OVERRIDE\n        local_pt_files = utils.weight_files(model_id, revision, \".bin\")\n\n    # No local pytorch weights\n    except utils.LocalEntryNotFoundError:\n        if extension == \".safetensors\":\n            logger.warning(\n                f\"No safetensors weights found for model {model_id} at revision {revision}. \"\n                f\"Downloading PyTorch weights.\"\n            )\n\n        # Try to see if there are pytorch weights on the hub\n        pt_filenames = utils.weight_hub_files(model_id, revision, \".bin\")\n        # Download pytorch weights\n        local_pt_files = utils.download_weights(pt_filenames, model_id, revision)\n\n    if auto_convert:\n        logger.warning(\n            f\"No safetensors weights found for model {model_id} at revision {revision}. \"\n            f\"Converting PyTorch weights to safetensors.\"\n        )\n\n        # Safetensors final filenames\n        local_st_files = [\n            p.parent / f\"{p.stem.lstrip('pytorch_')}.safetensors\"\n            for p in local_pt_files\n        ]\n        try:\n            import transformers\n            import json\n\n            if is_local_model:\n                config_filename = os.path.join(model_id, \"config.json\")\n            else:\n                config_filename = hf_hub_download(\n                    model_id, revision=revision, filename=\"config.json\"\n                )\n            with open(config_filename, \"r\") as f:\n                config = json.load(f)\n            architecture = config[\"architectures\"][0]\n\n            class_ = getattr(transformers, architecture)\n\n            # Name for this varible depends on transformers version.\n            discard_names = getattr(class_, \"_tied_weights_keys\", [])\n\n        except Exception as e:\n            discard_names = []\n        # Convert pytorch weights to safetensors\n        utils.convert_files(local_pt_files, local_st_files, discard_names)\n\n\n@app.command()\ndef quantize(\n    model_id: str,\n    output_dir: str,\n    revision: Optional[str] = None,\n    logger_level: str = \"INFO\",\n    json_output: bool = False,\n    trust_remote_code: bool = False,\n    upload_to_model_id: Optional[str] = None,\n    percdamp: float = 0.01,\n    act_order: bool = False,\n):\n    if revision is None:\n        revision = \"main\"\n    download_weights(\n        model_id=model_id,\n        revision=revision,\n        logger_level=logger_level,\n        json_output=json_output,\n    )\n    from text_generation_server.utils.gptq.quantize import quantize\n\n    quantize(\n        model_id=model_id,\n        bits=4,\n        groupsize=128,\n        output_dir=output_dir,\n        revision=revision,\n        trust_remote_code=trust_remote_code,\n        upload_to_model_id=upload_to_model_id,\n        percdamp=percdamp,\n        act_order=act_order,\n    )\n\n\nif __name__ == \"__main__\":\n    app()\n", "path": "server/text_generation_server/cli.py"}]}