We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
UnicodeDecodeError: 'charmap' codec can't decode byte 0x81 in position 125: character maps to <undefined>
### Describe the bug

That is probably locale-dependant thing, but some models complain on decoding error. Here's diff to fix:

```python
diff --git a/modules/models_settings.py b/modules/models_settings.py
index 8a4febd..90c7b69 100644
--- a/modules/models_settings.py
+++ b/modules/models_settings.py
@@ -102,7 +102,7 @@ def get_model_metadata(model):
     # Try to find the Jinja instruct template
     path = Path(f'{shared.args.model_dir}/{model}') / 'tokenizer_config.json'
     if path.exists():
-        metadata = json.loads(open(path, 'r').read())
+        metadata = json.loads(open(path, 'r', encoding='utf-8').read())
         if 'chat_template' in metadata:
             template = metadata['chat_template']
             for k in ['eos_token', 'bos_token']:
```

### Is there an existing issue for this?

- [X] I have searched the existing issues

### Reproduction

Eg. see https://huggingface.co/LoneStriker/deepseek-coder-33b-instruct-4.65bpw-h6-exl2/discussions/1#657daf9c9a9e347d4b9fc544

### Screenshot

_No response_

### Logs

```shell
See: https://huggingface.co/LoneStriker/deepseek-coder-33b-instruct-4.65bpw-h6-exl2/discussions/1#657daf9c9a9e347d4b9fc544
```


### System Info

```shell
windows
```


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `modules/models_settings.py`
Content:
```
1 import json
2 import re
3 from pathlib import Path
4 
5 import yaml
6 
7 from modules import chat, loaders, metadata_gguf, shared, ui
8 
9 
10 def get_fallback_settings():
11     return {
12         'wbits': 'None',
13         'groupsize': 'None',
14         'desc_act': False,
15         'model_type': 'None',
16         'max_seq_len': 2048,
17         'n_ctx': 2048,
18         'rope_freq_base': 0,
19         'compress_pos_emb': 1,
20         'truncation_length': shared.settings['truncation_length'],
21         'skip_special_tokens': shared.settings['skip_special_tokens'],
22         'custom_stopping_strings': shared.settings['custom_stopping_strings'],
23     }
24 
25 
26 def get_model_metadata(model):
27     model_settings = {}
28 
29     # Get settings from models/config.yaml and models/config-user.yaml
30     settings = shared.model_config
31     for pat in settings:
32         if re.match(pat.lower(), model.lower()):
33             for k in settings[pat]:
34                 model_settings[k] = settings[pat][k]
35 
36     path = Path(f'{shared.args.model_dir}/{model}/config.json')
37     if path.exists():
38         hf_metadata = json.loads(open(path, 'r').read())
39     else:
40         hf_metadata = None
41 
42     if 'loader' not in model_settings:
43         if hf_metadata is not None and 'quip_params' in hf_metadata:
44             model_settings['loader'] = 'QuIP#'
45         else:
46             loader = infer_loader(model, model_settings)
47             if 'wbits' in model_settings and type(model_settings['wbits']) is int and model_settings['wbits'] > 0:
48                 loader = 'AutoGPTQ'
49 
50             model_settings['loader'] = loader
51 
52     # GGUF metadata
53     if model_settings['loader'] in ['llama.cpp', 'llamacpp_HF', 'ctransformers']:
54         path = Path(f'{shared.args.model_dir}/{model}')
55         if path.is_file():
56             model_file = path
57         else:
58             model_file = list(path.glob('*.gguf'))[0]
59 
60         metadata = metadata_gguf.load_metadata(model_file)
61         if 'llama.context_length' in metadata:
62             model_settings['n_ctx'] = metadata['llama.context_length']
63         if 'llama.rope.scale_linear' in metadata:
64             model_settings['compress_pos_emb'] = metadata['llama.rope.scale_linear']
65         if 'llama.rope.freq_base' in metadata:
66             model_settings['rope_freq_base'] = metadata['llama.rope.freq_base']
67         if 'tokenizer.chat_template' in metadata:
68             template = metadata['tokenizer.chat_template']
69             eos_token = metadata['tokenizer.ggml.tokens'][metadata['tokenizer.ggml.eos_token_id']]
70             bos_token = metadata['tokenizer.ggml.tokens'][metadata['tokenizer.ggml.bos_token_id']]
71             template = template.replace('eos_token', "'{}'".format(eos_token))
72             template = template.replace('bos_token', "'{}'".format(bos_token))
73 
74             template = re.sub(r'raise_exception\([^)]*\)', "''", template)
75             model_settings['instruction_template'] = 'Custom (obtained from model metadata)'
76             model_settings['instruction_template_str'] = template
77 
78     else:
79         # Transformers metadata
80         if hf_metadata is not None:
81             metadata = json.loads(open(path, 'r').read())
82             if 'max_position_embeddings' in metadata:
83                 model_settings['truncation_length'] = metadata['max_position_embeddings']
84                 model_settings['max_seq_len'] = metadata['max_position_embeddings']
85 
86             if 'rope_theta' in metadata:
87                 model_settings['rope_freq_base'] = metadata['rope_theta']
88 
89             if 'rope_scaling' in metadata and type(metadata['rope_scaling']) is dict and all(key in metadata['rope_scaling'] for key in ('type', 'factor')):
90                 if metadata['rope_scaling']['type'] == 'linear':
91                     model_settings['compress_pos_emb'] = metadata['rope_scaling']['factor']
92 
93             if 'quantization_config' in metadata:
94                 if 'bits' in metadata['quantization_config']:
95                     model_settings['wbits'] = metadata['quantization_config']['bits']
96                 if 'group_size' in metadata['quantization_config']:
97                     model_settings['groupsize'] = metadata['quantization_config']['group_size']
98                 if 'desc_act' in metadata['quantization_config']:
99                     model_settings['desc_act'] = metadata['quantization_config']['desc_act']
100 
101         # Read AutoGPTQ metadata
102         path = Path(f'{shared.args.model_dir}/{model}/quantize_config.json')
103         if path.exists():
104             metadata = json.loads(open(path, 'r').read())
105             if 'bits' in metadata:
106                 model_settings['wbits'] = metadata['bits']
107             if 'group_size' in metadata:
108                 model_settings['groupsize'] = metadata['group_size']
109             if 'desc_act' in metadata:
110                 model_settings['desc_act'] = metadata['desc_act']
111 
112     # Try to find the Jinja instruct template
113     path = Path(f'{shared.args.model_dir}/{model}') / 'tokenizer_config.json'
114     if path.exists():
115         metadata = json.loads(open(path, 'r').read())
116         if 'chat_template' in metadata:
117             template = metadata['chat_template']
118             for k in ['eos_token', 'bos_token']:
119                 if k in metadata:
120                     value = metadata[k]
121                     if type(value) is dict:
122                         value = value['content']
123 
124                     template = template.replace(k, "'{}'".format(value))
125 
126             template = re.sub(r'raise_exception\([^)]*\)', "''", template)
127             model_settings['instruction_template'] = 'Custom (obtained from model metadata)'
128             model_settings['instruction_template_str'] = template
129 
130     if 'instruction_template' not in model_settings:
131         model_settings['instruction_template'] = 'Alpaca'
132 
133     if model_settings['instruction_template'] != 'Custom (obtained from model metadata)':
134         model_settings['instruction_template_str'] = chat.load_instruction_template(model_settings['instruction_template'])
135 
136     # Ignore rope_freq_base if set to the default value
137     if 'rope_freq_base' in model_settings and model_settings['rope_freq_base'] == 10000:
138         model_settings.pop('rope_freq_base')
139 
140     # Apply user settings from models/config-user.yaml
141     settings = shared.user_config
142     for pat in settings:
143         if re.match(pat.lower(), model.lower()):
144             for k in settings[pat]:
145                 model_settings[k] = settings[pat][k]
146 
147     return model_settings
148 
149 
150 def infer_loader(model_name, model_settings):
151     path_to_model = Path(f'{shared.args.model_dir}/{model_name}')
152     if not path_to_model.exists():
153         loader = None
154     elif (path_to_model / 'quantize_config.json').exists() or ('wbits' in model_settings and type(model_settings['wbits']) is int and model_settings['wbits'] > 0):
155         loader = 'ExLlama_HF'
156     elif (path_to_model / 'quant_config.json').exists() or re.match(r'.*-awq', model_name.lower()):
157         loader = 'AutoAWQ'
158     elif len(list(path_to_model.glob('*.gguf'))) > 0:
159         loader = 'llama.cpp'
160     elif re.match(r'.*\.gguf', model_name.lower()):
161         loader = 'llama.cpp'
162     elif re.match(r'.*rwkv.*\.pth', model_name.lower()):
163         loader = 'RWKV'
164     elif re.match(r'.*exl2', model_name.lower()):
165         loader = 'ExLlamav2_HF'
166     elif re.match(r'.*-hqq', model_name.lower()):
167         return 'HQQ'
168     else:
169         loader = 'Transformers'
170 
171     return loader
172 
173 
174 def update_model_parameters(state, initial=False):
175     '''
176     UI: update the command-line arguments based on the interface values
177     '''
178     elements = ui.list_model_elements()  # the names of the parameters
179     gpu_memories = []
180 
181     for i, element in enumerate(elements):
182         if element not in state:
183             continue
184 
185         value = state[element]
186         if element.startswith('gpu_memory'):
187             gpu_memories.append(value)
188             continue
189 
190         if initial and element in shared.provided_arguments:
191             continue
192 
193         # Setting null defaults
194         if element in ['wbits', 'groupsize', 'model_type'] and value == 'None':
195             value = vars(shared.args_defaults)[element]
196         elif element in ['cpu_memory'] and value == 0:
197             value = vars(shared.args_defaults)[element]
198 
199         # Making some simple conversions
200         if element in ['wbits', 'groupsize', 'pre_layer']:
201             value = int(value)
202         elif element == 'cpu_memory' and value is not None:
203             value = f"{value}MiB"
204 
205         if element in ['pre_layer']:
206             value = [value] if value > 0 else None
207 
208         setattr(shared.args, element, value)
209 
210     found_positive = False
211     for i in gpu_memories:
212         if i > 0:
213             found_positive = True
214             break
215 
216     if not (initial and vars(shared.args)['gpu_memory'] != vars(shared.args_defaults)['gpu_memory']):
217         if found_positive:
218             shared.args.gpu_memory = [f"{i}MiB" for i in gpu_memories]
219         else:
220             shared.args.gpu_memory = None
221 
222 
223 def apply_model_settings_to_state(model, state):
224     '''
225     UI: update the state variable with the model settings
226     '''
227     model_settings = get_model_metadata(model)
228     if 'loader' in model_settings:
229         loader = model_settings.pop('loader')
230 
231         # If the user is using an alternative loader for the same model type, let them keep using it
232         if not (loader == 'AutoGPTQ' and state['loader'] in ['GPTQ-for-LLaMa', 'ExLlama', 'ExLlama_HF', 'ExLlamav2', 'ExLlamav2_HF']) and not (loader == 'llama.cpp' and state['loader'] in ['llamacpp_HF', 'ctransformers']):
233             state['loader'] = loader
234 
235     for k in model_settings:
236         if k in state:
237             if k in ['wbits', 'groupsize']:
238                 state[k] = str(model_settings[k])
239             else:
240                 state[k] = model_settings[k]
241 
242     return state
243 
244 
245 def save_model_settings(model, state):
246     '''
247     Save the settings for this model to models/config-user.yaml
248     '''
249     if model == 'None':
250         yield ("Not saving the settings because no model is loaded.")
251         return
252 
253     with Path(f'{shared.args.model_dir}/config-user.yaml') as p:
254         if p.exists():
255             user_config = yaml.safe_load(open(p, 'r').read())
256         else:
257             user_config = {}
258 
259         model_regex = model + '$'  # For exact matches
260         if model_regex not in user_config:
261             user_config[model_regex] = {}
262 
263         for k in ui.list_model_elements():
264             if k == 'loader' or k in loaders.loaders_and_params[state['loader']]:
265                 user_config[model_regex][k] = state[k]
266 
267         shared.user_config = user_config
268 
269         output = yaml.dump(user_config, sort_keys=False)
270         with open(p, 'w') as f:
271             f.write(output)
272 
273         yield (f"Settings for `{model}` saved to `{p}`.")
274
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/modules/models_settings.py b/modules/models_settings.py
--- a/modules/models_settings.py
+++ b/modules/models_settings.py
@@ -35,7 +35,7 @@
 
     path = Path(f'{shared.args.model_dir}/{model}/config.json')
     if path.exists():
-        hf_metadata = json.loads(open(path, 'r').read())
+        hf_metadata = json.loads(open(path, 'r', encoding='utf-8').read())
     else:
         hf_metadata = None
 
@@ -78,7 +78,7 @@
     else:
         # Transformers metadata
         if hf_metadata is not None:
-            metadata = json.loads(open(path, 'r').read())
+            metadata = json.loads(open(path, 'r', encoding='utf-8').read())
             if 'max_position_embeddings' in metadata:
                 model_settings['truncation_length'] = metadata['max_position_embeddings']
                 model_settings['max_seq_len'] = metadata['max_position_embeddings']
@@ -101,7 +101,7 @@
         # Read AutoGPTQ metadata
         path = Path(f'{shared.args.model_dir}/{model}/quantize_config.json')
         if path.exists():
-            metadata = json.loads(open(path, 'r').read())
+            metadata = json.loads(open(path, 'r', encoding='utf-8').read())
             if 'bits' in metadata:
                 model_settings['wbits'] = metadata['bits']
             if 'group_size' in metadata:
@@ -112,7 +112,7 @@
     # Try to find the Jinja instruct template
     path = Path(f'{shared.args.model_dir}/{model}') / 'tokenizer_config.json'
     if path.exists():
-        metadata = json.loads(open(path, 'r').read())
+        metadata = json.loads(open(path, 'r', encoding='utf-8').read())
         if 'chat_template' in metadata:
             template = metadata['chat_template']
             for k in ['eos_token', 'bos_token']:

{"golden_diff": "diff --git a/modules/models_settings.py b/modules/models_settings.py\n--- a/modules/models_settings.py\n+++ b/modules/models_settings.py\n@@ -35,7 +35,7 @@\n \n     path = Path(f'{shared.args.model_dir}/{model}/config.json')\n     if path.exists():\n-        hf_metadata = json.loads(open(path, 'r').read())\n+        hf_metadata = json.loads(open(path, 'r', encoding='utf-8').read())\n     else:\n         hf_metadata = None\n \n@@ -78,7 +78,7 @@\n     else:\n         # Transformers metadata\n         if hf_metadata is not None:\n-            metadata = json.loads(open(path, 'r').read())\n+            metadata = json.loads(open(path, 'r', encoding='utf-8').read())\n             if 'max_position_embeddings' in metadata:\n                 model_settings['truncation_length'] = metadata['max_position_embeddings']\n                 model_settings['max_seq_len'] = metadata['max_position_embeddings']\n@@ -101,7 +101,7 @@\n         # Read AutoGPTQ metadata\n         path = Path(f'{shared.args.model_dir}/{model}/quantize_config.json')\n         if path.exists():\n-            metadata = json.loads(open(path, 'r').read())\n+            metadata = json.loads(open(path, 'r', encoding='utf-8').read())\n             if 'bits' in metadata:\n                 model_settings['wbits'] = metadata['bits']\n             if 'group_size' in metadata:\n@@ -112,7 +112,7 @@\n     # Try to find the Jinja instruct template\n     path = Path(f'{shared.args.model_dir}/{model}') / 'tokenizer_config.json'\n     if path.exists():\n-        metadata = json.loads(open(path, 'r').read())\n+        metadata = json.loads(open(path, 'r', encoding='utf-8').read())\n         if 'chat_template' in metadata:\n             template = metadata['chat_template']\n             for k in ['eos_token', 'bos_token']:\n", "issue": "UnicodeDecodeError: 'charmap' codec can't decode byte 0x81 in position 125: character maps to <undefined>\n### Describe the bug\n\nThat is probably locale-dependant thing, but some models complain on decoding error. Here's diff to fix:\r\n\r\n```python\r\ndiff --git a/modules/models_settings.py b/modules/models_settings.py\r\nindex 8a4febd..90c7b69 100644\r\n--- a/modules/models_settings.py\r\n+++ b/modules/models_settings.py\r\n@@ -102,7 +102,7 @@ def get_model_metadata(model):\r\n     # Try to find the Jinja instruct template\r\n     path = Path(f'{shared.args.model_dir}/{model}') / 'tokenizer_config.json'\r\n     if path.exists():\r\n-        metadata = json.loads(open(path, 'r').read())\r\n+        metadata = json.loads(open(path, 'r', encoding='utf-8').read())\r\n         if 'chat_template' in metadata:\r\n             template = metadata['chat_template']\r\n             for k in ['eos_token', 'bos_token']:\r\n```\n\n### Is there an existing issue for this?\n\n- [X] I have searched the existing issues\n\n### Reproduction\n\nEg. see https://huggingface.co/LoneStriker/deepseek-coder-33b-instruct-4.65bpw-h6-exl2/discussions/1#657daf9c9a9e347d4b9fc544\n\n### Screenshot\n\n_No response_\n\n### Logs\n\n```shell\nSee: https://huggingface.co/LoneStriker/deepseek-coder-33b-instruct-4.65bpw-h6-exl2/discussions/1#657daf9c9a9e347d4b9fc544\n```\n\n\n### System Info\n\n```shell\nwindows\n```\n\n", "before_files": [{"content": "import json\nimport re\nfrom pathlib import Path\n\nimport yaml\n\nfrom modules import chat, loaders, metadata_gguf, shared, ui\n\n\ndef get_fallback_settings():\n    return {\n        'wbits': 'None',\n        'groupsize': 'None',\n        'desc_act': False,\n        'model_type': 'None',\n        'max_seq_len': 2048,\n        'n_ctx': 2048,\n        'rope_freq_base': 0,\n        'compress_pos_emb': 1,\n        'truncation_length': shared.settings['truncation_length'],\n        'skip_special_tokens': shared.settings['skip_special_tokens'],\n        'custom_stopping_strings': shared.settings['custom_stopping_strings'],\n    }\n\n\ndef get_model_metadata(model):\n    model_settings = {}\n\n    # Get settings from models/config.yaml and models/config-user.yaml\n    settings = shared.model_config\n    for pat in settings:\n        if re.match(pat.lower(), model.lower()):\n            for k in settings[pat]:\n                model_settings[k] = settings[pat][k]\n\n    path = Path(f'{shared.args.model_dir}/{model}/config.json')\n    if path.exists():\n        hf_metadata = json.loads(open(path, 'r').read())\n    else:\n        hf_metadata = None\n\n    if 'loader' not in model_settings:\n        if hf_metadata is not None and 'quip_params' in hf_metadata:\n            model_settings['loader'] = 'QuIP#'\n        else:\n            loader = infer_loader(model, model_settings)\n            if 'wbits' in model_settings and type(model_settings['wbits']) is int and model_settings['wbits'] > 0:\n                loader = 'AutoGPTQ'\n\n            model_settings['loader'] = loader\n\n    # GGUF metadata\n    if model_settings['loader'] in ['llama.cpp', 'llamacpp_HF', 'ctransformers']:\n        path = Path(f'{shared.args.model_dir}/{model}')\n        if path.is_file():\n            model_file = path\n        else:\n            model_file = list(path.glob('*.gguf'))[0]\n\n        metadata = metadata_gguf.load_metadata(model_file)\n        if 'llama.context_length' in metadata:\n            model_settings['n_ctx'] = metadata['llama.context_length']\n        if 'llama.rope.scale_linear' in metadata:\n            model_settings['compress_pos_emb'] = metadata['llama.rope.scale_linear']\n        if 'llama.rope.freq_base' in metadata:\n            model_settings['rope_freq_base'] = metadata['llama.rope.freq_base']\n        if 'tokenizer.chat_template' in metadata:\n            template = metadata['tokenizer.chat_template']\n            eos_token = metadata['tokenizer.ggml.tokens'][metadata['tokenizer.ggml.eos_token_id']]\n            bos_token = metadata['tokenizer.ggml.tokens'][metadata['tokenizer.ggml.bos_token_id']]\n            template = template.replace('eos_token', \"'{}'\".format(eos_token))\n            template = template.replace('bos_token', \"'{}'\".format(bos_token))\n\n            template = re.sub(r'raise_exception\\([^)]*\\)', \"''\", template)\n            model_settings['instruction_template'] = 'Custom (obtained from model metadata)'\n            model_settings['instruction_template_str'] = template\n\n    else:\n        # Transformers metadata\n        if hf_metadata is not None:\n            metadata = json.loads(open(path, 'r').read())\n            if 'max_position_embeddings' in metadata:\n                model_settings['truncation_length'] = metadata['max_position_embeddings']\n                model_settings['max_seq_len'] = metadata['max_position_embeddings']\n\n            if 'rope_theta' in metadata:\n                model_settings['rope_freq_base'] = metadata['rope_theta']\n\n            if 'rope_scaling' in metadata and type(metadata['rope_scaling']) is dict and all(key in metadata['rope_scaling'] for key in ('type', 'factor')):\n                if metadata['rope_scaling']['type'] == 'linear':\n                    model_settings['compress_pos_emb'] = metadata['rope_scaling']['factor']\n\n            if 'quantization_config' in metadata:\n                if 'bits' in metadata['quantization_config']:\n                    model_settings['wbits'] = metadata['quantization_config']['bits']\n                if 'group_size' in metadata['quantization_config']:\n                    model_settings['groupsize'] = metadata['quantization_config']['group_size']\n                if 'desc_act' in metadata['quantization_config']:\n                    model_settings['desc_act'] = metadata['quantization_config']['desc_act']\n\n        # Read AutoGPTQ metadata\n        path = Path(f'{shared.args.model_dir}/{model}/quantize_config.json')\n        if path.exists():\n            metadata = json.loads(open(path, 'r').read())\n            if 'bits' in metadata:\n                model_settings['wbits'] = metadata['bits']\n            if 'group_size' in metadata:\n                model_settings['groupsize'] = metadata['group_size']\n            if 'desc_act' in metadata:\n                model_settings['desc_act'] = metadata['desc_act']\n\n    # Try to find the Jinja instruct template\n    path = Path(f'{shared.args.model_dir}/{model}') / 'tokenizer_config.json'\n    if path.exists():\n        metadata = json.loads(open(path, 'r').read())\n        if 'chat_template' in metadata:\n            template = metadata['chat_template']\n            for k in ['eos_token', 'bos_token']:\n                if k in metadata:\n                    value = metadata[k]\n                    if type(value) is dict:\n                        value = value['content']\n\n                    template = template.replace(k, \"'{}'\".format(value))\n\n            template = re.sub(r'raise_exception\\([^)]*\\)', \"''\", template)\n            model_settings['instruction_template'] = 'Custom (obtained from model metadata)'\n            model_settings['instruction_template_str'] = template\n\n    if 'instruction_template' not in model_settings:\n        model_settings['instruction_template'] = 'Alpaca'\n\n    if model_settings['instruction_template'] != 'Custom (obtained from model metadata)':\n        model_settings['instruction_template_str'] = chat.load_instruction_template(model_settings['instruction_template'])\n\n    # Ignore rope_freq_base if set to the default value\n    if 'rope_freq_base' in model_settings and model_settings['rope_freq_base'] == 10000:\n        model_settings.pop('rope_freq_base')\n\n    # Apply user settings from models/config-user.yaml\n    settings = shared.user_config\n    for pat in settings:\n        if re.match(pat.lower(), model.lower()):\n            for k in settings[pat]:\n                model_settings[k] = settings[pat][k]\n\n    return model_settings\n\n\ndef infer_loader(model_name, model_settings):\n    path_to_model = Path(f'{shared.args.model_dir}/{model_name}')\n    if not path_to_model.exists():\n        loader = None\n    elif (path_to_model / 'quantize_config.json').exists() or ('wbits' in model_settings and type(model_settings['wbits']) is int and model_settings['wbits'] > 0):\n        loader = 'ExLlama_HF'\n    elif (path_to_model / 'quant_config.json').exists() or re.match(r'.*-awq', model_name.lower()):\n        loader = 'AutoAWQ'\n    elif len(list(path_to_model.glob('*.gguf'))) > 0:\n        loader = 'llama.cpp'\n    elif re.match(r'.*\\.gguf', model_name.lower()):\n        loader = 'llama.cpp'\n    elif re.match(r'.*rwkv.*\\.pth', model_name.lower()):\n        loader = 'RWKV'\n    elif re.match(r'.*exl2', model_name.lower()):\n        loader = 'ExLlamav2_HF'\n    elif re.match(r'.*-hqq', model_name.lower()):\n        return 'HQQ'\n    else:\n        loader = 'Transformers'\n\n    return loader\n\n\ndef update_model_parameters(state, initial=False):\n    '''\n    UI: update the command-line arguments based on the interface values\n    '''\n    elements = ui.list_model_elements()  # the names of the parameters\n    gpu_memories = []\n\n    for i, element in enumerate(elements):\n        if element not in state:\n            continue\n\n        value = state[element]\n        if element.startswith('gpu_memory'):\n            gpu_memories.append(value)\n            continue\n\n        if initial and element in shared.provided_arguments:\n            continue\n\n        # Setting null defaults\n        if element in ['wbits', 'groupsize', 'model_type'] and value == 'None':\n            value = vars(shared.args_defaults)[element]\n        elif element in ['cpu_memory'] and value == 0:\n            value = vars(shared.args_defaults)[element]\n\n        # Making some simple conversions\n        if element in ['wbits', 'groupsize', 'pre_layer']:\n            value = int(value)\n        elif element == 'cpu_memory' and value is not None:\n            value = f\"{value}MiB\"\n\n        if element in ['pre_layer']:\n            value = [value] if value > 0 else None\n\n        setattr(shared.args, element, value)\n\n    found_positive = False\n    for i in gpu_memories:\n        if i > 0:\n            found_positive = True\n            break\n\n    if not (initial and vars(shared.args)['gpu_memory'] != vars(shared.args_defaults)['gpu_memory']):\n        if found_positive:\n            shared.args.gpu_memory = [f\"{i}MiB\" for i in gpu_memories]\n        else:\n            shared.args.gpu_memory = None\n\n\ndef apply_model_settings_to_state(model, state):\n    '''\n    UI: update the state variable with the model settings\n    '''\n    model_settings = get_model_metadata(model)\n    if 'loader' in model_settings:\n        loader = model_settings.pop('loader')\n\n        # If the user is using an alternative loader for the same model type, let them keep using it\n        if not (loader == 'AutoGPTQ' and state['loader'] in ['GPTQ-for-LLaMa', 'ExLlama', 'ExLlama_HF', 'ExLlamav2', 'ExLlamav2_HF']) and not (loader == 'llama.cpp' and state['loader'] in ['llamacpp_HF', 'ctransformers']):\n            state['loader'] = loader\n\n    for k in model_settings:\n        if k in state:\n            if k in ['wbits', 'groupsize']:\n                state[k] = str(model_settings[k])\n            else:\n                state[k] = model_settings[k]\n\n    return state\n\n\ndef save_model_settings(model, state):\n    '''\n    Save the settings for this model to models/config-user.yaml\n    '''\n    if model == 'None':\n        yield (\"Not saving the settings because no model is loaded.\")\n        return\n\n    with Path(f'{shared.args.model_dir}/config-user.yaml') as p:\n        if p.exists():\n            user_config = yaml.safe_load(open(p, 'r').read())\n        else:\n            user_config = {}\n\n        model_regex = model + '$'  # For exact matches\n        if model_regex not in user_config:\n            user_config[model_regex] = {}\n\n        for k in ui.list_model_elements():\n            if k == 'loader' or k in loaders.loaders_and_params[state['loader']]:\n                user_config[model_regex][k] = state[k]\n\n        shared.user_config = user_config\n\n        output = yaml.dump(user_config, sort_keys=False)\n        with open(p, 'w') as f:\n            f.write(output)\n\n        yield (f\"Settings for `{model}` saved to `{p}`.\")\n", "path": "modules/models_settings.py"}], "after_files": [{"content": "import json\nimport re\nfrom pathlib import Path\n\nimport yaml\n\nfrom modules import chat, loaders, metadata_gguf, shared, ui\n\n\ndef get_fallback_settings():\n    return {\n        'wbits': 'None',\n        'groupsize': 'None',\n        'desc_act': False,\n        'model_type': 'None',\n        'max_seq_len': 2048,\n        'n_ctx': 2048,\n        'rope_freq_base': 0,\n        'compress_pos_emb': 1,\n        'truncation_length': shared.settings['truncation_length'],\n        'skip_special_tokens': shared.settings['skip_special_tokens'],\n        'custom_stopping_strings': shared.settings['custom_stopping_strings'],\n    }\n\n\ndef get_model_metadata(model):\n    model_settings = {}\n\n    # Get settings from models/config.yaml and models/config-user.yaml\n    settings = shared.model_config\n    for pat in settings:\n        if re.match(pat.lower(), model.lower()):\n            for k in settings[pat]:\n                model_settings[k] = settings[pat][k]\n\n    path = Path(f'{shared.args.model_dir}/{model}/config.json')\n    if path.exists():\n        hf_metadata = json.loads(open(path, 'r', encoding='utf-8').read())\n    else:\n        hf_metadata = None\n\n    if 'loader' not in model_settings:\n        if hf_metadata is not None and 'quip_params' in hf_metadata:\n            model_settings['loader'] = 'QuIP#'\n        else:\n            loader = infer_loader(model, model_settings)\n            if 'wbits' in model_settings and type(model_settings['wbits']) is int and model_settings['wbits'] > 0:\n                loader = 'AutoGPTQ'\n\n            model_settings['loader'] = loader\n\n    # GGUF metadata\n    if model_settings['loader'] in ['llama.cpp', 'llamacpp_HF', 'ctransformers']:\n        path = Path(f'{shared.args.model_dir}/{model}')\n        if path.is_file():\n            model_file = path\n        else:\n            model_file = list(path.glob('*.gguf'))[0]\n\n        metadata = metadata_gguf.load_metadata(model_file)\n        if 'llama.context_length' in metadata:\n            model_settings['n_ctx'] = metadata['llama.context_length']\n        if 'llama.rope.scale_linear' in metadata:\n            model_settings['compress_pos_emb'] = metadata['llama.rope.scale_linear']\n        if 'llama.rope.freq_base' in metadata:\n            model_settings['rope_freq_base'] = metadata['llama.rope.freq_base']\n        if 'tokenizer.chat_template' in metadata:\n            template = metadata['tokenizer.chat_template']\n            eos_token = metadata['tokenizer.ggml.tokens'][metadata['tokenizer.ggml.eos_token_id']]\n            bos_token = metadata['tokenizer.ggml.tokens'][metadata['tokenizer.ggml.bos_token_id']]\n            template = template.replace('eos_token', \"'{}'\".format(eos_token))\n            template = template.replace('bos_token', \"'{}'\".format(bos_token))\n\n            template = re.sub(r'raise_exception\\([^)]*\\)', \"''\", template)\n            model_settings['instruction_template'] = 'Custom (obtained from model metadata)'\n            model_settings['instruction_template_str'] = template\n\n    else:\n        # Transformers metadata\n        if hf_metadata is not None:\n            metadata = json.loads(open(path, 'r', encoding='utf-8').read())\n            if 'max_position_embeddings' in metadata:\n                model_settings['truncation_length'] = metadata['max_position_embeddings']\n                model_settings['max_seq_len'] = metadata['max_position_embeddings']\n\n            if 'rope_theta' in metadata:\n                model_settings['rope_freq_base'] = metadata['rope_theta']\n\n            if 'rope_scaling' in metadata and type(metadata['rope_scaling']) is dict and all(key in metadata['rope_scaling'] for key in ('type', 'factor')):\n                if metadata['rope_scaling']['type'] == 'linear':\n                    model_settings['compress_pos_emb'] = metadata['rope_scaling']['factor']\n\n            if 'quantization_config' in metadata:\n                if 'bits' in metadata['quantization_config']:\n                    model_settings['wbits'] = metadata['quantization_config']['bits']\n                if 'group_size' in metadata['quantization_config']:\n                    model_settings['groupsize'] = metadata['quantization_config']['group_size']\n                if 'desc_act' in metadata['quantization_config']:\n                    model_settings['desc_act'] = metadata['quantization_config']['desc_act']\n\n        # Read AutoGPTQ metadata\n        path = Path(f'{shared.args.model_dir}/{model}/quantize_config.json')\n        if path.exists():\n            metadata = json.loads(open(path, 'r', encoding='utf-8').read())\n            if 'bits' in metadata:\n                model_settings['wbits'] = metadata['bits']\n            if 'group_size' in metadata:\n                model_settings['groupsize'] = metadata['group_size']\n            if 'desc_act' in metadata:\n                model_settings['desc_act'] = metadata['desc_act']\n\n    # Try to find the Jinja instruct template\n    path = Path(f'{shared.args.model_dir}/{model}') / 'tokenizer_config.json'\n    if path.exists():\n        metadata = json.loads(open(path, 'r', encoding='utf-8').read())\n        if 'chat_template' in metadata:\n            template = metadata['chat_template']\n            for k in ['eos_token', 'bos_token']:\n                if k in metadata:\n                    value = metadata[k]\n                    if type(value) is dict:\n                        value = value['content']\n\n                    template = template.replace(k, \"'{}'\".format(value))\n\n            template = re.sub(r'raise_exception\\([^)]*\\)', \"''\", template)\n            model_settings['instruction_template'] = 'Custom (obtained from model metadata)'\n            model_settings['instruction_template_str'] = template\n\n    if 'instruction_template' not in model_settings:\n        model_settings['instruction_template'] = 'Alpaca'\n\n    if model_settings['instruction_template'] != 'Custom (obtained from model metadata)':\n        model_settings['instruction_template_str'] = chat.load_instruction_template(model_settings['instruction_template'])\n\n    # Ignore rope_freq_base if set to the default value\n    if 'rope_freq_base' in model_settings and model_settings['rope_freq_base'] == 10000:\n        model_settings.pop('rope_freq_base')\n\n    # Apply user settings from models/config-user.yaml\n    settings = shared.user_config\n    for pat in settings:\n        if re.match(pat.lower(), model.lower()):\n            for k in settings[pat]:\n                model_settings[k] = settings[pat][k]\n\n    return model_settings\n\n\ndef infer_loader(model_name, model_settings):\n    path_to_model = Path(f'{shared.args.model_dir}/{model_name}')\n    if not path_to_model.exists():\n        loader = None\n    elif (path_to_model / 'quantize_config.json').exists() or ('wbits' in model_settings and type(model_settings['wbits']) is int and model_settings['wbits'] > 0):\n        loader = 'ExLlama_HF'\n    elif (path_to_model / 'quant_config.json').exists() or re.match(r'.*-awq', model_name.lower()):\n        loader = 'AutoAWQ'\n    elif len(list(path_to_model.glob('*.gguf'))) > 0:\n        loader = 'llama.cpp'\n    elif re.match(r'.*\\.gguf', model_name.lower()):\n        loader = 'llama.cpp'\n    elif re.match(r'.*rwkv.*\\.pth', model_name.lower()):\n        loader = 'RWKV'\n    elif re.match(r'.*exl2', model_name.lower()):\n        loader = 'ExLlamav2_HF'\n    elif re.match(r'.*-hqq', model_name.lower()):\n        return 'HQQ'\n    else:\n        loader = 'Transformers'\n\n    return loader\n\n\ndef update_model_parameters(state, initial=False):\n    '''\n    UI: update the command-line arguments based on the interface values\n    '''\n    elements = ui.list_model_elements()  # the names of the parameters\n    gpu_memories = []\n\n    for i, element in enumerate(elements):\n        if element not in state:\n            continue\n\n        value = state[element]\n        if element.startswith('gpu_memory'):\n            gpu_memories.append(value)\n            continue\n\n        if initial and element in shared.provided_arguments:\n            continue\n\n        # Setting null defaults\n        if element in ['wbits', 'groupsize', 'model_type'] and value == 'None':\n            value = vars(shared.args_defaults)[element]\n        elif element in ['cpu_memory'] and value == 0:\n            value = vars(shared.args_defaults)[element]\n\n        # Making some simple conversions\n        if element in ['wbits', 'groupsize', 'pre_layer']:\n            value = int(value)\n        elif element == 'cpu_memory' and value is not None:\n            value = f\"{value}MiB\"\n\n        if element in ['pre_layer']:\n            value = [value] if value > 0 else None\n\n        setattr(shared.args, element, value)\n\n    found_positive = False\n    for i in gpu_memories:\n        if i > 0:\n            found_positive = True\n            break\n\n    if not (initial and vars(shared.args)['gpu_memory'] != vars(shared.args_defaults)['gpu_memory']):\n        if found_positive:\n            shared.args.gpu_memory = [f\"{i}MiB\" for i in gpu_memories]\n        else:\n            shared.args.gpu_memory = None\n\n\ndef apply_model_settings_to_state(model, state):\n    '''\n    UI: update the state variable with the model settings\n    '''\n    model_settings = get_model_metadata(model)\n    if 'loader' in model_settings:\n        loader = model_settings.pop('loader')\n\n        # If the user is using an alternative loader for the same model type, let them keep using it\n        if not (loader == 'AutoGPTQ' and state['loader'] in ['GPTQ-for-LLaMa', 'ExLlama', 'ExLlama_HF', 'ExLlamav2', 'ExLlamav2_HF']) and not (loader == 'llama.cpp' and state['loader'] in ['llamacpp_HF', 'ctransformers']):\n            state['loader'] = loader\n\n    for k in model_settings:\n        if k in state:\n            if k in ['wbits', 'groupsize']:\n                state[k] = str(model_settings[k])\n            else:\n                state[k] = model_settings[k]\n\n    return state\n\n\ndef save_model_settings(model, state):\n    '''\n    Save the settings for this model to models/config-user.yaml\n    '''\n    if model == 'None':\n        yield (\"Not saving the settings because no model is loaded.\")\n        return\n\n    with Path(f'{shared.args.model_dir}/config-user.yaml') as p:\n        if p.exists():\n            user_config = yaml.safe_load(open(p, 'r').read())\n        else:\n            user_config = {}\n\n        model_regex = model + '$'  # For exact matches\n        if model_regex not in user_config:\n            user_config[model_regex] = {}\n\n        for k in ui.list_model_elements():\n            if k == 'loader' or k in loaders.loaders_and_params[state['loader']]:\n                user_config[model_regex][k] = state[k]\n\n        shared.user_config = user_config\n\n        output = yaml.dump(user_config, sort_keys=False)\n        with open(p, 'w') as f:\n            f.write(output)\n\n        yield (f\"Settings for `{model}` saved to `{p}`.\")\n", "path": "modules/models_settings.py"}]}