We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Missing language: pt-BR
<!-- PLEASE FILL THESE FIELDS, IT REALLY HELPS THE MAINTAINERS OF SearXNG -->

**Version of SearXNG, commit number if you are using on master branch and stipulate if you forked SearXNG**

1.0.0-950-dfdf71bf
(docker)
**How did you install SearXNG?**

Docker

**What happened?**
Language `pt-BR` is missing.
https://github.com/searxng/searxng/blob/master/searx/languages.py

It was present in a previous version.

Google supports this language (and some other engines) but it is not showing up on the list.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `searxng_extra/update/update_languages.py`
Content:
```
1 #!/usr/bin/env python
2 # SPDX-License-Identifier: AGPL-3.0-or-later
3 
4 # This script generates languages.py from intersecting each engine's supported languages.
5 #
6 # Output files: searx/data/engines_languages.json and searx/languages.py
7 
8 import json
9 from pathlib import Path
10 from pprint import pformat
11 from babel import Locale, UnknownLocaleError
12 from babel.languages import get_global
13 
14 from searx import settings, searx_dir
15 from searx.engines import load_engines, engines
16 from searx.network import set_timeout_for_thread
17 
18 # Output files.
19 engines_languages_file = Path(searx_dir) / 'data' / 'engines_languages.json'
20 languages_file = Path(searx_dir) / 'languages.py'
21 
22 
23 # Fetchs supported languages for each engine and writes json file with those.
24 def fetch_supported_languages():
25     set_timeout_for_thread(10.0)
26 
27     engines_languages = dict()
28     names = list(engines)
29     names.sort()
30 
31     for engine_name in names:
32         if hasattr(engines[engine_name], 'fetch_supported_languages'):
33             engines_languages[engine_name] = engines[engine_name].fetch_supported_languages()
34             print("fetched %s languages from engine %s" % (
35                 len(engines_languages[engine_name]), engine_name))
36             if type(engines_languages[engine_name]) == list:
37                 engines_languages[engine_name] = sorted(engines_languages[engine_name])
38 
39     # write json file
40     with open(engines_languages_file, 'w', encoding='utf-8') as f:
41         json.dump(engines_languages, f, indent=2, sort_keys=True)
42 
43     return engines_languages
44 
45 
46 # Get babel Locale object from lang_code if possible.
47 def get_locale(lang_code):
48     try:
49         locale = Locale.parse(lang_code, sep='-')
50         return locale
51     except (UnknownLocaleError, ValueError):
52         return None
53 
54 
55 # Join all language lists.
56 def join_language_lists(engines_languages):
57     language_list = dict()
58     for engine_name in engines_languages:
59         for lang_code in engines_languages[engine_name]:
60 
61             # apply custom fixes if necessary
62             if lang_code in getattr(engines[engine_name], 'language_aliases', {}).values():
63                 lang_code = next(lc for lc, alias in engines[engine_name].language_aliases.items()
64                                  if lang_code == alias)
65 
66             locale = get_locale(lang_code)
67 
68             # ensure that lang_code uses standard language and country codes
69             if locale and locale.territory:
70                 lang_code = "{lang}-{country}".format(lang=locale.language, country=locale.territory)
71             short_code = lang_code.split('-')[0]
72 
73             # add language without country if not in list
74             if short_code not in language_list:
75                 if locale:
76                     # get language's data from babel's Locale object
77                     language_name = locale.get_language_name().title()
78                     english_name = locale.english_name.split(' (')[0]
79                 elif short_code in engines_languages['wikipedia']:
80                     # get language's data from wikipedia if not known by babel
81                     language_name = engines_languages['wikipedia'][short_code]['name']
82                     english_name = engines_languages['wikipedia'][short_code]['english_name']
83                 else:
84                     language_name = None
85                     english_name = None
86 
87                 # add language to list
88                 language_list[short_code] = {'name': language_name,
89                                              'english_name': english_name,
90                                              'counter': set(),
91                                              'countries': dict()}
92 
93             # add language with country if not in list
94             if lang_code != short_code and lang_code not in language_list[short_code]['countries']:
95                 country_name = ''
96                 if locale:
97                     # get country name from babel's Locale object
98                     country_name = locale.get_territory_name()
99 
100                 language_list[short_code]['countries'][lang_code] = {'country_name': country_name,
101                                                                      'counter': set()}
102 
103             # count engine for both language_country combination and language alone
104             language_list[short_code]['counter'].add(engine_name)
105             if lang_code != short_code:
106                 language_list[short_code]['countries'][lang_code]['counter'].add(engine_name)
107 
108     return language_list
109 
110 
111 # Filter language list so it only includes the most supported languages and countries
112 def filter_language_list(all_languages):
113     min_engines_per_lang = 13
114     min_engines_per_country = 10
115     main_engines = [engine_name for engine_name in engines.keys()
116                     if 'general' in engines[engine_name].categories and
117                        engines[engine_name].supported_languages and
118                        not engines[engine_name].disabled]
119 
120     # filter list to include only languages supported by most engines or all default general engines
121     filtered_languages = {code: lang for code, lang
122                           in all_languages.items()
123                           if (len(lang['counter']) >= min_engines_per_lang or
124                               all(main_engine in lang['counter']
125                                   for main_engine in main_engines))}
126 
127     def _copy_lang_data(lang, country_name=None):
128         new_dict = dict()
129         new_dict['name'] = all_languages[lang]['name']
130         new_dict['english_name'] = all_languages[lang]['english_name']
131         if country_name:
132             new_dict['country_name'] = country_name
133         return new_dict
134 
135     # for each language get country codes supported by most engines or at least one country code
136     filtered_languages_with_countries = dict()
137     for lang, lang_data in filtered_languages.items():
138         countries = lang_data['countries']
139         filtered_countries = dict()
140 
141         # get language's country codes with enough supported engines
142         for lang_country, country_data in countries.items():
143             if len(country_data['counter']) >= min_engines_per_country:
144                 filtered_countries[lang_country] = _copy_lang_data(lang, country_data['country_name'])
145 
146         # add language without countries too if there's more than one country to choose from
147         if len(filtered_countries) > 1:
148             filtered_countries[lang] = _copy_lang_data(lang)
149         elif len(filtered_countries) == 1:
150             # if there's only one country per language, it's not necessary to show country name
151             lang_country = next(iter(filtered_countries))
152             filtered_countries[lang_country]['country_name'] = None
153 
154         # if no country has enough engines try to get most likely country code from babel
155         if not filtered_countries:
156             lang_country = None
157             subtags = get_global('likely_subtags').get(lang)
158             if subtags:
159                 country_code = subtags.split('_')[-1]
160                 if len(country_code) == 2:
161                     lang_country = "{lang}-{country}".format(lang=lang, country=country_code)
162 
163             if lang_country:
164                 filtered_countries[lang_country] = _copy_lang_data(lang)
165             else:
166                 filtered_countries[lang] = _copy_lang_data(lang)
167 
168         filtered_languages_with_countries.update(filtered_countries)
169 
170     return filtered_languages_with_countries
171 
172 
173 # Write languages.py.
174 def write_languages_file(languages):
175     file_headers = (
176         "# -*- coding: utf-8 -*-",
177         "# list of language codes",
178         "# this file is generated automatically by utils/fetch_languages.py",
179         "language_codes ="
180     )
181 
182     language_codes = tuple([
183         (
184             code,
185             languages[code]['name'].split(' (')[0],
186             languages[code].get('country_name') or '',
187             languages[code].get('english_name') or ''
188         ) for code in sorted(languages)
189     ])
190 
191     with open(languages_file, 'w') as new_file:
192         file_content = "{file_headers} \\\n{language_codes}".format(
193             file_headers='\n'.join(file_headers),
194             language_codes=pformat(language_codes, indent=4)
195         )
196         new_file.write(file_content)
197         new_file.close()
198 
199 
200 if __name__ == "__main__":
201     load_engines(settings['engines'])
202     engines_languages = fetch_supported_languages()
203     all_languages = join_language_lists(engines_languages)
204     filtered_languages = filter_language_list(all_languages)
205     write_languages_file(filtered_languages)
206
```

Path: `searx/languages.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 # list of language codes
3 # this file is generated automatically by utils/fetch_languages.py
4 language_codes = \
5 (   ('af-ZA', 'Afrikaans', '', 'Afrikaans'),
6     ('ar-EG', 'العربية', '', 'Arabic'),
7     ('be-BY', 'Беларуская', '', 'Belarusian'),
8     ('bg-BG', 'Български', '', 'Bulgarian'),
9     ('ca-ES', 'Català', '', 'Catalan'),
10     ('cs-CZ', 'Čeština', '', 'Czech'),
11     ('da-DK', 'Dansk', '', 'Danish'),
12     ('de', 'Deutsch', '', 'German'),
13     ('de-AT', 'Deutsch', 'Österreich', 'German'),
14     ('de-CH', 'Deutsch', 'Schweiz', 'German'),
15     ('de-DE', 'Deutsch', 'Deutschland', 'German'),
16     ('el-GR', 'Ελληνικά', '', 'Greek'),
17     ('en', 'English', '', 'English'),
18     ('en-AU', 'English', 'Australia', 'English'),
19     ('en-CA', 'English', 'Canada', 'English'),
20     ('en-GB', 'English', 'United Kingdom', 'English'),
21     ('en-IE', 'English', 'Ireland', 'English'),
22     ('en-NZ', 'English', 'New Zealand', 'English'),
23     ('en-US', 'English', 'United States', 'English'),
24     ('es', 'Español', '', 'Spanish'),
25     ('es-AR', 'Español', 'Argentina', 'Spanish'),
26     ('es-CL', 'Español', 'Chile', 'Spanish'),
27     ('es-ES', 'Español', 'España', 'Spanish'),
28     ('es-MX', 'Español', 'México', 'Spanish'),
29     ('et-EE', 'Eesti', '', 'Estonian'),
30     ('fa-IR', 'فارسی', '', 'Persian'),
31     ('fi-FI', 'Suomi', '', 'Finnish'),
32     ('fr', 'Français', '', 'French'),
33     ('fr-BE', 'Français', 'Belgique', 'French'),
34     ('fr-CA', 'Français', 'Canada', 'French'),
35     ('fr-CH', 'Français', 'Suisse', 'French'),
36     ('fr-FR', 'Français', 'France', 'French'),
37     ('he-IL', 'עברית', '', 'Hebrew'),
38     ('hr-HR', 'Hrvatski', '', 'Croatian'),
39     ('hu-HU', 'Magyar', '', 'Hungarian'),
40     ('id-ID', 'Indonesia', '', 'Indonesian'),
41     ('is-IS', 'Íslenska', '', 'Icelandic'),
42     ('it-IT', 'Italiano', '', 'Italian'),
43     ('ja-JP', '日本語', '', 'Japanese'),
44     ('ko-KR', '한국어', '', 'Korean'),
45     ('lt-LT', 'Lietuvių', '', 'Lithuanian'),
46     ('lv-LV', 'Latviešu', '', 'Latvian'),
47     ('nb-NO', 'Norsk Bokmål', '', 'Norwegian Bokmål'),
48     ('nl', 'Nederlands', '', 'Dutch'),
49     ('nl-BE', 'Nederlands', 'België', 'Dutch'),
50     ('nl-NL', 'Nederlands', 'Nederland', 'Dutch'),
51     ('pl-PL', 'Polski', '', 'Polish'),
52     ('pt-PT', 'Português', '', 'Portuguese'),
53     ('ro-RO', 'Română', '', 'Romanian'),
54     ('ru-RU', 'Русский', '', 'Russian'),
55     ('sk-SK', 'Slovenčina', '', 'Slovak'),
56     ('sl-SI', 'Slovenščina', '', 'Slovenian'),
57     ('sr-RS', 'Srpski', '', 'Serbian'),
58     ('sv-SE', 'Svenska', '', 'Swedish'),
59     ('sw-TZ', 'Kiswahili', '', 'Swahili'),
60     ('th-TH', 'ไทย', '', 'Thai'),
61     ('tr-TR', 'Türkçe', '', 'Turkish'),
62     ('uk-UA', 'Українська', '', 'Ukrainian'),
63     ('vi-VN', 'Tiếng Việt', '', 'Vietnamese'),
64     ('zh', '中文', '', 'Chinese'),
65     ('zh-CN', '中文', '中国', 'Chinese'),
66     ('zh-HK', '中文', '中國香港特別行政區', 'Chinese'),
67     ('zh-TW', '中文', '台灣', 'Chinese'))
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/searx/languages.py b/searx/languages.py
--- a/searx/languages.py
+++ b/searx/languages.py
@@ -19,13 +19,21 @@
     ('en-CA', 'English', 'Canada', 'English'),
     ('en-GB', 'English', 'United Kingdom', 'English'),
     ('en-IE', 'English', 'Ireland', 'English'),
+    ('en-IN', 'English', 'India', 'English'),
+    ('en-MY', 'English', 'Malaysia', 'English'),
     ('en-NZ', 'English', 'New Zealand', 'English'),
+    ('en-PH', 'English', 'Philippines', 'English'),
+    ('en-SG', 'English', 'Singapore', 'English'),
     ('en-US', 'English', 'United States', 'English'),
+    ('en-ZA', 'English', 'South Africa', 'English'),
     ('es', 'Español', '', 'Spanish'),
     ('es-AR', 'Español', 'Argentina', 'Spanish'),
     ('es-CL', 'Español', 'Chile', 'Spanish'),
+    ('es-CO', 'Español', 'Colombia', 'Spanish'),
     ('es-ES', 'Español', 'España', 'Spanish'),
     ('es-MX', 'Español', 'México', 'Spanish'),
+    ('es-PE', 'Español', 'Perú', 'Spanish'),
+    ('es-US', 'Español', 'Estados Unidos', 'Spanish'),
     ('et-EE', 'Eesti', '', 'Estonian'),
     ('fa-IR', 'فارسی', '', 'Persian'),
     ('fi-FI', 'Suomi', '', 'Finnish'),
@@ -49,12 +57,14 @@
     ('nl-BE', 'Nederlands', 'België', 'Dutch'),
     ('nl-NL', 'Nederlands', 'Nederland', 'Dutch'),
     ('pl-PL', 'Polski', '', 'Polish'),
-    ('pt-PT', 'Português', '', 'Portuguese'),
+    ('pt', 'Português', '', 'Portuguese'),
+    ('pt-BR', 'Português', 'Brasil', 'Portuguese'),
+    ('pt-PT', 'Português', 'Portugal', 'Portuguese'),
     ('ro-RO', 'Română', '', 'Romanian'),
     ('ru-RU', 'Русский', '', 'Russian'),
     ('sk-SK', 'Slovenčina', '', 'Slovak'),
     ('sl-SI', 'Slovenščina', '', 'Slovenian'),
-    ('sr-RS', 'Srpski', '', 'Serbian'),
+    ('sr-RS', 'Српски', '', 'Serbian'),
     ('sv-SE', 'Svenska', '', 'Swedish'),
     ('sw-TZ', 'Kiswahili', '', 'Swahili'),
     ('th-TH', 'ไทย', '', 'Thai'),
diff --git a/searxng_extra/update/update_languages.py b/searxng_extra/update/update_languages.py
--- a/searxng_extra/update/update_languages.py
+++ b/searxng_extra/update/update_languages.py
@@ -111,7 +111,7 @@
 # Filter language list so it only includes the most supported languages and countries
 def filter_language_list(all_languages):
     min_engines_per_lang = 13
-    min_engines_per_country = 10
+    min_engines_per_country = 7
     main_engines = [engine_name for engine_name in engines.keys()
                     if 'general' in engines[engine_name].categories and
                        engines[engine_name].supported_languages and

{"golden_diff": "diff --git a/searx/languages.py b/searx/languages.py\n--- a/searx/languages.py\n+++ b/searx/languages.py\n@@ -19,13 +19,21 @@\n     ('en-CA', 'English', 'Canada', 'English'),\n     ('en-GB', 'English', 'United Kingdom', 'English'),\n     ('en-IE', 'English', 'Ireland', 'English'),\n+    ('en-IN', 'English', 'India', 'English'),\n+    ('en-MY', 'English', 'Malaysia', 'English'),\n     ('en-NZ', 'English', 'New Zealand', 'English'),\n+    ('en-PH', 'English', 'Philippines', 'English'),\n+    ('en-SG', 'English', 'Singapore', 'English'),\n     ('en-US', 'English', 'United States', 'English'),\n+    ('en-ZA', 'English', 'South Africa', 'English'),\n     ('es', 'Espa\u00f1ol', '', 'Spanish'),\n     ('es-AR', 'Espa\u00f1ol', 'Argentina', 'Spanish'),\n     ('es-CL', 'Espa\u00f1ol', 'Chile', 'Spanish'),\n+    ('es-CO', 'Espa\u00f1ol', 'Colombia', 'Spanish'),\n     ('es-ES', 'Espa\u00f1ol', 'Espa\u00f1a', 'Spanish'),\n     ('es-MX', 'Espa\u00f1ol', 'M\u00e9xico', 'Spanish'),\n+    ('es-PE', 'Espa\u00f1ol', 'Per\u00fa', 'Spanish'),\n+    ('es-US', 'Espa\u00f1ol', 'Estados Unidos', 'Spanish'),\n     ('et-EE', 'Eesti', '', 'Estonian'),\n     ('fa-IR', '\u0641\u0627\u0631\u0633\u06cc', '', 'Persian'),\n     ('fi-FI', 'Suomi', '', 'Finnish'),\n@@ -49,12 +57,14 @@\n     ('nl-BE', 'Nederlands', 'Belgi\u00eb', 'Dutch'),\n     ('nl-NL', 'Nederlands', 'Nederland', 'Dutch'),\n     ('pl-PL', 'Polski', '', 'Polish'),\n-    ('pt-PT', 'Portugu\u00eas', '', 'Portuguese'),\n+    ('pt', 'Portugu\u00eas', '', 'Portuguese'),\n+    ('pt-BR', 'Portugu\u00eas', 'Brasil', 'Portuguese'),\n+    ('pt-PT', 'Portugu\u00eas', 'Portugal', 'Portuguese'),\n     ('ro-RO', 'Rom\u00e2n\u0103', '', 'Romanian'),\n     ('ru-RU', '\u0420\u0443\u0441\u0441\u043a\u0438\u0439', '', 'Russian'),\n     ('sk-SK', 'Sloven\u010dina', '', 'Slovak'),\n     ('sl-SI', 'Sloven\u0161\u010dina', '', 'Slovenian'),\n-    ('sr-RS', 'Srpski', '', 'Serbian'),\n+    ('sr-RS', '\u0421\u0440\u043f\u0441\u043a\u0438', '', 'Serbian'),\n     ('sv-SE', 'Svenska', '', 'Swedish'),\n     ('sw-TZ', 'Kiswahili', '', 'Swahili'),\n     ('th-TH', '\u0e44\u0e17\u0e22', '', 'Thai'),\ndiff --git a/searxng_extra/update/update_languages.py b/searxng_extra/update/update_languages.py\n--- a/searxng_extra/update/update_languages.py\n+++ b/searxng_extra/update/update_languages.py\n@@ -111,7 +111,7 @@\n # Filter language list so it only includes the most supported languages and countries\n def filter_language_list(all_languages):\n     min_engines_per_lang = 13\n-    min_engines_per_country = 10\n+    min_engines_per_country = 7\n     main_engines = [engine_name for engine_name in engines.keys()\n                     if 'general' in engines[engine_name].categories and\n                        engines[engine_name].supported_languages and\n", "issue": "Missing language: pt-BR\n<!-- PLEASE FILL THESE FIELDS, IT REALLY HELPS THE MAINTAINERS OF SearXNG -->\r\n\r\n**Version of SearXNG, commit number if you are using on master branch and stipulate if you forked SearXNG**\r\n\r\n1.0.0-950-dfdf71bf\r\n(docker)\r\n**How did you install SearXNG?**\r\n\r\nDocker\r\n\r\n**What happened?**\r\nLanguage `pt-BR` is missing.\r\nhttps://github.com/searxng/searxng/blob/master/searx/languages.py\r\n\r\nIt was present in a previous version.\r\n\r\nGoogle supports this language (and some other engines) but it is not showing up on the list.\r\n\n", "before_files": [{"content": "#!/usr/bin/env python\n# SPDX-License-Identifier: AGPL-3.0-or-later\n\n# This script generates languages.py from intersecting each engine's supported languages.\n#\n# Output files: searx/data/engines_languages.json and searx/languages.py\n\nimport json\nfrom pathlib import Path\nfrom pprint import pformat\nfrom babel import Locale, UnknownLocaleError\nfrom babel.languages import get_global\n\nfrom searx import settings, searx_dir\nfrom searx.engines import load_engines, engines\nfrom searx.network import set_timeout_for_thread\n\n# Output files.\nengines_languages_file = Path(searx_dir) / 'data' / 'engines_languages.json'\nlanguages_file = Path(searx_dir) / 'languages.py'\n\n\n# Fetchs supported languages for each engine and writes json file with those.\ndef fetch_supported_languages():\n    set_timeout_for_thread(10.0)\n\n    engines_languages = dict()\n    names = list(engines)\n    names.sort()\n\n    for engine_name in names:\n        if hasattr(engines[engine_name], 'fetch_supported_languages'):\n            engines_languages[engine_name] = engines[engine_name].fetch_supported_languages()\n            print(\"fetched %s languages from engine %s\" % (\n                len(engines_languages[engine_name]), engine_name))\n            if type(engines_languages[engine_name]) == list:\n                engines_languages[engine_name] = sorted(engines_languages[engine_name])\n\n    # write json file\n    with open(engines_languages_file, 'w', encoding='utf-8') as f:\n        json.dump(engines_languages, f, indent=2, sort_keys=True)\n\n    return engines_languages\n\n\n# Get babel Locale object from lang_code if possible.\ndef get_locale(lang_code):\n    try:\n        locale = Locale.parse(lang_code, sep='-')\n        return locale\n    except (UnknownLocaleError, ValueError):\n        return None\n\n\n# Join all language lists.\ndef join_language_lists(engines_languages):\n    language_list = dict()\n    for engine_name in engines_languages:\n        for lang_code in engines_languages[engine_name]:\n\n            # apply custom fixes if necessary\n            if lang_code in getattr(engines[engine_name], 'language_aliases', {}).values():\n                lang_code = next(lc for lc, alias in engines[engine_name].language_aliases.items()\n                                 if lang_code == alias)\n\n            locale = get_locale(lang_code)\n\n            # ensure that lang_code uses standard language and country codes\n            if locale and locale.territory:\n                lang_code = \"{lang}-{country}\".format(lang=locale.language, country=locale.territory)\n            short_code = lang_code.split('-')[0]\n\n            # add language without country if not in list\n            if short_code not in language_list:\n                if locale:\n                    # get language's data from babel's Locale object\n                    language_name = locale.get_language_name().title()\n                    english_name = locale.english_name.split(' (')[0]\n                elif short_code in engines_languages['wikipedia']:\n                    # get language's data from wikipedia if not known by babel\n                    language_name = engines_languages['wikipedia'][short_code]['name']\n                    english_name = engines_languages['wikipedia'][short_code]['english_name']\n                else:\n                    language_name = None\n                    english_name = None\n\n                # add language to list\n                language_list[short_code] = {'name': language_name,\n                                             'english_name': english_name,\n                                             'counter': set(),\n                                             'countries': dict()}\n\n            # add language with country if not in list\n            if lang_code != short_code and lang_code not in language_list[short_code]['countries']:\n                country_name = ''\n                if locale:\n                    # get country name from babel's Locale object\n                    country_name = locale.get_territory_name()\n\n                language_list[short_code]['countries'][lang_code] = {'country_name': country_name,\n                                                                     'counter': set()}\n\n            # count engine for both language_country combination and language alone\n            language_list[short_code]['counter'].add(engine_name)\n            if lang_code != short_code:\n                language_list[short_code]['countries'][lang_code]['counter'].add(engine_name)\n\n    return language_list\n\n\n# Filter language list so it only includes the most supported languages and countries\ndef filter_language_list(all_languages):\n    min_engines_per_lang = 13\n    min_engines_per_country = 10\n    main_engines = [engine_name for engine_name in engines.keys()\n                    if 'general' in engines[engine_name].categories and\n                       engines[engine_name].supported_languages and\n                       not engines[engine_name].disabled]\n\n    # filter list to include only languages supported by most engines or all default general engines\n    filtered_languages = {code: lang for code, lang\n                          in all_languages.items()\n                          if (len(lang['counter']) >= min_engines_per_lang or\n                              all(main_engine in lang['counter']\n                                  for main_engine in main_engines))}\n\n    def _copy_lang_data(lang, country_name=None):\n        new_dict = dict()\n        new_dict['name'] = all_languages[lang]['name']\n        new_dict['english_name'] = all_languages[lang]['english_name']\n        if country_name:\n            new_dict['country_name'] = country_name\n        return new_dict\n\n    # for each language get country codes supported by most engines or at least one country code\n    filtered_languages_with_countries = dict()\n    for lang, lang_data in filtered_languages.items():\n        countries = lang_data['countries']\n        filtered_countries = dict()\n\n        # get language's country codes with enough supported engines\n        for lang_country, country_data in countries.items():\n            if len(country_data['counter']) >= min_engines_per_country:\n                filtered_countries[lang_country] = _copy_lang_data(lang, country_data['country_name'])\n\n        # add language without countries too if there's more than one country to choose from\n        if len(filtered_countries) > 1:\n            filtered_countries[lang] = _copy_lang_data(lang)\n        elif len(filtered_countries) == 1:\n            # if there's only one country per language, it's not necessary to show country name\n            lang_country = next(iter(filtered_countries))\n            filtered_countries[lang_country]['country_name'] = None\n\n        # if no country has enough engines try to get most likely country code from babel\n        if not filtered_countries:\n            lang_country = None\n            subtags = get_global('likely_subtags').get(lang)\n            if subtags:\n                country_code = subtags.split('_')[-1]\n                if len(country_code) == 2:\n                    lang_country = \"{lang}-{country}\".format(lang=lang, country=country_code)\n\n            if lang_country:\n                filtered_countries[lang_country] = _copy_lang_data(lang)\n            else:\n                filtered_countries[lang] = _copy_lang_data(lang)\n\n        filtered_languages_with_countries.update(filtered_countries)\n\n    return filtered_languages_with_countries\n\n\n# Write languages.py.\ndef write_languages_file(languages):\n    file_headers = (\n        \"# -*- coding: utf-8 -*-\",\n        \"# list of language codes\",\n        \"# this file is generated automatically by utils/fetch_languages.py\",\n        \"language_codes =\"\n    )\n\n    language_codes = tuple([\n        (\n            code,\n            languages[code]['name'].split(' (')[0],\n            languages[code].get('country_name') or '',\n            languages[code].get('english_name') or ''\n        ) for code in sorted(languages)\n    ])\n\n    with open(languages_file, 'w') as new_file:\n        file_content = \"{file_headers} \\\\\\n{language_codes}\".format(\n            file_headers='\\n'.join(file_headers),\n            language_codes=pformat(language_codes, indent=4)\n        )\n        new_file.write(file_content)\n        new_file.close()\n\n\nif __name__ == \"__main__\":\n    load_engines(settings['engines'])\n    engines_languages = fetch_supported_languages()\n    all_languages = join_language_lists(engines_languages)\n    filtered_languages = filter_language_list(all_languages)\n    write_languages_file(filtered_languages)\n", "path": "searxng_extra/update/update_languages.py"}, {"content": "# -*- coding: utf-8 -*-\n# list of language codes\n# this file is generated automatically by utils/fetch_languages.py\nlanguage_codes = \\\n(   ('af-ZA', 'Afrikaans', '', 'Afrikaans'),\n    ('ar-EG', '\u0627\u0644\u0639\u0631\u0628\u064a\u0629', '', 'Arabic'),\n    ('be-BY', '\u0411\u0435\u043b\u0430\u0440\u0443\u0441\u043a\u0430\u044f', '', 'Belarusian'),\n    ('bg-BG', '\u0411\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438', '', 'Bulgarian'),\n    ('ca-ES', 'Catal\u00e0', '', 'Catalan'),\n    ('cs-CZ', '\u010ce\u0161tina', '', 'Czech'),\n    ('da-DK', 'Dansk', '', 'Danish'),\n    ('de', 'Deutsch', '', 'German'),\n    ('de-AT', 'Deutsch', '\u00d6sterreich', 'German'),\n    ('de-CH', 'Deutsch', 'Schweiz', 'German'),\n    ('de-DE', 'Deutsch', 'Deutschland', 'German'),\n    ('el-GR', '\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac', '', 'Greek'),\n    ('en', 'English', '', 'English'),\n    ('en-AU', 'English', 'Australia', 'English'),\n    ('en-CA', 'English', 'Canada', 'English'),\n    ('en-GB', 'English', 'United Kingdom', 'English'),\n    ('en-IE', 'English', 'Ireland', 'English'),\n    ('en-NZ', 'English', 'New Zealand', 'English'),\n    ('en-US', 'English', 'United States', 'English'),\n    ('es', 'Espa\u00f1ol', '', 'Spanish'),\n    ('es-AR', 'Espa\u00f1ol', 'Argentina', 'Spanish'),\n    ('es-CL', 'Espa\u00f1ol', 'Chile', 'Spanish'),\n    ('es-ES', 'Espa\u00f1ol', 'Espa\u00f1a', 'Spanish'),\n    ('es-MX', 'Espa\u00f1ol', 'M\u00e9xico', 'Spanish'),\n    ('et-EE', 'Eesti', '', 'Estonian'),\n    ('fa-IR', '\u0641\u0627\u0631\u0633\u06cc', '', 'Persian'),\n    ('fi-FI', 'Suomi', '', 'Finnish'),\n    ('fr', 'Fran\u00e7ais', '', 'French'),\n    ('fr-BE', 'Fran\u00e7ais', 'Belgique', 'French'),\n    ('fr-CA', 'Fran\u00e7ais', 'Canada', 'French'),\n    ('fr-CH', 'Fran\u00e7ais', 'Suisse', 'French'),\n    ('fr-FR', 'Fran\u00e7ais', 'France', 'French'),\n    ('he-IL', '\u05e2\u05d1\u05e8\u05d9\u05ea', '', 'Hebrew'),\n    ('hr-HR', 'Hrvatski', '', 'Croatian'),\n    ('hu-HU', 'Magyar', '', 'Hungarian'),\n    ('id-ID', 'Indonesia', '', 'Indonesian'),\n    ('is-IS', '\u00cdslenska', '', 'Icelandic'),\n    ('it-IT', 'Italiano', '', 'Italian'),\n    ('ja-JP', '\u65e5\u672c\u8a9e', '', 'Japanese'),\n    ('ko-KR', '\ud55c\uad6d\uc5b4', '', 'Korean'),\n    ('lt-LT', 'Lietuvi\u0173', '', 'Lithuanian'),\n    ('lv-LV', 'Latvie\u0161u', '', 'Latvian'),\n    ('nb-NO', 'Norsk Bokm\u00e5l', '', 'Norwegian Bokm\u00e5l'),\n    ('nl', 'Nederlands', '', 'Dutch'),\n    ('nl-BE', 'Nederlands', 'Belgi\u00eb', 'Dutch'),\n    ('nl-NL', 'Nederlands', 'Nederland', 'Dutch'),\n    ('pl-PL', 'Polski', '', 'Polish'),\n    ('pt-PT', 'Portugu\u00eas', '', 'Portuguese'),\n    ('ro-RO', 'Rom\u00e2n\u0103', '', 'Romanian'),\n    ('ru-RU', '\u0420\u0443\u0441\u0441\u043a\u0438\u0439', '', 'Russian'),\n    ('sk-SK', 'Sloven\u010dina', '', 'Slovak'),\n    ('sl-SI', 'Sloven\u0161\u010dina', '', 'Slovenian'),\n    ('sr-RS', 'Srpski', '', 'Serbian'),\n    ('sv-SE', 'Svenska', '', 'Swedish'),\n    ('sw-TZ', 'Kiswahili', '', 'Swahili'),\n    ('th-TH', '\u0e44\u0e17\u0e22', '', 'Thai'),\n    ('tr-TR', 'T\u00fcrk\u00e7e', '', 'Turkish'),\n    ('uk-UA', '\u0423\u043a\u0440\u0430\u0457\u043d\u0441\u044c\u043a\u0430', '', 'Ukrainian'),\n    ('vi-VN', 'Ti\u1ebfng Vi\u1ec7t', '', 'Vietnamese'),\n    ('zh', '\u4e2d\u6587', '', 'Chinese'),\n    ('zh-CN', '\u4e2d\u6587', '\u4e2d\u56fd', 'Chinese'),\n    ('zh-HK', '\u4e2d\u6587', '\u4e2d\u570b\u9999\u6e2f\u7279\u5225\u884c\u653f\u5340', 'Chinese'),\n    ('zh-TW', '\u4e2d\u6587', '\u53f0\u7063', 'Chinese'))", "path": "searx/languages.py"}], "after_files": [{"content": "#!/usr/bin/env python\n# SPDX-License-Identifier: AGPL-3.0-or-later\n\n# This script generates languages.py from intersecting each engine's supported languages.\n#\n# Output files: searx/data/engines_languages.json and searx/languages.py\n\nimport json\nfrom pathlib import Path\nfrom pprint import pformat\nfrom babel import Locale, UnknownLocaleError\nfrom babel.languages import get_global\n\nfrom searx import settings, searx_dir\nfrom searx.engines import load_engines, engines\nfrom searx.network import set_timeout_for_thread\n\n# Output files.\nengines_languages_file = Path(searx_dir) / 'data' / 'engines_languages.json'\nlanguages_file = Path(searx_dir) / 'languages.py'\n\n\n# Fetchs supported languages for each engine and writes json file with those.\ndef fetch_supported_languages():\n    set_timeout_for_thread(10.0)\n\n    engines_languages = dict()\n    names = list(engines)\n    names.sort()\n\n    for engine_name in names:\n        if hasattr(engines[engine_name], 'fetch_supported_languages'):\n            engines_languages[engine_name] = engines[engine_name].fetch_supported_languages()\n            print(\"fetched %s languages from engine %s\" % (\n                len(engines_languages[engine_name]), engine_name))\n            if type(engines_languages[engine_name]) == list:\n                engines_languages[engine_name] = sorted(engines_languages[engine_name])\n\n    # write json file\n    with open(engines_languages_file, 'w', encoding='utf-8') as f:\n        json.dump(engines_languages, f, indent=2, sort_keys=True)\n\n    return engines_languages\n\n\n# Get babel Locale object from lang_code if possible.\ndef get_locale(lang_code):\n    try:\n        locale = Locale.parse(lang_code, sep='-')\n        return locale\n    except (UnknownLocaleError, ValueError):\n        return None\n\n\n# Join all language lists.\ndef join_language_lists(engines_languages):\n    language_list = dict()\n    for engine_name in engines_languages:\n        for lang_code in engines_languages[engine_name]:\n\n            # apply custom fixes if necessary\n            if lang_code in getattr(engines[engine_name], 'language_aliases', {}).values():\n                lang_code = next(lc for lc, alias in engines[engine_name].language_aliases.items()\n                                 if lang_code == alias)\n\n            locale = get_locale(lang_code)\n\n            # ensure that lang_code uses standard language and country codes\n            if locale and locale.territory:\n                lang_code = \"{lang}-{country}\".format(lang=locale.language, country=locale.territory)\n            short_code = lang_code.split('-')[0]\n\n            # add language without country if not in list\n            if short_code not in language_list:\n                if locale:\n                    # get language's data from babel's Locale object\n                    language_name = locale.get_language_name().title()\n                    english_name = locale.english_name.split(' (')[0]\n                elif short_code in engines_languages['wikipedia']:\n                    # get language's data from wikipedia if not known by babel\n                    language_name = engines_languages['wikipedia'][short_code]['name']\n                    english_name = engines_languages['wikipedia'][short_code]['english_name']\n                else:\n                    language_name = None\n                    english_name = None\n\n                # add language to list\n                language_list[short_code] = {'name': language_name,\n                                             'english_name': english_name,\n                                             'counter': set(),\n                                             'countries': dict()}\n\n            # add language with country if not in list\n            if lang_code != short_code and lang_code not in language_list[short_code]['countries']:\n                country_name = ''\n                if locale:\n                    # get country name from babel's Locale object\n                    country_name = locale.get_territory_name()\n\n                language_list[short_code]['countries'][lang_code] = {'country_name': country_name,\n                                                                     'counter': set()}\n\n            # count engine for both language_country combination and language alone\n            language_list[short_code]['counter'].add(engine_name)\n            if lang_code != short_code:\n                language_list[short_code]['countries'][lang_code]['counter'].add(engine_name)\n\n    return language_list\n\n\n# Filter language list so it only includes the most supported languages and countries\ndef filter_language_list(all_languages):\n    min_engines_per_lang = 13\n    min_engines_per_country = 7\n    main_engines = [engine_name for engine_name in engines.keys()\n                    if 'general' in engines[engine_name].categories and\n                       engines[engine_name].supported_languages and\n                       not engines[engine_name].disabled]\n\n    # filter list to include only languages supported by most engines or all default general engines\n    filtered_languages = {code: lang for code, lang\n                          in all_languages.items()\n                          if (len(lang['counter']) >= min_engines_per_lang or\n                              all(main_engine in lang['counter']\n                                  for main_engine in main_engines))}\n\n    def _copy_lang_data(lang, country_name=None):\n        new_dict = dict()\n        new_dict['name'] = all_languages[lang]['name']\n        new_dict['english_name'] = all_languages[lang]['english_name']\n        if country_name:\n            new_dict['country_name'] = country_name\n        return new_dict\n\n    # for each language get country codes supported by most engines or at least one country code\n    filtered_languages_with_countries = dict()\n    for lang, lang_data in filtered_languages.items():\n        countries = lang_data['countries']\n        filtered_countries = dict()\n\n        # get language's country codes with enough supported engines\n        for lang_country, country_data in countries.items():\n            if len(country_data['counter']) >= min_engines_per_country:\n                filtered_countries[lang_country] = _copy_lang_data(lang, country_data['country_name'])\n\n        # add language without countries too if there's more than one country to choose from\n        if len(filtered_countries) > 1:\n            filtered_countries[lang] = _copy_lang_data(lang)\n        elif len(filtered_countries) == 1:\n            # if there's only one country per language, it's not necessary to show country name\n            lang_country = next(iter(filtered_countries))\n            filtered_countries[lang_country]['country_name'] = None\n\n        # if no country has enough engines try to get most likely country code from babel\n        if not filtered_countries:\n            lang_country = None\n            subtags = get_global('likely_subtags').get(lang)\n            if subtags:\n                country_code = subtags.split('_')[-1]\n                if len(country_code) == 2:\n                    lang_country = \"{lang}-{country}\".format(lang=lang, country=country_code)\n\n            if lang_country:\n                filtered_countries[lang_country] = _copy_lang_data(lang)\n            else:\n                filtered_countries[lang] = _copy_lang_data(lang)\n\n        filtered_languages_with_countries.update(filtered_countries)\n\n    return filtered_languages_with_countries\n\n\n# Write languages.py.\ndef write_languages_file(languages):\n    file_headers = (\n        \"# -*- coding: utf-8 -*-\",\n        \"# list of language codes\",\n        \"# this file is generated automatically by utils/fetch_languages.py\",\n        \"language_codes =\"\n    )\n\n    language_codes = tuple([\n        (\n            code,\n            languages[code]['name'].split(' (')[0],\n            languages[code].get('country_name') or '',\n            languages[code].get('english_name') or ''\n        ) for code in sorted(languages)\n    ])\n\n    with open(languages_file, 'w') as new_file:\n        file_content = \"{file_headers} \\\\\\n{language_codes}\".format(\n            file_headers='\\n'.join(file_headers),\n            language_codes=pformat(language_codes, indent=4)\n        )\n        new_file.write(file_content)\n        new_file.close()\n\n\nif __name__ == \"__main__\":\n    load_engines(settings['engines'])\n    engines_languages = fetch_supported_languages()\n    all_languages = join_language_lists(engines_languages)\n    filtered_languages = filter_language_list(all_languages)\n    write_languages_file(filtered_languages)\n", "path": "searxng_extra/update/update_languages.py"}, {"content": "# -*- coding: utf-8 -*-\n# list of language codes\n# this file is generated automatically by utils/fetch_languages.py\nlanguage_codes = \\\n(   ('af-ZA', 'Afrikaans', '', 'Afrikaans'),\n    ('ar-EG', '\u0627\u0644\u0639\u0631\u0628\u064a\u0629', '', 'Arabic'),\n    ('be-BY', '\u0411\u0435\u043b\u0430\u0440\u0443\u0441\u043a\u0430\u044f', '', 'Belarusian'),\n    ('bg-BG', '\u0411\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438', '', 'Bulgarian'),\n    ('ca-ES', 'Catal\u00e0', '', 'Catalan'),\n    ('cs-CZ', '\u010ce\u0161tina', '', 'Czech'),\n    ('da-DK', 'Dansk', '', 'Danish'),\n    ('de', 'Deutsch', '', 'German'),\n    ('de-AT', 'Deutsch', '\u00d6sterreich', 'German'),\n    ('de-CH', 'Deutsch', 'Schweiz', 'German'),\n    ('de-DE', 'Deutsch', 'Deutschland', 'German'),\n    ('el-GR', '\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac', '', 'Greek'),\n    ('en', 'English', '', 'English'),\n    ('en-AU', 'English', 'Australia', 'English'),\n    ('en-CA', 'English', 'Canada', 'English'),\n    ('en-GB', 'English', 'United Kingdom', 'English'),\n    ('en-IE', 'English', 'Ireland', 'English'),\n    ('en-IN', 'English', 'India', 'English'),\n    ('en-MY', 'English', 'Malaysia', 'English'),\n    ('en-NZ', 'English', 'New Zealand', 'English'),\n    ('en-PH', 'English', 'Philippines', 'English'),\n    ('en-SG', 'English', 'Singapore', 'English'),\n    ('en-US', 'English', 'United States', 'English'),\n    ('en-ZA', 'English', 'South Africa', 'English'),\n    ('es', 'Espa\u00f1ol', '', 'Spanish'),\n    ('es-AR', 'Espa\u00f1ol', 'Argentina', 'Spanish'),\n    ('es-CL', 'Espa\u00f1ol', 'Chile', 'Spanish'),\n    ('es-CO', 'Espa\u00f1ol', 'Colombia', 'Spanish'),\n    ('es-ES', 'Espa\u00f1ol', 'Espa\u00f1a', 'Spanish'),\n    ('es-MX', 'Espa\u00f1ol', 'M\u00e9xico', 'Spanish'),\n    ('es-PE', 'Espa\u00f1ol', 'Per\u00fa', 'Spanish'),\n    ('es-US', 'Espa\u00f1ol', 'Estados Unidos', 'Spanish'),\n    ('et-EE', 'Eesti', '', 'Estonian'),\n    ('fa-IR', '\u0641\u0627\u0631\u0633\u06cc', '', 'Persian'),\n    ('fi-FI', 'Suomi', '', 'Finnish'),\n    ('fr', 'Fran\u00e7ais', '', 'French'),\n    ('fr-BE', 'Fran\u00e7ais', 'Belgique', 'French'),\n    ('fr-CA', 'Fran\u00e7ais', 'Canada', 'French'),\n    ('fr-CH', 'Fran\u00e7ais', 'Suisse', 'French'),\n    ('fr-FR', 'Fran\u00e7ais', 'France', 'French'),\n    ('he-IL', '\u05e2\u05d1\u05e8\u05d9\u05ea', '', 'Hebrew'),\n    ('hr-HR', 'Hrvatski', '', 'Croatian'),\n    ('hu-HU', 'Magyar', '', 'Hungarian'),\n    ('id-ID', 'Indonesia', '', 'Indonesian'),\n    ('is-IS', '\u00cdslenska', '', 'Icelandic'),\n    ('it-IT', 'Italiano', '', 'Italian'),\n    ('ja-JP', '\u65e5\u672c\u8a9e', '', 'Japanese'),\n    ('ko-KR', '\ud55c\uad6d\uc5b4', '', 'Korean'),\n    ('lt-LT', 'Lietuvi\u0173', '', 'Lithuanian'),\n    ('lv-LV', 'Latvie\u0161u', '', 'Latvian'),\n    ('nb-NO', 'Norsk Bokm\u00e5l', '', 'Norwegian Bokm\u00e5l'),\n    ('nl', 'Nederlands', '', 'Dutch'),\n    ('nl-BE', 'Nederlands', 'Belgi\u00eb', 'Dutch'),\n    ('nl-NL', 'Nederlands', 'Nederland', 'Dutch'),\n    ('pl-PL', 'Polski', '', 'Polish'),\n    ('pt', 'Portugu\u00eas', '', 'Portuguese'),\n    ('pt-BR', 'Portugu\u00eas', 'Brasil', 'Portuguese'),\n    ('pt-PT', 'Portugu\u00eas', 'Portugal', 'Portuguese'),\n    ('ro-RO', 'Rom\u00e2n\u0103', '', 'Romanian'),\n    ('ru-RU', '\u0420\u0443\u0441\u0441\u043a\u0438\u0439', '', 'Russian'),\n    ('sk-SK', 'Sloven\u010dina', '', 'Slovak'),\n    ('sl-SI', 'Sloven\u0161\u010dina', '', 'Slovenian'),\n    ('sr-RS', '\u0421\u0440\u043f\u0441\u043a\u0438', '', 'Serbian'),\n    ('sv-SE', 'Svenska', '', 'Swedish'),\n    ('sw-TZ', 'Kiswahili', '', 'Swahili'),\n    ('th-TH', '\u0e44\u0e17\u0e22', '', 'Thai'),\n    ('tr-TR', 'T\u00fcrk\u00e7e', '', 'Turkish'),\n    ('uk-UA', '\u0423\u043a\u0440\u0430\u0457\u043d\u0441\u044c\u043a\u0430', '', 'Ukrainian'),\n    ('vi-VN', 'Ti\u1ebfng Vi\u1ec7t', '', 'Vietnamese'),\n    ('zh', '\u4e2d\u6587', '', 'Chinese'),\n    ('zh-CN', '\u4e2d\u6587', '\u4e2d\u56fd', 'Chinese'),\n    ('zh-HK', '\u4e2d\u6587', '\u4e2d\u570b\u9999\u6e2f\u7279\u5225\u884c\u653f\u5340', 'Chinese'),\n    ('zh-TW', '\u4e2d\u6587', '\u53f0\u7063', 'Chinese'))", "path": "searx/languages.py"}]}