We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Chardet fails on Slovenian characters
Preprocess Text fails with Slovenian stopword list. Seems like a chardet issue.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `orangecontrib/text/preprocess/filter.py`
Content:
```
1 import os
2 
3 import re
4 from gensim import corpora
5 from nltk.corpus import stopwords
6 
7 __all__ = ['BaseTokenFilter', 'StopwordsFilter', 'LexiconFilter', 'RegexpFilter', 'FrequencyFilter']
8 
9 
10 class BaseTokenFilter:
11     name = NotImplemented
12 
13     def __call__(self, corpus):
14         if len(corpus) == 0:
15             return corpus
16         if isinstance(corpus[0], str):
17             return self.filter(corpus)
18         return [self.filter(tokens) for tokens in corpus]
19 
20     def filter(self, tokens):
21         return list(filter(self.check, tokens))
22 
23     def check(self, token):
24         raise NotImplementedError
25 
26     def __str__(self):
27         return self.name
28 
29     def set_up(self):
30         """ A method for setting filters up before every __call__. """
31         pass
32 
33     def tear_down(self):
34         """ A method for cleaning up after every __call__. """
35         pass
36 
37 
38 class WordListMixin:
39     def __init__(self, word_list=None):
40         self.file_path = None
41         self.word_list = word_list or []
42 
43     def from_file(self, path):
44         self.file_path = path
45         if not path:
46             self.word_list = []
47         else:
48             with open(path) as f:
49                 self.word_list = set([line.strip() for line in f])
50 
51 
52 class StopwordsFilter(BaseTokenFilter, WordListMixin):
53     """ Remove tokens present in NLTK's language specific lists or a file. """
54     name = 'Stopwords'
55 
56     supported_languages = [file.capitalize() for file in os.listdir(stopwords._get_root())
57                            if file.islower()]
58 
59     def __init__(self, language='English', word_list=None):
60         WordListMixin.__init__(self, word_list)
61         super().__init__()
62         self.language = language
63 
64     @property
65     def language(self):
66         return self._language
67 
68     @language.setter
69     def language(self, value):
70         self._language = value
71         if not self._language:
72             self.stopwords = []
73         else:
74             self.stopwords = set(stopwords.words(self.language.lower()))
75 
76     def __str__(self):
77         config = ''
78         config += 'Language: {}, '.format(self.language.capitalize()) if self.language else ''
79         config += 'File: {}, '.format(self.file_path) if self.file_path else ''
80         return '{} ({})'.format(self.name, config.strip(', '))
81 
82     def check(self, token):
83         return token not in self.stopwords and token not in self.word_list
84 
85 
86 class LexiconFilter(BaseTokenFilter, WordListMixin):
87     """ Keep only tokens present in a file. """
88     name = 'Lexicon'
89 
90     def __init__(self, lexicon=None):
91         WordListMixin.__init__(self, word_list=lexicon)
92 
93     @property
94     def lexicon(self):
95         return self.word_list
96 
97     @lexicon.setter
98     def lexicon(self, value):
99         self.word_list = set(value)
100 
101     def check(self, token):
102         return not self.lexicon or token in self.lexicon
103 
104     def __str__(self):
105         return '{} ({})'.format(self.name, 'File: {}'.format(self.file_path))
106 
107 
108 class RegexpFilter(BaseTokenFilter):
109     """ Remove tokens matching this regular expressions. """
110     name = 'Regexp'
111 
112     def __init__(self, pattern=r'\.|,|:|!|\?'):
113         self._pattern = pattern
114         # Compiled Regexes are NOT deepcopy-able and hence to make Corpus deepcopy-able
115         # we cannot store then (due to Corpus also storing used_preprocessor for BoW compute values).
116         # To bypass the problem regex is compiled before every __call__ and discarded right after.
117         self.regex = None
118         self.set_up()
119 
120     @property
121     def pattern(self):
122         return self._pattern
123 
124     @pattern.setter
125     def pattern(self, value):
126         self._pattern = value
127         self.set_up()
128 
129     @staticmethod
130     def validate_regexp(regexp):
131         try:
132             re.compile(regexp)
133             return True
134         except re.error:
135             return False
136 
137     def check(self, token):
138         return not self.regex.match(token)
139 
140     def __str__(self):
141         return '{} ({})'.format(self.name, self.pattern)
142 
143     def set_up(self):
144         """ Compile Regex before the __call__. """
145         self.regex = re.compile(self.pattern)
146 
147     def tear_down(self):
148         """ Delete Regex after every __call__. """
149         self.regex = None
150 
151 
152 class FrequencyFilter(LexiconFilter):
153     """Remove tokens with document frequency outside this range;
154     use either absolute or relative frequency. """
155     name = 'Document frequency'
156 
157     def __init__(self, min_df=0., max_df=1., keep_n=None):
158         super().__init__()
159         self._corpus_len = 0
160         self.keep_n = keep_n
161         self._max_df = max_df
162         self._min_df = min_df
163 
164     def fit_filter(self, corpus):
165         self._corpus_len = len(corpus)
166         tokens = getattr(corpus, 'tokens', corpus)
167         dictionary = corpora.Dictionary(tokens)
168         dictionary.filter_extremes(self.min_df, self.max_df, self.keep_n)
169         self.lexicon = dictionary.token2id.keys()
170         return self(tokens), dictionary
171 
172     @property
173     def max_df(self):
174         if isinstance(self._max_df, int):
175             return self._max_df / self._corpus_len if self._corpus_len else 1.
176         else:
177             return self._max_df
178 
179     @max_df.setter
180     def max_df(self, value):
181         self._max_df = value
182 
183     @property
184     def min_df(self):
185         if isinstance(self._min_df, float):
186             return int(self._corpus_len * self._min_df) or 1
187         else:
188             return self._min_df
189 
190     @min_df.setter
191     def min_df(self, value):
192         self._min_df = value
193 
194     def __str__(self):
195         keep = ', keep {}'.format(self.keep_n) if self.keep_n else ''
196         return "{} (range [{}, {}]{})".format(self.name, self._min_df,
197                                               self._max_df, keep)
198 
199     def check(self, token):
200         return token in self.lexicon
201
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/orangecontrib/text/preprocess/filter.py b/orangecontrib/text/preprocess/filter.py
--- a/orangecontrib/text/preprocess/filter.py
+++ b/orangecontrib/text/preprocess/filter.py
@@ -1,6 +1,7 @@
 import os
-
 import re
+
+from Orange.data.io import detect_encoding
 from gensim import corpora
 from nltk.corpus import stopwords
 
@@ -45,7 +46,8 @@
         if not path:
             self.word_list = []
         else:
-            with open(path) as f:
+            enc = detect_encoding(path)
+            with open(path, encoding=enc) as f:
                 self.word_list = set([line.strip() for line in f])

{"golden_diff": "diff --git a/orangecontrib/text/preprocess/filter.py b/orangecontrib/text/preprocess/filter.py\n--- a/orangecontrib/text/preprocess/filter.py\n+++ b/orangecontrib/text/preprocess/filter.py\n@@ -1,6 +1,7 @@\n import os\n-\n import re\n+\n+from Orange.data.io import detect_encoding\n from gensim import corpora\n from nltk.corpus import stopwords\n \n@@ -45,7 +46,8 @@\n         if not path:\n             self.word_list = []\n         else:\n-            with open(path) as f:\n+            enc = detect_encoding(path)\n+            with open(path, encoding=enc) as f:\n                 self.word_list = set([line.strip() for line in f])\n", "issue": "Chardet fails on Slovenian characters\nPreprocess Text fails with Slovenian stopword list. Seems like a chardet issue.\n", "before_files": [{"content": "import os\n\nimport re\nfrom gensim import corpora\nfrom nltk.corpus import stopwords\n\n__all__ = ['BaseTokenFilter', 'StopwordsFilter', 'LexiconFilter', 'RegexpFilter', 'FrequencyFilter']\n\n\nclass BaseTokenFilter:\n    name = NotImplemented\n\n    def __call__(self, corpus):\n        if len(corpus) == 0:\n            return corpus\n        if isinstance(corpus[0], str):\n            return self.filter(corpus)\n        return [self.filter(tokens) for tokens in corpus]\n\n    def filter(self, tokens):\n        return list(filter(self.check, tokens))\n\n    def check(self, token):\n        raise NotImplementedError\n\n    def __str__(self):\n        return self.name\n\n    def set_up(self):\n        \"\"\" A method for setting filters up before every __call__. \"\"\"\n        pass\n\n    def tear_down(self):\n        \"\"\" A method for cleaning up after every __call__. \"\"\"\n        pass\n\n\nclass WordListMixin:\n    def __init__(self, word_list=None):\n        self.file_path = None\n        self.word_list = word_list or []\n\n    def from_file(self, path):\n        self.file_path = path\n        if not path:\n            self.word_list = []\n        else:\n            with open(path) as f:\n                self.word_list = set([line.strip() for line in f])\n\n\nclass StopwordsFilter(BaseTokenFilter, WordListMixin):\n    \"\"\" Remove tokens present in NLTK's language specific lists or a file. \"\"\"\n    name = 'Stopwords'\n\n    supported_languages = [file.capitalize() for file in os.listdir(stopwords._get_root())\n                           if file.islower()]\n\n    def __init__(self, language='English', word_list=None):\n        WordListMixin.__init__(self, word_list)\n        super().__init__()\n        self.language = language\n\n    @property\n    def language(self):\n        return self._language\n\n    @language.setter\n    def language(self, value):\n        self._language = value\n        if not self._language:\n            self.stopwords = []\n        else:\n            self.stopwords = set(stopwords.words(self.language.lower()))\n\n    def __str__(self):\n        config = ''\n        config += 'Language: {}, '.format(self.language.capitalize()) if self.language else ''\n        config += 'File: {}, '.format(self.file_path) if self.file_path else ''\n        return '{} ({})'.format(self.name, config.strip(', '))\n\n    def check(self, token):\n        return token not in self.stopwords and token not in self.word_list\n\n\nclass LexiconFilter(BaseTokenFilter, WordListMixin):\n    \"\"\" Keep only tokens present in a file. \"\"\"\n    name = 'Lexicon'\n\n    def __init__(self, lexicon=None):\n        WordListMixin.__init__(self, word_list=lexicon)\n\n    @property\n    def lexicon(self):\n        return self.word_list\n\n    @lexicon.setter\n    def lexicon(self, value):\n        self.word_list = set(value)\n\n    def check(self, token):\n        return not self.lexicon or token in self.lexicon\n\n    def __str__(self):\n        return '{} ({})'.format(self.name, 'File: {}'.format(self.file_path))\n\n\nclass RegexpFilter(BaseTokenFilter):\n    \"\"\" Remove tokens matching this regular expressions. \"\"\"\n    name = 'Regexp'\n\n    def __init__(self, pattern=r'\\.|,|:|!|\\?'):\n        self._pattern = pattern\n        # Compiled Regexes are NOT deepcopy-able and hence to make Corpus deepcopy-able\n        # we cannot store then (due to Corpus also storing used_preprocessor for BoW compute values).\n        # To bypass the problem regex is compiled before every __call__ and discarded right after.\n        self.regex = None\n        self.set_up()\n\n    @property\n    def pattern(self):\n        return self._pattern\n\n    @pattern.setter\n    def pattern(self, value):\n        self._pattern = value\n        self.set_up()\n\n    @staticmethod\n    def validate_regexp(regexp):\n        try:\n            re.compile(regexp)\n            return True\n        except re.error:\n            return False\n\n    def check(self, token):\n        return not self.regex.match(token)\n\n    def __str__(self):\n        return '{} ({})'.format(self.name, self.pattern)\n\n    def set_up(self):\n        \"\"\" Compile Regex before the __call__. \"\"\"\n        self.regex = re.compile(self.pattern)\n\n    def tear_down(self):\n        \"\"\" Delete Regex after every __call__. \"\"\"\n        self.regex = None\n\n\nclass FrequencyFilter(LexiconFilter):\n    \"\"\"Remove tokens with document frequency outside this range;\n    use either absolute or relative frequency. \"\"\"\n    name = 'Document frequency'\n\n    def __init__(self, min_df=0., max_df=1., keep_n=None):\n        super().__init__()\n        self._corpus_len = 0\n        self.keep_n = keep_n\n        self._max_df = max_df\n        self._min_df = min_df\n\n    def fit_filter(self, corpus):\n        self._corpus_len = len(corpus)\n        tokens = getattr(corpus, 'tokens', corpus)\n        dictionary = corpora.Dictionary(tokens)\n        dictionary.filter_extremes(self.min_df, self.max_df, self.keep_n)\n        self.lexicon = dictionary.token2id.keys()\n        return self(tokens), dictionary\n\n    @property\n    def max_df(self):\n        if isinstance(self._max_df, int):\n            return self._max_df / self._corpus_len if self._corpus_len else 1.\n        else:\n            return self._max_df\n\n    @max_df.setter\n    def max_df(self, value):\n        self._max_df = value\n\n    @property\n    def min_df(self):\n        if isinstance(self._min_df, float):\n            return int(self._corpus_len * self._min_df) or 1\n        else:\n            return self._min_df\n\n    @min_df.setter\n    def min_df(self, value):\n        self._min_df = value\n\n    def __str__(self):\n        keep = ', keep {}'.format(self.keep_n) if self.keep_n else ''\n        return \"{} (range [{}, {}]{})\".format(self.name, self._min_df,\n                                              self._max_df, keep)\n\n    def check(self, token):\n        return token in self.lexicon\n", "path": "orangecontrib/text/preprocess/filter.py"}], "after_files": [{"content": "import os\nimport re\n\nfrom Orange.data.io import detect_encoding\nfrom gensim import corpora\nfrom nltk.corpus import stopwords\n\n__all__ = ['BaseTokenFilter', 'StopwordsFilter', 'LexiconFilter', 'RegexpFilter', 'FrequencyFilter']\n\n\nclass BaseTokenFilter:\n    name = NotImplemented\n\n    def __call__(self, corpus):\n        if len(corpus) == 0:\n            return corpus\n        if isinstance(corpus[0], str):\n            return self.filter(corpus)\n        return [self.filter(tokens) for tokens in corpus]\n\n    def filter(self, tokens):\n        return list(filter(self.check, tokens))\n\n    def check(self, token):\n        raise NotImplementedError\n\n    def __str__(self):\n        return self.name\n\n    def set_up(self):\n        \"\"\" A method for setting filters up before every __call__. \"\"\"\n        pass\n\n    def tear_down(self):\n        \"\"\" A method for cleaning up after every __call__. \"\"\"\n        pass\n\n\nclass WordListMixin:\n    def __init__(self, word_list=None):\n        self.file_path = None\n        self.word_list = word_list or []\n\n    def from_file(self, path):\n        self.file_path = path\n        if not path:\n            self.word_list = []\n        else:\n            enc = detect_encoding(path)\n            with open(path, encoding=enc) as f:\n                self.word_list = set([line.strip() for line in f])\n\n\nclass StopwordsFilter(BaseTokenFilter, WordListMixin):\n    \"\"\" Remove tokens present in NLTK's language specific lists or a file. \"\"\"\n    name = 'Stopwords'\n\n    supported_languages = [file.capitalize() for file in os.listdir(stopwords._get_root())\n                           if file.islower()]\n\n    def __init__(self, language='English', word_list=None):\n        WordListMixin.__init__(self, word_list)\n        super().__init__()\n        self.language = language\n\n    @property\n    def language(self):\n        return self._language\n\n    @language.setter\n    def language(self, value):\n        self._language = value\n        if not self._language:\n            self.stopwords = []\n        else:\n            self.stopwords = set(stopwords.words(self.language.lower()))\n\n    def __str__(self):\n        config = ''\n        config += 'Language: {}, '.format(self.language.capitalize()) if self.language else ''\n        config += 'File: {}, '.format(self.file_path) if self.file_path else ''\n        return '{} ({})'.format(self.name, config.strip(', '))\n\n    def check(self, token):\n        return token not in self.stopwords and token not in self.word_list\n\n\nclass LexiconFilter(BaseTokenFilter, WordListMixin):\n    \"\"\" Keep only tokens present in a file. \"\"\"\n    name = 'Lexicon'\n\n    def __init__(self, lexicon=None):\n        WordListMixin.__init__(self, word_list=lexicon)\n\n    @property\n    def lexicon(self):\n        return self.word_list\n\n    @lexicon.setter\n    def lexicon(self, value):\n        self.word_list = set(value)\n\n    def check(self, token):\n        return not self.lexicon or token in self.lexicon\n\n    def __str__(self):\n        return '{} ({})'.format(self.name, 'File: {}'.format(self.file_path))\n\n\nclass RegexpFilter(BaseTokenFilter):\n    \"\"\" Remove tokens matching this regular expressions. \"\"\"\n    name = 'Regexp'\n\n    def __init__(self, pattern=r'\\.|,|:|!|\\?'):\n        self._pattern = pattern\n        # Compiled Regexes are NOT deepcopy-able and hence to make Corpus deepcopy-able\n        # we cannot store then (due to Corpus also storing used_preprocessor for BoW compute values).\n        # To bypass the problem regex is compiled before every __call__ and discarded right after.\n        self.regex = None\n        self.set_up()\n\n    @property\n    def pattern(self):\n        return self._pattern\n\n    @pattern.setter\n    def pattern(self, value):\n        self._pattern = value\n        self.set_up()\n\n    @staticmethod\n    def validate_regexp(regexp):\n        try:\n            re.compile(regexp)\n            return True\n        except re.error:\n            return False\n\n    def check(self, token):\n        return not self.regex.match(token)\n\n    def __str__(self):\n        return '{} ({})'.format(self.name, self.pattern)\n\n    def set_up(self):\n        \"\"\" Compile Regex before the __call__. \"\"\"\n        self.regex = re.compile(self.pattern)\n\n    def tear_down(self):\n        \"\"\" Delete Regex after every __call__. \"\"\"\n        self.regex = None\n\n\nclass FrequencyFilter(LexiconFilter):\n    \"\"\"Remove tokens with document frequency outside this range;\n    use either absolute or relative frequency. \"\"\"\n    name = 'Document frequency'\n\n    def __init__(self, min_df=0., max_df=1., keep_n=None):\n        super().__init__()\n        self._corpus_len = 0\n        self.keep_n = keep_n\n        self._max_df = max_df\n        self._min_df = min_df\n\n    def fit_filter(self, corpus):\n        self._corpus_len = len(corpus)\n        tokens = getattr(corpus, 'tokens', corpus)\n        dictionary = corpora.Dictionary(tokens)\n        dictionary.filter_extremes(self.min_df, self.max_df, self.keep_n)\n        self.lexicon = dictionary.token2id.keys()\n        return self(tokens), dictionary\n\n    @property\n    def max_df(self):\n        if isinstance(self._max_df, int):\n            return self._max_df / self._corpus_len if self._corpus_len else 1.\n        else:\n            return self._max_df\n\n    @max_df.setter\n    def max_df(self, value):\n        self._max_df = value\n\n    @property\n    def min_df(self):\n        if isinstance(self._min_df, float):\n            return int(self._corpus_len * self._min_df) or 1\n        else:\n            return self._min_df\n\n    @min_df.setter\n    def min_df(self, value):\n        self._min_df = value\n\n    def __str__(self):\n        keep = ', keep {}'.format(self.keep_n) if self.keep_n else ''\n        return \"{} (range [{}, {}]{})\".format(self.name, self._min_df,\n                                              self._max_df, keep)\n\n    def check(self, token):\n        return token in self.lexicon\n", "path": "orangecontrib/text/preprocess/filter.py"}]}