We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Preprocess Text: consider pos tags when filtering tokens
**Describe the bug**
If pos tags exists before filtering, final pos tags don't match the tokens. This is because tokens are filtered independently from pos tags and only zipped at the end.

**To Reproduce**
Steps to reproduce the behavior:
1. Corpus
2. Preprocess Text (have POS tagger before filtering on stopwords)
3. Check the result in Corpus Viewer

**Expected behavior**
If pos tags exist, they are filtered together with tokens.

**Orange version:**
3.30.dev

**Text add-on version:**
1.6.dev

**Screenshots**
<img width="509" alt="Screen Shot 2021-07-21 at 15 13 50" src="https://user-images.githubusercontent.com/12524972/126494455-90d78da1-d0d3-4a02-be65-e176268c4c73.png">

**Operating system:**
OSX

**Additional context**
/


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `orangecontrib/text/preprocess/filter.py`
Content:
```
1 from typing import List, Callable
2 import os
3 import re
4 
5 import numpy as np
6 from gensim import corpora
7 from nltk.corpus import stopwords
8 
9 from Orange.data.io import detect_encoding
10 from Orange.util import wrap_callback, dummy_callback
11 
12 from orangecontrib.text import Corpus
13 from orangecontrib.text.misc import wait_nltk_data
14 from orangecontrib.text.preprocess import TokenizedPreprocessor
15 
16 __all__ = ['BaseTokenFilter', 'StopwordsFilter', 'LexiconFilter',
17            'RegexpFilter', 'FrequencyFilter', 'MostFrequentTokensFilter',
18            'PosTagFilter']
19 
20 
21 class BaseTokenFilter(TokenizedPreprocessor):
22     def __call__(self, corpus: Corpus, callback: Callable = None) -> Corpus:
23         if callback is None:
24             callback = dummy_callback
25         corpus = super().__call__(corpus, wrap_callback(callback, end=0.2))
26         return self._filter_tokens(corpus, wrap_callback(callback, start=0.2))
27 
28     def _filter_tokens(self, corpus: Corpus, callback: Callable) -> Corpus:
29         callback(0, "Filtering...")
30         tokens = [self._preprocess(tokens) for tokens in corpus.tokens]
31         corpus.store_tokens(tokens)
32         return corpus
33 
34     def _preprocess(self, tokens: List) -> List:
35         return list(filter(self._check, tokens))
36 
37     def _check(self, token: str) -> bool:
38         raise NotImplementedError
39 
40 
41 class FileWordListMixin:
42     def __init__(self, path: str = None):
43         self._lexicon = self.from_file(path)
44 
45     @staticmethod
46     def from_file(path):
47         if not path:
48             return set()
49 
50         for encoding in ('utf-8', None, detect_encoding(path)):
51             try:
52                 with open(path, encoding=encoding) as f:
53                     return set(line.strip() for line in f)
54             except UnicodeDecodeError:
55                 continue
56         # No encoding worked, raise
57         raise UnicodeError("Couldn't determine file encoding")
58 
59 
60 class StopwordsFilter(BaseTokenFilter, FileWordListMixin):
61     """ Remove tokens present in NLTK's language specific lists or a file. """
62     name = 'Stopwords'
63 
64     @wait_nltk_data
65     def __init__(self, language='English', path: str = None):
66         super().__init__()
67         FileWordListMixin.__init__(self, path)
68         self.__stopwords = set(x.strip() for x in
69                                stopwords.words(language.lower())) \
70             if language else []
71 
72     @staticmethod
73     @wait_nltk_data
74     def supported_languages():
75         # get NLTK list of stopwords
76         stopwords_listdir = []
77         try:
78             stopwords_listdir = [file for file in
79                                  os.listdir(stopwords._get_root())
80                                  if file.islower()]
81         except LookupError:  # when no NLTK data is available
82             pass
83 
84         return sorted(file.capitalize() for file in stopwords_listdir)
85 
86     def _check(self, token):
87         return token not in self.__stopwords and token not in self._lexicon
88 
89 
90 class LexiconFilter(BaseTokenFilter, FileWordListMixin):
91     """ Keep only tokens present in a file. """
92     name = 'Lexicon'
93 
94     def _check(self, token):
95         return not self._lexicon or token in self._lexicon
96 
97 
98 class RegexpFilter(BaseTokenFilter):
99     """ Remove tokens matching this regular expressions. """
100     name = 'Regexp'
101 
102     def __init__(self, pattern=r'\.|,|:|!|\?'):
103         self._pattern = pattern
104         # Compiled Regexes are NOT deepcopy-able and hence to make Corpus deepcopy-able
105         # we cannot store then (due to Corpus also storing used_preprocessor for BoW compute values).
106         # To bypass the problem regex is compiled before every __call__ and discarded right after.
107         self.regex = None
108 
109     def __call__(self, corpus: Corpus, callback: Callable = None) -> Corpus:
110         self.regex = re.compile(self._pattern)
111         corpus = super().__call__(corpus, callback)
112         self.regex = None
113         return corpus
114 
115     @staticmethod
116     def validate_regexp(regexp):
117         try:
118             re.compile(regexp)
119             return True
120         except re.error:
121             return False
122 
123     def _check(self, token):
124         return not self.regex.match(token)
125 
126 
127 class FitDictionaryFilter(BaseTokenFilter):
128     def __init__(self):
129         self._lexicon = None
130         self._dictionary = None
131 
132     def __call__(self, corpus: Corpus, callback: Callable = None) -> Corpus:
133         if callback is None:
134             callback = dummy_callback
135         corpus = TokenizedPreprocessor.__call__(
136             self, corpus, wrap_callback(callback, end=0.2))
137         callback(0.2, "Fitting filter...")
138         self._fit(corpus)
139         return self._filter_tokens(corpus, wrap_callback(callback, start=0.6))
140 
141     def _fit(self, corpus: Corpus):
142         raise NotImplemented
143 
144     def _filter_tokens(self, corpus: Corpus, callback: Callable) -> Corpus:
145         callback(0, "Filtering...")
146         tokens = [self._preprocess(tokens) for tokens in corpus.tokens]
147         corpus.store_tokens(tokens, self._dictionary)
148         return corpus
149 
150     def _check(self, token):
151         assert self._lexicon is not None
152         assert self._dictionary is not None
153         return token in self._lexicon
154 
155 
156 class FrequencyFilter(FitDictionaryFilter):
157     """Remove tokens with document frequency outside this range;
158     use either absolute or relative frequency. """
159     name = 'Document frequency'
160 
161     def __init__(self, min_df=0., max_df=1.):
162         super().__init__()
163         self._corpus_len = 0
164         self._max_df = max_df
165         self._min_df = min_df
166 
167     def _fit(self, corpus: Corpus):
168         self._corpus_len = len(corpus)
169         self._dictionary = corpora.Dictionary(corpus.tokens)
170         self._dictionary.filter_extremes(self.min_df, self.max_df, None)
171         self._lexicon = set(self._dictionary.token2id.keys())
172 
173     @property
174     def max_df(self):
175         if isinstance(self._max_df, int):
176             return self._max_df / self._corpus_len if self._corpus_len else 1.
177         else:
178             return self._max_df
179 
180     @property
181     def min_df(self):
182         if isinstance(self._min_df, float):
183             return int(self._corpus_len * self._min_df) or 1
184         else:
185             return self._min_df
186 
187 
188 class MostFrequentTokensFilter(FitDictionaryFilter):
189     """Keep most frequent tokens."""
190     name = 'Most frequent tokens'
191 
192     def __init__(self, keep_n=None):
193         super().__init__()
194         self._keep_n = keep_n
195 
196     def _fit(self, corpus: Corpus):
197         self._dictionary = corpora.Dictionary(corpus.tokens)
198         self._dictionary.filter_extremes(0, 1, self._keep_n)
199         self._lexicon = set(self._dictionary.token2id.keys())
200 
201 
202 class PosTagFilter(BaseTokenFilter):
203     """Keep selected POS tags."""
204     name = 'POS tags'
205 
206     def __init__(self, tags=None):
207         self._tags = set(i.strip().upper() for i in tags.split(","))
208 
209     def __call__(self, corpus: Corpus, callback: Callable = None) -> Corpus:
210         if callback is None:
211             callback = dummy_callback
212         corpus = super().__call__(corpus, wrap_callback(callback, end=0.2))
213         return self._filter_tokens(corpus, wrap_callback(callback, start=0.2))
214 
215     @staticmethod
216     def validate_tags(tags):
217         # should we keep a dict of existing POS tags and compare them with
218         # input?
219         return len(tags.split(",")) > 0
220 
221     def _filter_tokens(self, corpus: Corpus, callback: Callable) -> Corpus:
222         if corpus.pos_tags is None:
223             return corpus
224         callback(0, "Filtering...")
225         filtered_tags = []
226         filtered_tokens = []
227         for tags, tokens in zip(corpus.pos_tags, corpus.tokens):
228             tmp_tags = []
229             tmp_tokens = []
230             for tag, token in zip(tags, tokens):
231                 # should we consider partial matches, i.e. "NN" for "NNS"?
232                 if tag in self._tags:
233                     tmp_tags.append(tag)
234                     tmp_tokens.append(token)
235             filtered_tags.append(tmp_tags)
236             filtered_tokens.append(tmp_tokens)
237         corpus.store_tokens(filtered_tokens)
238         corpus.pos_tags = filtered_tags
239         return corpus
240 
241     def _check(self, token: str) -> bool:
242         pass
243
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/orangecontrib/text/preprocess/filter.py b/orangecontrib/text/preprocess/filter.py
--- a/orangecontrib/text/preprocess/filter.py
+++ b/orangecontrib/text/preprocess/filter.py
@@ -1,3 +1,4 @@
+from itertools import compress
 from typing import List, Callable
 import os
 import re
@@ -25,14 +26,27 @@
         corpus = super().__call__(corpus, wrap_callback(callback, end=0.2))
         return self._filter_tokens(corpus, wrap_callback(callback, start=0.2))
 
-    def _filter_tokens(self, corpus: Corpus, callback: Callable) -> Corpus:
+    def _filter_tokens(self, corpus: Corpus, callback: Callable,
+                       dictionary=None) -> Corpus:
         callback(0, "Filtering...")
-        tokens = [self._preprocess(tokens) for tokens in corpus.tokens]
-        corpus.store_tokens(tokens)
+        filtered_tokens = []
+        filtered_tags = []
+        for i, tokens in enumerate(corpus.tokens):
+            filter_map = self._preprocess(tokens)
+            filtered_tokens.append(list(compress(tokens, filter_map)))
+            if corpus.pos_tags is not None:
+                filtered_tags.append(list(compress(corpus.pos_tags[i],
+                                                   filter_map)))
+        if dictionary is None:
+            corpus.store_tokens(filtered_tokens)
+        else:
+            corpus.store_tokens(filtered_tokens, dictionary)
+        if filtered_tags:
+            corpus.pos_tags = np.array(filtered_tags, dtype=object)
         return corpus
 
     def _preprocess(self, tokens: List) -> List:
-        return list(filter(self._check, tokens))
+        return [self._check(token) for token in tokens]
 
     def _check(self, token: str) -> bool:
         raise NotImplementedError
@@ -141,10 +155,10 @@
     def _fit(self, corpus: Corpus):
         raise NotImplemented
 
-    def _filter_tokens(self, corpus: Corpus, callback: Callable) -> Corpus:
-        callback(0, "Filtering...")
-        tokens = [self._preprocess(tokens) for tokens in corpus.tokens]
-        corpus.store_tokens(tokens, self._dictionary)
+    def _filter_tokens(self, corpus: Corpus, callback: Callable,
+                       dictionary=None) -> Corpus:
+        corpus = super()._filter_tokens(corpus, callback,
+                                        dictionary=self._dictionary)
         return corpus
 
     def _check(self, token):

{"golden_diff": "diff --git a/orangecontrib/text/preprocess/filter.py b/orangecontrib/text/preprocess/filter.py\n--- a/orangecontrib/text/preprocess/filter.py\n+++ b/orangecontrib/text/preprocess/filter.py\n@@ -1,3 +1,4 @@\n+from itertools import compress\n from typing import List, Callable\n import os\n import re\n@@ -25,14 +26,27 @@\n         corpus = super().__call__(corpus, wrap_callback(callback, end=0.2))\n         return self._filter_tokens(corpus, wrap_callback(callback, start=0.2))\n \n-    def _filter_tokens(self, corpus: Corpus, callback: Callable) -> Corpus:\n+    def _filter_tokens(self, corpus: Corpus, callback: Callable,\n+                       dictionary=None) -> Corpus:\n         callback(0, \"Filtering...\")\n-        tokens = [self._preprocess(tokens) for tokens in corpus.tokens]\n-        corpus.store_tokens(tokens)\n+        filtered_tokens = []\n+        filtered_tags = []\n+        for i, tokens in enumerate(corpus.tokens):\n+            filter_map = self._preprocess(tokens)\n+            filtered_tokens.append(list(compress(tokens, filter_map)))\n+            if corpus.pos_tags is not None:\n+                filtered_tags.append(list(compress(corpus.pos_tags[i],\n+                                                   filter_map)))\n+        if dictionary is None:\n+            corpus.store_tokens(filtered_tokens)\n+        else:\n+            corpus.store_tokens(filtered_tokens, dictionary)\n+        if filtered_tags:\n+            corpus.pos_tags = np.array(filtered_tags, dtype=object)\n         return corpus\n \n     def _preprocess(self, tokens: List) -> List:\n-        return list(filter(self._check, tokens))\n+        return [self._check(token) for token in tokens]\n \n     def _check(self, token: str) -> bool:\n         raise NotImplementedError\n@@ -141,10 +155,10 @@\n     def _fit(self, corpus: Corpus):\n         raise NotImplemented\n \n-    def _filter_tokens(self, corpus: Corpus, callback: Callable) -> Corpus:\n-        callback(0, \"Filtering...\")\n-        tokens = [self._preprocess(tokens) for tokens in corpus.tokens]\n-        corpus.store_tokens(tokens, self._dictionary)\n+    def _filter_tokens(self, corpus: Corpus, callback: Callable,\n+                       dictionary=None) -> Corpus:\n+        corpus = super()._filter_tokens(corpus, callback,\n+                                        dictionary=self._dictionary)\n         return corpus\n \n     def _check(self, token):\n", "issue": "Preprocess Text: consider pos tags when filtering tokens\n**Describe the bug**\r\nIf pos tags exists before filtering, final pos tags don't match the tokens. This is because tokens are filtered independently from pos tags and only zipped at the end.\r\n\r\n**To Reproduce**\r\nSteps to reproduce the behavior:\r\n1. Corpus\r\n2. Preprocess Text (have POS tagger before filtering on stopwords)\r\n3. Check the result in Corpus Viewer\r\n\r\n**Expected behavior**\r\nIf pos tags exist, they are filtered together with tokens.\r\n\r\n**Orange version:**\r\n3.30.dev\r\n\r\n**Text add-on version:**\r\n1.6.dev\r\n\r\n**Screenshots**\r\n<img width=\"509\" alt=\"Screen Shot 2021-07-21 at 15 13 50\" src=\"https://user-images.githubusercontent.com/12524972/126494455-90d78da1-d0d3-4a02-be65-e176268c4c73.png\">\r\n\r\n**Operating system:**\r\nOSX\r\n\r\n**Additional context**\r\n/\r\n\n", "before_files": [{"content": "from typing import List, Callable\nimport os\nimport re\n\nimport numpy as np\nfrom gensim import corpora\nfrom nltk.corpus import stopwords\n\nfrom Orange.data.io import detect_encoding\nfrom Orange.util import wrap_callback, dummy_callback\n\nfrom orangecontrib.text import Corpus\nfrom orangecontrib.text.misc import wait_nltk_data\nfrom orangecontrib.text.preprocess import TokenizedPreprocessor\n\n__all__ = ['BaseTokenFilter', 'StopwordsFilter', 'LexiconFilter',\n           'RegexpFilter', 'FrequencyFilter', 'MostFrequentTokensFilter',\n           'PosTagFilter']\n\n\nclass BaseTokenFilter(TokenizedPreprocessor):\n    def __call__(self, corpus: Corpus, callback: Callable = None) -> Corpus:\n        if callback is None:\n            callback = dummy_callback\n        corpus = super().__call__(corpus, wrap_callback(callback, end=0.2))\n        return self._filter_tokens(corpus, wrap_callback(callback, start=0.2))\n\n    def _filter_tokens(self, corpus: Corpus, callback: Callable) -> Corpus:\n        callback(0, \"Filtering...\")\n        tokens = [self._preprocess(tokens) for tokens in corpus.tokens]\n        corpus.store_tokens(tokens)\n        return corpus\n\n    def _preprocess(self, tokens: List) -> List:\n        return list(filter(self._check, tokens))\n\n    def _check(self, token: str) -> bool:\n        raise NotImplementedError\n\n\nclass FileWordListMixin:\n    def __init__(self, path: str = None):\n        self._lexicon = self.from_file(path)\n\n    @staticmethod\n    def from_file(path):\n        if not path:\n            return set()\n\n        for encoding in ('utf-8', None, detect_encoding(path)):\n            try:\n                with open(path, encoding=encoding) as f:\n                    return set(line.strip() for line in f)\n            except UnicodeDecodeError:\n                continue\n        # No encoding worked, raise\n        raise UnicodeError(\"Couldn't determine file encoding\")\n\n\nclass StopwordsFilter(BaseTokenFilter, FileWordListMixin):\n    \"\"\" Remove tokens present in NLTK's language specific lists or a file. \"\"\"\n    name = 'Stopwords'\n\n    @wait_nltk_data\n    def __init__(self, language='English', path: str = None):\n        super().__init__()\n        FileWordListMixin.__init__(self, path)\n        self.__stopwords = set(x.strip() for x in\n                               stopwords.words(language.lower())) \\\n            if language else []\n\n    @staticmethod\n    @wait_nltk_data\n    def supported_languages():\n        # get NLTK list of stopwords\n        stopwords_listdir = []\n        try:\n            stopwords_listdir = [file for file in\n                                 os.listdir(stopwords._get_root())\n                                 if file.islower()]\n        except LookupError:  # when no NLTK data is available\n            pass\n\n        return sorted(file.capitalize() for file in stopwords_listdir)\n\n    def _check(self, token):\n        return token not in self.__stopwords and token not in self._lexicon\n\n\nclass LexiconFilter(BaseTokenFilter, FileWordListMixin):\n    \"\"\" Keep only tokens present in a file. \"\"\"\n    name = 'Lexicon'\n\n    def _check(self, token):\n        return not self._lexicon or token in self._lexicon\n\n\nclass RegexpFilter(BaseTokenFilter):\n    \"\"\" Remove tokens matching this regular expressions. \"\"\"\n    name = 'Regexp'\n\n    def __init__(self, pattern=r'\\.|,|:|!|\\?'):\n        self._pattern = pattern\n        # Compiled Regexes are NOT deepcopy-able and hence to make Corpus deepcopy-able\n        # we cannot store then (due to Corpus also storing used_preprocessor for BoW compute values).\n        # To bypass the problem regex is compiled before every __call__ and discarded right after.\n        self.regex = None\n\n    def __call__(self, corpus: Corpus, callback: Callable = None) -> Corpus:\n        self.regex = re.compile(self._pattern)\n        corpus = super().__call__(corpus, callback)\n        self.regex = None\n        return corpus\n\n    @staticmethod\n    def validate_regexp(regexp):\n        try:\n            re.compile(regexp)\n            return True\n        except re.error:\n            return False\n\n    def _check(self, token):\n        return not self.regex.match(token)\n\n\nclass FitDictionaryFilter(BaseTokenFilter):\n    def __init__(self):\n        self._lexicon = None\n        self._dictionary = None\n\n    def __call__(self, corpus: Corpus, callback: Callable = None) -> Corpus:\n        if callback is None:\n            callback = dummy_callback\n        corpus = TokenizedPreprocessor.__call__(\n            self, corpus, wrap_callback(callback, end=0.2))\n        callback(0.2, \"Fitting filter...\")\n        self._fit(corpus)\n        return self._filter_tokens(corpus, wrap_callback(callback, start=0.6))\n\n    def _fit(self, corpus: Corpus):\n        raise NotImplemented\n\n    def _filter_tokens(self, corpus: Corpus, callback: Callable) -> Corpus:\n        callback(0, \"Filtering...\")\n        tokens = [self._preprocess(tokens) for tokens in corpus.tokens]\n        corpus.store_tokens(tokens, self._dictionary)\n        return corpus\n\n    def _check(self, token):\n        assert self._lexicon is not None\n        assert self._dictionary is not None\n        return token in self._lexicon\n\n\nclass FrequencyFilter(FitDictionaryFilter):\n    \"\"\"Remove tokens with document frequency outside this range;\n    use either absolute or relative frequency. \"\"\"\n    name = 'Document frequency'\n\n    def __init__(self, min_df=0., max_df=1.):\n        super().__init__()\n        self._corpus_len = 0\n        self._max_df = max_df\n        self._min_df = min_df\n\n    def _fit(self, corpus: Corpus):\n        self._corpus_len = len(corpus)\n        self._dictionary = corpora.Dictionary(corpus.tokens)\n        self._dictionary.filter_extremes(self.min_df, self.max_df, None)\n        self._lexicon = set(self._dictionary.token2id.keys())\n\n    @property\n    def max_df(self):\n        if isinstance(self._max_df, int):\n            return self._max_df / self._corpus_len if self._corpus_len else 1.\n        else:\n            return self._max_df\n\n    @property\n    def min_df(self):\n        if isinstance(self._min_df, float):\n            return int(self._corpus_len * self._min_df) or 1\n        else:\n            return self._min_df\n\n\nclass MostFrequentTokensFilter(FitDictionaryFilter):\n    \"\"\"Keep most frequent tokens.\"\"\"\n    name = 'Most frequent tokens'\n\n    def __init__(self, keep_n=None):\n        super().__init__()\n        self._keep_n = keep_n\n\n    def _fit(self, corpus: Corpus):\n        self._dictionary = corpora.Dictionary(corpus.tokens)\n        self._dictionary.filter_extremes(0, 1, self._keep_n)\n        self._lexicon = set(self._dictionary.token2id.keys())\n\n\nclass PosTagFilter(BaseTokenFilter):\n    \"\"\"Keep selected POS tags.\"\"\"\n    name = 'POS tags'\n\n    def __init__(self, tags=None):\n        self._tags = set(i.strip().upper() for i in tags.split(\",\"))\n\n    def __call__(self, corpus: Corpus, callback: Callable = None) -> Corpus:\n        if callback is None:\n            callback = dummy_callback\n        corpus = super().__call__(corpus, wrap_callback(callback, end=0.2))\n        return self._filter_tokens(corpus, wrap_callback(callback, start=0.2))\n\n    @staticmethod\n    def validate_tags(tags):\n        # should we keep a dict of existing POS tags and compare them with\n        # input?\n        return len(tags.split(\",\")) > 0\n\n    def _filter_tokens(self, corpus: Corpus, callback: Callable) -> Corpus:\n        if corpus.pos_tags is None:\n            return corpus\n        callback(0, \"Filtering...\")\n        filtered_tags = []\n        filtered_tokens = []\n        for tags, tokens in zip(corpus.pos_tags, corpus.tokens):\n            tmp_tags = []\n            tmp_tokens = []\n            for tag, token in zip(tags, tokens):\n                # should we consider partial matches, i.e. \"NN\" for \"NNS\"?\n                if tag in self._tags:\n                    tmp_tags.append(tag)\n                    tmp_tokens.append(token)\n            filtered_tags.append(tmp_tags)\n            filtered_tokens.append(tmp_tokens)\n        corpus.store_tokens(filtered_tokens)\n        corpus.pos_tags = filtered_tags\n        return corpus\n\n    def _check(self, token: str) -> bool:\n        pass\n", "path": "orangecontrib/text/preprocess/filter.py"}], "after_files": [{"content": "from itertools import compress\nfrom typing import List, Callable\nimport os\nimport re\n\nimport numpy as np\nfrom gensim import corpora\nfrom nltk.corpus import stopwords\n\nfrom Orange.data.io import detect_encoding\nfrom Orange.util import wrap_callback, dummy_callback\n\nfrom orangecontrib.text import Corpus\nfrom orangecontrib.text.misc import wait_nltk_data\nfrom orangecontrib.text.preprocess import TokenizedPreprocessor\n\n__all__ = ['BaseTokenFilter', 'StopwordsFilter', 'LexiconFilter',\n           'RegexpFilter', 'FrequencyFilter', 'MostFrequentTokensFilter',\n           'PosTagFilter']\n\n\nclass BaseTokenFilter(TokenizedPreprocessor):\n    def __call__(self, corpus: Corpus, callback: Callable = None) -> Corpus:\n        if callback is None:\n            callback = dummy_callback\n        corpus = super().__call__(corpus, wrap_callback(callback, end=0.2))\n        return self._filter_tokens(corpus, wrap_callback(callback, start=0.2))\n\n    def _filter_tokens(self, corpus: Corpus, callback: Callable,\n                       dictionary=None) -> Corpus:\n        callback(0, \"Filtering...\")\n        filtered_tokens = []\n        filtered_tags = []\n        for i, tokens in enumerate(corpus.tokens):\n            filter_map = self._preprocess(tokens)\n            filtered_tokens.append(list(compress(tokens, filter_map)))\n            if corpus.pos_tags is not None:\n                filtered_tags.append(list(compress(corpus.pos_tags[i],\n                                                   filter_map)))\n        if dictionary is None:\n            corpus.store_tokens(filtered_tokens)\n        else:\n            corpus.store_tokens(filtered_tokens, dictionary)\n        if filtered_tags:\n            corpus.pos_tags = np.array(filtered_tags, dtype=object)\n        return corpus\n\n    def _preprocess(self, tokens: List) -> List:\n        return [self._check(token) for token in tokens]\n\n    def _check(self, token: str) -> bool:\n        raise NotImplementedError\n\n\nclass FileWordListMixin:\n    def __init__(self, path: str = None):\n        self._lexicon = self.from_file(path)\n\n    @staticmethod\n    def from_file(path):\n        if not path:\n            return set()\n\n        for encoding in ('utf-8', None, detect_encoding(path)):\n            try:\n                with open(path, encoding=encoding) as f:\n                    return set(line.strip() for line in f)\n            except UnicodeDecodeError:\n                continue\n        # No encoding worked, raise\n        raise UnicodeError(\"Couldn't determine file encoding\")\n\n\nclass StopwordsFilter(BaseTokenFilter, FileWordListMixin):\n    \"\"\" Remove tokens present in NLTK's language specific lists or a file. \"\"\"\n    name = 'Stopwords'\n\n    @wait_nltk_data\n    def __init__(self, language='English', path: str = None):\n        super().__init__()\n        FileWordListMixin.__init__(self, path)\n        self.__stopwords = set(x.strip() for x in\n                               stopwords.words(language.lower())) \\\n            if language else []\n\n    @staticmethod\n    @wait_nltk_data\n    def supported_languages():\n        # get NLTK list of stopwords\n        stopwords_listdir = []\n        try:\n            stopwords_listdir = [file for file in\n                                 os.listdir(stopwords._get_root())\n                                 if file.islower()]\n        except LookupError:  # when no NLTK data is available\n            pass\n\n        return sorted(file.capitalize() for file in stopwords_listdir)\n\n    def _check(self, token):\n        return token not in self.__stopwords and token not in self._lexicon\n\n\nclass LexiconFilter(BaseTokenFilter, FileWordListMixin):\n    \"\"\" Keep only tokens present in a file. \"\"\"\n    name = 'Lexicon'\n\n    def _check(self, token):\n        return not self._lexicon or token in self._lexicon\n\n\nclass RegexpFilter(BaseTokenFilter):\n    \"\"\" Remove tokens matching this regular expressions. \"\"\"\n    name = 'Regexp'\n\n    def __init__(self, pattern=r'\\.|,|:|!|\\?'):\n        self._pattern = pattern\n        # Compiled Regexes are NOT deepcopy-able and hence to make Corpus deepcopy-able\n        # we cannot store then (due to Corpus also storing used_preprocessor for BoW compute values).\n        # To bypass the problem regex is compiled before every __call__ and discarded right after.\n        self.regex = None\n\n    def __call__(self, corpus: Corpus, callback: Callable = None) -> Corpus:\n        self.regex = re.compile(self._pattern)\n        corpus = super().__call__(corpus, callback)\n        self.regex = None\n        return corpus\n\n    @staticmethod\n    def validate_regexp(regexp):\n        try:\n            re.compile(regexp)\n            return True\n        except re.error:\n            return False\n\n    def _check(self, token):\n        return not self.regex.match(token)\n\n\nclass FitDictionaryFilter(BaseTokenFilter):\n    def __init__(self):\n        self._lexicon = None\n        self._dictionary = None\n\n    def __call__(self, corpus: Corpus, callback: Callable = None) -> Corpus:\n        if callback is None:\n            callback = dummy_callback\n        corpus = TokenizedPreprocessor.__call__(\n            self, corpus, wrap_callback(callback, end=0.2))\n        callback(0.2, \"Fitting filter...\")\n        self._fit(corpus)\n        return self._filter_tokens(corpus, wrap_callback(callback, start=0.6))\n\n    def _fit(self, corpus: Corpus):\n        raise NotImplemented\n\n    def _filter_tokens(self, corpus: Corpus, callback: Callable,\n                       dictionary=None) -> Corpus:\n        corpus = super()._filter_tokens(corpus, callback,\n                                        dictionary=self._dictionary)\n        return corpus\n\n    def _check(self, token):\n        assert self._lexicon is not None\n        assert self._dictionary is not None\n        return token in self._lexicon\n\n\nclass FrequencyFilter(FitDictionaryFilter):\n    \"\"\"Remove tokens with document frequency outside this range;\n    use either absolute or relative frequency. \"\"\"\n    name = 'Document frequency'\n\n    def __init__(self, min_df=0., max_df=1.):\n        super().__init__()\n        self._corpus_len = 0\n        self._max_df = max_df\n        self._min_df = min_df\n\n    def _fit(self, corpus: Corpus):\n        self._corpus_len = len(corpus)\n        self._dictionary = corpora.Dictionary(corpus.tokens)\n        self._dictionary.filter_extremes(self.min_df, self.max_df, None)\n        self._lexicon = set(self._dictionary.token2id.keys())\n\n    @property\n    def max_df(self):\n        if isinstance(self._max_df, int):\n            return self._max_df / self._corpus_len if self._corpus_len else 1.\n        else:\n            return self._max_df\n\n    @property\n    def min_df(self):\n        if isinstance(self._min_df, float):\n            return int(self._corpus_len * self._min_df) or 1\n        else:\n            return self._min_df\n\n\nclass MostFrequentTokensFilter(FitDictionaryFilter):\n    \"\"\"Keep most frequent tokens.\"\"\"\n    name = 'Most frequent tokens'\n\n    def __init__(self, keep_n=None):\n        super().__init__()\n        self._keep_n = keep_n\n\n    def _fit(self, corpus: Corpus):\n        self._dictionary = corpora.Dictionary(corpus.tokens)\n        self._dictionary.filter_extremes(0, 1, self._keep_n)\n        self._lexicon = set(self._dictionary.token2id.keys())\n\n\nclass PosTagFilter(BaseTokenFilter):\n    \"\"\"Keep selected POS tags.\"\"\"\n    name = 'POS tags'\n\n    def __init__(self, tags=None):\n        self._tags = set(i.strip().upper() for i in tags.split(\",\"))\n\n    def __call__(self, corpus: Corpus, callback: Callable = None) -> Corpus:\n        if callback is None:\n            callback = dummy_callback\n        corpus = super().__call__(corpus, wrap_callback(callback, end=0.2))\n        return self._filter_tokens(corpus, wrap_callback(callback, start=0.2))\n\n    @staticmethod\n    def validate_tags(tags):\n        # should we keep a dict of existing POS tags and compare them with\n        # input?\n        return len(tags.split(\",\")) > 0\n\n    def _filter_tokens(self, corpus: Corpus, callback: Callable) -> Corpus:\n        if corpus.pos_tags is None:\n            return corpus\n        callback(0, \"Filtering...\")\n        filtered_tags = []\n        filtered_tokens = []\n        for tags, tokens in zip(corpus.pos_tags, corpus.tokens):\n            tmp_tags = []\n            tmp_tokens = []\n            for tag, token in zip(tags, tokens):\n                # should we consider partial matches, i.e. \"NN\" for \"NNS\"?\n                if tag in self._tags:\n                    tmp_tags.append(tag)\n                    tmp_tokens.append(token)\n            filtered_tags.append(tmp_tags)\n            filtered_tokens.append(tmp_tokens)\n        corpus.store_tokens(filtered_tokens)\n        corpus.pos_tags = filtered_tags\n        return corpus\n\n    def _check(self, token: str) -> bool:\n        pass\n", "path": "orangecontrib/text/preprocess/filter.py"}]}