We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
KeyError: Popping a non-existing value from `_keywords`
OS X 10.11.1
Python 2.7.10
gensim 0.12.4

To reproduce this in Python shell:

```
>>> import gensim.summarization
>>> t = "Victor S. Sage Compare Sage 50c Editions Find accounting software that's right for your business Every product comes with anytime, anywhere online access; automatic updates; access to unlimited support; access to built-in credit card processing and payroll; and advanced reporting. Three solutions for your business 1 user From $249/year Buy now Free Trial 1-5 users From $299/year Buy now Free Trial 3-40 users From $1,199/year Buy now Free Trial Essential Accounting Accounts payable, accounts receivable, cash management check check check open check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check Advanced Accounting Automated tasks, audit trail, budgeting, change order processing check check open check check check check check check check check check check check check check check check check check check check check check check check check check check check check In-depth Accounting Fast processing, industry-specific features, workflow management check open check check check check check check check check check check check Disclaimers open * This product is backed by a no-risk guarantee for first-time Sage 50 customers. If, within 60 days of purchase, you are not convinced that Sage 50 is the best accounting program for your business, we will refund your money (less and rebate you have received for this purchase). Dated proof of purchase and return of product is required. For details, call 877-481-0341."
>>> import gensim.summarization
>>> keywords = gensim.summarization.keywords(t, pos_filter=[], ratio=0.2, lemmatize=True, scores=True)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python2.7/site-packages/gensim/summarization/keywords.py", line 229, in keywords
    combined_keywords = _get_combined_keywords(keywords, text.split())
  File "/usr/local/lib/python2.7/site-packages/gensim/summarization/keywords.py", line 171, in _get_combined_keywords
    _keywords.pop(keyword)
KeyError: u'check'
```


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `gensim/summarization/keywords.py`
Content:
```
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3 #
4 # Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html
5 
6 from gensim.summarization.pagerank_weighted import pagerank_weighted as _pagerank
7 from gensim.summarization.textcleaner import clean_text_by_word as _clean_text_by_word
8 from gensim.summarization.textcleaner import tokenize_by_word as _tokenize_by_word
9 from gensim.summarization.commons import build_graph as _build_graph
10 from gensim.summarization.commons import remove_unreachable_nodes as _remove_unreachable_nodes
11 from gensim.utils import to_unicode
12 from itertools import combinations as _combinations
13 from six.moves.queue import Queue as _Queue
14 from six.moves import xrange
15 from six import iteritems
16 
17 
18 WINDOW_SIZE = 2
19 
20 """
21 Check tags in http://www.clips.ua.ac.be/pages/mbsp-tags and use only first two letters
22 Example: filter for nouns and adjectives:
23 INCLUDING_FILTER = ['NN', 'JJ']
24 """
25 INCLUDING_FILTER = ['NN', 'JJ']
26 EXCLUDING_FILTER = []
27 
28 
29 def _get_pos_filters():
30     return frozenset(INCLUDING_FILTER), frozenset(EXCLUDING_FILTER)
31 
32 
33 def _get_words_for_graph(tokens, pos_filter):
34     if pos_filter is None:
35         include_filters, exclude_filters = _get_pos_filters()
36     else:
37         include_filters = set(pos_filter)
38         exclude_filters = frozenset([])
39     if include_filters and exclude_filters:
40         raise ValueError("Can't use both include and exclude filters, should use only one")
41 
42     result = []
43     for word, unit in iteritems(tokens):
44         if exclude_filters and unit.tag in exclude_filters:
45             continue
46         if (include_filters and unit.tag in include_filters) or not include_filters or not unit.tag:
47             result.append(unit.token)
48     return result
49 
50 
51 def _get_first_window(split_text):
52     return split_text[:WINDOW_SIZE]
53 
54 
55 def _set_graph_edge(graph, tokens, word_a, word_b):
56     if word_a in tokens and word_b in tokens:
57         lemma_a = tokens[word_a].token
58         lemma_b = tokens[word_b].token
59         edge = (lemma_a, lemma_b)
60 
61         if graph.has_node(lemma_a) and graph.has_node(lemma_b) and not graph.has_edge(edge):
62             graph.add_edge(edge)
63 
64 
65 def _process_first_window(graph, tokens, split_text):
66     first_window = _get_first_window(split_text)
67     for word_a, word_b in _combinations(first_window, 2):
68         _set_graph_edge(graph, tokens, word_a, word_b)
69 
70 
71 def _init_queue(split_text):
72     queue = _Queue()
73     first_window = _get_first_window(split_text)
74     for word in first_window[1:]:
75         queue.put(word)
76     return queue
77 
78 
79 def _process_word(graph, tokens, queue, word):
80     for word_to_compare in _queue_iterator(queue):
81         _set_graph_edge(graph, tokens, word, word_to_compare)
82 
83 
84 def _update_queue(queue, word):
85     queue.get()
86     queue.put(word)
87     assert queue.qsize() == (WINDOW_SIZE - 1)
88 
89 
90 def _process_text(graph, tokens, split_text):
91     queue = _init_queue(split_text)
92     for i in xrange(WINDOW_SIZE, len(split_text)):
93         word = split_text[i]
94         _process_word(graph, tokens, queue, word)
95         _update_queue(queue, word)
96 
97 
98 def _queue_iterator(queue):
99     iterations = queue.qsize()
100     for i in xrange(iterations):
101         var = queue.get()
102         yield var
103         queue.put(var)
104 
105 
106 def _set_graph_edges(graph, tokens, split_text):
107     _process_first_window(graph, tokens, split_text)
108     _process_text(graph, tokens, split_text)
109 
110 
111 def _extract_tokens(lemmas, scores, ratio, words):
112     lemmas.sort(key=lambda s: scores[s], reverse=True)
113 
114     # If no "words" option is selected, the number of sentences is
115     # reduced by the provided ratio, else, the ratio is ignored.
116     length = len(lemmas) * ratio if words is None else words
117     return [(scores[lemmas[i]], lemmas[i],) for i in range(int(length))]
118 
119 
120 def _lemmas_to_words(tokens):
121     lemma_to_word = {}
122     for word, unit in iteritems(tokens):
123         lemma = unit.token
124         if lemma in lemma_to_word:
125             lemma_to_word[lemma].append(word)
126         else:
127             lemma_to_word[lemma] = [word]
128     return lemma_to_word
129 
130 
131 def _get_keywords_with_score(extracted_lemmas, lemma_to_word):
132     """
133     :param extracted_lemmas:list of tuples
134     :param lemma_to_word: dict of {lemma:list of words}
135     :return: dict of {keyword:score}
136     """
137     keywords = {}
138     for score, lemma in extracted_lemmas:
139         keyword_list = lemma_to_word[lemma]
140         for keyword in keyword_list:
141             keywords[keyword] = score
142     return keywords
143 
144 
145 def _strip_word(word):
146     stripped_word_list = list(_tokenize_by_word(word))
147     return stripped_word_list[0] if stripped_word_list else ""
148 
149 
150 def _get_combined_keywords(_keywords, split_text):
151     """
152     :param keywords:dict of keywords:scores
153     :param split_text: list of strings
154     :return: combined_keywords:list
155     """
156     result = []
157     _keywords = _keywords.copy()
158     len_text = len(split_text)
159     for i in xrange(len_text):
160         word = _strip_word(split_text[i])
161         if word in _keywords:
162             combined_word = [word]
163             if i + 1 == len_text:
164                 result.append(word)   # appends last word if keyword and doesn't iterate
165             for j in xrange(i + 1, len_text):
166                 other_word = _strip_word(split_text[j])
167                 if other_word in _keywords and other_word == split_text[j]:
168                     combined_word.append(other_word)
169                 else:
170                     for keyword in combined_word:
171                         _keywords.pop(keyword)
172                     result.append(" ".join(combined_word))
173                     break
174     return result
175 
176 
177 def _get_average_score(concept, _keywords):
178     word_list = concept.split()
179     word_counter = 0
180     total = 0
181     for word in word_list:
182         total += _keywords[word]
183         word_counter += 1
184     return total / word_counter
185 
186 
187 def _format_results(_keywords, combined_keywords, split, scores):
188     """
189     :param keywords:dict of keywords:scores
190     :param combined_keywords:list of word/s
191     """
192     combined_keywords.sort(key=lambda w: _get_average_score(w, _keywords), reverse=True)
193     if scores:
194         return [(word, _get_average_score(word, _keywords)) for word in combined_keywords]
195     if split:
196         return combined_keywords
197     return "\n".join(combined_keywords)
198 
199 
200 def keywords(text, ratio=0.2, words=None, split=False, scores=False, pos_filter=['NN', 'JJ'], lemmatize=False):
201     # Gets a dict of word -> lemma
202     text = to_unicode(text)
203     tokens = _clean_text_by_word(text)
204     split_text = list(_tokenize_by_word(text))
205 
206     # Creates the graph and adds the edges
207     graph = _build_graph(_get_words_for_graph(tokens, pos_filter))
208     _set_graph_edges(graph, tokens, split_text)
209     del split_text  # It's no longer used
210 
211     _remove_unreachable_nodes(graph)
212 
213     # Ranks the tokens using the PageRank algorithm. Returns dict of lemma -> score
214     pagerank_scores = _pagerank(graph)
215 
216     extracted_lemmas = _extract_tokens(graph.nodes(), pagerank_scores, ratio, words)
217 
218     # The results can be polluted by many variations of the same word
219     if lemmatize:
220         lemmas_to_word = {}
221         for word, unit in iteritems(tokens):
222             lemmas_to_word[unit.token] = [word]
223     else:
224         lemmas_to_word = _lemmas_to_words(tokens)
225 
226     keywords = _get_keywords_with_score(extracted_lemmas, lemmas_to_word)
227 
228     # text.split() to keep numbers and punctuation marks, so separeted concepts are not combined
229     combined_keywords = _get_combined_keywords(keywords, text.split())
230 
231     return _format_results(keywords, combined_keywords, split, scores)
232 
233 
234 def get_graph(text):
235     tokens = _clean_text_by_word(text)
236     split_text = list(_tokenize_by_word(text))
237 
238     graph = _build_graph(_get_words_for_graph(tokens))
239     _set_graph_edges(graph, tokens, split_text)
240 
241     return graph
242
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/gensim/summarization/keywords.py b/gensim/summarization/keywords.py
--- a/gensim/summarization/keywords.py
+++ b/gensim/summarization/keywords.py
@@ -164,7 +164,7 @@
                 result.append(word)   # appends last word if keyword and doesn't iterate
             for j in xrange(i + 1, len_text):
                 other_word = _strip_word(split_text[j])
-                if other_word in _keywords and other_word == split_text[j]:
+                if other_word in _keywords and other_word == split_text[j] and not other_word in combined_word:
                     combined_word.append(other_word)
                 else:
                     for keyword in combined_word:

{"golden_diff": "diff --git a/gensim/summarization/keywords.py b/gensim/summarization/keywords.py\n--- a/gensim/summarization/keywords.py\n+++ b/gensim/summarization/keywords.py\n@@ -164,7 +164,7 @@\n                 result.append(word)   # appends last word if keyword and doesn't iterate\n             for j in xrange(i + 1, len_text):\n                 other_word = _strip_word(split_text[j])\n-                if other_word in _keywords and other_word == split_text[j]:\n+                if other_word in _keywords and other_word == split_text[j] and not other_word in combined_word:\n                     combined_word.append(other_word)\n                 else:\n                     for keyword in combined_word:\n", "issue": "KeyError: Popping a non-existing value from `_keywords`\nOS X 10.11.1\nPython 2.7.10\ngensim 0.12.4\n\nTo reproduce this in Python shell:\n\n```\n>>> import gensim.summarization\n>>> t = \"Victor S. Sage Compare Sage 50c Editions Find accounting software that's right for your business Every product comes with anytime, anywhere online access; automatic updates; access to unlimited support; access to built-in credit card processing and payroll; and advanced reporting. Three solutions for your business 1 user From $249/year Buy now Free Trial 1-5 users From $299/year Buy now Free Trial 3-40 users From $1,199/year Buy now Free Trial Essential Accounting Accounts payable, accounts receivable, cash management check check check open check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check check Advanced Accounting Automated tasks, audit trail, budgeting, change order processing check check open check check check check check check check check check check check check check check check check check check check check check check check check check check check check In-depth Accounting Fast processing, industry-specific features, workflow management check open check check check check check check check check check check check Disclaimers open * This product is backed by a no-risk guarantee for first-time Sage 50 customers. If, within 60 days of purchase, you are not convinced that Sage 50 is the best accounting program for your business, we will refund your money (less and rebate you have received for this purchase). Dated proof of purchase and return of product is required. For details, call 877-481-0341.\"\n>>> import gensim.summarization\n>>> keywords = gensim.summarization.keywords(t, pos_filter=[], ratio=0.2, lemmatize=True, scores=True)\nTraceback (most recent call last):\n  File \"<stdin>\", line 1, in <module>\n  File \"/usr/local/lib/python2.7/site-packages/gensim/summarization/keywords.py\", line 229, in keywords\n    combined_keywords = _get_combined_keywords(keywords, text.split())\n  File \"/usr/local/lib/python2.7/site-packages/gensim/summarization/keywords.py\", line 171, in _get_combined_keywords\n    _keywords.pop(keyword)\nKeyError: u'check'\n```\n\n", "before_files": [{"content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n#\n# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html\n\nfrom gensim.summarization.pagerank_weighted import pagerank_weighted as _pagerank\nfrom gensim.summarization.textcleaner import clean_text_by_word as _clean_text_by_word\nfrom gensim.summarization.textcleaner import tokenize_by_word as _tokenize_by_word\nfrom gensim.summarization.commons import build_graph as _build_graph\nfrom gensim.summarization.commons import remove_unreachable_nodes as _remove_unreachable_nodes\nfrom gensim.utils import to_unicode\nfrom itertools import combinations as _combinations\nfrom six.moves.queue import Queue as _Queue\nfrom six.moves import xrange\nfrom six import iteritems\n\n\nWINDOW_SIZE = 2\n\n\"\"\"\nCheck tags in http://www.clips.ua.ac.be/pages/mbsp-tags and use only first two letters\nExample: filter for nouns and adjectives:\nINCLUDING_FILTER = ['NN', 'JJ']\n\"\"\"\nINCLUDING_FILTER = ['NN', 'JJ']\nEXCLUDING_FILTER = []\n\n\ndef _get_pos_filters():\n    return frozenset(INCLUDING_FILTER), frozenset(EXCLUDING_FILTER)\n\n\ndef _get_words_for_graph(tokens, pos_filter):\n    if pos_filter is None:\n        include_filters, exclude_filters = _get_pos_filters()\n    else:\n        include_filters = set(pos_filter)\n        exclude_filters = frozenset([])\n    if include_filters and exclude_filters:\n        raise ValueError(\"Can't use both include and exclude filters, should use only one\")\n\n    result = []\n    for word, unit in iteritems(tokens):\n        if exclude_filters and unit.tag in exclude_filters:\n            continue\n        if (include_filters and unit.tag in include_filters) or not include_filters or not unit.tag:\n            result.append(unit.token)\n    return result\n\n\ndef _get_first_window(split_text):\n    return split_text[:WINDOW_SIZE]\n\n\ndef _set_graph_edge(graph, tokens, word_a, word_b):\n    if word_a in tokens and word_b in tokens:\n        lemma_a = tokens[word_a].token\n        lemma_b = tokens[word_b].token\n        edge = (lemma_a, lemma_b)\n\n        if graph.has_node(lemma_a) and graph.has_node(lemma_b) and not graph.has_edge(edge):\n            graph.add_edge(edge)\n\n\ndef _process_first_window(graph, tokens, split_text):\n    first_window = _get_first_window(split_text)\n    for word_a, word_b in _combinations(first_window, 2):\n        _set_graph_edge(graph, tokens, word_a, word_b)\n\n\ndef _init_queue(split_text):\n    queue = _Queue()\n    first_window = _get_first_window(split_text)\n    for word in first_window[1:]:\n        queue.put(word)\n    return queue\n\n\ndef _process_word(graph, tokens, queue, word):\n    for word_to_compare in _queue_iterator(queue):\n        _set_graph_edge(graph, tokens, word, word_to_compare)\n\n\ndef _update_queue(queue, word):\n    queue.get()\n    queue.put(word)\n    assert queue.qsize() == (WINDOW_SIZE - 1)\n\n\ndef _process_text(graph, tokens, split_text):\n    queue = _init_queue(split_text)\n    for i in xrange(WINDOW_SIZE, len(split_text)):\n        word = split_text[i]\n        _process_word(graph, tokens, queue, word)\n        _update_queue(queue, word)\n\n\ndef _queue_iterator(queue):\n    iterations = queue.qsize()\n    for i in xrange(iterations):\n        var = queue.get()\n        yield var\n        queue.put(var)\n\n\ndef _set_graph_edges(graph, tokens, split_text):\n    _process_first_window(graph, tokens, split_text)\n    _process_text(graph, tokens, split_text)\n\n\ndef _extract_tokens(lemmas, scores, ratio, words):\n    lemmas.sort(key=lambda s: scores[s], reverse=True)\n\n    # If no \"words\" option is selected, the number of sentences is\n    # reduced by the provided ratio, else, the ratio is ignored.\n    length = len(lemmas) * ratio if words is None else words\n    return [(scores[lemmas[i]], lemmas[i],) for i in range(int(length))]\n\n\ndef _lemmas_to_words(tokens):\n    lemma_to_word = {}\n    for word, unit in iteritems(tokens):\n        lemma = unit.token\n        if lemma in lemma_to_word:\n            lemma_to_word[lemma].append(word)\n        else:\n            lemma_to_word[lemma] = [word]\n    return lemma_to_word\n\n\ndef _get_keywords_with_score(extracted_lemmas, lemma_to_word):\n    \"\"\"\n    :param extracted_lemmas:list of tuples\n    :param lemma_to_word: dict of {lemma:list of words}\n    :return: dict of {keyword:score}\n    \"\"\"\n    keywords = {}\n    for score, lemma in extracted_lemmas:\n        keyword_list = lemma_to_word[lemma]\n        for keyword in keyword_list:\n            keywords[keyword] = score\n    return keywords\n\n\ndef _strip_word(word):\n    stripped_word_list = list(_tokenize_by_word(word))\n    return stripped_word_list[0] if stripped_word_list else \"\"\n\n\ndef _get_combined_keywords(_keywords, split_text):\n    \"\"\"\n    :param keywords:dict of keywords:scores\n    :param split_text: list of strings\n    :return: combined_keywords:list\n    \"\"\"\n    result = []\n    _keywords = _keywords.copy()\n    len_text = len(split_text)\n    for i in xrange(len_text):\n        word = _strip_word(split_text[i])\n        if word in _keywords:\n            combined_word = [word]\n            if i + 1 == len_text:\n                result.append(word)   # appends last word if keyword and doesn't iterate\n            for j in xrange(i + 1, len_text):\n                other_word = _strip_word(split_text[j])\n                if other_word in _keywords and other_word == split_text[j]:\n                    combined_word.append(other_word)\n                else:\n                    for keyword in combined_word:\n                        _keywords.pop(keyword)\n                    result.append(\" \".join(combined_word))\n                    break\n    return result\n\n\ndef _get_average_score(concept, _keywords):\n    word_list = concept.split()\n    word_counter = 0\n    total = 0\n    for word in word_list:\n        total += _keywords[word]\n        word_counter += 1\n    return total / word_counter\n\n\ndef _format_results(_keywords, combined_keywords, split, scores):\n    \"\"\"\n    :param keywords:dict of keywords:scores\n    :param combined_keywords:list of word/s\n    \"\"\"\n    combined_keywords.sort(key=lambda w: _get_average_score(w, _keywords), reverse=True)\n    if scores:\n        return [(word, _get_average_score(word, _keywords)) for word in combined_keywords]\n    if split:\n        return combined_keywords\n    return \"\\n\".join(combined_keywords)\n\n\ndef keywords(text, ratio=0.2, words=None, split=False, scores=False, pos_filter=['NN', 'JJ'], lemmatize=False):\n    # Gets a dict of word -> lemma\n    text = to_unicode(text)\n    tokens = _clean_text_by_word(text)\n    split_text = list(_tokenize_by_word(text))\n\n    # Creates the graph and adds the edges\n    graph = _build_graph(_get_words_for_graph(tokens, pos_filter))\n    _set_graph_edges(graph, tokens, split_text)\n    del split_text  # It's no longer used\n\n    _remove_unreachable_nodes(graph)\n\n    # Ranks the tokens using the PageRank algorithm. Returns dict of lemma -> score\n    pagerank_scores = _pagerank(graph)\n\n    extracted_lemmas = _extract_tokens(graph.nodes(), pagerank_scores, ratio, words)\n\n    # The results can be polluted by many variations of the same word\n    if lemmatize:\n        lemmas_to_word = {}\n        for word, unit in iteritems(tokens):\n            lemmas_to_word[unit.token] = [word]\n    else:\n        lemmas_to_word = _lemmas_to_words(tokens)\n\n    keywords = _get_keywords_with_score(extracted_lemmas, lemmas_to_word)\n\n    # text.split() to keep numbers and punctuation marks, so separeted concepts are not combined\n    combined_keywords = _get_combined_keywords(keywords, text.split())\n\n    return _format_results(keywords, combined_keywords, split, scores)\n\n\ndef get_graph(text):\n    tokens = _clean_text_by_word(text)\n    split_text = list(_tokenize_by_word(text))\n\n    graph = _build_graph(_get_words_for_graph(tokens))\n    _set_graph_edges(graph, tokens, split_text)\n\n    return graph\n", "path": "gensim/summarization/keywords.py"}], "after_files": [{"content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n#\n# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html\n\nfrom gensim.summarization.pagerank_weighted import pagerank_weighted as _pagerank\nfrom gensim.summarization.textcleaner import clean_text_by_word as _clean_text_by_word\nfrom gensim.summarization.textcleaner import tokenize_by_word as _tokenize_by_word\nfrom gensim.summarization.commons import build_graph as _build_graph\nfrom gensim.summarization.commons import remove_unreachable_nodes as _remove_unreachable_nodes\nfrom gensim.utils import to_unicode\nfrom itertools import combinations as _combinations\nfrom six.moves.queue import Queue as _Queue\nfrom six.moves import xrange\nfrom six import iteritems\n\n\nWINDOW_SIZE = 2\n\n\"\"\"\nCheck tags in http://www.clips.ua.ac.be/pages/mbsp-tags and use only first two letters\nExample: filter for nouns and adjectives:\nINCLUDING_FILTER = ['NN', 'JJ']\n\"\"\"\nINCLUDING_FILTER = ['NN', 'JJ']\nEXCLUDING_FILTER = []\n\n\ndef _get_pos_filters():\n    return frozenset(INCLUDING_FILTER), frozenset(EXCLUDING_FILTER)\n\n\ndef _get_words_for_graph(tokens, pos_filter):\n    if pos_filter is None:\n        include_filters, exclude_filters = _get_pos_filters()\n    else:\n        include_filters = set(pos_filter)\n        exclude_filters = frozenset([])\n    if include_filters and exclude_filters:\n        raise ValueError(\"Can't use both include and exclude filters, should use only one\")\n\n    result = []\n    for word, unit in iteritems(tokens):\n        if exclude_filters and unit.tag in exclude_filters:\n            continue\n        if (include_filters and unit.tag in include_filters) or not include_filters or not unit.tag:\n            result.append(unit.token)\n    return result\n\n\ndef _get_first_window(split_text):\n    return split_text[:WINDOW_SIZE]\n\n\ndef _set_graph_edge(graph, tokens, word_a, word_b):\n    if word_a in tokens and word_b in tokens:\n        lemma_a = tokens[word_a].token\n        lemma_b = tokens[word_b].token\n        edge = (lemma_a, lemma_b)\n\n        if graph.has_node(lemma_a) and graph.has_node(lemma_b) and not graph.has_edge(edge):\n            graph.add_edge(edge)\n\n\ndef _process_first_window(graph, tokens, split_text):\n    first_window = _get_first_window(split_text)\n    for word_a, word_b in _combinations(first_window, 2):\n        _set_graph_edge(graph, tokens, word_a, word_b)\n\n\ndef _init_queue(split_text):\n    queue = _Queue()\n    first_window = _get_first_window(split_text)\n    for word in first_window[1:]:\n        queue.put(word)\n    return queue\n\n\ndef _process_word(graph, tokens, queue, word):\n    for word_to_compare in _queue_iterator(queue):\n        _set_graph_edge(graph, tokens, word, word_to_compare)\n\n\ndef _update_queue(queue, word):\n    queue.get()\n    queue.put(word)\n    assert queue.qsize() == (WINDOW_SIZE - 1)\n\n\ndef _process_text(graph, tokens, split_text):\n    queue = _init_queue(split_text)\n    for i in xrange(WINDOW_SIZE, len(split_text)):\n        word = split_text[i]\n        _process_word(graph, tokens, queue, word)\n        _update_queue(queue, word)\n\n\ndef _queue_iterator(queue):\n    iterations = queue.qsize()\n    for i in xrange(iterations):\n        var = queue.get()\n        yield var\n        queue.put(var)\n\n\ndef _set_graph_edges(graph, tokens, split_text):\n    _process_first_window(graph, tokens, split_text)\n    _process_text(graph, tokens, split_text)\n\n\ndef _extract_tokens(lemmas, scores, ratio, words):\n    lemmas.sort(key=lambda s: scores[s], reverse=True)\n\n    # If no \"words\" option is selected, the number of sentences is\n    # reduced by the provided ratio, else, the ratio is ignored.\n    length = len(lemmas) * ratio if words is None else words\n    return [(scores[lemmas[i]], lemmas[i],) for i in range(int(length))]\n\n\ndef _lemmas_to_words(tokens):\n    lemma_to_word = {}\n    for word, unit in iteritems(tokens):\n        lemma = unit.token\n        if lemma in lemma_to_word:\n            lemma_to_word[lemma].append(word)\n        else:\n            lemma_to_word[lemma] = [word]\n    return lemma_to_word\n\n\ndef _get_keywords_with_score(extracted_lemmas, lemma_to_word):\n    \"\"\"\n    :param extracted_lemmas:list of tuples\n    :param lemma_to_word: dict of {lemma:list of words}\n    :return: dict of {keyword:score}\n    \"\"\"\n    keywords = {}\n    for score, lemma in extracted_lemmas:\n        keyword_list = lemma_to_word[lemma]\n        for keyword in keyword_list:\n            keywords[keyword] = score\n    return keywords\n\n\ndef _strip_word(word):\n    stripped_word_list = list(_tokenize_by_word(word))\n    return stripped_word_list[0] if stripped_word_list else \"\"\n\n\ndef _get_combined_keywords(_keywords, split_text):\n    \"\"\"\n    :param keywords:dict of keywords:scores\n    :param split_text: list of strings\n    :return: combined_keywords:list\n    \"\"\"\n    result = []\n    _keywords = _keywords.copy()\n    len_text = len(split_text)\n    for i in xrange(len_text):\n        word = _strip_word(split_text[i])\n        if word in _keywords:\n            combined_word = [word]\n            if i + 1 == len_text:\n                result.append(word)   # appends last word if keyword and doesn't iterate\n            for j in xrange(i + 1, len_text):\n                other_word = _strip_word(split_text[j])\n                if other_word in _keywords and other_word == split_text[j] and not other_word in combined_word:\n                    combined_word.append(other_word)\n                else:\n                    for keyword in combined_word:\n                        _keywords.pop(keyword)\n                    result.append(\" \".join(combined_word))\n                    break\n    return result\n\n\ndef _get_average_score(concept, _keywords):\n    word_list = concept.split()\n    word_counter = 0\n    total = 0\n    for word in word_list:\n        total += _keywords[word]\n        word_counter += 1\n    return total / word_counter\n\n\ndef _format_results(_keywords, combined_keywords, split, scores):\n    \"\"\"\n    :param keywords:dict of keywords:scores\n    :param combined_keywords:list of word/s\n    \"\"\"\n    combined_keywords.sort(key=lambda w: _get_average_score(w, _keywords), reverse=True)\n    if scores:\n        return [(word, _get_average_score(word, _keywords)) for word in combined_keywords]\n    if split:\n        return combined_keywords\n    return \"\\n\".join(combined_keywords)\n\n\ndef keywords(text, ratio=0.2, words=None, split=False, scores=False, pos_filter=['NN', 'JJ'], lemmatize=False):\n    # Gets a dict of word -> lemma\n    text = to_unicode(text)\n    tokens = _clean_text_by_word(text)\n    split_text = list(_tokenize_by_word(text))\n\n    # Creates the graph and adds the edges\n    graph = _build_graph(_get_words_for_graph(tokens, pos_filter))\n    _set_graph_edges(graph, tokens, split_text)\n    del split_text  # It's no longer used\n\n    _remove_unreachable_nodes(graph)\n\n    # Ranks the tokens using the PageRank algorithm. Returns dict of lemma -> score\n    pagerank_scores = _pagerank(graph)\n\n    extracted_lemmas = _extract_tokens(graph.nodes(), pagerank_scores, ratio, words)\n\n    # The results can be polluted by many variations of the same word\n    if lemmatize:\n        lemmas_to_word = {}\n        for word, unit in iteritems(tokens):\n            lemmas_to_word[unit.token] = [word]\n    else:\n        lemmas_to_word = _lemmas_to_words(tokens)\n\n    keywords = _get_keywords_with_score(extracted_lemmas, lemmas_to_word)\n\n    # text.split() to keep numbers and punctuation marks, so separeted concepts are not combined\n    combined_keywords = _get_combined_keywords(keywords, text.split())\n\n    return _format_results(keywords, combined_keywords, split, scores)\n\n\ndef get_graph(text):\n    tokens = _clean_text_by_word(text)\n    split_text = list(_tokenize_by_word(text))\n\n    graph = _build_graph(_get_words_for_graph(tokens))\n    _set_graph_edges(graph, tokens, split_text)\n\n    return graph\n", "path": "gensim/summarization/keywords.py"}]}