We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
dict.iteritems() usage breaks "keywords" function in python 3
Calling `gensim.summarization.keywords` in python3 results in `AttributeError: 'dict' object has no attribute 'iteritems'`.

Looks like there are offenders [here](https://github.com/piskvorky/gensim/blob/485d024e119d0ff818ea6651042748a3bff21081/gensim/summarization/keywords.py#L35) and [here](https://github.com/piskvorky/gensim/blob/485d024e119d0ff818ea6651042748a3bff21081/gensim/summarization/keywords.py#L114). I assume these should be fixed with six.iteritems?


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `gensim/summarization/keywords.py`
Content:
```
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3 #
4 # Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html
5 
6 from gensim.summarization.pagerank_weighted import pagerank_weighted as _pagerank
7 from gensim.summarization.textcleaner import clean_text_by_word as _clean_text_by_word
8 from gensim.summarization.textcleaner import tokenize_by_word as _tokenize_by_word
9 from gensim.summarization.commons import build_graph as _build_graph
10 from gensim.summarization.commons import remove_unreachable_nodes as _remove_unreachable_nodes
11 from itertools import combinations as _combinations
12 from six.moves.queue import Queue as _Queue
13 from six.moves import xrange
14 
15 
16 WINDOW_SIZE = 2
17 
18 """Check tags in http://www.clips.ua.ac.be/pages/mbsp-tags and use only first two letters
19 Example: filter for nouns and adjectives:
20 INCLUDING_FILTER = ['NN', 'JJ']"""
21 INCLUDING_FILTER = ['NN', 'JJ']
22 EXCLUDING_FILTER = []
23 
24 
25 def _get_pos_filters():
26     return frozenset(INCLUDING_FILTER), frozenset(EXCLUDING_FILTER)
27 
28 
29 def _get_words_for_graph(tokens):
30     include_filters, exclude_filters = _get_pos_filters()
31     if include_filters and exclude_filters:
32         raise ValueError("Can't use both include and exclude filters, should use only one")
33 
34     result = []
35     for word, unit in tokens.iteritems():
36         if exclude_filters and unit.tag in exclude_filters:
37             continue
38         if (include_filters and unit.tag in include_filters) or not include_filters or not unit.tag:
39             result.append(unit.token)
40     return result
41 
42 
43 def _get_first_window(split_text):
44     return split_text[:WINDOW_SIZE]
45 
46 
47 def _set_graph_edge(graph, tokens, word_a, word_b):
48     if word_a in tokens and word_b in tokens:
49         lemma_a = tokens[word_a].token
50         lemma_b = tokens[word_b].token
51         edge = (lemma_a, lemma_b)
52 
53         if graph.has_node(lemma_a) and graph.has_node(lemma_b) and not graph.has_edge(edge):
54             graph.add_edge(edge)
55 
56 
57 def _process_first_window(graph, tokens, split_text):
58     first_window = _get_first_window(split_text)
59     for word_a, word_b in _combinations(first_window, 2):
60         _set_graph_edge(graph, tokens, word_a, word_b)
61 
62 
63 def _init_queue(split_text):
64     queue = _Queue()
65     first_window = _get_first_window(split_text)
66     for word in first_window[1:]:
67         queue.put(word)
68     return queue
69 
70 
71 def _process_word(graph, tokens, queue, word):
72     for word_to_compare in _queue_iterator(queue):
73         _set_graph_edge(graph, tokens, word, word_to_compare)
74 
75 
76 def _update_queue(queue, word):
77     queue.get()
78     queue.put(word)
79     assert queue.qsize() == (WINDOW_SIZE - 1)
80 
81 
82 def _process_text(graph, tokens, split_text):
83     queue = _init_queue(split_text)
84     for i in xrange(WINDOW_SIZE, len(split_text)):
85         word = split_text[i]
86         _process_word(graph, tokens, queue, word)
87         _update_queue(queue, word)
88 
89 
90 def _queue_iterator(queue):
91     iterations = queue.qsize()
92     for i in xrange(iterations):
93         var = queue.get()
94         yield var
95         queue.put(var)
96 
97 
98 def _set_graph_edges(graph, tokens, split_text):
99     _process_first_window(graph, tokens, split_text)
100     _process_text(graph, tokens, split_text)
101 
102 
103 def _extract_tokens(lemmas, scores, ratio, words):
104     lemmas.sort(key=lambda s: scores[s], reverse=True)
105 
106     # If no "words" option is selected, the number of sentences is
107     # reduced by the provided ratio, else, the ratio is ignored.
108     length = len(lemmas) * ratio if words is None else words
109     return [(scores[lemmas[i]], lemmas[i],) for i in range(int(length))]
110 
111 
112 def _lemmas_to_words(tokens):
113     lemma_to_word = {}
114     for word, unit in tokens.iteritems():
115         lemma = unit.token
116         if lemma in lemma_to_word:
117             lemma_to_word[lemma].append(word)
118         else:
119             lemma_to_word[lemma] = [word]
120     return lemma_to_word
121 
122 
123 def _get_keywords_with_score(extracted_lemmas, lemma_to_word):
124     """
125     :param extracted_lemmas:list of tuples
126     :param lemma_to_word: dict of {lemma:list of words}
127     :return: dict of {keyword:score}
128     """
129     keywords = {}
130     for score, lemma in extracted_lemmas:
131         keyword_list = lemma_to_word[lemma]
132         for keyword in keyword_list:
133             keywords[keyword] = score
134     return keywords
135 
136 
137 def _strip_word(word):
138     stripped_word_list = list(_tokenize_by_word(word))
139     return stripped_word_list[0] if stripped_word_list else ""
140 
141 
142 def _get_combined_keywords(_keywords, split_text):
143     """
144     :param keywords:dict of keywords:scores
145     :param split_text: list of strings
146     :return: combined_keywords:list
147     """
148     result = []
149     _keywords = _keywords.copy()
150     len_text = len(split_text)
151     for i in xrange(len_text):
152         word = _strip_word(split_text[i])
153         if word in _keywords:
154             combined_word = [word]
155             if i + 1 == len_text:
156                 result.append(word)   # appends last word if keyword and doesn't iterate
157             for j in xrange(i + 1, len_text):
158                 other_word = _strip_word(split_text[j])
159                 if other_word in _keywords and other_word == split_text[j].decode("utf-8"):
160                     combined_word.append(other_word)
161                 else:
162                     for keyword in combined_word:
163                         _keywords.pop(keyword)
164                     result.append(" ".join(combined_word))
165                     break
166     return result
167 
168 
169 def _get_average_score(concept, _keywords):
170     word_list = concept.split()
171     word_counter = 0
172     total = 0
173     for word in word_list:
174         total += _keywords[word]
175         word_counter += 1
176     return total / word_counter
177 
178 
179 def _format_results(_keywords, combined_keywords, split, scores):
180     """
181     :param keywords:dict of keywords:scores
182     :param combined_keywords:list of word/s
183     """
184     combined_keywords.sort(key=lambda w: _get_average_score(w, _keywords), reverse=True)
185     if scores:
186         return [(word, _get_average_score(word, _keywords)) for word in combined_keywords]
187     if split:
188         return combined_keywords
189     return "\n".join(combined_keywords)
190 
191 
192 def keywords(text, ratio=0.2, words=None, split=False, scores=False):
193     # Gets a dict of word -> lemma
194     tokens = _clean_text_by_word(text)
195     split_text = list(_tokenize_by_word(text))
196 
197     # Creates the graph and adds the edges
198     graph = _build_graph(_get_words_for_graph(tokens))
199     _set_graph_edges(graph, tokens, split_text)
200     del split_text  # It's no longer used
201 
202     _remove_unreachable_nodes(graph)
203 
204     # Ranks the tokens using the PageRank algorithm. Returns dict of lemma -> score
205     pagerank_scores = _pagerank(graph)
206 
207     extracted_lemmas = _extract_tokens(graph.nodes(), pagerank_scores, ratio, words)
208 
209     lemmas_to_word = _lemmas_to_words(tokens)
210     keywords = _get_keywords_with_score(extracted_lemmas, lemmas_to_word)
211 
212     # text.split() to keep numbers and punctuation marks, so separeted concepts are not combined
213     combined_keywords = _get_combined_keywords(keywords, text.split())
214 
215     return _format_results(keywords, combined_keywords, split, scores)
216 
217 
218 def get_graph(text):
219     tokens = _clean_text_by_word(text)
220     split_text = list(_tokenize_by_word(text))
221 
222     graph = _build_graph(_get_words_for_graph(tokens))
223     _set_graph_edges(graph, tokens, split_text)
224 
225     return graph
226
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/gensim/summarization/keywords.py b/gensim/summarization/keywords.py
--- a/gensim/summarization/keywords.py
+++ b/gensim/summarization/keywords.py
@@ -8,9 +8,11 @@
 from gensim.summarization.textcleaner import tokenize_by_word as _tokenize_by_word
 from gensim.summarization.commons import build_graph as _build_graph
 from gensim.summarization.commons import remove_unreachable_nodes as _remove_unreachable_nodes
+from gensim.utils import to_unicode
 from itertools import combinations as _combinations
 from six.moves.queue import Queue as _Queue
 from six.moves import xrange
+from six import iteritems
 
 
 WINDOW_SIZE = 2
@@ -32,7 +34,7 @@
         raise ValueError("Can't use both include and exclude filters, should use only one")
 
     result = []
-    for word, unit in tokens.iteritems():
+    for word, unit in iteritems(tokens):
         if exclude_filters and unit.tag in exclude_filters:
             continue
         if (include_filters and unit.tag in include_filters) or not include_filters or not unit.tag:
@@ -111,7 +113,7 @@
 
 def _lemmas_to_words(tokens):
     lemma_to_word = {}
-    for word, unit in tokens.iteritems():
+    for word, unit in iteritems(tokens):
         lemma = unit.token
         if lemma in lemma_to_word:
             lemma_to_word[lemma].append(word)
@@ -156,7 +158,7 @@
                 result.append(word)   # appends last word if keyword and doesn't iterate
             for j in xrange(i + 1, len_text):
                 other_word = _strip_word(split_text[j])
-                if other_word in _keywords and other_word == split_text[j].decode("utf-8"):
+                if other_word in _keywords and other_word == split_text[j]:
                     combined_word.append(other_word)
                 else:
                     for keyword in combined_word:
@@ -191,6 +193,7 @@
 
 def keywords(text, ratio=0.2, words=None, split=False, scores=False):
     # Gets a dict of word -> lemma
+    text = to_unicode(text)
     tokens = _clean_text_by_word(text)
     split_text = list(_tokenize_by_word(text))

{"golden_diff": "diff --git a/gensim/summarization/keywords.py b/gensim/summarization/keywords.py\n--- a/gensim/summarization/keywords.py\n+++ b/gensim/summarization/keywords.py\n@@ -8,9 +8,11 @@\n from gensim.summarization.textcleaner import tokenize_by_word as _tokenize_by_word\n from gensim.summarization.commons import build_graph as _build_graph\n from gensim.summarization.commons import remove_unreachable_nodes as _remove_unreachable_nodes\n+from gensim.utils import to_unicode\n from itertools import combinations as _combinations\n from six.moves.queue import Queue as _Queue\n from six.moves import xrange\n+from six import iteritems\n \n \n WINDOW_SIZE = 2\n@@ -32,7 +34,7 @@\n         raise ValueError(\"Can't use both include and exclude filters, should use only one\")\n \n     result = []\n-    for word, unit in tokens.iteritems():\n+    for word, unit in iteritems(tokens):\n         if exclude_filters and unit.tag in exclude_filters:\n             continue\n         if (include_filters and unit.tag in include_filters) or not include_filters or not unit.tag:\n@@ -111,7 +113,7 @@\n \n def _lemmas_to_words(tokens):\n     lemma_to_word = {}\n-    for word, unit in tokens.iteritems():\n+    for word, unit in iteritems(tokens):\n         lemma = unit.token\n         if lemma in lemma_to_word:\n             lemma_to_word[lemma].append(word)\n@@ -156,7 +158,7 @@\n                 result.append(word)   # appends last word if keyword and doesn't iterate\n             for j in xrange(i + 1, len_text):\n                 other_word = _strip_word(split_text[j])\n-                if other_word in _keywords and other_word == split_text[j].decode(\"utf-8\"):\n+                if other_word in _keywords and other_word == split_text[j]:\n                     combined_word.append(other_word)\n                 else:\n                     for keyword in combined_word:\n@@ -191,6 +193,7 @@\n \n def keywords(text, ratio=0.2, words=None, split=False, scores=False):\n     # Gets a dict of word -> lemma\n+    text = to_unicode(text)\n     tokens = _clean_text_by_word(text)\n     split_text = list(_tokenize_by_word(text))\n", "issue": "dict.iteritems() usage breaks \"keywords\" function in python 3\nCalling `gensim.summarization.keywords` in python3 results in `AttributeError: 'dict' object has no attribute 'iteritems'`.\n\nLooks like there are offenders [here](https://github.com/piskvorky/gensim/blob/485d024e119d0ff818ea6651042748a3bff21081/gensim/summarization/keywords.py#L35) and [here](https://github.com/piskvorky/gensim/blob/485d024e119d0ff818ea6651042748a3bff21081/gensim/summarization/keywords.py#L114). I assume these should be fixed with six.iteritems?\n\n", "before_files": [{"content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n#\n# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html\n\nfrom gensim.summarization.pagerank_weighted import pagerank_weighted as _pagerank\nfrom gensim.summarization.textcleaner import clean_text_by_word as _clean_text_by_word\nfrom gensim.summarization.textcleaner import tokenize_by_word as _tokenize_by_word\nfrom gensim.summarization.commons import build_graph as _build_graph\nfrom gensim.summarization.commons import remove_unreachable_nodes as _remove_unreachable_nodes\nfrom itertools import combinations as _combinations\nfrom six.moves.queue import Queue as _Queue\nfrom six.moves import xrange\n\n\nWINDOW_SIZE = 2\n\n\"\"\"Check tags in http://www.clips.ua.ac.be/pages/mbsp-tags and use only first two letters\nExample: filter for nouns and adjectives:\nINCLUDING_FILTER = ['NN', 'JJ']\"\"\"\nINCLUDING_FILTER = ['NN', 'JJ']\nEXCLUDING_FILTER = []\n\n\ndef _get_pos_filters():\n    return frozenset(INCLUDING_FILTER), frozenset(EXCLUDING_FILTER)\n\n\ndef _get_words_for_graph(tokens):\n    include_filters, exclude_filters = _get_pos_filters()\n    if include_filters and exclude_filters:\n        raise ValueError(\"Can't use both include and exclude filters, should use only one\")\n\n    result = []\n    for word, unit in tokens.iteritems():\n        if exclude_filters and unit.tag in exclude_filters:\n            continue\n        if (include_filters and unit.tag in include_filters) or not include_filters or not unit.tag:\n            result.append(unit.token)\n    return result\n\n\ndef _get_first_window(split_text):\n    return split_text[:WINDOW_SIZE]\n\n\ndef _set_graph_edge(graph, tokens, word_a, word_b):\n    if word_a in tokens and word_b in tokens:\n        lemma_a = tokens[word_a].token\n        lemma_b = tokens[word_b].token\n        edge = (lemma_a, lemma_b)\n\n        if graph.has_node(lemma_a) and graph.has_node(lemma_b) and not graph.has_edge(edge):\n            graph.add_edge(edge)\n\n\ndef _process_first_window(graph, tokens, split_text):\n    first_window = _get_first_window(split_text)\n    for word_a, word_b in _combinations(first_window, 2):\n        _set_graph_edge(graph, tokens, word_a, word_b)\n\n\ndef _init_queue(split_text):\n    queue = _Queue()\n    first_window = _get_first_window(split_text)\n    for word in first_window[1:]:\n        queue.put(word)\n    return queue\n\n\ndef _process_word(graph, tokens, queue, word):\n    for word_to_compare in _queue_iterator(queue):\n        _set_graph_edge(graph, tokens, word, word_to_compare)\n\n\ndef _update_queue(queue, word):\n    queue.get()\n    queue.put(word)\n    assert queue.qsize() == (WINDOW_SIZE - 1)\n\n\ndef _process_text(graph, tokens, split_text):\n    queue = _init_queue(split_text)\n    for i in xrange(WINDOW_SIZE, len(split_text)):\n        word = split_text[i]\n        _process_word(graph, tokens, queue, word)\n        _update_queue(queue, word)\n\n\ndef _queue_iterator(queue):\n    iterations = queue.qsize()\n    for i in xrange(iterations):\n        var = queue.get()\n        yield var\n        queue.put(var)\n\n\ndef _set_graph_edges(graph, tokens, split_text):\n    _process_first_window(graph, tokens, split_text)\n    _process_text(graph, tokens, split_text)\n\n\ndef _extract_tokens(lemmas, scores, ratio, words):\n    lemmas.sort(key=lambda s: scores[s], reverse=True)\n\n    # If no \"words\" option is selected, the number of sentences is\n    # reduced by the provided ratio, else, the ratio is ignored.\n    length = len(lemmas) * ratio if words is None else words\n    return [(scores[lemmas[i]], lemmas[i],) for i in range(int(length))]\n\n\ndef _lemmas_to_words(tokens):\n    lemma_to_word = {}\n    for word, unit in tokens.iteritems():\n        lemma = unit.token\n        if lemma in lemma_to_word:\n            lemma_to_word[lemma].append(word)\n        else:\n            lemma_to_word[lemma] = [word]\n    return lemma_to_word\n\n\ndef _get_keywords_with_score(extracted_lemmas, lemma_to_word):\n    \"\"\"\n    :param extracted_lemmas:list of tuples\n    :param lemma_to_word: dict of {lemma:list of words}\n    :return: dict of {keyword:score}\n    \"\"\"\n    keywords = {}\n    for score, lemma in extracted_lemmas:\n        keyword_list = lemma_to_word[lemma]\n        for keyword in keyword_list:\n            keywords[keyword] = score\n    return keywords\n\n\ndef _strip_word(word):\n    stripped_word_list = list(_tokenize_by_word(word))\n    return stripped_word_list[0] if stripped_word_list else \"\"\n\n\ndef _get_combined_keywords(_keywords, split_text):\n    \"\"\"\n    :param keywords:dict of keywords:scores\n    :param split_text: list of strings\n    :return: combined_keywords:list\n    \"\"\"\n    result = []\n    _keywords = _keywords.copy()\n    len_text = len(split_text)\n    for i in xrange(len_text):\n        word = _strip_word(split_text[i])\n        if word in _keywords:\n            combined_word = [word]\n            if i + 1 == len_text:\n                result.append(word)   # appends last word if keyword and doesn't iterate\n            for j in xrange(i + 1, len_text):\n                other_word = _strip_word(split_text[j])\n                if other_word in _keywords and other_word == split_text[j].decode(\"utf-8\"):\n                    combined_word.append(other_word)\n                else:\n                    for keyword in combined_word:\n                        _keywords.pop(keyword)\n                    result.append(\" \".join(combined_word))\n                    break\n    return result\n\n\ndef _get_average_score(concept, _keywords):\n    word_list = concept.split()\n    word_counter = 0\n    total = 0\n    for word in word_list:\n        total += _keywords[word]\n        word_counter += 1\n    return total / word_counter\n\n\ndef _format_results(_keywords, combined_keywords, split, scores):\n    \"\"\"\n    :param keywords:dict of keywords:scores\n    :param combined_keywords:list of word/s\n    \"\"\"\n    combined_keywords.sort(key=lambda w: _get_average_score(w, _keywords), reverse=True)\n    if scores:\n        return [(word, _get_average_score(word, _keywords)) for word in combined_keywords]\n    if split:\n        return combined_keywords\n    return \"\\n\".join(combined_keywords)\n\n\ndef keywords(text, ratio=0.2, words=None, split=False, scores=False):\n    # Gets a dict of word -> lemma\n    tokens = _clean_text_by_word(text)\n    split_text = list(_tokenize_by_word(text))\n\n    # Creates the graph and adds the edges\n    graph = _build_graph(_get_words_for_graph(tokens))\n    _set_graph_edges(graph, tokens, split_text)\n    del split_text  # It's no longer used\n\n    _remove_unreachable_nodes(graph)\n\n    # Ranks the tokens using the PageRank algorithm. Returns dict of lemma -> score\n    pagerank_scores = _pagerank(graph)\n\n    extracted_lemmas = _extract_tokens(graph.nodes(), pagerank_scores, ratio, words)\n\n    lemmas_to_word = _lemmas_to_words(tokens)\n    keywords = _get_keywords_with_score(extracted_lemmas, lemmas_to_word)\n\n    # text.split() to keep numbers and punctuation marks, so separeted concepts are not combined\n    combined_keywords = _get_combined_keywords(keywords, text.split())\n\n    return _format_results(keywords, combined_keywords, split, scores)\n\n\ndef get_graph(text):\n    tokens = _clean_text_by_word(text)\n    split_text = list(_tokenize_by_word(text))\n\n    graph = _build_graph(_get_words_for_graph(tokens))\n    _set_graph_edges(graph, tokens, split_text)\n\n    return graph\n", "path": "gensim/summarization/keywords.py"}], "after_files": [{"content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n#\n# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html\n\nfrom gensim.summarization.pagerank_weighted import pagerank_weighted as _pagerank\nfrom gensim.summarization.textcleaner import clean_text_by_word as _clean_text_by_word\nfrom gensim.summarization.textcleaner import tokenize_by_word as _tokenize_by_word\nfrom gensim.summarization.commons import build_graph as _build_graph\nfrom gensim.summarization.commons import remove_unreachable_nodes as _remove_unreachable_nodes\nfrom gensim.utils import to_unicode\nfrom itertools import combinations as _combinations\nfrom six.moves.queue import Queue as _Queue\nfrom six.moves import xrange\nfrom six import iteritems\n\n\nWINDOW_SIZE = 2\n\n\"\"\"Check tags in http://www.clips.ua.ac.be/pages/mbsp-tags and use only first two letters\nExample: filter for nouns and adjectives:\nINCLUDING_FILTER = ['NN', 'JJ']\"\"\"\nINCLUDING_FILTER = ['NN', 'JJ']\nEXCLUDING_FILTER = []\n\n\ndef _get_pos_filters():\n    return frozenset(INCLUDING_FILTER), frozenset(EXCLUDING_FILTER)\n\n\ndef _get_words_for_graph(tokens):\n    include_filters, exclude_filters = _get_pos_filters()\n    if include_filters and exclude_filters:\n        raise ValueError(\"Can't use both include and exclude filters, should use only one\")\n\n    result = []\n    for word, unit in iteritems(tokens):\n        if exclude_filters and unit.tag in exclude_filters:\n            continue\n        if (include_filters and unit.tag in include_filters) or not include_filters or not unit.tag:\n            result.append(unit.token)\n    return result\n\n\ndef _get_first_window(split_text):\n    return split_text[:WINDOW_SIZE]\n\n\ndef _set_graph_edge(graph, tokens, word_a, word_b):\n    if word_a in tokens and word_b in tokens:\n        lemma_a = tokens[word_a].token\n        lemma_b = tokens[word_b].token\n        edge = (lemma_a, lemma_b)\n\n        if graph.has_node(lemma_a) and graph.has_node(lemma_b) and not graph.has_edge(edge):\n            graph.add_edge(edge)\n\n\ndef _process_first_window(graph, tokens, split_text):\n    first_window = _get_first_window(split_text)\n    for word_a, word_b in _combinations(first_window, 2):\n        _set_graph_edge(graph, tokens, word_a, word_b)\n\n\ndef _init_queue(split_text):\n    queue = _Queue()\n    first_window = _get_first_window(split_text)\n    for word in first_window[1:]:\n        queue.put(word)\n    return queue\n\n\ndef _process_word(graph, tokens, queue, word):\n    for word_to_compare in _queue_iterator(queue):\n        _set_graph_edge(graph, tokens, word, word_to_compare)\n\n\ndef _update_queue(queue, word):\n    queue.get()\n    queue.put(word)\n    assert queue.qsize() == (WINDOW_SIZE - 1)\n\n\ndef _process_text(graph, tokens, split_text):\n    queue = _init_queue(split_text)\n    for i in xrange(WINDOW_SIZE, len(split_text)):\n        word = split_text[i]\n        _process_word(graph, tokens, queue, word)\n        _update_queue(queue, word)\n\n\ndef _queue_iterator(queue):\n    iterations = queue.qsize()\n    for i in xrange(iterations):\n        var = queue.get()\n        yield var\n        queue.put(var)\n\n\ndef _set_graph_edges(graph, tokens, split_text):\n    _process_first_window(graph, tokens, split_text)\n    _process_text(graph, tokens, split_text)\n\n\ndef _extract_tokens(lemmas, scores, ratio, words):\n    lemmas.sort(key=lambda s: scores[s], reverse=True)\n\n    # If no \"words\" option is selected, the number of sentences is\n    # reduced by the provided ratio, else, the ratio is ignored.\n    length = len(lemmas) * ratio if words is None else words\n    return [(scores[lemmas[i]], lemmas[i],) for i in range(int(length))]\n\n\ndef _lemmas_to_words(tokens):\n    lemma_to_word = {}\n    for word, unit in iteritems(tokens):\n        lemma = unit.token\n        if lemma in lemma_to_word:\n            lemma_to_word[lemma].append(word)\n        else:\n            lemma_to_word[lemma] = [word]\n    return lemma_to_word\n\n\ndef _get_keywords_with_score(extracted_lemmas, lemma_to_word):\n    \"\"\"\n    :param extracted_lemmas:list of tuples\n    :param lemma_to_word: dict of {lemma:list of words}\n    :return: dict of {keyword:score}\n    \"\"\"\n    keywords = {}\n    for score, lemma in extracted_lemmas:\n        keyword_list = lemma_to_word[lemma]\n        for keyword in keyword_list:\n            keywords[keyword] = score\n    return keywords\n\n\ndef _strip_word(word):\n    stripped_word_list = list(_tokenize_by_word(word))\n    return stripped_word_list[0] if stripped_word_list else \"\"\n\n\ndef _get_combined_keywords(_keywords, split_text):\n    \"\"\"\n    :param keywords:dict of keywords:scores\n    :param split_text: list of strings\n    :return: combined_keywords:list\n    \"\"\"\n    result = []\n    _keywords = _keywords.copy()\n    len_text = len(split_text)\n    for i in xrange(len_text):\n        word = _strip_word(split_text[i])\n        if word in _keywords:\n            combined_word = [word]\n            if i + 1 == len_text:\n                result.append(word)   # appends last word if keyword and doesn't iterate\n            for j in xrange(i + 1, len_text):\n                other_word = _strip_word(split_text[j])\n                if other_word in _keywords and other_word == split_text[j]:\n                    combined_word.append(other_word)\n                else:\n                    for keyword in combined_word:\n                        _keywords.pop(keyword)\n                    result.append(\" \".join(combined_word))\n                    break\n    return result\n\n\ndef _get_average_score(concept, _keywords):\n    word_list = concept.split()\n    word_counter = 0\n    total = 0\n    for word in word_list:\n        total += _keywords[word]\n        word_counter += 1\n    return total / word_counter\n\n\ndef _format_results(_keywords, combined_keywords, split, scores):\n    \"\"\"\n    :param keywords:dict of keywords:scores\n    :param combined_keywords:list of word/s\n    \"\"\"\n    combined_keywords.sort(key=lambda w: _get_average_score(w, _keywords), reverse=True)\n    if scores:\n        return [(word, _get_average_score(word, _keywords)) for word in combined_keywords]\n    if split:\n        return combined_keywords\n    return \"\\n\".join(combined_keywords)\n\n\ndef keywords(text, ratio=0.2, words=None, split=False, scores=False):\n    # Gets a dict of word -> lemma\n    text = to_unicode(text)\n    tokens = _clean_text_by_word(text)\n    split_text = list(_tokenize_by_word(text))\n\n    # Creates the graph and adds the edges\n    graph = _build_graph(_get_words_for_graph(tokens))\n    _set_graph_edges(graph, tokens, split_text)\n    del split_text  # It's no longer used\n\n    _remove_unreachable_nodes(graph)\n\n    # Ranks the tokens using the PageRank algorithm. Returns dict of lemma -> score\n    pagerank_scores = _pagerank(graph)\n\n    extracted_lemmas = _extract_tokens(graph.nodes(), pagerank_scores, ratio, words)\n\n    lemmas_to_word = _lemmas_to_words(tokens)\n    keywords = _get_keywords_with_score(extracted_lemmas, lemmas_to_word)\n\n    # text.split() to keep numbers and punctuation marks, so separeted concepts are not combined\n    combined_keywords = _get_combined_keywords(keywords, text.split())\n\n    return _format_results(keywords, combined_keywords, split, scores)\n\n\ndef get_graph(text):\n    tokens = _clean_text_by_word(text)\n    split_text = list(_tokenize_by_word(text))\n\n    graph = _build_graph(_get_words_for_graph(tokens))\n    _set_graph_edges(graph, tokens, split_text)\n\n    return graph\n", "path": "gensim/summarization/keywords.py"}]}