We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
BM25 does not support generator as corpus
``__init__`` method in BM25 class takes a "list of list of str" as corpus instead of a generator. More precisely, this is what it looks like right now:

```
def __init__(self, corpus):
        """
        Parameters
        ----------
        corpus : list of list of str
            Given corpus.
        """
        self.corpus_size = len(corpus)
        self.avgdl = 0
        self.doc_freqs = []
        self.idf = {}
        self.doc_len = []
        self._initialize(corpus)
```
As we know, considering a generator instead of a list would be great to handle large collections of documents that do not fit in memory.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `gensim/summarization/bm25.py`
Content:
```
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3 #
4 # Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html
5 
6 """This module contains function of computing rank scores for documents in
7 corpus and helper class `BM25` used in calculations. Original algorithm
8 descibed in [1]_, also you may check Wikipedia page [2]_.
9 
10 
11 .. [1] Robertson, Stephen; Zaragoza, Hugo (2009).  The Probabilistic Relevance Framework: BM25 and Beyond,
12        http://www.staff.city.ac.uk/~sb317/papers/foundations_bm25_review.pdf
13 .. [2] Okapi BM25 on Wikipedia, https://en.wikipedia.org/wiki/Okapi_BM25
14 
15 
16 
17 Examples
18 --------
19 
20 .. sourcecode:: pycon
21 
22     >>> from gensim.summarization.bm25 import get_bm25_weights
23     >>> corpus = [
24     ...     ["black", "cat", "white", "cat"],
25     ...     ["cat", "outer", "space"],
26     ...     ["wag", "dog"]
27     ... ]
28     >>> result = get_bm25_weights(corpus, n_jobs=-1)
29 
30 
31 Data:
32 -----
33 .. data:: PARAM_K1 - Free smoothing parameter for BM25.
34 .. data:: PARAM_B - Free smoothing parameter for BM25.
35 .. data:: EPSILON - Constant used for negative idf of document in corpus.
36 
37 """
38 
39 
40 import math
41 from six import iteritems
42 from six.moves import range
43 from functools import partial
44 from multiprocessing import Pool
45 from ..utils import effective_n_jobs
46 
47 PARAM_K1 = 1.5
48 PARAM_B = 0.75
49 EPSILON = 0.25
50 
51 
52 class BM25(object):
53     """Implementation of Best Matching 25 ranking function.
54 
55     Attributes
56     ----------
57     corpus_size : int
58         Size of corpus (number of documents).
59     avgdl : float
60         Average length of document in `corpus`.
61     doc_freqs : list of dicts of int
62         Dictionary with terms frequencies for each document in `corpus`. Words used as keys and frequencies as values.
63     idf : dict
64         Dictionary with inversed documents frequencies for whole `corpus`. Words used as keys and frequencies as values.
65     doc_len : list of int
66         List of document lengths.
67     """
68 
69     def __init__(self, corpus):
70         """
71         Parameters
72         ----------
73         corpus : list of list of str
74             Given corpus.
75 
76         """
77         self.corpus_size = len(corpus)
78         self.avgdl = 0
79         self.doc_freqs = []
80         self.idf = {}
81         self.doc_len = []
82         self._initialize(corpus)
83 
84     def _initialize(self, corpus):
85         """Calculates frequencies of terms in documents and in corpus. Also computes inverse document frequencies."""
86         nd = {}  # word -> number of documents with word
87         num_doc = 0
88         for document in corpus:
89             self.doc_len.append(len(document))
90             num_doc += len(document)
91 
92             frequencies = {}
93             for word in document:
94                 if word not in frequencies:
95                     frequencies[word] = 0
96                 frequencies[word] += 1
97             self.doc_freqs.append(frequencies)
98 
99             for word, freq in iteritems(frequencies):
100                 if word not in nd:
101                     nd[word] = 0
102                 nd[word] += 1
103 
104         self.avgdl = float(num_doc) / self.corpus_size
105         # collect idf sum to calculate an average idf for epsilon value
106         idf_sum = 0
107         # collect words with negative idf to set them a special epsilon value.
108         # idf can be negative if word is contained in more than half of documents
109         negative_idfs = []
110         for word, freq in iteritems(nd):
111             idf = math.log(self.corpus_size - freq + 0.5) - math.log(freq + 0.5)
112             self.idf[word] = idf
113             idf_sum += idf
114             if idf < 0:
115                 negative_idfs.append(word)
116         self.average_idf = float(idf_sum) / len(self.idf)
117 
118         eps = EPSILON * self.average_idf
119         for word in negative_idfs:
120             self.idf[word] = eps
121 
122     def get_score(self, document, index):
123         """Computes BM25 score of given `document` in relation to item of corpus selected by `index`.
124 
125         Parameters
126         ----------
127         document : list of str
128             Document to be scored.
129         index : int
130             Index of document in corpus selected to score with `document`.
131 
132         Returns
133         -------
134         float
135             BM25 score.
136 
137         """
138         score = 0
139         doc_freqs = self.doc_freqs[index]
140         for word in document:
141             if word not in doc_freqs:
142                 continue
143             score += (self.idf[word] * doc_freqs[word] * (PARAM_K1 + 1)
144                       / (doc_freqs[word] + PARAM_K1 * (1 - PARAM_B + PARAM_B * self.doc_len[index] / self.avgdl)))
145         return score
146 
147     def get_scores(self, document):
148         """Computes and returns BM25 scores of given `document` in relation to
149         every item in corpus.
150 
151         Parameters
152         ----------
153         document : list of str
154             Document to be scored.
155 
156         Returns
157         -------
158         list of float
159             BM25 scores.
160 
161         """
162         scores = [self.get_score(document, index) for index in range(self.corpus_size)]
163         return scores
164 
165     def get_scores_bow(self, document):
166         """Computes and returns BM25 scores of given `document` in relation to
167         every item in corpus.
168 
169         Parameters
170         ----------
171         document : list of str
172             Document to be scored.
173 
174         Returns
175         -------
176         list of float
177             BM25 scores.
178 
179         """
180         scores = []
181         for index in range(self.corpus_size):
182             score = self.get_score(document, index)
183             if score > 0:
184                 scores.append((index, score))
185         return scores
186 
187 
188 def _get_scores_bow(bm25, document):
189     """Helper function for retrieving bm25 scores of given `document` in parallel
190     in relation to every item in corpus.
191 
192     Parameters
193     ----------
194     bm25 : BM25 object
195         BM25 object fitted on the corpus where documents are retrieved.
196     document : list of str
197         Document to be scored.
198 
199     Returns
200     -------
201     list of (index, float)
202         BM25 scores in a bag of weights format.
203 
204     """
205     return bm25.get_scores_bow(document)
206 
207 
208 def _get_scores(bm25, document):
209     """Helper function for retrieving bm25 scores of given `document` in parallel
210     in relation to every item in corpus.
211 
212     Parameters
213     ----------
214     bm25 : BM25 object
215         BM25 object fitted on the corpus where documents are retrieved.
216     document : list of str
217         Document to be scored.
218 
219     Returns
220     -------
221     list of float
222         BM25 scores.
223 
224     """
225     return bm25.get_scores(document)
226 
227 
228 def iter_bm25_bow(corpus, n_jobs=1):
229     """Yield BM25 scores (weights) of documents in corpus.
230     Each document has to be weighted with every document in given corpus.
231 
232     Parameters
233     ----------
234     corpus : list of list of str
235         Corpus of documents.
236     n_jobs : int
237         The number of processes to use for computing bm25.
238 
239     Yields
240     -------
241     list of (index, float)
242         BM25 scores in bag of weights format.
243 
244     Examples
245     --------
246     .. sourcecode:: pycon
247 
248         >>> from gensim.summarization.bm25 import iter_bm25_weights
249         >>> corpus = [
250         ...     ["black", "cat", "white", "cat"],
251         ...     ["cat", "outer", "space"],
252         ...     ["wag", "dog"]
253         ... ]
254         >>> result = iter_bm25_weights(corpus, n_jobs=-1)
255 
256     """
257     bm25 = BM25(corpus)
258 
259     n_processes = effective_n_jobs(n_jobs)
260     if n_processes == 1:
261         for doc in corpus:
262             yield bm25.get_scores_bow(doc)
263         return
264 
265     get_score = partial(_get_scores_bow, bm25)
266     pool = Pool(n_processes)
267 
268     for bow in pool.imap(get_score, corpus):
269         yield bow
270     pool.close()
271     pool.join()
272 
273 
274 def get_bm25_weights(corpus, n_jobs=1):
275     """Returns BM25 scores (weights) of documents in corpus.
276     Each document has to be weighted with every document in given corpus.
277 
278     Parameters
279     ----------
280     corpus : list of list of str
281         Corpus of documents.
282     n_jobs : int
283         The number of processes to use for computing bm25.
284 
285     Returns
286     -------
287     list of list of float
288         BM25 scores.
289 
290     Examples
291     --------
292     .. sourcecode:: pycon
293 
294         >>> from gensim.summarization.bm25 import get_bm25_weights
295         >>> corpus = [
296         ...     ["black", "cat", "white", "cat"],
297         ...     ["cat", "outer", "space"],
298         ...     ["wag", "dog"]
299         ... ]
300         >>> result = get_bm25_weights(corpus, n_jobs=-1)
301 
302     """
303     bm25 = BM25(corpus)
304 
305     n_processes = effective_n_jobs(n_jobs)
306     if n_processes == 1:
307         weights = [bm25.get_scores(doc) for doc in corpus]
308         return weights
309 
310     get_score = partial(_get_scores, bm25)
311     pool = Pool(n_processes)
312     weights = pool.map(get_score, corpus)
313     pool.close()
314     pool.join()
315     return weights
316
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/gensim/summarization/bm25.py b/gensim/summarization/bm25.py
--- a/gensim/summarization/bm25.py
+++ b/gensim/summarization/bm25.py
@@ -74,7 +74,7 @@
             Given corpus.
 
         """
-        self.corpus_size = len(corpus)
+        self.corpus_size = 0
         self.avgdl = 0
         self.doc_freqs = []
         self.idf = {}
@@ -86,6 +86,7 @@
         nd = {}  # word -> number of documents with word
         num_doc = 0
         for document in corpus:
+            self.corpus_size += 1
             self.doc_len.append(len(document))
             num_doc += len(document)

{"golden_diff": "diff --git a/gensim/summarization/bm25.py b/gensim/summarization/bm25.py\n--- a/gensim/summarization/bm25.py\n+++ b/gensim/summarization/bm25.py\n@@ -74,7 +74,7 @@\n             Given corpus.\n \n         \"\"\"\n-        self.corpus_size = len(corpus)\n+        self.corpus_size = 0\n         self.avgdl = 0\n         self.doc_freqs = []\n         self.idf = {}\n@@ -86,6 +86,7 @@\n         nd = {}  # word -> number of documents with word\n         num_doc = 0\n         for document in corpus:\n+            self.corpus_size += 1\n             self.doc_len.append(len(document))\n             num_doc += len(document)\n", "issue": "BM25 does not support generator as corpus\n``__init__`` method in BM25 class takes a \"list of list of str\" as corpus instead of a generator. More precisely, this is what it looks like right now:\r\n\r\n```\r\ndef __init__(self, corpus):\r\n        \"\"\"\r\n        Parameters\r\n        ----------\r\n        corpus : list of list of str\r\n            Given corpus.\r\n        \"\"\"\r\n        self.corpus_size = len(corpus)\r\n        self.avgdl = 0\r\n        self.doc_freqs = []\r\n        self.idf = {}\r\n        self.doc_len = []\r\n        self._initialize(corpus)\r\n```\r\nAs we know, considering a generator instead of a list would be great to handle large collections of documents that do not fit in memory.\n", "before_files": [{"content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n#\n# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html\n\n\"\"\"This module contains function of computing rank scores for documents in\ncorpus and helper class `BM25` used in calculations. Original algorithm\ndescibed in [1]_, also you may check Wikipedia page [2]_.\n\n\n.. [1] Robertson, Stephen; Zaragoza, Hugo (2009).  The Probabilistic Relevance Framework: BM25 and Beyond,\n       http://www.staff.city.ac.uk/~sb317/papers/foundations_bm25_review.pdf\n.. [2] Okapi BM25 on Wikipedia, https://en.wikipedia.org/wiki/Okapi_BM25\n\n\n\nExamples\n--------\n\n.. sourcecode:: pycon\n\n    >>> from gensim.summarization.bm25 import get_bm25_weights\n    >>> corpus = [\n    ...     [\"black\", \"cat\", \"white\", \"cat\"],\n    ...     [\"cat\", \"outer\", \"space\"],\n    ...     [\"wag\", \"dog\"]\n    ... ]\n    >>> result = get_bm25_weights(corpus, n_jobs=-1)\n\n\nData:\n-----\n.. data:: PARAM_K1 - Free smoothing parameter for BM25.\n.. data:: PARAM_B - Free smoothing parameter for BM25.\n.. data:: EPSILON - Constant used for negative idf of document in corpus.\n\n\"\"\"\n\n\nimport math\nfrom six import iteritems\nfrom six.moves import range\nfrom functools import partial\nfrom multiprocessing import Pool\nfrom ..utils import effective_n_jobs\n\nPARAM_K1 = 1.5\nPARAM_B = 0.75\nEPSILON = 0.25\n\n\nclass BM25(object):\n    \"\"\"Implementation of Best Matching 25 ranking function.\n\n    Attributes\n    ----------\n    corpus_size : int\n        Size of corpus (number of documents).\n    avgdl : float\n        Average length of document in `corpus`.\n    doc_freqs : list of dicts of int\n        Dictionary with terms frequencies for each document in `corpus`. Words used as keys and frequencies as values.\n    idf : dict\n        Dictionary with inversed documents frequencies for whole `corpus`. Words used as keys and frequencies as values.\n    doc_len : list of int\n        List of document lengths.\n    \"\"\"\n\n    def __init__(self, corpus):\n        \"\"\"\n        Parameters\n        ----------\n        corpus : list of list of str\n            Given corpus.\n\n        \"\"\"\n        self.corpus_size = len(corpus)\n        self.avgdl = 0\n        self.doc_freqs = []\n        self.idf = {}\n        self.doc_len = []\n        self._initialize(corpus)\n\n    def _initialize(self, corpus):\n        \"\"\"Calculates frequencies of terms in documents and in corpus. Also computes inverse document frequencies.\"\"\"\n        nd = {}  # word -> number of documents with word\n        num_doc = 0\n        for document in corpus:\n            self.doc_len.append(len(document))\n            num_doc += len(document)\n\n            frequencies = {}\n            for word in document:\n                if word not in frequencies:\n                    frequencies[word] = 0\n                frequencies[word] += 1\n            self.doc_freqs.append(frequencies)\n\n            for word, freq in iteritems(frequencies):\n                if word not in nd:\n                    nd[word] = 0\n                nd[word] += 1\n\n        self.avgdl = float(num_doc) / self.corpus_size\n        # collect idf sum to calculate an average idf for epsilon value\n        idf_sum = 0\n        # collect words with negative idf to set them a special epsilon value.\n        # idf can be negative if word is contained in more than half of documents\n        negative_idfs = []\n        for word, freq in iteritems(nd):\n            idf = math.log(self.corpus_size - freq + 0.5) - math.log(freq + 0.5)\n            self.idf[word] = idf\n            idf_sum += idf\n            if idf < 0:\n                negative_idfs.append(word)\n        self.average_idf = float(idf_sum) / len(self.idf)\n\n        eps = EPSILON * self.average_idf\n        for word in negative_idfs:\n            self.idf[word] = eps\n\n    def get_score(self, document, index):\n        \"\"\"Computes BM25 score of given `document` in relation to item of corpus selected by `index`.\n\n        Parameters\n        ----------\n        document : list of str\n            Document to be scored.\n        index : int\n            Index of document in corpus selected to score with `document`.\n\n        Returns\n        -------\n        float\n            BM25 score.\n\n        \"\"\"\n        score = 0\n        doc_freqs = self.doc_freqs[index]\n        for word in document:\n            if word not in doc_freqs:\n                continue\n            score += (self.idf[word] * doc_freqs[word] * (PARAM_K1 + 1)\n                      / (doc_freqs[word] + PARAM_K1 * (1 - PARAM_B + PARAM_B * self.doc_len[index] / self.avgdl)))\n        return score\n\n    def get_scores(self, document):\n        \"\"\"Computes and returns BM25 scores of given `document` in relation to\n        every item in corpus.\n\n        Parameters\n        ----------\n        document : list of str\n            Document to be scored.\n\n        Returns\n        -------\n        list of float\n            BM25 scores.\n\n        \"\"\"\n        scores = [self.get_score(document, index) for index in range(self.corpus_size)]\n        return scores\n\n    def get_scores_bow(self, document):\n        \"\"\"Computes and returns BM25 scores of given `document` in relation to\n        every item in corpus.\n\n        Parameters\n        ----------\n        document : list of str\n            Document to be scored.\n\n        Returns\n        -------\n        list of float\n            BM25 scores.\n\n        \"\"\"\n        scores = []\n        for index in range(self.corpus_size):\n            score = self.get_score(document, index)\n            if score > 0:\n                scores.append((index, score))\n        return scores\n\n\ndef _get_scores_bow(bm25, document):\n    \"\"\"Helper function for retrieving bm25 scores of given `document` in parallel\n    in relation to every item in corpus.\n\n    Parameters\n    ----------\n    bm25 : BM25 object\n        BM25 object fitted on the corpus where documents are retrieved.\n    document : list of str\n        Document to be scored.\n\n    Returns\n    -------\n    list of (index, float)\n        BM25 scores in a bag of weights format.\n\n    \"\"\"\n    return bm25.get_scores_bow(document)\n\n\ndef _get_scores(bm25, document):\n    \"\"\"Helper function for retrieving bm25 scores of given `document` in parallel\n    in relation to every item in corpus.\n\n    Parameters\n    ----------\n    bm25 : BM25 object\n        BM25 object fitted on the corpus where documents are retrieved.\n    document : list of str\n        Document to be scored.\n\n    Returns\n    -------\n    list of float\n        BM25 scores.\n\n    \"\"\"\n    return bm25.get_scores(document)\n\n\ndef iter_bm25_bow(corpus, n_jobs=1):\n    \"\"\"Yield BM25 scores (weights) of documents in corpus.\n    Each document has to be weighted with every document in given corpus.\n\n    Parameters\n    ----------\n    corpus : list of list of str\n        Corpus of documents.\n    n_jobs : int\n        The number of processes to use for computing bm25.\n\n    Yields\n    -------\n    list of (index, float)\n        BM25 scores in bag of weights format.\n\n    Examples\n    --------\n    .. sourcecode:: pycon\n\n        >>> from gensim.summarization.bm25 import iter_bm25_weights\n        >>> corpus = [\n        ...     [\"black\", \"cat\", \"white\", \"cat\"],\n        ...     [\"cat\", \"outer\", \"space\"],\n        ...     [\"wag\", \"dog\"]\n        ... ]\n        >>> result = iter_bm25_weights(corpus, n_jobs=-1)\n\n    \"\"\"\n    bm25 = BM25(corpus)\n\n    n_processes = effective_n_jobs(n_jobs)\n    if n_processes == 1:\n        for doc in corpus:\n            yield bm25.get_scores_bow(doc)\n        return\n\n    get_score = partial(_get_scores_bow, bm25)\n    pool = Pool(n_processes)\n\n    for bow in pool.imap(get_score, corpus):\n        yield bow\n    pool.close()\n    pool.join()\n\n\ndef get_bm25_weights(corpus, n_jobs=1):\n    \"\"\"Returns BM25 scores (weights) of documents in corpus.\n    Each document has to be weighted with every document in given corpus.\n\n    Parameters\n    ----------\n    corpus : list of list of str\n        Corpus of documents.\n    n_jobs : int\n        The number of processes to use for computing bm25.\n\n    Returns\n    -------\n    list of list of float\n        BM25 scores.\n\n    Examples\n    --------\n    .. sourcecode:: pycon\n\n        >>> from gensim.summarization.bm25 import get_bm25_weights\n        >>> corpus = [\n        ...     [\"black\", \"cat\", \"white\", \"cat\"],\n        ...     [\"cat\", \"outer\", \"space\"],\n        ...     [\"wag\", \"dog\"]\n        ... ]\n        >>> result = get_bm25_weights(corpus, n_jobs=-1)\n\n    \"\"\"\n    bm25 = BM25(corpus)\n\n    n_processes = effective_n_jobs(n_jobs)\n    if n_processes == 1:\n        weights = [bm25.get_scores(doc) for doc in corpus]\n        return weights\n\n    get_score = partial(_get_scores, bm25)\n    pool = Pool(n_processes)\n    weights = pool.map(get_score, corpus)\n    pool.close()\n    pool.join()\n    return weights\n", "path": "gensim/summarization/bm25.py"}], "after_files": [{"content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n#\n# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html\n\n\"\"\"This module contains function of computing rank scores for documents in\ncorpus and helper class `BM25` used in calculations. Original algorithm\ndescibed in [1]_, also you may check Wikipedia page [2]_.\n\n\n.. [1] Robertson, Stephen; Zaragoza, Hugo (2009).  The Probabilistic Relevance Framework: BM25 and Beyond,\n       http://www.staff.city.ac.uk/~sb317/papers/foundations_bm25_review.pdf\n.. [2] Okapi BM25 on Wikipedia, https://en.wikipedia.org/wiki/Okapi_BM25\n\n\n\nExamples\n--------\n\n.. sourcecode:: pycon\n\n    >>> from gensim.summarization.bm25 import get_bm25_weights\n    >>> corpus = [\n    ...     [\"black\", \"cat\", \"white\", \"cat\"],\n    ...     [\"cat\", \"outer\", \"space\"],\n    ...     [\"wag\", \"dog\"]\n    ... ]\n    >>> result = get_bm25_weights(corpus, n_jobs=-1)\n\n\nData:\n-----\n.. data:: PARAM_K1 - Free smoothing parameter for BM25.\n.. data:: PARAM_B - Free smoothing parameter for BM25.\n.. data:: EPSILON - Constant used for negative idf of document in corpus.\n\n\"\"\"\n\n\nimport math\nfrom six import iteritems\nfrom six.moves import range\nfrom functools import partial\nfrom multiprocessing import Pool\nfrom ..utils import effective_n_jobs\n\nPARAM_K1 = 1.5\nPARAM_B = 0.75\nEPSILON = 0.25\n\n\nclass BM25(object):\n    \"\"\"Implementation of Best Matching 25 ranking function.\n\n    Attributes\n    ----------\n    corpus_size : int\n        Size of corpus (number of documents).\n    avgdl : float\n        Average length of document in `corpus`.\n    doc_freqs : list of dicts of int\n        Dictionary with terms frequencies for each document in `corpus`. Words used as keys and frequencies as values.\n    idf : dict\n        Dictionary with inversed documents frequencies for whole `corpus`. Words used as keys and frequencies as values.\n    doc_len : list of int\n        List of document lengths.\n    \"\"\"\n\n    def __init__(self, corpus):\n        \"\"\"\n        Parameters\n        ----------\n        corpus : list of list of str\n            Given corpus.\n\n        \"\"\"\n        self.corpus_size = 0\n        self.avgdl = 0\n        self.doc_freqs = []\n        self.idf = {}\n        self.doc_len = []\n        self._initialize(corpus)\n\n    def _initialize(self, corpus):\n        \"\"\"Calculates frequencies of terms in documents and in corpus. Also computes inverse document frequencies.\"\"\"\n        nd = {}  # word -> number of documents with word\n        num_doc = 0\n        for document in corpus:\n            self.corpus_size += 1\n            self.doc_len.append(len(document))\n            num_doc += len(document)\n\n            frequencies = {}\n            for word in document:\n                if word not in frequencies:\n                    frequencies[word] = 0\n                frequencies[word] += 1\n            self.doc_freqs.append(frequencies)\n\n            for word, freq in iteritems(frequencies):\n                if word not in nd:\n                    nd[word] = 0\n                nd[word] += 1\n\n        self.avgdl = float(num_doc) / self.corpus_size\n        # collect idf sum to calculate an average idf for epsilon value\n        idf_sum = 0\n        # collect words with negative idf to set them a special epsilon value.\n        # idf can be negative if word is contained in more than half of documents\n        negative_idfs = []\n        for word, freq in iteritems(nd):\n            idf = math.log(self.corpus_size - freq + 0.5) - math.log(freq + 0.5)\n            self.idf[word] = idf\n            idf_sum += idf\n            if idf < 0:\n                negative_idfs.append(word)\n        self.average_idf = float(idf_sum) / len(self.idf)\n\n        eps = EPSILON * self.average_idf\n        for word in negative_idfs:\n            self.idf[word] = eps\n\n    def get_score(self, document, index):\n        \"\"\"Computes BM25 score of given `document` in relation to item of corpus selected by `index`.\n\n        Parameters\n        ----------\n        document : list of str\n            Document to be scored.\n        index : int\n            Index of document in corpus selected to score with `document`.\n\n        Returns\n        -------\n        float\n            BM25 score.\n\n        \"\"\"\n        score = 0\n        doc_freqs = self.doc_freqs[index]\n        for word in document:\n            if word not in doc_freqs:\n                continue\n            score += (self.idf[word] * doc_freqs[word] * (PARAM_K1 + 1)\n                      / (doc_freqs[word] + PARAM_K1 * (1 - PARAM_B + PARAM_B * self.doc_len[index] / self.avgdl)))\n        return score\n\n    def get_scores(self, document):\n        \"\"\"Computes and returns BM25 scores of given `document` in relation to\n        every item in corpus.\n\n        Parameters\n        ----------\n        document : list of str\n            Document to be scored.\n\n        Returns\n        -------\n        list of float\n            BM25 scores.\n\n        \"\"\"\n        scores = [self.get_score(document, index) for index in range(self.corpus_size)]\n        return scores\n\n    def get_scores_bow(self, document):\n        \"\"\"Computes and returns BM25 scores of given `document` in relation to\n        every item in corpus.\n\n        Parameters\n        ----------\n        document : list of str\n            Document to be scored.\n\n        Returns\n        -------\n        list of float\n            BM25 scores.\n\n        \"\"\"\n        scores = []\n        for index in range(self.corpus_size):\n            score = self.get_score(document, index)\n            if score > 0:\n                scores.append((index, score))\n        return scores\n\n\ndef _get_scores_bow(bm25, document):\n    \"\"\"Helper function for retrieving bm25 scores of given `document` in parallel\n    in relation to every item in corpus.\n\n    Parameters\n    ----------\n    bm25 : BM25 object\n        BM25 object fitted on the corpus where documents are retrieved.\n    document : list of str\n        Document to be scored.\n\n    Returns\n    -------\n    list of (index, float)\n        BM25 scores in a bag of weights format.\n\n    \"\"\"\n    return bm25.get_scores_bow(document)\n\n\ndef _get_scores(bm25, document):\n    \"\"\"Helper function for retrieving bm25 scores of given `document` in parallel\n    in relation to every item in corpus.\n\n    Parameters\n    ----------\n    bm25 : BM25 object\n        BM25 object fitted on the corpus where documents are retrieved.\n    document : list of str\n        Document to be scored.\n\n    Returns\n    -------\n    list of float\n        BM25 scores.\n\n    \"\"\"\n    return bm25.get_scores(document)\n\n\ndef iter_bm25_bow(corpus, n_jobs=1):\n    \"\"\"Yield BM25 scores (weights) of documents in corpus.\n    Each document has to be weighted with every document in given corpus.\n\n    Parameters\n    ----------\n    corpus : list of list of str\n        Corpus of documents.\n    n_jobs : int\n        The number of processes to use for computing bm25.\n\n    Yields\n    -------\n    list of (index, float)\n        BM25 scores in bag of weights format.\n\n    Examples\n    --------\n    .. sourcecode:: pycon\n\n        >>> from gensim.summarization.bm25 import iter_bm25_weights\n        >>> corpus = [\n        ...     [\"black\", \"cat\", \"white\", \"cat\"],\n        ...     [\"cat\", \"outer\", \"space\"],\n        ...     [\"wag\", \"dog\"]\n        ... ]\n        >>> result = iter_bm25_weights(corpus, n_jobs=-1)\n\n    \"\"\"\n    bm25 = BM25(corpus)\n\n    n_processes = effective_n_jobs(n_jobs)\n    if n_processes == 1:\n        for doc in corpus:\n            yield bm25.get_scores_bow(doc)\n        return\n\n    get_score = partial(_get_scores_bow, bm25)\n    pool = Pool(n_processes)\n\n    for bow in pool.imap(get_score, corpus):\n        yield bow\n    pool.close()\n    pool.join()\n\n\ndef get_bm25_weights(corpus, n_jobs=1):\n    \"\"\"Returns BM25 scores (weights) of documents in corpus.\n    Each document has to be weighted with every document in given corpus.\n\n    Parameters\n    ----------\n    corpus : list of list of str\n        Corpus of documents.\n    n_jobs : int\n        The number of processes to use for computing bm25.\n\n    Returns\n    -------\n    list of list of float\n        BM25 scores.\n\n    Examples\n    --------\n    .. sourcecode:: pycon\n\n        >>> from gensim.summarization.bm25 import get_bm25_weights\n        >>> corpus = [\n        ...     [\"black\", \"cat\", \"white\", \"cat\"],\n        ...     [\"cat\", \"outer\", \"space\"],\n        ...     [\"wag\", \"dog\"]\n        ... ]\n        >>> result = get_bm25_weights(corpus, n_jobs=-1)\n\n    \"\"\"\n    bm25 = BM25(corpus)\n\n    n_processes = effective_n_jobs(n_jobs)\n    if n_processes == 1:\n        weights = [bm25.get_scores(doc) for doc in corpus]\n        return weights\n\n    get_score = partial(_get_scores, bm25)\n    pool = Pool(n_processes)\n    weights = pool.map(get_score, corpus)\n    pool.close()\n    pool.join()\n    return weights\n", "path": "gensim/summarization/bm25.py"}]}