We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
`datatype` parameter in `load_word2vec_format` doesn't work as expected
#### Description
Using `datatype=np.float64` in a `KeyedVectors.load_word2vec_call` doesn't work as expected, the loaded floats seem to lose precision. The datatype for `syn0` is still `float64` though, so it seems that they are cast to float32 first while loading, then cast to float64 when creating the array.

#### Steps/Code/Corpus to Reproduce
Using this file - 
[test.kv.txt](https://github.com/RaRe-Technologies/gensim/files/1434953/test.kv.txt)

```python
from gensim.models.keyedvectors import KeyedVectors
import numpy as np

kv = KeyedVectors.load_word2vec_format('test.kv.txt', datatype=np.float64)
print(kv['horse.n.01'][0] == -0.0008546282343595379)
# False
print(kv['horse.n.01'].dtype)
# float64
```
#### Expected Results
```python
print(kv['horse.n.01'][0] == -0.0008546282343595379)
# True
```
#### Actual Results
```python
print(kv['horse.n.01'][0] == -0.0008546282343595379)
# False
```

Looking at the code and making a quick hack [here](https://github.com/RaRe-Technologies/gensim/blob/develop/gensim/models/keyedvectors.py#L253), changing..
```
word, weights = parts[0], [REAL(x) for x in parts[1:]]
```
to..
```
word, weights = parts[0], [datatype(x) for x in parts[1:]]
```
..leads to the correct result. However, I imagine there are other cases to be covered as well.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `gensim/models/utils_any2vec.py`
Content:
```
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3 #
4 # Author: Shiva Manne <s.manne@rare-technologies.com>
5 # Copyright (C) 2018 RaRe Technologies s.r.o.
6 
7 """This module contains various general functions useful for any2vec models."""
8 
9 import logging
10 import numpy as np
11 from gensim import utils
12 
13 from numpy import zeros, dtype, float32 as REAL, ascontiguousarray, fromstring
14 
15 from six.moves import xrange
16 from six import iteritems
17 
18 logger = logging.getLogger(__name__)
19 
20 
21 def _compute_ngrams(word, min_n, max_n):
22     """Returns the list of all possible ngrams for a given word.
23 
24     Parameters
25     ----------
26     word : str
27         The word whose ngrams need to be computed
28     min_n : int
29         minimum character length of the ngrams
30     max_n : int
31         maximum character length of the ngrams
32 
33     Returns
34     -------
35     :obj:`list` of :obj:`str`
36         List of character ngrams
37 
38     """
39     BOW, EOW = ('<', '>')  # Used by FastText to attach to all words as prefix and suffix
40     extended_word = BOW + word + EOW
41     ngrams = []
42     for ngram_length in range(min_n, min(len(extended_word), max_n) + 1):
43         for i in range(0, len(extended_word) - ngram_length + 1):
44             ngrams.append(extended_word[i:i + ngram_length])
45     return ngrams
46 
47 
48 def _ft_hash(string):
49     """Reproduces [hash method](https://github.com/facebookresearch/fastText/blob/master/src/dictionary.cc)
50     used in [1]_.
51 
52     Parameter
53     ---------
54     string : str
55         The string whose hash needs to be calculated
56 
57     Returns
58     -------
59     int
60         The hash of the string
61 
62     """
63     # Runtime warnings for integer overflow are raised, this is expected behaviour. These warnings are suppressed.
64     old_settings = np.seterr(all='ignore')
65     h = np.uint32(2166136261)
66     for c in string:
67         h = h ^ np.uint32(ord(c))
68         h = h * np.uint32(16777619)
69     np.seterr(**old_settings)
70     return h
71 
72 
73 def _save_word2vec_format(fname, vocab, vectors, fvocab=None, binary=False, total_vec=None):
74         """Store the input-hidden weight matrix in the same format used by the original
75         C word2vec-tool, for compatibility.
76 
77         Parameters
78         ----------
79         fname : str
80             The file path used to save the vectors in
81         vocab : dict
82             The vocabulary of words
83         vectors : numpy.array
84             The vectors to be stored
85         fvocab : str
86             Optional file path used to save the vocabulary
87         binary : bool
88             If True, the data wil be saved in binary word2vec format, else it will be saved in plain text.
89         total_vec :  int
90             Optional parameter to explicitly specify total no. of vectors
91             (in case word vectors are appended with document vectors afterwards)
92 
93         """
94         if not (vocab or vectors):
95             raise RuntimeError("no input")
96         if total_vec is None:
97             total_vec = len(vocab)
98         vector_size = vectors.shape[1]
99         if fvocab is not None:
100             logger.info("storing vocabulary in %s", fvocab)
101             with utils.smart_open(fvocab, 'wb') as vout:
102                 for word, vocab_ in sorted(iteritems(vocab), key=lambda item: -item[1].count):
103                     vout.write(utils.to_utf8("%s %s\n" % (word, vocab_.count)))
104         logger.info("storing %sx%s projection weights into %s", total_vec, vector_size, fname)
105         assert (len(vocab), vector_size) == vectors.shape
106         with utils.smart_open(fname, 'wb') as fout:
107             fout.write(utils.to_utf8("%s %s\n" % (total_vec, vector_size)))
108             # store in sorted order: most frequent words at the top
109             for word, vocab_ in sorted(iteritems(vocab), key=lambda item: -item[1].count):
110                 row = vectors[vocab_.index]
111                 if binary:
112                     fout.write(utils.to_utf8(word) + b" " + row.tostring())
113                 else:
114                     fout.write(utils.to_utf8("%s %s\n" % (word, ' '.join("%f" % val for val in row))))
115 
116 
117 def _load_word2vec_format(cls, fname, fvocab=None, binary=False, encoding='utf8', unicode_errors='strict',
118                           limit=None, datatype=REAL):
119     """Load the input-hidden weight matrix from the original C word2vec-tool format.
120 
121     Note that the information stored in the file is incomplete (the binary tree is missing),
122     so while you can query for word similarity etc., you cannot continue training
123     with a model loaded this way.
124 
125     Parameters
126     ----------
127     fname : str
128         The file path to the saved word2vec-format file.
129     fvocab : str
130             Optional file path to the vocabulary.Word counts are read from `fvocab` filename,
131             if set (this is the file generated by `-save-vocab` flag of the original C tool).
132     binary : bool
133         If True, indicates whether the data is in binary word2vec format.
134     encoding : str
135         If you trained the C model using non-utf8 encoding for words, specify that
136         encoding in `encoding`.
137     unicode_errors : str
138         default 'strict', is a string suitable to be passed as the `errors`
139         argument to the unicode() (Python 2.x) or str() (Python 3.x) function. If your source
140         file may include word tokens truncated in the middle of a multibyte unicode character
141         (as is common from the original word2vec.c tool), 'ignore' or 'replace' may help.
142     limit : int
143         Sets a maximum number of word-vectors to read from the file. The default,
144         None, means read all.
145     datatype : :class: `numpy.float*`
146         (Experimental) Can coerce dimensions to a non-default float type (such
147         as np.float16) to save memory. (Such types may result in much slower bulk operations
148         or incompatibility with optimized routines.)
149 
150     Returns
151     -------
152     :obj: `cls`
153         Returns the loaded model as an instance of :class: `cls`.
154 
155     """
156     from gensim.models.keyedvectors import Vocab
157     counts = None
158     if fvocab is not None:
159         logger.info("loading word counts from %s", fvocab)
160         counts = {}
161         with utils.smart_open(fvocab) as fin:
162             for line in fin:
163                 word, count = utils.to_unicode(line).strip().split()
164                 counts[word] = int(count)
165 
166     logger.info("loading projection weights from %s", fname)
167     with utils.smart_open(fname) as fin:
168         header = utils.to_unicode(fin.readline(), encoding=encoding)
169         vocab_size, vector_size = (int(x) for x in header.split())  # throws for invalid file format
170         if limit:
171             vocab_size = min(vocab_size, limit)
172         result = cls(vector_size)
173         result.vector_size = vector_size
174         result.vectors = zeros((vocab_size, vector_size), dtype=datatype)
175 
176         def add_word(word, weights):
177             word_id = len(result.vocab)
178             if word in result.vocab:
179                 logger.warning("duplicate word '%s' in %s, ignoring all but first", word, fname)
180                 return
181             if counts is None:
182                 # most common scenario: no vocab file given. just make up some bogus counts, in descending order
183                 result.vocab[word] = Vocab(index=word_id, count=vocab_size - word_id)
184             elif word in counts:
185                 # use count from the vocab file
186                 result.vocab[word] = Vocab(index=word_id, count=counts[word])
187             else:
188                 # vocab file given, but word is missing -- set count to None (TODO: or raise?)
189                 logger.warning("vocabulary file is incomplete: '%s' is missing", word)
190                 result.vocab[word] = Vocab(index=word_id, count=None)
191             result.vectors[word_id] = weights
192             result.index2word.append(word)
193 
194         if binary:
195             binary_len = dtype(REAL).itemsize * vector_size
196             for _ in xrange(vocab_size):
197                 # mixed text and binary: read text first, then binary
198                 word = []
199                 while True:
200                     ch = fin.read(1)
201                     if ch == b' ':
202                         break
203                     if ch == b'':
204                         raise EOFError("unexpected end of input; is count incorrect or file otherwise damaged?")
205                     if ch != b'\n':  # ignore newlines in front of words (some binary files have)
206                         word.append(ch)
207                 word = utils.to_unicode(b''.join(word), encoding=encoding, errors=unicode_errors)
208                 weights = fromstring(fin.read(binary_len), dtype=REAL)
209                 add_word(word, weights)
210         else:
211             for line_no in xrange(vocab_size):
212                 line = fin.readline()
213                 if line == b'':
214                     raise EOFError("unexpected end of input; is count incorrect or file otherwise damaged?")
215                 parts = utils.to_unicode(line.rstrip(), encoding=encoding, errors=unicode_errors).split(" ")
216                 if len(parts) != vector_size + 1:
217                     raise ValueError("invalid vector on line %s (is this really the text format?)" % line_no)
218                 word, weights = parts[0], [REAL(x) for x in parts[1:]]
219                 add_word(word, weights)
220     if result.vectors.shape[0] != len(result.vocab):
221         logger.info(
222             "duplicate words detected, shrinking matrix size from %i to %i",
223             result.vectors.shape[0], len(result.vocab)
224         )
225         result.vectors = ascontiguousarray(result.vectors[: len(result.vocab)])
226     assert (len(result.vocab), vector_size) == result.vectors.shape
227 
228     logger.info("loaded %s matrix from %s", result.vectors.shape, fname)
229     return result
230
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/gensim/models/utils_any2vec.py b/gensim/models/utils_any2vec.py
--- a/gensim/models/utils_any2vec.py
+++ b/gensim/models/utils_any2vec.py
@@ -109,9 +109,10 @@
             for word, vocab_ in sorted(iteritems(vocab), key=lambda item: -item[1].count):
                 row = vectors[vocab_.index]
                 if binary:
+                    row = row.astype(REAL)
                     fout.write(utils.to_utf8(word) + b" " + row.tostring())
                 else:
-                    fout.write(utils.to_utf8("%s %s\n" % (word, ' '.join("%f" % val for val in row))))
+                    fout.write(utils.to_utf8("%s %s\n" % (word, ' '.join(repr(val) for val in row))))
 
 
 def _load_word2vec_format(cls, fname, fvocab=None, binary=False, encoding='utf8', unicode_errors='strict',
@@ -205,7 +206,7 @@
                     if ch != b'\n':  # ignore newlines in front of words (some binary files have)
                         word.append(ch)
                 word = utils.to_unicode(b''.join(word), encoding=encoding, errors=unicode_errors)
-                weights = fromstring(fin.read(binary_len), dtype=REAL)
+                weights = fromstring(fin.read(binary_len), dtype=REAL).astype(datatype)
                 add_word(word, weights)
         else:
             for line_no in xrange(vocab_size):
@@ -215,7 +216,7 @@
                 parts = utils.to_unicode(line.rstrip(), encoding=encoding, errors=unicode_errors).split(" ")
                 if len(parts) != vector_size + 1:
                     raise ValueError("invalid vector on line %s (is this really the text format?)" % line_no)
-                word, weights = parts[0], [REAL(x) for x in parts[1:]]
+                word, weights = parts[0], [datatype(x) for x in parts[1:]]
                 add_word(word, weights)
     if result.vectors.shape[0] != len(result.vocab):
         logger.info(

{"golden_diff": "diff --git a/gensim/models/utils_any2vec.py b/gensim/models/utils_any2vec.py\n--- a/gensim/models/utils_any2vec.py\n+++ b/gensim/models/utils_any2vec.py\n@@ -109,9 +109,10 @@\n             for word, vocab_ in sorted(iteritems(vocab), key=lambda item: -item[1].count):\n                 row = vectors[vocab_.index]\n                 if binary:\n+                    row = row.astype(REAL)\n                     fout.write(utils.to_utf8(word) + b\" \" + row.tostring())\n                 else:\n-                    fout.write(utils.to_utf8(\"%s %s\\n\" % (word, ' '.join(\"%f\" % val for val in row))))\n+                    fout.write(utils.to_utf8(\"%s %s\\n\" % (word, ' '.join(repr(val) for val in row))))\n \n \n def _load_word2vec_format(cls, fname, fvocab=None, binary=False, encoding='utf8', unicode_errors='strict',\n@@ -205,7 +206,7 @@\n                     if ch != b'\\n':  # ignore newlines in front of words (some binary files have)\n                         word.append(ch)\n                 word = utils.to_unicode(b''.join(word), encoding=encoding, errors=unicode_errors)\n-                weights = fromstring(fin.read(binary_len), dtype=REAL)\n+                weights = fromstring(fin.read(binary_len), dtype=REAL).astype(datatype)\n                 add_word(word, weights)\n         else:\n             for line_no in xrange(vocab_size):\n@@ -215,7 +216,7 @@\n                 parts = utils.to_unicode(line.rstrip(), encoding=encoding, errors=unicode_errors).split(\" \")\n                 if len(parts) != vector_size + 1:\n                     raise ValueError(\"invalid vector on line %s (is this really the text format?)\" % line_no)\n-                word, weights = parts[0], [REAL(x) for x in parts[1:]]\n+                word, weights = parts[0], [datatype(x) for x in parts[1:]]\n                 add_word(word, weights)\n     if result.vectors.shape[0] != len(result.vocab):\n         logger.info(\n", "issue": "`datatype` parameter in `load_word2vec_format` doesn't work as expected\n#### Description\r\nUsing `datatype=np.float64` in a `KeyedVectors.load_word2vec_call` doesn't work as expected, the loaded floats seem to lose precision. The datatype for `syn0` is still `float64` though, so it seems that they are cast to float32 first while loading, then cast to float64 when creating the array.\r\n\r\n#### Steps/Code/Corpus to Reproduce\r\nUsing this file - \r\n[test.kv.txt](https://github.com/RaRe-Technologies/gensim/files/1434953/test.kv.txt)\r\n\r\n```python\r\nfrom gensim.models.keyedvectors import KeyedVectors\r\nimport numpy as np\r\n\r\nkv = KeyedVectors.load_word2vec_format('test.kv.txt', datatype=np.float64)\r\nprint(kv['horse.n.01'][0] == -0.0008546282343595379)\r\n# False\r\nprint(kv['horse.n.01'].dtype)\r\n# float64\r\n```\r\n#### Expected Results\r\n```python\r\nprint(kv['horse.n.01'][0] == -0.0008546282343595379)\r\n# True\r\n```\r\n#### Actual Results\r\n```python\r\nprint(kv['horse.n.01'][0] == -0.0008546282343595379)\r\n# False\r\n```\r\n\r\nLooking at the code and making a quick hack [here](https://github.com/RaRe-Technologies/gensim/blob/develop/gensim/models/keyedvectors.py#L253), changing..\r\n```\r\nword, weights = parts[0], [REAL(x) for x in parts[1:]]\r\n```\r\nto..\r\n```\r\nword, weights = parts[0], [datatype(x) for x in parts[1:]]\r\n```\r\n..leads to the correct result. However, I imagine there are other cases to be covered as well.\n", "before_files": [{"content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n#\n# Author: Shiva Manne <s.manne@rare-technologies.com>\n# Copyright (C) 2018 RaRe Technologies s.r.o.\n\n\"\"\"This module contains various general functions useful for any2vec models.\"\"\"\n\nimport logging\nimport numpy as np\nfrom gensim import utils\n\nfrom numpy import zeros, dtype, float32 as REAL, ascontiguousarray, fromstring\n\nfrom six.moves import xrange\nfrom six import iteritems\n\nlogger = logging.getLogger(__name__)\n\n\ndef _compute_ngrams(word, min_n, max_n):\n    \"\"\"Returns the list of all possible ngrams for a given word.\n\n    Parameters\n    ----------\n    word : str\n        The word whose ngrams need to be computed\n    min_n : int\n        minimum character length of the ngrams\n    max_n : int\n        maximum character length of the ngrams\n\n    Returns\n    -------\n    :obj:`list` of :obj:`str`\n        List of character ngrams\n\n    \"\"\"\n    BOW, EOW = ('<', '>')  # Used by FastText to attach to all words as prefix and suffix\n    extended_word = BOW + word + EOW\n    ngrams = []\n    for ngram_length in range(min_n, min(len(extended_word), max_n) + 1):\n        for i in range(0, len(extended_word) - ngram_length + 1):\n            ngrams.append(extended_word[i:i + ngram_length])\n    return ngrams\n\n\ndef _ft_hash(string):\n    \"\"\"Reproduces [hash method](https://github.com/facebookresearch/fastText/blob/master/src/dictionary.cc)\n    used in [1]_.\n\n    Parameter\n    ---------\n    string : str\n        The string whose hash needs to be calculated\n\n    Returns\n    -------\n    int\n        The hash of the string\n\n    \"\"\"\n    # Runtime warnings for integer overflow are raised, this is expected behaviour. These warnings are suppressed.\n    old_settings = np.seterr(all='ignore')\n    h = np.uint32(2166136261)\n    for c in string:\n        h = h ^ np.uint32(ord(c))\n        h = h * np.uint32(16777619)\n    np.seterr(**old_settings)\n    return h\n\n\ndef _save_word2vec_format(fname, vocab, vectors, fvocab=None, binary=False, total_vec=None):\n        \"\"\"Store the input-hidden weight matrix in the same format used by the original\n        C word2vec-tool, for compatibility.\n\n        Parameters\n        ----------\n        fname : str\n            The file path used to save the vectors in\n        vocab : dict\n            The vocabulary of words\n        vectors : numpy.array\n            The vectors to be stored\n        fvocab : str\n            Optional file path used to save the vocabulary\n        binary : bool\n            If True, the data wil be saved in binary word2vec format, else it will be saved in plain text.\n        total_vec :  int\n            Optional parameter to explicitly specify total no. of vectors\n            (in case word vectors are appended with document vectors afterwards)\n\n        \"\"\"\n        if not (vocab or vectors):\n            raise RuntimeError(\"no input\")\n        if total_vec is None:\n            total_vec = len(vocab)\n        vector_size = vectors.shape[1]\n        if fvocab is not None:\n            logger.info(\"storing vocabulary in %s\", fvocab)\n            with utils.smart_open(fvocab, 'wb') as vout:\n                for word, vocab_ in sorted(iteritems(vocab), key=lambda item: -item[1].count):\n                    vout.write(utils.to_utf8(\"%s %s\\n\" % (word, vocab_.count)))\n        logger.info(\"storing %sx%s projection weights into %s\", total_vec, vector_size, fname)\n        assert (len(vocab), vector_size) == vectors.shape\n        with utils.smart_open(fname, 'wb') as fout:\n            fout.write(utils.to_utf8(\"%s %s\\n\" % (total_vec, vector_size)))\n            # store in sorted order: most frequent words at the top\n            for word, vocab_ in sorted(iteritems(vocab), key=lambda item: -item[1].count):\n                row = vectors[vocab_.index]\n                if binary:\n                    fout.write(utils.to_utf8(word) + b\" \" + row.tostring())\n                else:\n                    fout.write(utils.to_utf8(\"%s %s\\n\" % (word, ' '.join(\"%f\" % val for val in row))))\n\n\ndef _load_word2vec_format(cls, fname, fvocab=None, binary=False, encoding='utf8', unicode_errors='strict',\n                          limit=None, datatype=REAL):\n    \"\"\"Load the input-hidden weight matrix from the original C word2vec-tool format.\n\n    Note that the information stored in the file is incomplete (the binary tree is missing),\n    so while you can query for word similarity etc., you cannot continue training\n    with a model loaded this way.\n\n    Parameters\n    ----------\n    fname : str\n        The file path to the saved word2vec-format file.\n    fvocab : str\n            Optional file path to the vocabulary.Word counts are read from `fvocab` filename,\n            if set (this is the file generated by `-save-vocab` flag of the original C tool).\n    binary : bool\n        If True, indicates whether the data is in binary word2vec format.\n    encoding : str\n        If you trained the C model using non-utf8 encoding for words, specify that\n        encoding in `encoding`.\n    unicode_errors : str\n        default 'strict', is a string suitable to be passed as the `errors`\n        argument to the unicode() (Python 2.x) or str() (Python 3.x) function. If your source\n        file may include word tokens truncated in the middle of a multibyte unicode character\n        (as is common from the original word2vec.c tool), 'ignore' or 'replace' may help.\n    limit : int\n        Sets a maximum number of word-vectors to read from the file. The default,\n        None, means read all.\n    datatype : :class: `numpy.float*`\n        (Experimental) Can coerce dimensions to a non-default float type (such\n        as np.float16) to save memory. (Such types may result in much slower bulk operations\n        or incompatibility with optimized routines.)\n\n    Returns\n    -------\n    :obj: `cls`\n        Returns the loaded model as an instance of :class: `cls`.\n\n    \"\"\"\n    from gensim.models.keyedvectors import Vocab\n    counts = None\n    if fvocab is not None:\n        logger.info(\"loading word counts from %s\", fvocab)\n        counts = {}\n        with utils.smart_open(fvocab) as fin:\n            for line in fin:\n                word, count = utils.to_unicode(line).strip().split()\n                counts[word] = int(count)\n\n    logger.info(\"loading projection weights from %s\", fname)\n    with utils.smart_open(fname) as fin:\n        header = utils.to_unicode(fin.readline(), encoding=encoding)\n        vocab_size, vector_size = (int(x) for x in header.split())  # throws for invalid file format\n        if limit:\n            vocab_size = min(vocab_size, limit)\n        result = cls(vector_size)\n        result.vector_size = vector_size\n        result.vectors = zeros((vocab_size, vector_size), dtype=datatype)\n\n        def add_word(word, weights):\n            word_id = len(result.vocab)\n            if word in result.vocab:\n                logger.warning(\"duplicate word '%s' in %s, ignoring all but first\", word, fname)\n                return\n            if counts is None:\n                # most common scenario: no vocab file given. just make up some bogus counts, in descending order\n                result.vocab[word] = Vocab(index=word_id, count=vocab_size - word_id)\n            elif word in counts:\n                # use count from the vocab file\n                result.vocab[word] = Vocab(index=word_id, count=counts[word])\n            else:\n                # vocab file given, but word is missing -- set count to None (TODO: or raise?)\n                logger.warning(\"vocabulary file is incomplete: '%s' is missing\", word)\n                result.vocab[word] = Vocab(index=word_id, count=None)\n            result.vectors[word_id] = weights\n            result.index2word.append(word)\n\n        if binary:\n            binary_len = dtype(REAL).itemsize * vector_size\n            for _ in xrange(vocab_size):\n                # mixed text and binary: read text first, then binary\n                word = []\n                while True:\n                    ch = fin.read(1)\n                    if ch == b' ':\n                        break\n                    if ch == b'':\n                        raise EOFError(\"unexpected end of input; is count incorrect or file otherwise damaged?\")\n                    if ch != b'\\n':  # ignore newlines in front of words (some binary files have)\n                        word.append(ch)\n                word = utils.to_unicode(b''.join(word), encoding=encoding, errors=unicode_errors)\n                weights = fromstring(fin.read(binary_len), dtype=REAL)\n                add_word(word, weights)\n        else:\n            for line_no in xrange(vocab_size):\n                line = fin.readline()\n                if line == b'':\n                    raise EOFError(\"unexpected end of input; is count incorrect or file otherwise damaged?\")\n                parts = utils.to_unicode(line.rstrip(), encoding=encoding, errors=unicode_errors).split(\" \")\n                if len(parts) != vector_size + 1:\n                    raise ValueError(\"invalid vector on line %s (is this really the text format?)\" % line_no)\n                word, weights = parts[0], [REAL(x) for x in parts[1:]]\n                add_word(word, weights)\n    if result.vectors.shape[0] != len(result.vocab):\n        logger.info(\n            \"duplicate words detected, shrinking matrix size from %i to %i\",\n            result.vectors.shape[0], len(result.vocab)\n        )\n        result.vectors = ascontiguousarray(result.vectors[: len(result.vocab)])\n    assert (len(result.vocab), vector_size) == result.vectors.shape\n\n    logger.info(\"loaded %s matrix from %s\", result.vectors.shape, fname)\n    return result\n", "path": "gensim/models/utils_any2vec.py"}], "after_files": [{"content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n#\n# Author: Shiva Manne <s.manne@rare-technologies.com>\n# Copyright (C) 2018 RaRe Technologies s.r.o.\n\n\"\"\"This module contains various general functions useful for any2vec models.\"\"\"\n\nimport logging\nimport numpy as np\nfrom gensim import utils\n\nfrom numpy import zeros, dtype, float32 as REAL, ascontiguousarray, fromstring\n\nfrom six.moves import xrange\nfrom six import iteritems\n\nlogger = logging.getLogger(__name__)\n\n\ndef _compute_ngrams(word, min_n, max_n):\n    \"\"\"Returns the list of all possible ngrams for a given word.\n\n    Parameters\n    ----------\n    word : str\n        The word whose ngrams need to be computed\n    min_n : int\n        minimum character length of the ngrams\n    max_n : int\n        maximum character length of the ngrams\n\n    Returns\n    -------\n    :obj:`list` of :obj:`str`\n        List of character ngrams\n\n    \"\"\"\n    BOW, EOW = ('<', '>')  # Used by FastText to attach to all words as prefix and suffix\n    extended_word = BOW + word + EOW\n    ngrams = []\n    for ngram_length in range(min_n, min(len(extended_word), max_n) + 1):\n        for i in range(0, len(extended_word) - ngram_length + 1):\n            ngrams.append(extended_word[i:i + ngram_length])\n    return ngrams\n\n\ndef _ft_hash(string):\n    \"\"\"Reproduces [hash method](https://github.com/facebookresearch/fastText/blob/master/src/dictionary.cc)\n    used in [1]_.\n\n    Parameter\n    ---------\n    string : str\n        The string whose hash needs to be calculated\n\n    Returns\n    -------\n    int\n        The hash of the string\n\n    \"\"\"\n    # Runtime warnings for integer overflow are raised, this is expected behaviour. These warnings are suppressed.\n    old_settings = np.seterr(all='ignore')\n    h = np.uint32(2166136261)\n    for c in string:\n        h = h ^ np.uint32(ord(c))\n        h = h * np.uint32(16777619)\n    np.seterr(**old_settings)\n    return h\n\n\ndef _save_word2vec_format(fname, vocab, vectors, fvocab=None, binary=False, total_vec=None):\n        \"\"\"Store the input-hidden weight matrix in the same format used by the original\n        C word2vec-tool, for compatibility.\n\n        Parameters\n        ----------\n        fname : str\n            The file path used to save the vectors in\n        vocab : dict\n            The vocabulary of words\n        vectors : numpy.array\n            The vectors to be stored\n        fvocab : str\n            Optional file path used to save the vocabulary\n        binary : bool\n            If True, the data wil be saved in binary word2vec format, else it will be saved in plain text.\n        total_vec :  int\n            Optional parameter to explicitly specify total no. of vectors\n            (in case word vectors are appended with document vectors afterwards)\n\n        \"\"\"\n        if not (vocab or vectors):\n            raise RuntimeError(\"no input\")\n        if total_vec is None:\n            total_vec = len(vocab)\n        vector_size = vectors.shape[1]\n        if fvocab is not None:\n            logger.info(\"storing vocabulary in %s\", fvocab)\n            with utils.smart_open(fvocab, 'wb') as vout:\n                for word, vocab_ in sorted(iteritems(vocab), key=lambda item: -item[1].count):\n                    vout.write(utils.to_utf8(\"%s %s\\n\" % (word, vocab_.count)))\n        logger.info(\"storing %sx%s projection weights into %s\", total_vec, vector_size, fname)\n        assert (len(vocab), vector_size) == vectors.shape\n        with utils.smart_open(fname, 'wb') as fout:\n            fout.write(utils.to_utf8(\"%s %s\\n\" % (total_vec, vector_size)))\n            # store in sorted order: most frequent words at the top\n            for word, vocab_ in sorted(iteritems(vocab), key=lambda item: -item[1].count):\n                row = vectors[vocab_.index]\n                if binary:\n                    row = row.astype(REAL)\n                    fout.write(utils.to_utf8(word) + b\" \" + row.tostring())\n                else:\n                    fout.write(utils.to_utf8(\"%s %s\\n\" % (word, ' '.join(repr(val) for val in row))))\n\n\ndef _load_word2vec_format(cls, fname, fvocab=None, binary=False, encoding='utf8', unicode_errors='strict',\n                          limit=None, datatype=REAL):\n    \"\"\"Load the input-hidden weight matrix from the original C word2vec-tool format.\n\n    Note that the information stored in the file is incomplete (the binary tree is missing),\n    so while you can query for word similarity etc., you cannot continue training\n    with a model loaded this way.\n\n    Parameters\n    ----------\n    fname : str\n        The file path to the saved word2vec-format file.\n    fvocab : str\n            Optional file path to the vocabulary.Word counts are read from `fvocab` filename,\n            if set (this is the file generated by `-save-vocab` flag of the original C tool).\n    binary : bool\n        If True, indicates whether the data is in binary word2vec format.\n    encoding : str\n        If you trained the C model using non-utf8 encoding for words, specify that\n        encoding in `encoding`.\n    unicode_errors : str\n        default 'strict', is a string suitable to be passed as the `errors`\n        argument to the unicode() (Python 2.x) or str() (Python 3.x) function. If your source\n        file may include word tokens truncated in the middle of a multibyte unicode character\n        (as is common from the original word2vec.c tool), 'ignore' or 'replace' may help.\n    limit : int\n        Sets a maximum number of word-vectors to read from the file. The default,\n        None, means read all.\n    datatype : :class: `numpy.float*`\n        (Experimental) Can coerce dimensions to a non-default float type (such\n        as np.float16) to save memory. (Such types may result in much slower bulk operations\n        or incompatibility with optimized routines.)\n\n    Returns\n    -------\n    :obj: `cls`\n        Returns the loaded model as an instance of :class: `cls`.\n\n    \"\"\"\n    from gensim.models.keyedvectors import Vocab\n    counts = None\n    if fvocab is not None:\n        logger.info(\"loading word counts from %s\", fvocab)\n        counts = {}\n        with utils.smart_open(fvocab) as fin:\n            for line in fin:\n                word, count = utils.to_unicode(line).strip().split()\n                counts[word] = int(count)\n\n    logger.info(\"loading projection weights from %s\", fname)\n    with utils.smart_open(fname) as fin:\n        header = utils.to_unicode(fin.readline(), encoding=encoding)\n        vocab_size, vector_size = (int(x) for x in header.split())  # throws for invalid file format\n        if limit:\n            vocab_size = min(vocab_size, limit)\n        result = cls(vector_size)\n        result.vector_size = vector_size\n        result.vectors = zeros((vocab_size, vector_size), dtype=datatype)\n\n        def add_word(word, weights):\n            word_id = len(result.vocab)\n            if word in result.vocab:\n                logger.warning(\"duplicate word '%s' in %s, ignoring all but first\", word, fname)\n                return\n            if counts is None:\n                # most common scenario: no vocab file given. just make up some bogus counts, in descending order\n                result.vocab[word] = Vocab(index=word_id, count=vocab_size - word_id)\n            elif word in counts:\n                # use count from the vocab file\n                result.vocab[word] = Vocab(index=word_id, count=counts[word])\n            else:\n                # vocab file given, but word is missing -- set count to None (TODO: or raise?)\n                logger.warning(\"vocabulary file is incomplete: '%s' is missing\", word)\n                result.vocab[word] = Vocab(index=word_id, count=None)\n            result.vectors[word_id] = weights\n            result.index2word.append(word)\n\n        if binary:\n            binary_len = dtype(REAL).itemsize * vector_size\n            for _ in xrange(vocab_size):\n                # mixed text and binary: read text first, then binary\n                word = []\n                while True:\n                    ch = fin.read(1)\n                    if ch == b' ':\n                        break\n                    if ch == b'':\n                        raise EOFError(\"unexpected end of input; is count incorrect or file otherwise damaged?\")\n                    if ch != b'\\n':  # ignore newlines in front of words (some binary files have)\n                        word.append(ch)\n                word = utils.to_unicode(b''.join(word), encoding=encoding, errors=unicode_errors)\n                weights = fromstring(fin.read(binary_len), dtype=REAL).astype(datatype)\n                add_word(word, weights)\n        else:\n            for line_no in xrange(vocab_size):\n                line = fin.readline()\n                if line == b'':\n                    raise EOFError(\"unexpected end of input; is count incorrect or file otherwise damaged?\")\n                parts = utils.to_unicode(line.rstrip(), encoding=encoding, errors=unicode_errors).split(\" \")\n                if len(parts) != vector_size + 1:\n                    raise ValueError(\"invalid vector on line %s (is this really the text format?)\" % line_no)\n                word, weights = parts[0], [datatype(x) for x in parts[1:]]\n                add_word(word, weights)\n    if result.vectors.shape[0] != len(result.vocab):\n        logger.info(\n            \"duplicate words detected, shrinking matrix size from %i to %i\",\n            result.vectors.shape[0], len(result.vocab)\n        )\n        result.vectors = ascontiguousarray(result.vectors[: len(result.vocab)])\n    assert (len(result.vocab), vector_size) == result.vectors.shape\n\n    logger.info(\"loaded %s matrix from %s\", result.vectors.shape, fname)\n    return result\n", "path": "gensim/models/utils_any2vec.py"}]}