We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Make NLP dependencies optional
Some dependencies are only needed in some images, we should make some dependencies optional so that they are only installed when necessary.

For example, Spacy is the biggest dependency, but it is currently not really used by any image.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `setup.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 # This Source Code Form is subject to the terms of the Mozilla Public
3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 # You can obtain one at http://mozilla.org/MPL/2.0/.
5 
6 import os
7 
8 from setuptools import find_packages, setup
9 
10 here = os.path.dirname(__file__)
11 
12 
13 def read_requirements(file_):
14     requires = []
15     links = []
16     with open(os.path.join(here, file_)) as f:
17         for line in f.readlines():
18             line = line.strip()
19 
20             if line.startswith("https://"):
21                 links.append(line + "-1.0.0")
22                 extras = ""
23                 if "[" in line:
24                     extras = "[" + line.split("[")[1].split("]")[0] + "]"
25                 line = line.split("#")[1].split("egg=")[1] + extras
26             elif line == "" or line.startswith("#") or line.startswith("-"):
27                 continue
28             line = line.split("#")[0].strip()
29             requires.append(line)
30 
31     return sorted(list(set(requires))), links
32 
33 
34 install_requires, dependency_links = read_requirements("requirements.txt")
35 
36 
37 with open(os.path.join(here, "VERSION")) as f:
38     version = f.read().strip()
39 
40 setup(
41     name="bugbug",
42     version=version,
43     description="ML tools for Mozilla projects",
44     author="Marco Castelluccio",
45     author_email="mcastelluccio@mozilla.com",
46     install_requires=install_requires,
47     dependency_links=dependency_links,
48     packages=find_packages(exclude=["contrib", "docs", "tests"]),
49     include_package_data=True,
50     license="MPL2",
51     entry_points={
52         "console_scripts": [
53             "bugbug-data-commits = scripts.commit_retriever:main",
54             "bugbug-data-bugzilla = scripts.bug_retriever:main",
55             "bugbug-train = scripts.trainer:main",
56         ]
57     },
58 )
59
```

Path: `bugbug/nlp.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 # This Source Code Form is subject to the terms of the Mozilla Public
3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 # You can obtain one at http://mozilla.org/MPL/2.0/.
5 
6 from collections import defaultdict
7 from functools import lru_cache
8 
9 import numpy as np
10 import spacy
11 from gensim.models import KeyedVectors
12 from sklearn.base import BaseEstimator, TransformerMixin
13 from sklearn.feature_extraction.text import TfidfVectorizer
14 from spacy.tokenizer import Tokenizer
15 
16 nlp = spacy.load("en_core_web_sm")
17 
18 
19 def spacy_token_lemmatizer(text):
20     if len(text) > nlp.max_length:
21         text = text[: nlp.max_length - 1]
22     doc = nlp(text)
23     return [token.lemma_ for token in doc]
24 
25 
26 class SpacyVectorizer(TfidfVectorizer):
27     def __init__(self, *args, **kwargs):
28         super().__init__(tokenizer=spacy_token_lemmatizer, *args, **kwargs)
29 
30 
31 @lru_cache()
32 def get_word_embeddings():
33     word_embeddings = KeyedVectors.load_word2vec_format("wiki-news-300d-1M-subword.vec")
34     word_embeddings.init_sims(replace=True)
35     return word_embeddings
36 
37 
38 class MeanEmbeddingTransformer(BaseEstimator, TransformerMixin):
39     def __init__(self):
40         self.model = get_word_embeddings()
41         self.dim = len(self.model["if"])
42 
43     def fit(self, x, y=None):
44         return self
45 
46     def transform(self, data):
47         tokenizer = Tokenizer(nlp.vocab)
48         return np.array(
49             [
50                 np.mean(
51                     [
52                         self.model[w.text.lower()]
53                         for w in words
54                         if w.text.lower() in self.model
55                     ]
56                     or [np.zeros(self.dim)],
57                     axis=0,
58                 )
59                 for words in tokenizer.pipe(data)
60             ]
61         )
62 
63     def get_feature_names(self):
64         return np.array([f"_{i}" for i in range(self.dim)], dtype=object)
65 
66 
67 class TfidfMeanEmbeddingTransformer(MeanEmbeddingTransformer):
68     def __init__(self):
69         super().__init__()
70         self.word2weight = None
71 
72     def fit(self, X, y=None):
73         tfidf = TfidfVectorizer(analyzer=lambda x: x)
74         tfidf.fit(X)
75 
76         # If a word was never seen, it must be at least as infrequent as any of the known words.
77         # So, the default idf is the max of known idfs.
78         max_idf = max(tfidf.idf_)
79         self.word2weight = defaultdict(
80             lambda: max_idf, [(w, tfidf.idf_[i]) for w, i in tfidf.vocabulary_.items()]
81         )
82 
83         return self
84 
85     def transform(self, data):
86         tokenizer = Tokenizer(nlp.vocab)
87         return np.array(
88             [
89                 np.mean(
90                     [
91                         self.model[w.text.lower()] * self.word2weight[w.text.lower()]
92                         for w in words
93                         if w.text.lower() in self.model
94                     ]
95                     or [np.zeros(self.dim)],
96                     axis=0,
97                 )
98                 for words in tokenizer.pipe(data)
99             ]
100         )
101
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/bugbug/nlp.py b/bugbug/nlp.py
--- a/bugbug/nlp.py
+++ b/bugbug/nlp.py
@@ -3,17 +3,41 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import sys
 from collections import defaultdict
 from functools import lru_cache
 
 import numpy as np
-import spacy
-from gensim.models import KeyedVectors
 from sklearn.base import BaseEstimator, TransformerMixin
 from sklearn.feature_extraction.text import TfidfVectorizer
-from spacy.tokenizer import Tokenizer
 
-nlp = spacy.load("en_core_web_sm")
+HAS_OPTIONAL_DEPENDENCIES = False
+
+try:
+    import spacy
+
+    from spacy.tokenizer import Tokenizer
+    from gensim.models import KeyedVectors
+
+    HAS_OPTIONAL_DEPENDENCIES = True
+except ImportError:
+    pass
+
+try:
+    if HAS_OPTIONAL_DEPENDENCIES:
+        nlp = spacy.load("en_core_web_sm")
+except OSError:
+    msg = (
+        "Spacy model is missing, install it with: "
+        f"{sys.executable} -m spacy download en_core_web_sm"
+    )
+    print(msg, file=sys.stderr)
+
+OPT_MSG_MISSING = (
+    "Optional dependencies are missing, install them with: pip install bugbug[nlp]\n"
+    "You might need also to download the models with: "
+    f"{sys.executable} -m spacy download en_core_web_sm"
+)
 
 
 def spacy_token_lemmatizer(text):
@@ -25,6 +49,11 @@
 
 class SpacyVectorizer(TfidfVectorizer):
     def __init__(self, *args, **kwargs):
+
+        # Detect when the Spacy optional dependency is missing
+        if not HAS_OPTIONAL_DEPENDENCIES:
+            raise NotImplementedError(OPT_MSG_MISSING)
+
         super().__init__(tokenizer=spacy_token_lemmatizer, *args, **kwargs)
 
 
@@ -37,6 +66,10 @@
 
 class MeanEmbeddingTransformer(BaseEstimator, TransformerMixin):
     def __init__(self):
+        # Detect when the Gensim optional dependency are missing
+        if not HAS_OPTIONAL_DEPENDENCIES:
+            raise NotImplementedError(OPT_MSG_MISSING)
+
         self.model = get_word_embeddings()
         self.dim = len(self.model["if"])
 
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -37,6 +37,20 @@
 with open(os.path.join(here, "VERSION")) as f:
     version = f.read().strip()
 
+# Read the extra requirements
+extras = ["nlp"]
+
+extras_require = {}
+
+for extra in extras:
+    extras_install, extra_links = read_requirements("extra-%s-requirements.txt" % extra)
+
+    # Merge the dependency links
+    dependency_links.extend(extra_links)
+
+    extras_require[extra] = extras_install
+
+
 setup(
     name="bugbug",
     version=version,
@@ -44,6 +58,7 @@
     author="Marco Castelluccio",
     author_email="mcastelluccio@mozilla.com",
     install_requires=install_requires,
+    extras_require=extras_require,
     dependency_links=dependency_links,
     packages=find_packages(exclude=["contrib", "docs", "tests"]),
     include_package_data=True,
@@ -55,4 +70,9 @@
             "bugbug-train = scripts.trainer:main",
         ]
     },
+    classifiers=[
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3 :: Only",
+        "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
+    ],
 )

{"golden_diff": "diff --git a/bugbug/nlp.py b/bugbug/nlp.py\n--- a/bugbug/nlp.py\n+++ b/bugbug/nlp.py\n@@ -3,17 +3,41 @@\n # License, v. 2.0. If a copy of the MPL was not distributed with this file,\n # You can obtain one at http://mozilla.org/MPL/2.0/.\n \n+import sys\n from collections import defaultdict\n from functools import lru_cache\n \n import numpy as np\n-import spacy\n-from gensim.models import KeyedVectors\n from sklearn.base import BaseEstimator, TransformerMixin\n from sklearn.feature_extraction.text import TfidfVectorizer\n-from spacy.tokenizer import Tokenizer\n \n-nlp = spacy.load(\"en_core_web_sm\")\n+HAS_OPTIONAL_DEPENDENCIES = False\n+\n+try:\n+    import spacy\n+\n+    from spacy.tokenizer import Tokenizer\n+    from gensim.models import KeyedVectors\n+\n+    HAS_OPTIONAL_DEPENDENCIES = True\n+except ImportError:\n+    pass\n+\n+try:\n+    if HAS_OPTIONAL_DEPENDENCIES:\n+        nlp = spacy.load(\"en_core_web_sm\")\n+except OSError:\n+    msg = (\n+        \"Spacy model is missing, install it with: \"\n+        f\"{sys.executable} -m spacy download en_core_web_sm\"\n+    )\n+    print(msg, file=sys.stderr)\n+\n+OPT_MSG_MISSING = (\n+    \"Optional dependencies are missing, install them with: pip install bugbug[nlp]\\n\"\n+    \"You might need also to download the models with: \"\n+    f\"{sys.executable} -m spacy download en_core_web_sm\"\n+)\n \n \n def spacy_token_lemmatizer(text):\n@@ -25,6 +49,11 @@\n \n class SpacyVectorizer(TfidfVectorizer):\n     def __init__(self, *args, **kwargs):\n+\n+        # Detect when the Spacy optional dependency is missing\n+        if not HAS_OPTIONAL_DEPENDENCIES:\n+            raise NotImplementedError(OPT_MSG_MISSING)\n+\n         super().__init__(tokenizer=spacy_token_lemmatizer, *args, **kwargs)\n \n \n@@ -37,6 +66,10 @@\n \n class MeanEmbeddingTransformer(BaseEstimator, TransformerMixin):\n     def __init__(self):\n+        # Detect when the Gensim optional dependency are missing\n+        if not HAS_OPTIONAL_DEPENDENCIES:\n+            raise NotImplementedError(OPT_MSG_MISSING)\n+\n         self.model = get_word_embeddings()\n         self.dim = len(self.model[\"if\"])\n \ndiff --git a/setup.py b/setup.py\n--- a/setup.py\n+++ b/setup.py\n@@ -37,6 +37,20 @@\n with open(os.path.join(here, \"VERSION\")) as f:\n     version = f.read().strip()\n \n+# Read the extra requirements\n+extras = [\"nlp\"]\n+\n+extras_require = {}\n+\n+for extra in extras:\n+    extras_install, extra_links = read_requirements(\"extra-%s-requirements.txt\" % extra)\n+\n+    # Merge the dependency links\n+    dependency_links.extend(extra_links)\n+\n+    extras_require[extra] = extras_install\n+\n+\n setup(\n     name=\"bugbug\",\n     version=version,\n@@ -44,6 +58,7 @@\n     author=\"Marco Castelluccio\",\n     author_email=\"mcastelluccio@mozilla.com\",\n     install_requires=install_requires,\n+    extras_require=extras_require,\n     dependency_links=dependency_links,\n     packages=find_packages(exclude=[\"contrib\", \"docs\", \"tests\"]),\n     include_package_data=True,\n@@ -55,4 +70,9 @@\n             \"bugbug-train = scripts.trainer:main\",\n         ]\n     },\n+    classifiers=[\n+        \"Programming Language :: Python :: 3.6\",\n+        \"Programming Language :: Python :: 3 :: Only\",\n+        \"License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)\",\n+    ],\n )\n", "issue": "Make NLP dependencies optional\nSome dependencies are only needed in some images, we should make some dependencies optional so that they are only installed when necessary.\r\n\r\nFor example, Spacy is the biggest dependency, but it is currently not really used by any image.\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport os\n\nfrom setuptools import find_packages, setup\n\nhere = os.path.dirname(__file__)\n\n\ndef read_requirements(file_):\n    requires = []\n    links = []\n    with open(os.path.join(here, file_)) as f:\n        for line in f.readlines():\n            line = line.strip()\n\n            if line.startswith(\"https://\"):\n                links.append(line + \"-1.0.0\")\n                extras = \"\"\n                if \"[\" in line:\n                    extras = \"[\" + line.split(\"[\")[1].split(\"]\")[0] + \"]\"\n                line = line.split(\"#\")[1].split(\"egg=\")[1] + extras\n            elif line == \"\" or line.startswith(\"#\") or line.startswith(\"-\"):\n                continue\n            line = line.split(\"#\")[0].strip()\n            requires.append(line)\n\n    return sorted(list(set(requires))), links\n\n\ninstall_requires, dependency_links = read_requirements(\"requirements.txt\")\n\n\nwith open(os.path.join(here, \"VERSION\")) as f:\n    version = f.read().strip()\n\nsetup(\n    name=\"bugbug\",\n    version=version,\n    description=\"ML tools for Mozilla projects\",\n    author=\"Marco Castelluccio\",\n    author_email=\"mcastelluccio@mozilla.com\",\n    install_requires=install_requires,\n    dependency_links=dependency_links,\n    packages=find_packages(exclude=[\"contrib\", \"docs\", \"tests\"]),\n    include_package_data=True,\n    license=\"MPL2\",\n    entry_points={\n        \"console_scripts\": [\n            \"bugbug-data-commits = scripts.commit_retriever:main\",\n            \"bugbug-data-bugzilla = scripts.bug_retriever:main\",\n            \"bugbug-train = scripts.trainer:main\",\n        ]\n    },\n)\n", "path": "setup.py"}, {"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nfrom collections import defaultdict\nfrom functools import lru_cache\n\nimport numpy as np\nimport spacy\nfrom gensim.models import KeyedVectors\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom spacy.tokenizer import Tokenizer\n\nnlp = spacy.load(\"en_core_web_sm\")\n\n\ndef spacy_token_lemmatizer(text):\n    if len(text) > nlp.max_length:\n        text = text[: nlp.max_length - 1]\n    doc = nlp(text)\n    return [token.lemma_ for token in doc]\n\n\nclass SpacyVectorizer(TfidfVectorizer):\n    def __init__(self, *args, **kwargs):\n        super().__init__(tokenizer=spacy_token_lemmatizer, *args, **kwargs)\n\n\n@lru_cache()\ndef get_word_embeddings():\n    word_embeddings = KeyedVectors.load_word2vec_format(\"wiki-news-300d-1M-subword.vec\")\n    word_embeddings.init_sims(replace=True)\n    return word_embeddings\n\n\nclass MeanEmbeddingTransformer(BaseEstimator, TransformerMixin):\n    def __init__(self):\n        self.model = get_word_embeddings()\n        self.dim = len(self.model[\"if\"])\n\n    def fit(self, x, y=None):\n        return self\n\n    def transform(self, data):\n        tokenizer = Tokenizer(nlp.vocab)\n        return np.array(\n            [\n                np.mean(\n                    [\n                        self.model[w.text.lower()]\n                        for w in words\n                        if w.text.lower() in self.model\n                    ]\n                    or [np.zeros(self.dim)],\n                    axis=0,\n                )\n                for words in tokenizer.pipe(data)\n            ]\n        )\n\n    def get_feature_names(self):\n        return np.array([f\"_{i}\" for i in range(self.dim)], dtype=object)\n\n\nclass TfidfMeanEmbeddingTransformer(MeanEmbeddingTransformer):\n    def __init__(self):\n        super().__init__()\n        self.word2weight = None\n\n    def fit(self, X, y=None):\n        tfidf = TfidfVectorizer(analyzer=lambda x: x)\n        tfidf.fit(X)\n\n        # If a word was never seen, it must be at least as infrequent as any of the known words.\n        # So, the default idf is the max of known idfs.\n        max_idf = max(tfidf.idf_)\n        self.word2weight = defaultdict(\n            lambda: max_idf, [(w, tfidf.idf_[i]) for w, i in tfidf.vocabulary_.items()]\n        )\n\n        return self\n\n    def transform(self, data):\n        tokenizer = Tokenizer(nlp.vocab)\n        return np.array(\n            [\n                np.mean(\n                    [\n                        self.model[w.text.lower()] * self.word2weight[w.text.lower()]\n                        for w in words\n                        if w.text.lower() in self.model\n                    ]\n                    or [np.zeros(self.dim)],\n                    axis=0,\n                )\n                for words in tokenizer.pipe(data)\n            ]\n        )\n", "path": "bugbug/nlp.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport os\n\nfrom setuptools import find_packages, setup\n\nhere = os.path.dirname(__file__)\n\n\ndef read_requirements(file_):\n    requires = []\n    links = []\n    with open(os.path.join(here, file_)) as f:\n        for line in f.readlines():\n            line = line.strip()\n\n            if line.startswith(\"https://\"):\n                links.append(line + \"-1.0.0\")\n                extras = \"\"\n                if \"[\" in line:\n                    extras = \"[\" + line.split(\"[\")[1].split(\"]\")[0] + \"]\"\n                line = line.split(\"#\")[1].split(\"egg=\")[1] + extras\n            elif line == \"\" or line.startswith(\"#\") or line.startswith(\"-\"):\n                continue\n            line = line.split(\"#\")[0].strip()\n            requires.append(line)\n\n    return sorted(list(set(requires))), links\n\n\ninstall_requires, dependency_links = read_requirements(\"requirements.txt\")\n\n\nwith open(os.path.join(here, \"VERSION\")) as f:\n    version = f.read().strip()\n\n# Read the extra requirements\nextras = [\"nlp\"]\n\nextras_require = {}\n\nfor extra in extras:\n    extras_install, extra_links = read_requirements(\"extra-%s-requirements.txt\" % extra)\n\n    # Merge the dependency links\n    dependency_links.extend(extra_links)\n\n    extras_require[extra] = extras_install\n\n\nsetup(\n    name=\"bugbug\",\n    version=version,\n    description=\"ML tools for Mozilla projects\",\n    author=\"Marco Castelluccio\",\n    author_email=\"mcastelluccio@mozilla.com\",\n    install_requires=install_requires,\n    extras_require=extras_require,\n    dependency_links=dependency_links,\n    packages=find_packages(exclude=[\"contrib\", \"docs\", \"tests\"]),\n    include_package_data=True,\n    license=\"MPL2\",\n    entry_points={\n        \"console_scripts\": [\n            \"bugbug-data-commits = scripts.commit_retriever:main\",\n            \"bugbug-data-bugzilla = scripts.bug_retriever:main\",\n            \"bugbug-train = scripts.trainer:main\",\n        ]\n    },\n    classifiers=[\n        \"Programming Language :: Python :: 3.6\",\n        \"Programming Language :: Python :: 3 :: Only\",\n        \"License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)\",\n    ],\n)\n", "path": "setup.py"}, {"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport sys\nfrom collections import defaultdict\nfrom functools import lru_cache\n\nimport numpy as np\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nHAS_OPTIONAL_DEPENDENCIES = False\n\ntry:\n    import spacy\n\n    from spacy.tokenizer import Tokenizer\n    from gensim.models import KeyedVectors\n\n    HAS_OPTIONAL_DEPENDENCIES = True\nexcept ImportError:\n    pass\n\ntry:\n    if HAS_OPTIONAL_DEPENDENCIES:\n        nlp = spacy.load(\"en_core_web_sm\")\nexcept OSError:\n    msg = (\n        \"Spacy model is missing, install it with: \"\n        f\"{sys.executable} -m spacy download en_core_web_sm\"\n    )\n    print(msg, file=sys.stderr)\n\nOPT_MSG_MISSING = (\n    \"Optional dependencies are missing, install them with: pip install bugbug[nlp]\\n\"\n    \"You might need also to download the models with: \"\n    f\"{sys.executable} -m spacy download en_core_web_sm\"\n)\n\n\ndef spacy_token_lemmatizer(text):\n    if len(text) > nlp.max_length:\n        text = text[: nlp.max_length - 1]\n    doc = nlp(text)\n    return [token.lemma_ for token in doc]\n\n\nclass SpacyVectorizer(TfidfVectorizer):\n    def __init__(self, *args, **kwargs):\n\n        # Detect when the Spacy optional dependency is missing\n        if not HAS_OPTIONAL_DEPENDENCIES:\n            raise NotImplementedError(OPT_MSG_MISSING)\n\n        super().__init__(tokenizer=spacy_token_lemmatizer, *args, **kwargs)\n\n\n@lru_cache()\ndef get_word_embeddings():\n    word_embeddings = KeyedVectors.load_word2vec_format(\"wiki-news-300d-1M-subword.vec\")\n    word_embeddings.init_sims(replace=True)\n    return word_embeddings\n\n\nclass MeanEmbeddingTransformer(BaseEstimator, TransformerMixin):\n    def __init__(self):\n        # Detect when the Gensim optional dependency are missing\n        if not HAS_OPTIONAL_DEPENDENCIES:\n            raise NotImplementedError(OPT_MSG_MISSING)\n\n        self.model = get_word_embeddings()\n        self.dim = len(self.model[\"if\"])\n\n    def fit(self, x, y=None):\n        return self\n\n    def transform(self, data):\n        tokenizer = Tokenizer(nlp.vocab)\n        return np.array(\n            [\n                np.mean(\n                    [\n                        self.model[w.text.lower()]\n                        for w in words\n                        if w.text.lower() in self.model\n                    ]\n                    or [np.zeros(self.dim)],\n                    axis=0,\n                )\n                for words in tokenizer.pipe(data)\n            ]\n        )\n\n    def get_feature_names(self):\n        return np.array([f\"_{i}\" for i in range(self.dim)], dtype=object)\n\n\nclass TfidfMeanEmbeddingTransformer(MeanEmbeddingTransformer):\n    def __init__(self):\n        super().__init__()\n        self.word2weight = None\n\n    def fit(self, X, y=None):\n        tfidf = TfidfVectorizer(analyzer=lambda x: x)\n        tfidf.fit(X)\n\n        # If a word was never seen, it must be at least as infrequent as any of the known words.\n        # So, the default idf is the max of known idfs.\n        max_idf = max(tfidf.idf_)\n        self.word2weight = defaultdict(\n            lambda: max_idf, [(w, tfidf.idf_[i]) for w, i in tfidf.vocabulary_.items()]\n        )\n\n        return self\n\n    def transform(self, data):\n        tokenizer = Tokenizer(nlp.vocab)\n        return np.array(\n            [\n                np.mean(\n                    [\n                        self.model[w.text.lower()] * self.word2weight[w.text.lower()]\n                        for w in words\n                        if w.text.lower() in self.model\n                    ]\n                    or [np.zeros(self.dim)],\n                    axis=0,\n                )\n                for words in tokenizer.pipe(data)\n            ]\n        )\n", "path": "bugbug/nlp.py"}]}