We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Critical Vulnerability in np.load with NumPy v1.16 and earlier
There is a critical vulnerability with NumPy v1.16 and earlier that affects the new YB datasets module:

https://www.bleepingcomputer.com/news/security/numpy-is-awaiting-fix-for-critical-remote-code-execution-bug/

This does not affect any Yellowbrick user for version 0.9.1 or earlier and we will not release version 1.0 without a patch for this bug. When NumPy 1.17 is released (if it contains the fix), we will mark our minimum NumPy requirement to that version. 

Currently, in the `develop` branch, we do use `np.load` when [loading a numpy dataset](https://github.com/DistrictDataLabs/yellowbrick/blob/develop/yellowbrick/datasets/base.py#L195), e.g. if Pandas is not available. We should update this to `np.load(allow_pickle=False)` as per the recommendation of the post above. Note that we do [ensure data downloaded from our repository matches an expected signature](https://github.com/DistrictDataLabs/yellowbrick/blob/708274289d66d9265f7ded03e3445bc2bd70f46e/yellowbrick/datasets/download.py#L106), which minimizes but does not eliminate the risk to Yellowbrick users. 

Thanks @theagent for bringing this to our attention!

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `yellowbrick/datasets/base.py`
Content:
```
1 # yellowbrick.datasets.base
2 # Loading utilities for the yellowbrick datasets.
3 #
4 # Author:   Rebecca Bilbro <rbilbro@districtdatalabs.com>
5 # Author:   Benjamin Bengfort <bbengfort@districtdatalabs.com>
6 # Author:   Raul Peralta <raulpl25@gmail.com>
7 # Created: Thu Jul 26 13:53:01 2018 -0400
8 #
9 # ID: base.py [] benjamin@bengfort.com $
10 
11 """
12 Loading utilities for the yellowbrick datasets.
13 """
14 
15 ##########################################################################
16 ## Imports
17 ##########################################################################
18 
19 import os
20 import json
21 import numpy as np
22 
23 from .download import download_data
24 from .path import find_dataset_path, dataset_exists
25 
26 from yellowbrick.exceptions import DatasetsError
27 from yellowbrick.utils.decorators import memoized
28 
29 try:
30     import pandas as pd
31 except ImportError:
32     pd = None
33 
34 
35 ##########################################################################
36 ## Dataset Object
37 ##########################################################################
38 
39 class BaseDataset(object):
40     """
41     Base functionality for Dataset and Corpus objects.
42     """
43 
44     def __init__(self, name, url=None, signature=None, data_home=None):
45         self.url = url
46         self.name = name
47         self.data_home = data_home
48         self.signature = signature
49 
50         # Check if the dataset exists, and if not - download it!
51         if not dataset_exists(self.name, data_home=data_home):
52             self.download()
53 
54     def download(self, replace=False):
55         """
56         Download the dataset from the hosted Yellowbrick data store and save
57         it to the location specified by ``get_data_home``. The downloader
58         verifies the download completed successfully and safely by comparing
59         the expected signature with the SHA 256 signature of the downloaded
60         archive file.
61 
62         Parameters
63         ----------
64         replace : bool, default: False
65             If the data archive already exists, replace the dataset. If this is
66             False and the dataset exists, an exception is raised.
67         """
68         download_data(
69             self.url, self.signature, data_home=self.data_home,
70             replace=replace, extract=True
71         )
72 
73     def contents(self):
74         """
75         Contents returns a list of the files in the data directory.
76         """
77         data = find_dataset_path(
78             self.name, data_home=self.data_home, ext=None
79         )
80         return os.listdir(data)
81 
82     @memoized
83     def README(self):
84         """
85         Returns the contents of the README.md file that describes the dataset
86         in detail and contains attribution information.
87         """
88         path = find_dataset_path(
89             self.name, data_home=self.data_home, fname="README.md"
90         )
91         with open(path, 'r') as f:
92             return f.read()
93 
94     @memoized
95     def meta(self):
96         """
97         Returns the contents of the meta.json file that describes important
98         attributes about the dataset and modifies the behavior of the loader.
99         """
100         path = find_dataset_path(
101             self.name, data_home=self.data_home, fname="meta.json", raises=False
102         )
103         if path is None:
104             return None
105 
106         with open(path, 'r') as f:
107             return json.load(f)
108 
109     @memoized
110     def citation(self):
111         """
112         Returns the contents of the citation.bib file that describes the source
113         and provenance of the dataset or to cite for academic work.
114         """
115         path = find_dataset_path(
116             self.name, data_home=self.data_home, fname="meta.json", raises=False
117         )
118         if path is None:
119             return None
120 
121         with open(path, 'r') as f:
122             return f.read()
123 
124 
125 class Dataset(BaseDataset):
126     """
127     Datasets contain a reference to data on disk and provide utilities for
128     quickly loading files and objects into a variety of formats. The most
129     common use of the Dataset object is to load example datasets provided by
130     Yellowbrick to run the examples in the documentation.
131 
132     The dataset by default will return the data as a numpy array, however if
133     Pandas is installed, it is possible to access the data as a DataFrame and
134     Series object. In either case, the data is represented by a features table,
135     X and a target vector, y.
136 
137     Parameters
138     ----------
139     name : str
140         The name of the dataset; should either be a folder in data home or
141         specified in the yellowbrick.datasets.DATASETS variable. This name is
142         used to perform all lookups and identify the dataset externally.
143 
144     data_home : str, optional
145         The path on disk where data is stored. If not passed in, it is looked
146         up from YELLOWBRICK_DATA or the default returned by ``get_data_home``.
147 
148     url : str, optional
149         The web location where the archive file of the dataset can be
150         downloaded from.
151 
152     signature : str, optional
153         The signature of the data archive file, used to verify that the latest
154         version of the data has been downloaded and that the download hasn't
155         been corrupted or modified in anyway.
156     """
157 
158     def to_data(self):
159         """
160         Returns the data contained in the dataset as X and y where X is the
161         features matrix and y is the target vector. If pandas is installed,
162         the data will be returned as DataFrame and Series objects. Otherwise,
163         the data will be returned as two numpy arrays.
164 
165         Returns
166         -------
167         X : array-like with shape (n_instances, n_features)
168             A pandas DataFrame or numpy array describing the instance features.
169 
170         y : array-like with shape (n_instances,)
171             A pandas Series or numpy array describing the target vector.
172         """
173         if pd is not None:
174             return self.to_pandas()
175         return self.to_numpy()
176 
177     def to_numpy(self):
178         """
179         Returns the dataset as two numpy arrays: X and y.
180 
181         Returns
182         -------
183         X : array-like with shape (n_instances, n_features)
184             A numpy array describing the instance features.
185 
186         y : array-like with shape (n_instances,)
187             A numpy array describing the target vector.
188         """
189         path = find_dataset_path(self.name, ext=".npz", data_home=self.data_home)
190         with np.load(path) as npf:
191             if "X" not in npf or "y" not in npf:
192                 raise DatasetsError((
193                     "the downloaded dataset was improperly packaged without numpy arrays "
194                     "- please report this bug to the Yellowbrick maintainers!"
195                 ))
196 
197             # TODO: How to handle the case where y is None?
198             return npf["X"], npf["y"]
199 
200     def to_pandas(self):
201         """
202         Returns the dataset as two pandas objects: X and y.
203 
204         Returns
205         -------
206         X : DataFrame with shape (n_instances, n_features)
207             A pandas DataFrame containing feature data and named columns.
208 
209         y : Series with shape (n_instances,)
210             A pandas Series containing target data and an index that matches
211             the feature DataFrame index.
212         """
213         # Ensure the metadata is valid before continuing
214         if self.meta is None:
215             raise DatasetsError((
216                 "the downloaded dataset was improperly packaged without meta.json "
217                 "- please report this bug to the Yellowbrick maintainers!"
218             ))
219 
220         if "features" not in self.meta or "target" not in self.meta:
221             raise DatasetsError((
222                 "the downloaded dataset was improperly packaged without features "
223                 "or target - please report this bug to the Yellowbrick maintainers!"
224             ))
225 
226         # Load data frame and return features and target
227         # TODO: Return y as None if there is no self.meta["target"]
228         df = self.to_dataframe()
229         return df[self.meta["features"]], df[self.meta["target"]]
230 
231 
232     def to_dataframe(self):
233         """
234         Returns the entire dataset as a single pandas DataFrame.
235 
236         Returns
237         -------
238         df : DataFrame with shape (n_instances, n_columns)
239             A pandas DataFrame containing the complete original data table
240             including all targets (specified by the meta data) and all
241             features (including those that might have been filtered out).
242         """
243         if pd is None:
244             raise DatasetsError(
245                 "pandas is required to load DataFrame, it can be installed with pip"
246             )
247 
248         path = find_dataset_path(self.name, ext=".csv.gz", data_home=self.data_home)
249         return pd.read_csv(path, compression="gzip")
250 
251 
252 class Corpus(BaseDataset):
253     """
254     Corpus datasets contain a reference to documents on disk and provide
255     utilities for quickly loading text data for use in machine learning
256     workflows. The most common use of the corpus is to load the text analysis
257     examples from the Yellowbrick documentation.
258 
259     Parameters
260     ----------
261     name : str
262         The name of the corpus; should either be a folder in data home or
263         specified in the yellowbrick.datasets.DATASETS variable. This name is
264         used to perform all lookups and identify the corpus externally.
265 
266     data_home : str, optional
267         The path on disk where data is stored. If not passed in, it is looked
268         up from YELLOWBRICK_DATA or the default returned by ``get_data_home``.
269 
270     url : str, optional
271         The web location where the archive file of the corpus can be
272         downloaded from.
273 
274     signature : str, optional
275         The signature of the data archive file, used to verify that the latest
276         version of the data has been downloaded and that the download hasn't
277         been corrupted or modified in anyway.
278     """
279 
280     @memoized
281     def root(self):
282         """
283         Discovers and caches the root directory of the corpus.
284         """
285         return find_dataset_path(self.name, data_home=self.data_home, ext=None)
286 
287     @memoized
288     def labels(self):
289         """
290         Return the unique labels assigned to the documents.
291         """
292         return [
293             name for name in os.listdir(self.root)
294             if os.path.isdir(os.path.join(self.root, name))
295         ]
296 
297     @property
298     def files(self):
299         """
300         Returns the list of file names for all documents.
301         """
302         return [
303             os.path.join(self.root, label, name)
304             for label in self.labels
305             for name in os.listdir(os.path.join(self.root, label))
306         ]
307 
308     @property
309     def data(self):
310         """
311         Read all of the documents from disk into an in-memory list.
312         """
313         def read(path):
314             with open(path, 'r', encoding='UTF-8') as f:
315                 return f.read()
316 
317         return [
318             read(f) for f in self.files
319         ]
320 
321     @property
322     def target(self):
323         """
324         Returns the label associated with each item in data.
325         """
326         return [
327             os.path.basename(os.path.dirname(f)) for f in self.files
328         ]
329
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/yellowbrick/datasets/base.py b/yellowbrick/datasets/base.py
--- a/yellowbrick/datasets/base.py
+++ b/yellowbrick/datasets/base.py
@@ -187,7 +187,7 @@
             A numpy array describing the target vector.
         """
         path = find_dataset_path(self.name, ext=".npz", data_home=self.data_home)
-        with np.load(path) as npf:
+        with np.load(path, allow_pickle=False) as npf:
             if "X" not in npf or "y" not in npf:
                 raise DatasetsError((
                     "the downloaded dataset was improperly packaged without numpy arrays "

{"golden_diff": "diff --git a/yellowbrick/datasets/base.py b/yellowbrick/datasets/base.py\n--- a/yellowbrick/datasets/base.py\n+++ b/yellowbrick/datasets/base.py\n@@ -187,7 +187,7 @@\n             A numpy array describing the target vector.\n         \"\"\"\n         path = find_dataset_path(self.name, ext=\".npz\", data_home=self.data_home)\n-        with np.load(path) as npf:\n+        with np.load(path, allow_pickle=False) as npf:\n             if \"X\" not in npf or \"y\" not in npf:\n                 raise DatasetsError((\n                     \"the downloaded dataset was improperly packaged without numpy arrays \"\n", "issue": "Critical Vulnerability in np.load with NumPy v1.16 and earlier\nThere is a critical vulnerability with NumPy v1.16 and earlier that affects the new YB datasets module:\r\n\r\nhttps://www.bleepingcomputer.com/news/security/numpy-is-awaiting-fix-for-critical-remote-code-execution-bug/\r\n\r\nThis does not affect any Yellowbrick user for version 0.9.1 or earlier and we will not release version 1.0 without a patch for this bug. When NumPy 1.17 is released (if it contains the fix), we will mark our minimum NumPy requirement to that version. \r\n\r\nCurrently, in the `develop` branch, we do use `np.load` when [loading a numpy dataset](https://github.com/DistrictDataLabs/yellowbrick/blob/develop/yellowbrick/datasets/base.py#L195), e.g. if Pandas is not available. We should update this to `np.load(allow_pickle=False)` as per the recommendation of the post above. Note that we do [ensure data downloaded from our repository matches an expected signature](https://github.com/DistrictDataLabs/yellowbrick/blob/708274289d66d9265f7ded03e3445bc2bd70f46e/yellowbrick/datasets/download.py#L106), which minimizes but does not eliminate the risk to Yellowbrick users. \r\n\r\nThanks @theagent for bringing this to our attention!\n", "before_files": [{"content": "# yellowbrick.datasets.base\n# Loading utilities for the yellowbrick datasets.\n#\n# Author:   Rebecca Bilbro <rbilbro@districtdatalabs.com>\n# Author:   Benjamin Bengfort <bbengfort@districtdatalabs.com>\n# Author:   Raul Peralta <raulpl25@gmail.com>\n# Created: Thu Jul 26 13:53:01 2018 -0400\n#\n# ID: base.py [] benjamin@bengfort.com $\n\n\"\"\"\nLoading utilities for the yellowbrick datasets.\n\"\"\"\n\n##########################################################################\n## Imports\n##########################################################################\n\nimport os\nimport json\nimport numpy as np\n\nfrom .download import download_data\nfrom .path import find_dataset_path, dataset_exists\n\nfrom yellowbrick.exceptions import DatasetsError\nfrom yellowbrick.utils.decorators import memoized\n\ntry:\n    import pandas as pd\nexcept ImportError:\n    pd = None\n\n\n##########################################################################\n## Dataset Object\n##########################################################################\n\nclass BaseDataset(object):\n    \"\"\"\n    Base functionality for Dataset and Corpus objects.\n    \"\"\"\n\n    def __init__(self, name, url=None, signature=None, data_home=None):\n        self.url = url\n        self.name = name\n        self.data_home = data_home\n        self.signature = signature\n\n        # Check if the dataset exists, and if not - download it!\n        if not dataset_exists(self.name, data_home=data_home):\n            self.download()\n\n    def download(self, replace=False):\n        \"\"\"\n        Download the dataset from the hosted Yellowbrick data store and save\n        it to the location specified by ``get_data_home``. The downloader\n        verifies the download completed successfully and safely by comparing\n        the expected signature with the SHA 256 signature of the downloaded\n        archive file.\n\n        Parameters\n        ----------\n        replace : bool, default: False\n            If the data archive already exists, replace the dataset. If this is\n            False and the dataset exists, an exception is raised.\n        \"\"\"\n        download_data(\n            self.url, self.signature, data_home=self.data_home,\n            replace=replace, extract=True\n        )\n\n    def contents(self):\n        \"\"\"\n        Contents returns a list of the files in the data directory.\n        \"\"\"\n        data = find_dataset_path(\n            self.name, data_home=self.data_home, ext=None\n        )\n        return os.listdir(data)\n\n    @memoized\n    def README(self):\n        \"\"\"\n        Returns the contents of the README.md file that describes the dataset\n        in detail and contains attribution information.\n        \"\"\"\n        path = find_dataset_path(\n            self.name, data_home=self.data_home, fname=\"README.md\"\n        )\n        with open(path, 'r') as f:\n            return f.read()\n\n    @memoized\n    def meta(self):\n        \"\"\"\n        Returns the contents of the meta.json file that describes important\n        attributes about the dataset and modifies the behavior of the loader.\n        \"\"\"\n        path = find_dataset_path(\n            self.name, data_home=self.data_home, fname=\"meta.json\", raises=False\n        )\n        if path is None:\n            return None\n\n        with open(path, 'r') as f:\n            return json.load(f)\n\n    @memoized\n    def citation(self):\n        \"\"\"\n        Returns the contents of the citation.bib file that describes the source\n        and provenance of the dataset or to cite for academic work.\n        \"\"\"\n        path = find_dataset_path(\n            self.name, data_home=self.data_home, fname=\"meta.json\", raises=False\n        )\n        if path is None:\n            return None\n\n        with open(path, 'r') as f:\n            return f.read()\n\n\nclass Dataset(BaseDataset):\n    \"\"\"\n    Datasets contain a reference to data on disk and provide utilities for\n    quickly loading files and objects into a variety of formats. The most\n    common use of the Dataset object is to load example datasets provided by\n    Yellowbrick to run the examples in the documentation.\n\n    The dataset by default will return the data as a numpy array, however if\n    Pandas is installed, it is possible to access the data as a DataFrame and\n    Series object. In either case, the data is represented by a features table,\n    X and a target vector, y.\n\n    Parameters\n    ----------\n    name : str\n        The name of the dataset; should either be a folder in data home or\n        specified in the yellowbrick.datasets.DATASETS variable. This name is\n        used to perform all lookups and identify the dataset externally.\n\n    data_home : str, optional\n        The path on disk where data is stored. If not passed in, it is looked\n        up from YELLOWBRICK_DATA or the default returned by ``get_data_home``.\n\n    url : str, optional\n        The web location where the archive file of the dataset can be\n        downloaded from.\n\n    signature : str, optional\n        The signature of the data archive file, used to verify that the latest\n        version of the data has been downloaded and that the download hasn't\n        been corrupted or modified in anyway.\n    \"\"\"\n\n    def to_data(self):\n        \"\"\"\n        Returns the data contained in the dataset as X and y where X is the\n        features matrix and y is the target vector. If pandas is installed,\n        the data will be returned as DataFrame and Series objects. Otherwise,\n        the data will be returned as two numpy arrays.\n\n        Returns\n        -------\n        X : array-like with shape (n_instances, n_features)\n            A pandas DataFrame or numpy array describing the instance features.\n\n        y : array-like with shape (n_instances,)\n            A pandas Series or numpy array describing the target vector.\n        \"\"\"\n        if pd is not None:\n            return self.to_pandas()\n        return self.to_numpy()\n\n    def to_numpy(self):\n        \"\"\"\n        Returns the dataset as two numpy arrays: X and y.\n\n        Returns\n        -------\n        X : array-like with shape (n_instances, n_features)\n            A numpy array describing the instance features.\n\n        y : array-like with shape (n_instances,)\n            A numpy array describing the target vector.\n        \"\"\"\n        path = find_dataset_path(self.name, ext=\".npz\", data_home=self.data_home)\n        with np.load(path) as npf:\n            if \"X\" not in npf or \"y\" not in npf:\n                raise DatasetsError((\n                    \"the downloaded dataset was improperly packaged without numpy arrays \"\n                    \"- please report this bug to the Yellowbrick maintainers!\"\n                ))\n\n            # TODO: How to handle the case where y is None?\n            return npf[\"X\"], npf[\"y\"]\n\n    def to_pandas(self):\n        \"\"\"\n        Returns the dataset as two pandas objects: X and y.\n\n        Returns\n        -------\n        X : DataFrame with shape (n_instances, n_features)\n            A pandas DataFrame containing feature data and named columns.\n\n        y : Series with shape (n_instances,)\n            A pandas Series containing target data and an index that matches\n            the feature DataFrame index.\n        \"\"\"\n        # Ensure the metadata is valid before continuing\n        if self.meta is None:\n            raise DatasetsError((\n                \"the downloaded dataset was improperly packaged without meta.json \"\n                \"- please report this bug to the Yellowbrick maintainers!\"\n            ))\n\n        if \"features\" not in self.meta or \"target\" not in self.meta:\n            raise DatasetsError((\n                \"the downloaded dataset was improperly packaged without features \"\n                \"or target - please report this bug to the Yellowbrick maintainers!\"\n            ))\n\n        # Load data frame and return features and target\n        # TODO: Return y as None if there is no self.meta[\"target\"]\n        df = self.to_dataframe()\n        return df[self.meta[\"features\"]], df[self.meta[\"target\"]]\n\n\n    def to_dataframe(self):\n        \"\"\"\n        Returns the entire dataset as a single pandas DataFrame.\n\n        Returns\n        -------\n        df : DataFrame with shape (n_instances, n_columns)\n            A pandas DataFrame containing the complete original data table\n            including all targets (specified by the meta data) and all\n            features (including those that might have been filtered out).\n        \"\"\"\n        if pd is None:\n            raise DatasetsError(\n                \"pandas is required to load DataFrame, it can be installed with pip\"\n            )\n\n        path = find_dataset_path(self.name, ext=\".csv.gz\", data_home=self.data_home)\n        return pd.read_csv(path, compression=\"gzip\")\n\n\nclass Corpus(BaseDataset):\n    \"\"\"\n    Corpus datasets contain a reference to documents on disk and provide\n    utilities for quickly loading text data for use in machine learning\n    workflows. The most common use of the corpus is to load the text analysis\n    examples from the Yellowbrick documentation.\n\n    Parameters\n    ----------\n    name : str\n        The name of the corpus; should either be a folder in data home or\n        specified in the yellowbrick.datasets.DATASETS variable. This name is\n        used to perform all lookups and identify the corpus externally.\n\n    data_home : str, optional\n        The path on disk where data is stored. If not passed in, it is looked\n        up from YELLOWBRICK_DATA or the default returned by ``get_data_home``.\n\n    url : str, optional\n        The web location where the archive file of the corpus can be\n        downloaded from.\n\n    signature : str, optional\n        The signature of the data archive file, used to verify that the latest\n        version of the data has been downloaded and that the download hasn't\n        been corrupted or modified in anyway.\n    \"\"\"\n\n    @memoized\n    def root(self):\n        \"\"\"\n        Discovers and caches the root directory of the corpus.\n        \"\"\"\n        return find_dataset_path(self.name, data_home=self.data_home, ext=None)\n\n    @memoized\n    def labels(self):\n        \"\"\"\n        Return the unique labels assigned to the documents.\n        \"\"\"\n        return [\n            name for name in os.listdir(self.root)\n            if os.path.isdir(os.path.join(self.root, name))\n        ]\n\n    @property\n    def files(self):\n        \"\"\"\n        Returns the list of file names for all documents.\n        \"\"\"\n        return [\n            os.path.join(self.root, label, name)\n            for label in self.labels\n            for name in os.listdir(os.path.join(self.root, label))\n        ]\n\n    @property\n    def data(self):\n        \"\"\"\n        Read all of the documents from disk into an in-memory list.\n        \"\"\"\n        def read(path):\n            with open(path, 'r', encoding='UTF-8') as f:\n                return f.read()\n\n        return [\n            read(f) for f in self.files\n        ]\n\n    @property\n    def target(self):\n        \"\"\"\n        Returns the label associated with each item in data.\n        \"\"\"\n        return [\n            os.path.basename(os.path.dirname(f)) for f in self.files\n        ]\n", "path": "yellowbrick/datasets/base.py"}], "after_files": [{"content": "# yellowbrick.datasets.base\n# Loading utilities for the yellowbrick datasets.\n#\n# Author:   Rebecca Bilbro <rbilbro@districtdatalabs.com>\n# Author:   Benjamin Bengfort <bbengfort@districtdatalabs.com>\n# Author:   Raul Peralta <raulpl25@gmail.com>\n# Created: Thu Jul 26 13:53:01 2018 -0400\n#\n# ID: base.py [] benjamin@bengfort.com $\n\n\"\"\"\nLoading utilities for the yellowbrick datasets.\n\"\"\"\n\n##########################################################################\n## Imports\n##########################################################################\n\nimport os\nimport json\nimport numpy as np\n\nfrom .download import download_data\nfrom .path import find_dataset_path, dataset_exists\n\nfrom yellowbrick.exceptions import DatasetsError\nfrom yellowbrick.utils.decorators import memoized\n\ntry:\n    import pandas as pd\nexcept ImportError:\n    pd = None\n\n\n##########################################################################\n## Dataset Object\n##########################################################################\n\nclass BaseDataset(object):\n    \"\"\"\n    Base functionality for Dataset and Corpus objects.\n    \"\"\"\n\n    def __init__(self, name, url=None, signature=None, data_home=None):\n        self.url = url\n        self.name = name\n        self.data_home = data_home\n        self.signature = signature\n\n        # Check if the dataset exists, and if not - download it!\n        if not dataset_exists(self.name, data_home=data_home):\n            self.download()\n\n    def download(self, replace=False):\n        \"\"\"\n        Download the dataset from the hosted Yellowbrick data store and save\n        it to the location specified by ``get_data_home``. The downloader\n        verifies the download completed successfully and safely by comparing\n        the expected signature with the SHA 256 signature of the downloaded\n        archive file.\n\n        Parameters\n        ----------\n        replace : bool, default: False\n            If the data archive already exists, replace the dataset. If this is\n            False and the dataset exists, an exception is raised.\n        \"\"\"\n        download_data(\n            self.url, self.signature, data_home=self.data_home,\n            replace=replace, extract=True\n        )\n\n    def contents(self):\n        \"\"\"\n        Contents returns a list of the files in the data directory.\n        \"\"\"\n        data = find_dataset_path(\n            self.name, data_home=self.data_home, ext=None\n        )\n        return os.listdir(data)\n\n    @memoized\n    def README(self):\n        \"\"\"\n        Returns the contents of the README.md file that describes the dataset\n        in detail and contains attribution information.\n        \"\"\"\n        path = find_dataset_path(\n            self.name, data_home=self.data_home, fname=\"README.md\"\n        )\n        with open(path, 'r') as f:\n            return f.read()\n\n    @memoized\n    def meta(self):\n        \"\"\"\n        Returns the contents of the meta.json file that describes important\n        attributes about the dataset and modifies the behavior of the loader.\n        \"\"\"\n        path = find_dataset_path(\n            self.name, data_home=self.data_home, fname=\"meta.json\", raises=False\n        )\n        if path is None:\n            return None\n\n        with open(path, 'r') as f:\n            return json.load(f)\n\n    @memoized\n    def citation(self):\n        \"\"\"\n        Returns the contents of the citation.bib file that describes the source\n        and provenance of the dataset or to cite for academic work.\n        \"\"\"\n        path = find_dataset_path(\n            self.name, data_home=self.data_home, fname=\"meta.json\", raises=False\n        )\n        if path is None:\n            return None\n\n        with open(path, 'r') as f:\n            return f.read()\n\n\nclass Dataset(BaseDataset):\n    \"\"\"\n    Datasets contain a reference to data on disk and provide utilities for\n    quickly loading files and objects into a variety of formats. The most\n    common use of the Dataset object is to load example datasets provided by\n    Yellowbrick to run the examples in the documentation.\n\n    The dataset by default will return the data as a numpy array, however if\n    Pandas is installed, it is possible to access the data as a DataFrame and\n    Series object. In either case, the data is represented by a features table,\n    X and a target vector, y.\n\n    Parameters\n    ----------\n    name : str\n        The name of the dataset; should either be a folder in data home or\n        specified in the yellowbrick.datasets.DATASETS variable. This name is\n        used to perform all lookups and identify the dataset externally.\n\n    data_home : str, optional\n        The path on disk where data is stored. If not passed in, it is looked\n        up from YELLOWBRICK_DATA or the default returned by ``get_data_home``.\n\n    url : str, optional\n        The web location where the archive file of the dataset can be\n        downloaded from.\n\n    signature : str, optional\n        The signature of the data archive file, used to verify that the latest\n        version of the data has been downloaded and that the download hasn't\n        been corrupted or modified in anyway.\n    \"\"\"\n\n    def to_data(self):\n        \"\"\"\n        Returns the data contained in the dataset as X and y where X is the\n        features matrix and y is the target vector. If pandas is installed,\n        the data will be returned as DataFrame and Series objects. Otherwise,\n        the data will be returned as two numpy arrays.\n\n        Returns\n        -------\n        X : array-like with shape (n_instances, n_features)\n            A pandas DataFrame or numpy array describing the instance features.\n\n        y : array-like with shape (n_instances,)\n            A pandas Series or numpy array describing the target vector.\n        \"\"\"\n        if pd is not None:\n            return self.to_pandas()\n        return self.to_numpy()\n\n    def to_numpy(self):\n        \"\"\"\n        Returns the dataset as two numpy arrays: X and y.\n\n        Returns\n        -------\n        X : array-like with shape (n_instances, n_features)\n            A numpy array describing the instance features.\n\n        y : array-like with shape (n_instances,)\n            A numpy array describing the target vector.\n        \"\"\"\n        path = find_dataset_path(self.name, ext=\".npz\", data_home=self.data_home)\n        with np.load(path, allow_pickle=False) as npf:\n            if \"X\" not in npf or \"y\" not in npf:\n                raise DatasetsError((\n                    \"the downloaded dataset was improperly packaged without numpy arrays \"\n                    \"- please report this bug to the Yellowbrick maintainers!\"\n                ))\n\n            # TODO: How to handle the case where y is None?\n            return npf[\"X\"], npf[\"y\"]\n\n    def to_pandas(self):\n        \"\"\"\n        Returns the dataset as two pandas objects: X and y.\n\n        Returns\n        -------\n        X : DataFrame with shape (n_instances, n_features)\n            A pandas DataFrame containing feature data and named columns.\n\n        y : Series with shape (n_instances,)\n            A pandas Series containing target data and an index that matches\n            the feature DataFrame index.\n        \"\"\"\n        # Ensure the metadata is valid before continuing\n        if self.meta is None:\n            raise DatasetsError((\n                \"the downloaded dataset was improperly packaged without meta.json \"\n                \"- please report this bug to the Yellowbrick maintainers!\"\n            ))\n\n        if \"features\" not in self.meta or \"target\" not in self.meta:\n            raise DatasetsError((\n                \"the downloaded dataset was improperly packaged without features \"\n                \"or target - please report this bug to the Yellowbrick maintainers!\"\n            ))\n\n        # Load data frame and return features and target\n        # TODO: Return y as None if there is no self.meta[\"target\"]\n        df = self.to_dataframe()\n        return df[self.meta[\"features\"]], df[self.meta[\"target\"]]\n\n\n    def to_dataframe(self):\n        \"\"\"\n        Returns the entire dataset as a single pandas DataFrame.\n\n        Returns\n        -------\n        df : DataFrame with shape (n_instances, n_columns)\n            A pandas DataFrame containing the complete original data table\n            including all targets (specified by the meta data) and all\n            features (including those that might have been filtered out).\n        \"\"\"\n        if pd is None:\n            raise DatasetsError(\n                \"pandas is required to load DataFrame, it can be installed with pip\"\n            )\n\n        path = find_dataset_path(self.name, ext=\".csv.gz\", data_home=self.data_home)\n        return pd.read_csv(path, compression=\"gzip\")\n\n\nclass Corpus(BaseDataset):\n    \"\"\"\n    Corpus datasets contain a reference to documents on disk and provide\n    utilities for quickly loading text data for use in machine learning\n    workflows. The most common use of the corpus is to load the text analysis\n    examples from the Yellowbrick documentation.\n\n    Parameters\n    ----------\n    name : str\n        The name of the corpus; should either be a folder in data home or\n        specified in the yellowbrick.datasets.DATASETS variable. This name is\n        used to perform all lookups and identify the corpus externally.\n\n    data_home : str, optional\n        The path on disk where data is stored. If not passed in, it is looked\n        up from YELLOWBRICK_DATA or the default returned by ``get_data_home``.\n\n    url : str, optional\n        The web location where the archive file of the corpus can be\n        downloaded from.\n\n    signature : str, optional\n        The signature of the data archive file, used to verify that the latest\n        version of the data has been downloaded and that the download hasn't\n        been corrupted or modified in anyway.\n    \"\"\"\n\n    @memoized\n    def root(self):\n        \"\"\"\n        Discovers and caches the root directory of the corpus.\n        \"\"\"\n        return find_dataset_path(self.name, data_home=self.data_home, ext=None)\n\n    @memoized\n    def labels(self):\n        \"\"\"\n        Return the unique labels assigned to the documents.\n        \"\"\"\n        return [\n            name for name in os.listdir(self.root)\n            if os.path.isdir(os.path.join(self.root, name))\n        ]\n\n    @property\n    def files(self):\n        \"\"\"\n        Returns the list of file names for all documents.\n        \"\"\"\n        return [\n            os.path.join(self.root, label, name)\n            for label in self.labels\n            for name in os.listdir(os.path.join(self.root, label))\n        ]\n\n    @property\n    def data(self):\n        \"\"\"\n        Read all of the documents from disk into an in-memory list.\n        \"\"\"\n        def read(path):\n            with open(path, 'r', encoding='UTF-8') as f:\n                return f.read()\n\n        return [\n            read(f) for f in self.files\n        ]\n\n    @property\n    def target(self):\n        \"\"\"\n        Returns the label associated with each item in data.\n        \"\"\"\n        return [\n            os.path.basename(os.path.dirname(f)) for f in self.files\n        ]\n", "path": "yellowbrick/datasets/base.py"}]}