We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Iterate bugs only once when training, not twice
Currently we are iterating them once for getting y, and once for getting X. We should just iterate once.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `bugbug/utils.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 # This Source Code Form is subject to the terms of the Mozilla Public
3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 # You can obtain one at http://mozilla.org/MPL/2.0/.
5 
6 import os
7 
8 import numpy as np
9 import taskcluster
10 from sklearn.base import BaseEstimator, TransformerMixin
11 from sklearn.compose import ColumnTransformer
12 from sklearn.preprocessing import OrdinalEncoder
13 
14 TASKCLUSTER_DEFAULT_URL = "https://taskcluster.net"
15 
16 
17 def numpy_to_dict(array):
18     return {name: array[name].squeeze(axis=1) for name in array.dtype.names}
19 
20 
21 class StructuredColumnTransformer(ColumnTransformer):
22     def _hstack(self, Xs):
23         result = super()._hstack(Xs)
24 
25         transformer_names = (name for name, transformer, column in self.transformers_)
26         types = []
27         for i, (f, transformer_name) in enumerate(zip(Xs, transformer_names)):
28             types.append((transformer_name, result.dtype, (f.shape[1],)))
29 
30         return result.todense().view(np.dtype(types))
31 
32 
33 class DictExtractor(BaseEstimator, TransformerMixin):
34     def __init__(self, key):
35         self.key = key
36 
37     def fit(self, x, y=None):
38         return self
39 
40     def transform(self, data):
41         return np.array([elem[self.key] for elem in data]).reshape(-1, 1)
42 
43 
44 class MissingOrdinalEncoder(OrdinalEncoder):
45     """
46     Ordinal encoder that ignores missing values encountered after training.
47     Workaround for issue: scikit-learn/scikit-learn#11997
48     """
49 
50     def fit(self, X, y=None):
51         self._categories = self.categories
52         self._fit(X, handle_unknown="ignore")
53         return self
54 
55     def transform(self, X):
56         X_int, _ = self._transform(X, handle_unknown="ignore")
57         return X_int.astype(self.dtype, copy=False)
58 
59 
60 def get_taskcluster_options():
61     """
62     Helper to get the Taskcluster setup options
63     according to current environment (local or Taskcluster)
64     """
65     options = taskcluster.optionsFromEnvironment()
66     proxy_url = os.environ.get("TASKCLUSTER_PROXY_URL")
67 
68     if proxy_url is not None:
69         # Always use proxy url when available
70         options["rootUrl"] = proxy_url
71 
72     if "rootUrl" not in options:
73         # Always have a value in root url
74         options["rootUrl"] = TASKCLUSTER_DEFAULT_URL
75 
76     return options
77 
78 
79 def get_secret(secret_id):
80     """ Return the secret value
81     """
82     env_variable_name = f"BUGBUG_{secret_id}"
83 
84     # Try in the environment first
85     secret_from_env = os.environ.get(env_variable_name)
86 
87     if secret_from_env:
88         return secret_from_env
89 
90     # If not in env, try with TC if we have the secret id
91     tc_secret_id = os.environ.get("TC_SECRET_ID")
92 
93     if tc_secret_id:
94         secrets = taskcluster.Secrets(get_taskcluster_options())
95         secret_bucket = secrets.get(tc_secret_id)
96 
97         return secret_bucket["secret"][secret_id]
98 
99     else:
100         raise ValueError("Failed to find secret {}".format(secret_id))
101
```

Path: `bugbug/model.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 # This Source Code Form is subject to the terms of the Mozilla Public
3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 # You can obtain one at http://mozilla.org/MPL/2.0/.
5 
6 import numpy as np
7 import shap
8 from imblearn.metrics import classification_report_imbalanced
9 from imblearn.pipeline import make_pipeline
10 from sklearn import metrics
11 from sklearn.externals import joblib
12 from sklearn.feature_extraction.text import TfidfVectorizer
13 from sklearn.model_selection import cross_validate, train_test_split
14 
15 from bugbug import bugzilla, repository
16 from bugbug.nlp import SpacyVectorizer
17 
18 
19 class Model:
20     def __init__(self, lemmatization=False):
21         if lemmatization:
22             self.text_vectorizer = SpacyVectorizer
23         else:
24             self.text_vectorizer = TfidfVectorizer
25 
26         self.cross_validation_enabled = True
27         self.sampler = None
28 
29         self.calculate_importance = True
30 
31     @property
32     def le(self):
33         """Classifier agnostic getter for the label encoder property"""
34         try:
35             return self.clf._le
36         except AttributeError:
37             return self.clf.le_
38 
39     def get_feature_names(self):
40         return []
41 
42     def get_important_features(self, cutoff, shap_values):
43         # Calculate the values that represent the fraction of the model output variability attributable
44         # to each feature across the whole dataset.
45         shap_sums = shap_values.sum(0)
46         abs_shap_sums = np.abs(shap_values).sum(0)
47         rel_shap_sums = abs_shap_sums / abs_shap_sums.sum()
48 
49         cut_off_value = cutoff * np.amax(rel_shap_sums)
50 
51         # Get indices of features that pass the cut off value
52         top_feature_indices = np.where(rel_shap_sums >= cut_off_value)[0]
53         # Get the importance values of the top features from their indices
54         top_features = np.take(rel_shap_sums, top_feature_indices)
55         # Gets the sign of the importance from shap_sums as boolean
56         is_positive = (np.take(shap_sums, top_feature_indices)) >= 0
57         # Stack the importance, indices and shap_sums in a 2D array
58         top_features = np.column_stack((top_features, top_feature_indices, is_positive))
59         # Sort the array (in decreasing order of importance values)
60         top_features = top_features[top_features[:, 0].argsort()][::-1]
61 
62         return top_features
63 
64     def train(self, importance_cutoff=0.15):
65         classes, class_names = self.get_labels()
66         class_names = sorted(list(class_names), reverse=True)
67 
68         # Get items, filtering out those for which we have no labels.
69         def trainable_items_gen():
70             return (
71                 item for item in self.items_gen(classes) if self.get_id(item) in classes
72             )
73 
74         # Calculate labels.
75         y = np.array([classes[self.get_id(item)] for item in trainable_items_gen()])
76 
77         # Extract features from the items.
78         X = self.extraction_pipeline.fit_transform(trainable_items_gen())
79 
80         print(f"X: {X.shape}, y: {y.shape}")
81 
82         # Split dataset in training and test.
83         X_train, X_test, y_train, y_test = train_test_split(
84             X, y, test_size=0.1, random_state=0
85         )
86         if self.sampler is not None:
87             pipeline = make_pipeline(self.sampler, self.clf)
88         else:
89             pipeline = self.clf
90 
91         # Use k-fold cross validation to evaluate results.
92         if self.cross_validation_enabled:
93             scorings = ["accuracy"]
94             if len(class_names) == 2:
95                 scorings += ["precision", "recall"]
96 
97             scores = cross_validate(pipeline, X_train, y_train, scoring=scorings, cv=5)
98 
99             print("Cross Validation scores:")
100             for scoring in scorings:
101                 score = scores[f"test_{scoring}"]
102                 print(
103                     f"{scoring.capitalize()}: f{score.mean()} (+/- {score.std() * 2})"
104                 )
105 
106         # Training on the resampled dataset if sampler is provided.
107         if self.sampler is not None:
108             X_train, y_train = self.sampler.fit_resample(X_train, y_train)
109 
110         print(f"X_train: {X_train.shape}, y_train: {y_train.shape}")
111         print(f"X_test: {X_test.shape}, y_test: {y_test.shape}")
112 
113         self.clf.fit(X_train, y_train)
114 
115         feature_names = self.get_feature_names()
116         if self.calculate_importance and len(feature_names):
117             explainer = shap.TreeExplainer(self.clf)
118             shap_values = explainer.shap_values(X_train)
119 
120             # TODO: Actually implement feature importance visualization for multiclass problems.
121             if isinstance(shap_values, list):
122                 shap_values = np.sum(np.abs(shap_values), axis=0)
123 
124             important_features = self.get_important_features(
125                 importance_cutoff, shap_values
126             )
127 
128             print(f"\nTop {len(important_features)} Features:")
129             for i, [importance, index, is_positive] in enumerate(important_features):
130                 print(
131                     f'{i + 1}. \'{feature_names[int(index)]}\' ({"+" if (is_positive) else "-"}{importance})'
132                 )
133 
134         # Evaluate results on the test set.
135         y_pred = self.clf.predict(X_test)
136 
137         print(f"No confidence threshold - {len(y_test)} classified")
138         print(metrics.confusion_matrix(y_test, y_pred, labels=class_names))
139         print(classification_report_imbalanced(y_test, y_pred, labels=class_names))
140 
141         # Evaluate results on the test set for some confidence thresholds.
142         for confidence_threshold in [0.6, 0.7, 0.8, 0.9]:
143             y_pred_probas = self.clf.predict_proba(X_test)
144 
145             y_test_filter = []
146             y_pred_filter = []
147             for i in range(0, len(y_test)):
148                 argmax = np.argmax(y_pred_probas[i])
149                 if y_pred_probas[i][argmax] < confidence_threshold:
150                     continue
151 
152                 y_test_filter.append(y_test[i])
153                 y_pred_filter.append(argmax)
154 
155             y_pred_filter = self.le.inverse_transform(y_pred_filter)
156 
157             print(
158                 f"\nConfidence threshold > {confidence_threshold} - {len(y_test_filter)} classified"
159             )
160             print(
161                 metrics.confusion_matrix(
162                     y_test_filter, y_pred_filter, labels=class_names
163                 )
164             )
165             print(
166                 classification_report_imbalanced(
167                     y_test_filter, y_pred_filter, labels=class_names
168                 )
169             )
170 
171         joblib.dump(self, self.__class__.__name__.lower())
172 
173     @staticmethod
174     def load(model_file_name):
175         return joblib.load(model_file_name)
176 
177     def overwrite_classes(self, items, classes, probabilities):
178         return classes
179 
180     def classify(
181         self, items, probabilities=False, importances=False, importance_cutoff=0.15
182     ):
183         assert items is not None
184         assert (
185             self.extraction_pipeline is not None and self.clf is not None
186         ), "The module needs to be initialized first"
187 
188         if not isinstance(items, list):
189             items = [items]
190 
191         assert isinstance(items[0], dict) or isinstance(items[0], tuple)
192 
193         X = self.extraction_pipeline.transform(items)
194         if probabilities:
195             classes = self.clf.predict_proba(X)
196         else:
197             classes = self.clf.predict(X)
198 
199         classes = self.overwrite_classes(items, classes, probabilities)
200 
201         if importances:
202             explainer = shap.TreeExplainer(self.clf)
203             shap_values = explainer.shap_values(X)
204 
205             # TODO: Actually implement feature importance visualization for multiclass problems.
206             if isinstance(shap_values, list):
207                 shap_values = np.sum(np.abs(shap_values), axis=0)
208 
209             importances = self.get_important_features(importance_cutoff, shap_values)
210 
211             return classes, importances
212 
213         return classes
214 
215     def check(self):
216         """ Subclasses can implement their own check, the base model doesn't
217         check anything at the moment
218         """
219         return True
220 
221 
222 class BugModel(Model):
223     def get_id(self, bug):
224         return bug["id"]
225 
226     def items_gen(self, classes):
227         return (bug for bug in bugzilla.get_bugs())
228 
229 
230 class CommitModel(Model):
231     def get_id(self, commit):
232         return commit["node"]
233 
234     def items_gen(self, classes):
235         return (commit for commit in repository.get_commits())
236 
237 
238 class BugCoupleModel(Model):
239     def get_id(self, bug):
240         return bug[0]["id"], bug[1]["id"]
241 
242     def items_gen(self, classes):
243         bugs = {}
244         for bug in bugzilla.get_bugs():
245             bugs[bug["id"]] = bug
246 
247         return ((bugs[bug_id1], bugs[bug_id2]) for bug_id1, bug_id2 in classes)
248
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/bugbug/model.py b/bugbug/model.py
--- a/bugbug/model.py
+++ b/bugbug/model.py
@@ -14,6 +14,7 @@
 
 from bugbug import bugzilla, repository
 from bugbug.nlp import SpacyVectorizer
+from bugbug.utils import split_tuple_iterator
 
 
 class Model:
@@ -65,17 +66,14 @@
         classes, class_names = self.get_labels()
         class_names = sorted(list(class_names), reverse=True)
 
-        # Get items, filtering out those for which we have no labels.
-        def trainable_items_gen():
-            return (
-                item for item in self.items_gen(classes) if self.get_id(item) in classes
-            )
-
-        # Calculate labels.
-        y = np.array([classes[self.get_id(item)] for item in trainable_items_gen()])
+        # Get items and labels, filtering out those for which we have no labels.
+        X_iter, y_iter = split_tuple_iterator(self.items_gen(classes))
 
         # Extract features from the items.
-        X = self.extraction_pipeline.fit_transform(trainable_items_gen())
+        X = self.extraction_pipeline.fit_transform(X_iter)
+
+        # Calculate labels.
+        y = np.array(y_iter)
 
         print(f"X: {X.shape}, y: {y.shape}")
 
@@ -220,28 +218,28 @@
 
 
 class BugModel(Model):
-    def get_id(self, bug):
-        return bug["id"]
-
     def items_gen(self, classes):
-        return (bug for bug in bugzilla.get_bugs())
+        for bug in bugzilla.get_bugs():
+            if bug["id"] not in classes:
+                continue
 
+            yield bug, classes[bug["id"]]
 
-class CommitModel(Model):
-    def get_id(self, commit):
-        return commit["node"]
 
+class CommitModel(Model):
     def items_gen(self, classes):
-        return (commit for commit in repository.get_commits())
+        for commit in repository.get_commits():
+            if commit["node"] not in classes:
+                continue
 
+            yield commit, classes[commit["node"]]
 
-class BugCoupleModel(Model):
-    def get_id(self, bug):
-        return bug[0]["id"], bug[1]["id"]
 
+class BugCoupleModel(Model):
     def items_gen(self, classes):
         bugs = {}
         for bug in bugzilla.get_bugs():
             bugs[bug["id"]] = bug
 
-        return ((bugs[bug_id1], bugs[bug_id2]) for bug_id1, bug_id2 in classes)
+        for (bug_id1, bug_id2), label in classes.items():
+            yield (bugs[bug_id1], bugs[bug_id2]), label
diff --git a/bugbug/utils.py b/bugbug/utils.py
--- a/bugbug/utils.py
+++ b/bugbug/utils.py
@@ -3,6 +3,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import collections
 import os
 
 import numpy as np
@@ -14,6 +15,17 @@
 TASKCLUSTER_DEFAULT_URL = "https://taskcluster.net"
 
 
+def split_tuple_iterator(iterable):
+    q = collections.deque()
+
+    def first_iter():
+        for first, second in iterable:
+            yield first
+            q.append(second)
+
+    return first_iter(), q
+
+
 def numpy_to_dict(array):
     return {name: array[name].squeeze(axis=1) for name in array.dtype.names}

{"golden_diff": "diff --git a/bugbug/model.py b/bugbug/model.py\n--- a/bugbug/model.py\n+++ b/bugbug/model.py\n@@ -14,6 +14,7 @@\n \n from bugbug import bugzilla, repository\n from bugbug.nlp import SpacyVectorizer\n+from bugbug.utils import split_tuple_iterator\n \n \n class Model:\n@@ -65,17 +66,14 @@\n         classes, class_names = self.get_labels()\n         class_names = sorted(list(class_names), reverse=True)\n \n-        # Get items, filtering out those for which we have no labels.\n-        def trainable_items_gen():\n-            return (\n-                item for item in self.items_gen(classes) if self.get_id(item) in classes\n-            )\n-\n-        # Calculate labels.\n-        y = np.array([classes[self.get_id(item)] for item in trainable_items_gen()])\n+        # Get items and labels, filtering out those for which we have no labels.\n+        X_iter, y_iter = split_tuple_iterator(self.items_gen(classes))\n \n         # Extract features from the items.\n-        X = self.extraction_pipeline.fit_transform(trainable_items_gen())\n+        X = self.extraction_pipeline.fit_transform(X_iter)\n+\n+        # Calculate labels.\n+        y = np.array(y_iter)\n \n         print(f\"X: {X.shape}, y: {y.shape}\")\n \n@@ -220,28 +218,28 @@\n \n \n class BugModel(Model):\n-    def get_id(self, bug):\n-        return bug[\"id\"]\n-\n     def items_gen(self, classes):\n-        return (bug for bug in bugzilla.get_bugs())\n+        for bug in bugzilla.get_bugs():\n+            if bug[\"id\"] not in classes:\n+                continue\n \n+            yield bug, classes[bug[\"id\"]]\n \n-class CommitModel(Model):\n-    def get_id(self, commit):\n-        return commit[\"node\"]\n \n+class CommitModel(Model):\n     def items_gen(self, classes):\n-        return (commit for commit in repository.get_commits())\n+        for commit in repository.get_commits():\n+            if commit[\"node\"] not in classes:\n+                continue\n \n+            yield commit, classes[commit[\"node\"]]\n \n-class BugCoupleModel(Model):\n-    def get_id(self, bug):\n-        return bug[0][\"id\"], bug[1][\"id\"]\n \n+class BugCoupleModel(Model):\n     def items_gen(self, classes):\n         bugs = {}\n         for bug in bugzilla.get_bugs():\n             bugs[bug[\"id\"]] = bug\n \n-        return ((bugs[bug_id1], bugs[bug_id2]) for bug_id1, bug_id2 in classes)\n+        for (bug_id1, bug_id2), label in classes.items():\n+            yield (bugs[bug_id1], bugs[bug_id2]), label\ndiff --git a/bugbug/utils.py b/bugbug/utils.py\n--- a/bugbug/utils.py\n+++ b/bugbug/utils.py\n@@ -3,6 +3,7 @@\n # License, v. 2.0. If a copy of the MPL was not distributed with this file,\n # You can obtain one at http://mozilla.org/MPL/2.0/.\n \n+import collections\n import os\n \n import numpy as np\n@@ -14,6 +15,17 @@\n TASKCLUSTER_DEFAULT_URL = \"https://taskcluster.net\"\n \n \n+def split_tuple_iterator(iterable):\n+    q = collections.deque()\n+\n+    def first_iter():\n+        for first, second in iterable:\n+            yield first\n+            q.append(second)\n+\n+    return first_iter(), q\n+\n+\n def numpy_to_dict(array):\n     return {name: array[name].squeeze(axis=1) for name in array.dtype.names}\n", "issue": "Iterate bugs only once when training, not twice\nCurrently we are iterating them once for getting y, and once for getting X. We should just iterate once.\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport os\n\nimport numpy as np\nimport taskcluster\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import OrdinalEncoder\n\nTASKCLUSTER_DEFAULT_URL = \"https://taskcluster.net\"\n\n\ndef numpy_to_dict(array):\n    return {name: array[name].squeeze(axis=1) for name in array.dtype.names}\n\n\nclass StructuredColumnTransformer(ColumnTransformer):\n    def _hstack(self, Xs):\n        result = super()._hstack(Xs)\n\n        transformer_names = (name for name, transformer, column in self.transformers_)\n        types = []\n        for i, (f, transformer_name) in enumerate(zip(Xs, transformer_names)):\n            types.append((transformer_name, result.dtype, (f.shape[1],)))\n\n        return result.todense().view(np.dtype(types))\n\n\nclass DictExtractor(BaseEstimator, TransformerMixin):\n    def __init__(self, key):\n        self.key = key\n\n    def fit(self, x, y=None):\n        return self\n\n    def transform(self, data):\n        return np.array([elem[self.key] for elem in data]).reshape(-1, 1)\n\n\nclass MissingOrdinalEncoder(OrdinalEncoder):\n    \"\"\"\n    Ordinal encoder that ignores missing values encountered after training.\n    Workaround for issue: scikit-learn/scikit-learn#11997\n    \"\"\"\n\n    def fit(self, X, y=None):\n        self._categories = self.categories\n        self._fit(X, handle_unknown=\"ignore\")\n        return self\n\n    def transform(self, X):\n        X_int, _ = self._transform(X, handle_unknown=\"ignore\")\n        return X_int.astype(self.dtype, copy=False)\n\n\ndef get_taskcluster_options():\n    \"\"\"\n    Helper to get the Taskcluster setup options\n    according to current environment (local or Taskcluster)\n    \"\"\"\n    options = taskcluster.optionsFromEnvironment()\n    proxy_url = os.environ.get(\"TASKCLUSTER_PROXY_URL\")\n\n    if proxy_url is not None:\n        # Always use proxy url when available\n        options[\"rootUrl\"] = proxy_url\n\n    if \"rootUrl\" not in options:\n        # Always have a value in root url\n        options[\"rootUrl\"] = TASKCLUSTER_DEFAULT_URL\n\n    return options\n\n\ndef get_secret(secret_id):\n    \"\"\" Return the secret value\n    \"\"\"\n    env_variable_name = f\"BUGBUG_{secret_id}\"\n\n    # Try in the environment first\n    secret_from_env = os.environ.get(env_variable_name)\n\n    if secret_from_env:\n        return secret_from_env\n\n    # If not in env, try with TC if we have the secret id\n    tc_secret_id = os.environ.get(\"TC_SECRET_ID\")\n\n    if tc_secret_id:\n        secrets = taskcluster.Secrets(get_taskcluster_options())\n        secret_bucket = secrets.get(tc_secret_id)\n\n        return secret_bucket[\"secret\"][secret_id]\n\n    else:\n        raise ValueError(\"Failed to find secret {}\".format(secret_id))\n", "path": "bugbug/utils.py"}, {"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport numpy as np\nimport shap\nfrom imblearn.metrics import classification_report_imbalanced\nfrom imblearn.pipeline import make_pipeline\nfrom sklearn import metrics\nfrom sklearn.externals import joblib\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.model_selection import cross_validate, train_test_split\n\nfrom bugbug import bugzilla, repository\nfrom bugbug.nlp import SpacyVectorizer\n\n\nclass Model:\n    def __init__(self, lemmatization=False):\n        if lemmatization:\n            self.text_vectorizer = SpacyVectorizer\n        else:\n            self.text_vectorizer = TfidfVectorizer\n\n        self.cross_validation_enabled = True\n        self.sampler = None\n\n        self.calculate_importance = True\n\n    @property\n    def le(self):\n        \"\"\"Classifier agnostic getter for the label encoder property\"\"\"\n        try:\n            return self.clf._le\n        except AttributeError:\n            return self.clf.le_\n\n    def get_feature_names(self):\n        return []\n\n    def get_important_features(self, cutoff, shap_values):\n        # Calculate the values that represent the fraction of the model output variability attributable\n        # to each feature across the whole dataset.\n        shap_sums = shap_values.sum(0)\n        abs_shap_sums = np.abs(shap_values).sum(0)\n        rel_shap_sums = abs_shap_sums / abs_shap_sums.sum()\n\n        cut_off_value = cutoff * np.amax(rel_shap_sums)\n\n        # Get indices of features that pass the cut off value\n        top_feature_indices = np.where(rel_shap_sums >= cut_off_value)[0]\n        # Get the importance values of the top features from their indices\n        top_features = np.take(rel_shap_sums, top_feature_indices)\n        # Gets the sign of the importance from shap_sums as boolean\n        is_positive = (np.take(shap_sums, top_feature_indices)) >= 0\n        # Stack the importance, indices and shap_sums in a 2D array\n        top_features = np.column_stack((top_features, top_feature_indices, is_positive))\n        # Sort the array (in decreasing order of importance values)\n        top_features = top_features[top_features[:, 0].argsort()][::-1]\n\n        return top_features\n\n    def train(self, importance_cutoff=0.15):\n        classes, class_names = self.get_labels()\n        class_names = sorted(list(class_names), reverse=True)\n\n        # Get items, filtering out those for which we have no labels.\n        def trainable_items_gen():\n            return (\n                item for item in self.items_gen(classes) if self.get_id(item) in classes\n            )\n\n        # Calculate labels.\n        y = np.array([classes[self.get_id(item)] for item in trainable_items_gen()])\n\n        # Extract features from the items.\n        X = self.extraction_pipeline.fit_transform(trainable_items_gen())\n\n        print(f\"X: {X.shape}, y: {y.shape}\")\n\n        # Split dataset in training and test.\n        X_train, X_test, y_train, y_test = train_test_split(\n            X, y, test_size=0.1, random_state=0\n        )\n        if self.sampler is not None:\n            pipeline = make_pipeline(self.sampler, self.clf)\n        else:\n            pipeline = self.clf\n\n        # Use k-fold cross validation to evaluate results.\n        if self.cross_validation_enabled:\n            scorings = [\"accuracy\"]\n            if len(class_names) == 2:\n                scorings += [\"precision\", \"recall\"]\n\n            scores = cross_validate(pipeline, X_train, y_train, scoring=scorings, cv=5)\n\n            print(\"Cross Validation scores:\")\n            for scoring in scorings:\n                score = scores[f\"test_{scoring}\"]\n                print(\n                    f\"{scoring.capitalize()}: f{score.mean()} (+/- {score.std() * 2})\"\n                )\n\n        # Training on the resampled dataset if sampler is provided.\n        if self.sampler is not None:\n            X_train, y_train = self.sampler.fit_resample(X_train, y_train)\n\n        print(f\"X_train: {X_train.shape}, y_train: {y_train.shape}\")\n        print(f\"X_test: {X_test.shape}, y_test: {y_test.shape}\")\n\n        self.clf.fit(X_train, y_train)\n\n        feature_names = self.get_feature_names()\n        if self.calculate_importance and len(feature_names):\n            explainer = shap.TreeExplainer(self.clf)\n            shap_values = explainer.shap_values(X_train)\n\n            # TODO: Actually implement feature importance visualization for multiclass problems.\n            if isinstance(shap_values, list):\n                shap_values = np.sum(np.abs(shap_values), axis=0)\n\n            important_features = self.get_important_features(\n                importance_cutoff, shap_values\n            )\n\n            print(f\"\\nTop {len(important_features)} Features:\")\n            for i, [importance, index, is_positive] in enumerate(important_features):\n                print(\n                    f'{i + 1}. \\'{feature_names[int(index)]}\\' ({\"+\" if (is_positive) else \"-\"}{importance})'\n                )\n\n        # Evaluate results on the test set.\n        y_pred = self.clf.predict(X_test)\n\n        print(f\"No confidence threshold - {len(y_test)} classified\")\n        print(metrics.confusion_matrix(y_test, y_pred, labels=class_names))\n        print(classification_report_imbalanced(y_test, y_pred, labels=class_names))\n\n        # Evaluate results on the test set for some confidence thresholds.\n        for confidence_threshold in [0.6, 0.7, 0.8, 0.9]:\n            y_pred_probas = self.clf.predict_proba(X_test)\n\n            y_test_filter = []\n            y_pred_filter = []\n            for i in range(0, len(y_test)):\n                argmax = np.argmax(y_pred_probas[i])\n                if y_pred_probas[i][argmax] < confidence_threshold:\n                    continue\n\n                y_test_filter.append(y_test[i])\n                y_pred_filter.append(argmax)\n\n            y_pred_filter = self.le.inverse_transform(y_pred_filter)\n\n            print(\n                f\"\\nConfidence threshold > {confidence_threshold} - {len(y_test_filter)} classified\"\n            )\n            print(\n                metrics.confusion_matrix(\n                    y_test_filter, y_pred_filter, labels=class_names\n                )\n            )\n            print(\n                classification_report_imbalanced(\n                    y_test_filter, y_pred_filter, labels=class_names\n                )\n            )\n\n        joblib.dump(self, self.__class__.__name__.lower())\n\n    @staticmethod\n    def load(model_file_name):\n        return joblib.load(model_file_name)\n\n    def overwrite_classes(self, items, classes, probabilities):\n        return classes\n\n    def classify(\n        self, items, probabilities=False, importances=False, importance_cutoff=0.15\n    ):\n        assert items is not None\n        assert (\n            self.extraction_pipeline is not None and self.clf is not None\n        ), \"The module needs to be initialized first\"\n\n        if not isinstance(items, list):\n            items = [items]\n\n        assert isinstance(items[0], dict) or isinstance(items[0], tuple)\n\n        X = self.extraction_pipeline.transform(items)\n        if probabilities:\n            classes = self.clf.predict_proba(X)\n        else:\n            classes = self.clf.predict(X)\n\n        classes = self.overwrite_classes(items, classes, probabilities)\n\n        if importances:\n            explainer = shap.TreeExplainer(self.clf)\n            shap_values = explainer.shap_values(X)\n\n            # TODO: Actually implement feature importance visualization for multiclass problems.\n            if isinstance(shap_values, list):\n                shap_values = np.sum(np.abs(shap_values), axis=0)\n\n            importances = self.get_important_features(importance_cutoff, shap_values)\n\n            return classes, importances\n\n        return classes\n\n    def check(self):\n        \"\"\" Subclasses can implement their own check, the base model doesn't\n        check anything at the moment\n        \"\"\"\n        return True\n\n\nclass BugModel(Model):\n    def get_id(self, bug):\n        return bug[\"id\"]\n\n    def items_gen(self, classes):\n        return (bug for bug in bugzilla.get_bugs())\n\n\nclass CommitModel(Model):\n    def get_id(self, commit):\n        return commit[\"node\"]\n\n    def items_gen(self, classes):\n        return (commit for commit in repository.get_commits())\n\n\nclass BugCoupleModel(Model):\n    def get_id(self, bug):\n        return bug[0][\"id\"], bug[1][\"id\"]\n\n    def items_gen(self, classes):\n        bugs = {}\n        for bug in bugzilla.get_bugs():\n            bugs[bug[\"id\"]] = bug\n\n        return ((bugs[bug_id1], bugs[bug_id2]) for bug_id1, bug_id2 in classes)\n", "path": "bugbug/model.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport collections\nimport os\n\nimport numpy as np\nimport taskcluster\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import OrdinalEncoder\n\nTASKCLUSTER_DEFAULT_URL = \"https://taskcluster.net\"\n\n\ndef split_tuple_iterator(iterable):\n    q = collections.deque()\n\n    def first_iter():\n        for first, second in iterable:\n            yield first\n            q.append(second)\n\n    return first_iter(), q\n\n\ndef numpy_to_dict(array):\n    return {name: array[name].squeeze(axis=1) for name in array.dtype.names}\n\n\nclass StructuredColumnTransformer(ColumnTransformer):\n    def _hstack(self, Xs):\n        result = super()._hstack(Xs)\n\n        transformer_names = (name for name, transformer, column in self.transformers_)\n        types = []\n        for i, (f, transformer_name) in enumerate(zip(Xs, transformer_names)):\n            types.append((transformer_name, result.dtype, (f.shape[1],)))\n\n        return result.todense().view(np.dtype(types))\n\n\nclass DictExtractor(BaseEstimator, TransformerMixin):\n    def __init__(self, key):\n        self.key = key\n\n    def fit(self, x, y=None):\n        return self\n\n    def transform(self, data):\n        return np.array([elem[self.key] for elem in data]).reshape(-1, 1)\n\n\nclass MissingOrdinalEncoder(OrdinalEncoder):\n    \"\"\"\n    Ordinal encoder that ignores missing values encountered after training.\n    Workaround for issue: scikit-learn/scikit-learn#11997\n    \"\"\"\n\n    def fit(self, X, y=None):\n        self._categories = self.categories\n        self._fit(X, handle_unknown=\"ignore\")\n        return self\n\n    def transform(self, X):\n        X_int, _ = self._transform(X, handle_unknown=\"ignore\")\n        return X_int.astype(self.dtype, copy=False)\n\n\ndef get_taskcluster_options():\n    \"\"\"\n    Helper to get the Taskcluster setup options\n    according to current environment (local or Taskcluster)\n    \"\"\"\n    options = taskcluster.optionsFromEnvironment()\n    proxy_url = os.environ.get(\"TASKCLUSTER_PROXY_URL\")\n\n    if proxy_url is not None:\n        # Always use proxy url when available\n        options[\"rootUrl\"] = proxy_url\n\n    if \"rootUrl\" not in options:\n        # Always have a value in root url\n        options[\"rootUrl\"] = TASKCLUSTER_DEFAULT_URL\n\n    return options\n\n\ndef get_secret(secret_id):\n    \"\"\" Return the secret value\n    \"\"\"\n    env_variable_name = f\"BUGBUG_{secret_id}\"\n\n    # Try in the environment first\n    secret_from_env = os.environ.get(env_variable_name)\n\n    if secret_from_env:\n        return secret_from_env\n\n    # If not in env, try with TC if we have the secret id\n    tc_secret_id = os.environ.get(\"TC_SECRET_ID\")\n\n    if tc_secret_id:\n        secrets = taskcluster.Secrets(get_taskcluster_options())\n        secret_bucket = secrets.get(tc_secret_id)\n\n        return secret_bucket[\"secret\"][secret_id]\n\n    else:\n        raise ValueError(\"Failed to find secret {}\".format(secret_id))\n", "path": "bugbug/utils.py"}, {"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport numpy as np\nimport shap\nfrom imblearn.metrics import classification_report_imbalanced\nfrom imblearn.pipeline import make_pipeline\nfrom sklearn import metrics\nfrom sklearn.externals import joblib\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.model_selection import cross_validate, train_test_split\n\nfrom bugbug import bugzilla, repository\nfrom bugbug.nlp import SpacyVectorizer\nfrom bugbug.utils import split_tuple_iterator\n\n\nclass Model:\n    def __init__(self, lemmatization=False):\n        if lemmatization:\n            self.text_vectorizer = SpacyVectorizer\n        else:\n            self.text_vectorizer = TfidfVectorizer\n\n        self.cross_validation_enabled = True\n        self.sampler = None\n\n        self.calculate_importance = True\n\n    @property\n    def le(self):\n        \"\"\"Classifier agnostic getter for the label encoder property\"\"\"\n        try:\n            return self.clf._le\n        except AttributeError:\n            return self.clf.le_\n\n    def get_feature_names(self):\n        return []\n\n    def get_important_features(self, cutoff, shap_values):\n        # Calculate the values that represent the fraction of the model output variability attributable\n        # to each feature across the whole dataset.\n        shap_sums = shap_values.sum(0)\n        abs_shap_sums = np.abs(shap_values).sum(0)\n        rel_shap_sums = abs_shap_sums / abs_shap_sums.sum()\n\n        cut_off_value = cutoff * np.amax(rel_shap_sums)\n\n        # Get indices of features that pass the cut off value\n        top_feature_indices = np.where(rel_shap_sums >= cut_off_value)[0]\n        # Get the importance values of the top features from their indices\n        top_features = np.take(rel_shap_sums, top_feature_indices)\n        # Gets the sign of the importance from shap_sums as boolean\n        is_positive = (np.take(shap_sums, top_feature_indices)) >= 0\n        # Stack the importance, indices and shap_sums in a 2D array\n        top_features = np.column_stack((top_features, top_feature_indices, is_positive))\n        # Sort the array (in decreasing order of importance values)\n        top_features = top_features[top_features[:, 0].argsort()][::-1]\n\n        return top_features\n\n    def train(self, importance_cutoff=0.15):\n        classes, class_names = self.get_labels()\n        class_names = sorted(list(class_names), reverse=True)\n\n        # Get items and labels, filtering out those for which we have no labels.\n        X_iter, y_iter = split_tuple_iterator(self.items_gen(classes))\n\n        # Extract features from the items.\n        X = self.extraction_pipeline.fit_transform(X_iter)\n\n        # Calculate labels.\n        y = np.array(y_iter)\n\n        print(f\"X: {X.shape}, y: {y.shape}\")\n\n        # Split dataset in training and test.\n        X_train, X_test, y_train, y_test = train_test_split(\n            X, y, test_size=0.1, random_state=0\n        )\n        if self.sampler is not None:\n            pipeline = make_pipeline(self.sampler, self.clf)\n        else:\n            pipeline = self.clf\n\n        # Use k-fold cross validation to evaluate results.\n        if self.cross_validation_enabled:\n            scorings = [\"accuracy\"]\n            if len(class_names) == 2:\n                scorings += [\"precision\", \"recall\"]\n\n            scores = cross_validate(pipeline, X_train, y_train, scoring=scorings, cv=5)\n\n            print(\"Cross Validation scores:\")\n            for scoring in scorings:\n                score = scores[f\"test_{scoring}\"]\n                print(\n                    f\"{scoring.capitalize()}: f{score.mean()} (+/- {score.std() * 2})\"\n                )\n\n        # Training on the resampled dataset if sampler is provided.\n        if self.sampler is not None:\n            X_train, y_train = self.sampler.fit_resample(X_train, y_train)\n\n        print(f\"X_train: {X_train.shape}, y_train: {y_train.shape}\")\n        print(f\"X_test: {X_test.shape}, y_test: {y_test.shape}\")\n\n        self.clf.fit(X_train, y_train)\n\n        feature_names = self.get_feature_names()\n        if self.calculate_importance and len(feature_names):\n            explainer = shap.TreeExplainer(self.clf)\n            shap_values = explainer.shap_values(X_train)\n\n            # TODO: Actually implement feature importance visualization for multiclass problems.\n            if isinstance(shap_values, list):\n                shap_values = np.sum(np.abs(shap_values), axis=0)\n\n            important_features = self.get_important_features(\n                importance_cutoff, shap_values\n            )\n\n            print(f\"\\nTop {len(important_features)} Features:\")\n            for i, [importance, index, is_positive] in enumerate(important_features):\n                print(\n                    f'{i + 1}. \\'{feature_names[int(index)]}\\' ({\"+\" if (is_positive) else \"-\"}{importance})'\n                )\n\n        # Evaluate results on the test set.\n        y_pred = self.clf.predict(X_test)\n\n        print(f\"No confidence threshold - {len(y_test)} classified\")\n        print(metrics.confusion_matrix(y_test, y_pred, labels=class_names))\n        print(classification_report_imbalanced(y_test, y_pred, labels=class_names))\n\n        # Evaluate results on the test set for some confidence thresholds.\n        for confidence_threshold in [0.6, 0.7, 0.8, 0.9]:\n            y_pred_probas = self.clf.predict_proba(X_test)\n\n            y_test_filter = []\n            y_pred_filter = []\n            for i in range(0, len(y_test)):\n                argmax = np.argmax(y_pred_probas[i])\n                if y_pred_probas[i][argmax] < confidence_threshold:\n                    continue\n\n                y_test_filter.append(y_test[i])\n                y_pred_filter.append(argmax)\n\n            y_pred_filter = self.le.inverse_transform(y_pred_filter)\n\n            print(\n                f\"\\nConfidence threshold > {confidence_threshold} - {len(y_test_filter)} classified\"\n            )\n            print(\n                metrics.confusion_matrix(\n                    y_test_filter, y_pred_filter, labels=class_names\n                )\n            )\n            print(\n                classification_report_imbalanced(\n                    y_test_filter, y_pred_filter, labels=class_names\n                )\n            )\n\n        joblib.dump(self, self.__class__.__name__.lower())\n\n    @staticmethod\n    def load(model_file_name):\n        return joblib.load(model_file_name)\n\n    def overwrite_classes(self, items, classes, probabilities):\n        return classes\n\n    def classify(\n        self, items, probabilities=False, importances=False, importance_cutoff=0.15\n    ):\n        assert items is not None\n        assert (\n            self.extraction_pipeline is not None and self.clf is not None\n        ), \"The module needs to be initialized first\"\n\n        if not isinstance(items, list):\n            items = [items]\n\n        assert isinstance(items[0], dict) or isinstance(items[0], tuple)\n\n        X = self.extraction_pipeline.transform(items)\n        if probabilities:\n            classes = self.clf.predict_proba(X)\n        else:\n            classes = self.clf.predict(X)\n\n        classes = self.overwrite_classes(items, classes, probabilities)\n\n        if importances:\n            explainer = shap.TreeExplainer(self.clf)\n            shap_values = explainer.shap_values(X)\n\n            # TODO: Actually implement feature importance visualization for multiclass problems.\n            if isinstance(shap_values, list):\n                shap_values = np.sum(np.abs(shap_values), axis=0)\n\n            importances = self.get_important_features(importance_cutoff, shap_values)\n\n            return classes, importances\n\n        return classes\n\n    def check(self):\n        \"\"\" Subclasses can implement their own check, the base model doesn't\n        check anything at the moment\n        \"\"\"\n        return True\n\n\nclass BugModel(Model):\n    def items_gen(self, classes):\n        for bug in bugzilla.get_bugs():\n            if bug[\"id\"] not in classes:\n                continue\n\n            yield bug, classes[bug[\"id\"]]\n\n\nclass CommitModel(Model):\n    def items_gen(self, classes):\n        for commit in repository.get_commits():\n            if commit[\"node\"] not in classes:\n                continue\n\n            yield commit, classes[commit[\"node\"]]\n\n\nclass BugCoupleModel(Model):\n    def items_gen(self, classes):\n        bugs = {}\n        for bug in bugzilla.get_bugs():\n            bugs[bug[\"id\"]] = bug\n\n        for (bug_id1, bug_id2), label in classes.items():\n            yield (bugs[bug_id1], bugs[bug_id2]), label\n", "path": "bugbug/model.py"}]}