We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Add 'files' to regressor features
I've removed it in 66c496bb25b69ca4b42333ee365755f49faf62fd so that I could enable feature importance for the model.
In order to add it back, we need to implement something similar to TfIdfVectorizer's min_df parameter and only consider files that are most often modified.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `bugbug/models/regressor.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 # This Source Code Form is subject to the terms of the Mozilla Public
3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 # You can obtain one at http://mozilla.org/MPL/2.0/.
5 
6 from datetime import datetime
7 
8 import dateutil.parser
9 import xgboost
10 from dateutil.relativedelta import relativedelta
11 from imblearn.under_sampling import RandomUnderSampler
12 from sklearn.compose import ColumnTransformer
13 from sklearn.feature_extraction import DictVectorizer
14 from sklearn.pipeline import Pipeline
15 
16 from bugbug import commit_features, feature_cleanup, labels, repository
17 from bugbug.model import CommitModel
18 
19 
20 class RegressorModel(CommitModel):
21     def __init__(self, lemmatization=False):
22         CommitModel.__init__(self, lemmatization)
23 
24         self.sampler = RandomUnderSampler(random_state=0)
25 
26         feature_extractors = [
27             commit_features.file_size(),
28             commit_features.test_added(),
29             commit_features.added(),
30             commit_features.deleted(),
31             commit_features.test_deleted(),
32             commit_features.author_experience(),
33             commit_features.reviewer_experience(),
34             commit_features.reviewers_num(),
35             commit_features.component_touched_prev(),
36             commit_features.directory_touched_prev(),
37             commit_features.file_touched_prev(),
38             commit_features.types(),
39             commit_features.components(),
40             commit_features.components_modified_num(),
41             commit_features.directories(),
42             commit_features.directories_modified_num(),
43             commit_features.files_modified_num(),
44         ]
45 
46         cleanup_functions = [
47             feature_cleanup.fileref(),
48             feature_cleanup.url(),
49             feature_cleanup.synonyms(),
50         ]
51 
52         self.extraction_pipeline = Pipeline(
53             [
54                 (
55                     "commit_extractor",
56                     commit_features.CommitExtractor(
57                         feature_extractors, cleanup_functions
58                     ),
59                 ),
60                 (
61                     "union",
62                     ColumnTransformer(
63                         [
64                             ("data", DictVectorizer(), "data"),
65                             ("desc", self.text_vectorizer(min_df=0.0001), "desc"),
66                         ]
67                     ),
68                 ),
69             ]
70         )
71 
72         self.clf = xgboost.XGBClassifier(n_jobs=16)
73         self.clf.set_params(predictor="cpu_predictor")
74 
75     def get_labels(self):
76         classes = {}
77 
78         regressors = set(r[0] for r in labels.get_labels("regressor"))
79 
80         for commit_data in repository.get_commits():
81             if commit_data["ever_backedout"]:
82                 continue
83 
84             node = commit_data["node"]
85             if node in regressors:
86                 classes[node] = 1
87             else:
88                 push_date = dateutil.parser.parse(commit_data["pushdate"])
89 
90                 # The labels we have are only from 2016-11-01.
91                 # TODO: Automate collection of labels and somehow remove this check.
92                 if push_date < datetime(2016, 11, 1):
93                     continue
94 
95                 # We remove the last 6 months, as there could be regressions which haven't been filed yet.
96                 if push_date > datetime.utcnow() - relativedelta(months=6):
97                     continue
98 
99                 classes[node] = 0
100 
101         print(
102             "{} commits caused regressions".format(
103                 sum(1 for label in classes.values() if label == 1)
104             )
105         )
106 
107         print(
108             "{} commits did not cause regressions".format(
109                 sum(1 for label in classes.values() if label == 0)
110             )
111         )
112 
113         return classes, [0, 1]
114 
115     def get_feature_names(self):
116         return self.extraction_pipeline.named_steps["union"].get_feature_names()
117
```

Path: `bugbug/commit_features.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 # This Source Code Form is subject to the terms of the Mozilla Public
3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 # You can obtain one at http://mozilla.org/MPL/2.0/.
5 
6 import pandas as pd
7 from sklearn.base import BaseEstimator, TransformerMixin
8 
9 EXPERIENCE_TIMESPAN = 90
10 EXPERIENCE_TIMESPAN_TEXT = f"{EXPERIENCE_TIMESPAN}_days"
11 EXPERIENCE_TIMESPAN_TEXT_USER = f"{EXPERIENCE_TIMESPAN} days"
12 
13 
14 class files_modified_num(object):
15     name = "# of modified files"
16 
17     def __call__(self, commit, **kwargs):
18         return commit["files_modified_num"]
19 
20 
21 class file_size(object):
22     name = "File size"
23 
24     def __call__(self, commit, **kwargs):
25         return {
26             "sum": commit["total_file_size"],
27             "avg": commit["average_file_size"],
28             "max": commit["maximum_file_size"],
29             "min": commit["minimum_file_size"],
30         }
31 
32 
33 class added(object):
34     name = "# of lines added"
35 
36     def __call__(self, commit, **kwargs):
37         return commit["added"]
38 
39 
40 class test_added(object):
41     name = "# of lines added in tests"
42 
43     def __call__(self, commit, **kwargs):
44         return commit["test_added"]
45 
46 
47 class deleted(object):
48     name = "# of lines deleted"
49 
50     def __call__(self, commit, **kwargs):
51         return commit["deleted"]
52 
53 
54 class test_deleted(object):
55     name = "# of lines deleted in tests"
56 
57     def __call__(self, commit, **kwargs):
58         return commit["test_deleted"]
59 
60 
61 def get_exps(exp_type, commit):
62     items_key = f"{exp_type}s" if exp_type != "directory" else "directories"
63     items_num = len(commit[items_key])
64 
65     return {
66         "sum": commit[f"touched_prev_total_{exp_type}_sum"],
67         "max": commit[f"touched_prev_total_{exp_type}_max"],
68         "min": commit[f"touched_prev_total_{exp_type}_min"],
69         "avg": commit[f"touched_prev_total_{exp_type}_sum"] / items_num
70         if items_num > 0
71         else 0,
72         "sum backout": commit[f"touched_prev_total_{exp_type}_backout_sum"],
73         "max backout": commit[f"touched_prev_total_{exp_type}_backout_max"],
74         "min backout": commit[f"touched_prev_total_{exp_type}_backout_min"],
75         "avg backout": commit[f"touched_prev_total_{exp_type}_backout_sum"] / items_num
76         if items_num > 0
77         else 0,
78         f"sum {EXPERIENCE_TIMESPAN_TEXT_USER}": commit[
79             f"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_sum"
80         ],
81         f"max {EXPERIENCE_TIMESPAN_TEXT_USER}": commit[
82             f"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_max"
83         ],
84         f"min {EXPERIENCE_TIMESPAN_TEXT_USER}": commit[
85             f"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_min"
86         ],
87         f"avg {EXPERIENCE_TIMESPAN_TEXT_USER}": commit[
88             f"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_sum"
89         ]
90         / items_num
91         if items_num > 0
92         else 0,
93         f"sum {EXPERIENCE_TIMESPAN_TEXT_USER} backout": commit[
94             f"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_backout_sum"
95         ],
96         f"max {EXPERIENCE_TIMESPAN_TEXT_USER} backout": commit[
97             f"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_backout_max"
98         ],
99         f"min {EXPERIENCE_TIMESPAN_TEXT_USER} backout": commit[
100             f"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_backout_min"
101         ],
102         f"avg {EXPERIENCE_TIMESPAN_TEXT_USER} backout": commit[
103             f"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_backout_sum"
104         ]
105         / items_num
106         if items_num > 0
107         else 0,
108     }
109 
110 
111 class author_experience(object):
112     name = "Author experience"
113 
114     def __call__(self, commit, **kwargs):
115         return {
116             "total": commit["touched_prev_total_author_sum"],
117             EXPERIENCE_TIMESPAN_TEXT_USER: commit[
118                 f"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_author_sum"
119             ],
120             "total backouts": commit["touched_prev_total_author_backout_sum"],
121             f"{EXPERIENCE_TIMESPAN_TEXT_USER} backouts": commit[
122                 f"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_author_backout_sum"
123             ],
124             "seniority": commit["seniority_author"] / 86400,
125         }
126 
127 
128 class reviewer_experience(object):
129     name = "Reviewer experience"
130 
131     def __call__(self, commit, **kwargs):
132         return get_exps("reviewer", commit)
133 
134 
135 class reviewers_num(object):
136     name = "# of reviewers"
137 
138     def __call__(self, commit, **kwargs):
139         return len(commit["reviewers"])
140 
141 
142 class components(object):
143     def __call__(self, commit, **kwargs):
144         return commit["components"]
145 
146 
147 class components_modified_num(object):
148     name = "# of components modified"
149 
150     def __call__(self, commit, **kwargs):
151         return len(commit["components"])
152 
153 
154 class component_touched_prev(object):
155     name = "# of times the components were touched before"
156 
157     def __call__(self, commit, **kwargs):
158         return get_exps("component", commit)
159 
160 
161 class directories(object):
162     def __call__(self, commit, **kwargs):
163         return commit["directories"]
164 
165 
166 class directories_modified_num(object):
167     name = "# of directories modified"
168 
169     def __call__(self, commit, **kwargs):
170         return len(commit["directories"])
171 
172 
173 class directory_touched_prev(object):
174     name = "# of times the directories were touched before"
175 
176     def __call__(self, commit, **kwargs):
177         return get_exps("directory", commit)
178 
179 
180 class files(object):
181     def __call__(self, commit, **kwargs):
182         return commit["files"]
183 
184 
185 class file_touched_prev(object):
186     name = "# of times the files were touched before"
187 
188     def __call__(self, commit, **kwargs):
189         return get_exps("file", commit)
190 
191 
192 class types(object):
193     name = "file types"
194 
195     def __call__(self, commit, **kwargs):
196         return commit["types"]
197 
198 
199 class CommitExtractor(BaseEstimator, TransformerMixin):
200     def __init__(self, feature_extractors, cleanup_functions):
201         self.feature_extractors = feature_extractors
202         self.cleanup_functions = cleanup_functions
203 
204     def fit(self, x, y=None):
205         return self
206 
207     def transform(self, commits):
208         results = []
209 
210         for commit in commits:
211             data = {}
212 
213             for feature_extractor in self.feature_extractors:
214                 if "bug_features" in feature_extractor.__module__:
215                     if not commit["bug"]:
216                         continue
217 
218                     res = feature_extractor(commit["bug"])
219                 else:
220                     res = feature_extractor(commit)
221 
222                 if res is None:
223                     continue
224 
225                 if hasattr(feature_extractor, "name"):
226                     feature_extractor_name = feature_extractor.name
227                 else:
228                     feature_extractor_name = feature_extractor.__class__.__name__
229 
230                 if isinstance(res, dict):
231                     for key, value in res.items():
232                         data[f"{feature_extractor_name} ({key})"] = value
233                     continue
234 
235                 if isinstance(res, list):
236                     for item in res:
237                         data[f"{item} in {feature_extractor_name}"] = "True"
238                     continue
239 
240                 if isinstance(res, bool):
241                     res = str(res)
242 
243                 data[feature_extractor_name] = res
244 
245             # TODO: Try simply using all possible fields instead of extracting features manually.
246 
247             for cleanup_function in self.cleanup_functions:
248                 commit["desc"] = cleanup_function(commit["desc"])
249 
250             result = {"data": data, "desc": commit["desc"]}
251 
252             results.append(result)
253 
254         return pd.DataFrame(results)
255
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/bugbug/commit_features.py b/bugbug/commit_features.py
--- a/bugbug/commit_features.py
+++ b/bugbug/commit_features.py
@@ -3,6 +3,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from collections import defaultdict
+
 import pandas as pd
 from sklearn.base import BaseEstimator, TransformerMixin
 
@@ -178,8 +180,23 @@
 
 
 class files(object):
+    def __init__(self, min_freq=0.00003):
+        self.min_freq = min_freq
+
+    def fit(self, commits):
+        self.count = defaultdict(int)
+
+        for commit in commits:
+            for f in commit["files"]:
+                self.count[f] += 1
+        self.total_files = sum(self.count.values())
+
     def __call__(self, commit, **kwargs):
-        return commit["files"]
+        return [
+            f
+            for f in commit["files"]
+            if (self.count[f] / self.total_files) > self.min_freq
+        ]
 
 
 class file_touched_prev(object):
@@ -202,6 +219,10 @@
         self.cleanup_functions = cleanup_functions
 
     def fit(self, x, y=None):
+        for feature in self.feature_extractors:
+            if hasattr(feature, "fit"):
+                feature.fit(x)
+
         return self
 
     def transform(self, commits):
diff --git a/bugbug/models/regressor.py b/bugbug/models/regressor.py
--- a/bugbug/models/regressor.py
+++ b/bugbug/models/regressor.py
@@ -36,6 +36,7 @@
             commit_features.directory_touched_prev(),
             commit_features.file_touched_prev(),
             commit_features.types(),
+            commit_features.files(),
             commit_features.components(),
             commit_features.components_modified_num(),
             commit_features.directories(),

{"golden_diff": "diff --git a/bugbug/commit_features.py b/bugbug/commit_features.py\n--- a/bugbug/commit_features.py\n+++ b/bugbug/commit_features.py\n@@ -3,6 +3,8 @@\n # License, v. 2.0. If a copy of the MPL was not distributed with this file,\n # You can obtain one at http://mozilla.org/MPL/2.0/.\n \n+from collections import defaultdict\n+\n import pandas as pd\n from sklearn.base import BaseEstimator, TransformerMixin\n \n@@ -178,8 +180,23 @@\n \n \n class files(object):\n+    def __init__(self, min_freq=0.00003):\n+        self.min_freq = min_freq\n+\n+    def fit(self, commits):\n+        self.count = defaultdict(int)\n+\n+        for commit in commits:\n+            for f in commit[\"files\"]:\n+                self.count[f] += 1\n+        self.total_files = sum(self.count.values())\n+\n     def __call__(self, commit, **kwargs):\n-        return commit[\"files\"]\n+        return [\n+            f\n+            for f in commit[\"files\"]\n+            if (self.count[f] / self.total_files) > self.min_freq\n+        ]\n \n \n class file_touched_prev(object):\n@@ -202,6 +219,10 @@\n         self.cleanup_functions = cleanup_functions\n \n     def fit(self, x, y=None):\n+        for feature in self.feature_extractors:\n+            if hasattr(feature, \"fit\"):\n+                feature.fit(x)\n+\n         return self\n \n     def transform(self, commits):\ndiff --git a/bugbug/models/regressor.py b/bugbug/models/regressor.py\n--- a/bugbug/models/regressor.py\n+++ b/bugbug/models/regressor.py\n@@ -36,6 +36,7 @@\n             commit_features.directory_touched_prev(),\n             commit_features.file_touched_prev(),\n             commit_features.types(),\n+            commit_features.files(),\n             commit_features.components(),\n             commit_features.components_modified_num(),\n             commit_features.directories(),\n", "issue": "Add 'files' to regressor features\nI've removed it in 66c496bb25b69ca4b42333ee365755f49faf62fd so that I could enable feature importance for the model.\r\nIn order to add it back, we need to implement something similar to TfIdfVectorizer's min_df parameter and only consider files that are most often modified.\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nfrom datetime import datetime\n\nimport dateutil.parser\nimport xgboost\nfrom dateutil.relativedelta import relativedelta\nfrom imblearn.under_sampling import RandomUnderSampler\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.feature_extraction import DictVectorizer\nfrom sklearn.pipeline import Pipeline\n\nfrom bugbug import commit_features, feature_cleanup, labels, repository\nfrom bugbug.model import CommitModel\n\n\nclass RegressorModel(CommitModel):\n    def __init__(self, lemmatization=False):\n        CommitModel.__init__(self, lemmatization)\n\n        self.sampler = RandomUnderSampler(random_state=0)\n\n        feature_extractors = [\n            commit_features.file_size(),\n            commit_features.test_added(),\n            commit_features.added(),\n            commit_features.deleted(),\n            commit_features.test_deleted(),\n            commit_features.author_experience(),\n            commit_features.reviewer_experience(),\n            commit_features.reviewers_num(),\n            commit_features.component_touched_prev(),\n            commit_features.directory_touched_prev(),\n            commit_features.file_touched_prev(),\n            commit_features.types(),\n            commit_features.components(),\n            commit_features.components_modified_num(),\n            commit_features.directories(),\n            commit_features.directories_modified_num(),\n            commit_features.files_modified_num(),\n        ]\n\n        cleanup_functions = [\n            feature_cleanup.fileref(),\n            feature_cleanup.url(),\n            feature_cleanup.synonyms(),\n        ]\n\n        self.extraction_pipeline = Pipeline(\n            [\n                (\n                    \"commit_extractor\",\n                    commit_features.CommitExtractor(\n                        feature_extractors, cleanup_functions\n                    ),\n                ),\n                (\n                    \"union\",\n                    ColumnTransformer(\n                        [\n                            (\"data\", DictVectorizer(), \"data\"),\n                            (\"desc\", self.text_vectorizer(min_df=0.0001), \"desc\"),\n                        ]\n                    ),\n                ),\n            ]\n        )\n\n        self.clf = xgboost.XGBClassifier(n_jobs=16)\n        self.clf.set_params(predictor=\"cpu_predictor\")\n\n    def get_labels(self):\n        classes = {}\n\n        regressors = set(r[0] for r in labels.get_labels(\"regressor\"))\n\n        for commit_data in repository.get_commits():\n            if commit_data[\"ever_backedout\"]:\n                continue\n\n            node = commit_data[\"node\"]\n            if node in regressors:\n                classes[node] = 1\n            else:\n                push_date = dateutil.parser.parse(commit_data[\"pushdate\"])\n\n                # The labels we have are only from 2016-11-01.\n                # TODO: Automate collection of labels and somehow remove this check.\n                if push_date < datetime(2016, 11, 1):\n                    continue\n\n                # We remove the last 6 months, as there could be regressions which haven't been filed yet.\n                if push_date > datetime.utcnow() - relativedelta(months=6):\n                    continue\n\n                classes[node] = 0\n\n        print(\n            \"{} commits caused regressions\".format(\n                sum(1 for label in classes.values() if label == 1)\n            )\n        )\n\n        print(\n            \"{} commits did not cause regressions\".format(\n                sum(1 for label in classes.values() if label == 0)\n            )\n        )\n\n        return classes, [0, 1]\n\n    def get_feature_names(self):\n        return self.extraction_pipeline.named_steps[\"union\"].get_feature_names()\n", "path": "bugbug/models/regressor.py"}, {"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport pandas as pd\nfrom sklearn.base import BaseEstimator, TransformerMixin\n\nEXPERIENCE_TIMESPAN = 90\nEXPERIENCE_TIMESPAN_TEXT = f\"{EXPERIENCE_TIMESPAN}_days\"\nEXPERIENCE_TIMESPAN_TEXT_USER = f\"{EXPERIENCE_TIMESPAN} days\"\n\n\nclass files_modified_num(object):\n    name = \"# of modified files\"\n\n    def __call__(self, commit, **kwargs):\n        return commit[\"files_modified_num\"]\n\n\nclass file_size(object):\n    name = \"File size\"\n\n    def __call__(self, commit, **kwargs):\n        return {\n            \"sum\": commit[\"total_file_size\"],\n            \"avg\": commit[\"average_file_size\"],\n            \"max\": commit[\"maximum_file_size\"],\n            \"min\": commit[\"minimum_file_size\"],\n        }\n\n\nclass added(object):\n    name = \"# of lines added\"\n\n    def __call__(self, commit, **kwargs):\n        return commit[\"added\"]\n\n\nclass test_added(object):\n    name = \"# of lines added in tests\"\n\n    def __call__(self, commit, **kwargs):\n        return commit[\"test_added\"]\n\n\nclass deleted(object):\n    name = \"# of lines deleted\"\n\n    def __call__(self, commit, **kwargs):\n        return commit[\"deleted\"]\n\n\nclass test_deleted(object):\n    name = \"# of lines deleted in tests\"\n\n    def __call__(self, commit, **kwargs):\n        return commit[\"test_deleted\"]\n\n\ndef get_exps(exp_type, commit):\n    items_key = f\"{exp_type}s\" if exp_type != \"directory\" else \"directories\"\n    items_num = len(commit[items_key])\n\n    return {\n        \"sum\": commit[f\"touched_prev_total_{exp_type}_sum\"],\n        \"max\": commit[f\"touched_prev_total_{exp_type}_max\"],\n        \"min\": commit[f\"touched_prev_total_{exp_type}_min\"],\n        \"avg\": commit[f\"touched_prev_total_{exp_type}_sum\"] / items_num\n        if items_num > 0\n        else 0,\n        \"sum backout\": commit[f\"touched_prev_total_{exp_type}_backout_sum\"],\n        \"max backout\": commit[f\"touched_prev_total_{exp_type}_backout_max\"],\n        \"min backout\": commit[f\"touched_prev_total_{exp_type}_backout_min\"],\n        \"avg backout\": commit[f\"touched_prev_total_{exp_type}_backout_sum\"] / items_num\n        if items_num > 0\n        else 0,\n        f\"sum {EXPERIENCE_TIMESPAN_TEXT_USER}\": commit[\n            f\"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_sum\"\n        ],\n        f\"max {EXPERIENCE_TIMESPAN_TEXT_USER}\": commit[\n            f\"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_max\"\n        ],\n        f\"min {EXPERIENCE_TIMESPAN_TEXT_USER}\": commit[\n            f\"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_min\"\n        ],\n        f\"avg {EXPERIENCE_TIMESPAN_TEXT_USER}\": commit[\n            f\"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_sum\"\n        ]\n        / items_num\n        if items_num > 0\n        else 0,\n        f\"sum {EXPERIENCE_TIMESPAN_TEXT_USER} backout\": commit[\n            f\"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_backout_sum\"\n        ],\n        f\"max {EXPERIENCE_TIMESPAN_TEXT_USER} backout\": commit[\n            f\"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_backout_max\"\n        ],\n        f\"min {EXPERIENCE_TIMESPAN_TEXT_USER} backout\": commit[\n            f\"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_backout_min\"\n        ],\n        f\"avg {EXPERIENCE_TIMESPAN_TEXT_USER} backout\": commit[\n            f\"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_backout_sum\"\n        ]\n        / items_num\n        if items_num > 0\n        else 0,\n    }\n\n\nclass author_experience(object):\n    name = \"Author experience\"\n\n    def __call__(self, commit, **kwargs):\n        return {\n            \"total\": commit[\"touched_prev_total_author_sum\"],\n            EXPERIENCE_TIMESPAN_TEXT_USER: commit[\n                f\"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_author_sum\"\n            ],\n            \"total backouts\": commit[\"touched_prev_total_author_backout_sum\"],\n            f\"{EXPERIENCE_TIMESPAN_TEXT_USER} backouts\": commit[\n                f\"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_author_backout_sum\"\n            ],\n            \"seniority\": commit[\"seniority_author\"] / 86400,\n        }\n\n\nclass reviewer_experience(object):\n    name = \"Reviewer experience\"\n\n    def __call__(self, commit, **kwargs):\n        return get_exps(\"reviewer\", commit)\n\n\nclass reviewers_num(object):\n    name = \"# of reviewers\"\n\n    def __call__(self, commit, **kwargs):\n        return len(commit[\"reviewers\"])\n\n\nclass components(object):\n    def __call__(self, commit, **kwargs):\n        return commit[\"components\"]\n\n\nclass components_modified_num(object):\n    name = \"# of components modified\"\n\n    def __call__(self, commit, **kwargs):\n        return len(commit[\"components\"])\n\n\nclass component_touched_prev(object):\n    name = \"# of times the components were touched before\"\n\n    def __call__(self, commit, **kwargs):\n        return get_exps(\"component\", commit)\n\n\nclass directories(object):\n    def __call__(self, commit, **kwargs):\n        return commit[\"directories\"]\n\n\nclass directories_modified_num(object):\n    name = \"# of directories modified\"\n\n    def __call__(self, commit, **kwargs):\n        return len(commit[\"directories\"])\n\n\nclass directory_touched_prev(object):\n    name = \"# of times the directories were touched before\"\n\n    def __call__(self, commit, **kwargs):\n        return get_exps(\"directory\", commit)\n\n\nclass files(object):\n    def __call__(self, commit, **kwargs):\n        return commit[\"files\"]\n\n\nclass file_touched_prev(object):\n    name = \"# of times the files were touched before\"\n\n    def __call__(self, commit, **kwargs):\n        return get_exps(\"file\", commit)\n\n\nclass types(object):\n    name = \"file types\"\n\n    def __call__(self, commit, **kwargs):\n        return commit[\"types\"]\n\n\nclass CommitExtractor(BaseEstimator, TransformerMixin):\n    def __init__(self, feature_extractors, cleanup_functions):\n        self.feature_extractors = feature_extractors\n        self.cleanup_functions = cleanup_functions\n\n    def fit(self, x, y=None):\n        return self\n\n    def transform(self, commits):\n        results = []\n\n        for commit in commits:\n            data = {}\n\n            for feature_extractor in self.feature_extractors:\n                if \"bug_features\" in feature_extractor.__module__:\n                    if not commit[\"bug\"]:\n                        continue\n\n                    res = feature_extractor(commit[\"bug\"])\n                else:\n                    res = feature_extractor(commit)\n\n                if res is None:\n                    continue\n\n                if hasattr(feature_extractor, \"name\"):\n                    feature_extractor_name = feature_extractor.name\n                else:\n                    feature_extractor_name = feature_extractor.__class__.__name__\n\n                if isinstance(res, dict):\n                    for key, value in res.items():\n                        data[f\"{feature_extractor_name} ({key})\"] = value\n                    continue\n\n                if isinstance(res, list):\n                    for item in res:\n                        data[f\"{item} in {feature_extractor_name}\"] = \"True\"\n                    continue\n\n                if isinstance(res, bool):\n                    res = str(res)\n\n                data[feature_extractor_name] = res\n\n            # TODO: Try simply using all possible fields instead of extracting features manually.\n\n            for cleanup_function in self.cleanup_functions:\n                commit[\"desc\"] = cleanup_function(commit[\"desc\"])\n\n            result = {\"data\": data, \"desc\": commit[\"desc\"]}\n\n            results.append(result)\n\n        return pd.DataFrame(results)\n", "path": "bugbug/commit_features.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nfrom datetime import datetime\n\nimport dateutil.parser\nimport xgboost\nfrom dateutil.relativedelta import relativedelta\nfrom imblearn.under_sampling import RandomUnderSampler\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.feature_extraction import DictVectorizer\nfrom sklearn.pipeline import Pipeline\n\nfrom bugbug import commit_features, feature_cleanup, labels, repository\nfrom bugbug.model import CommitModel\n\n\nclass RegressorModel(CommitModel):\n    def __init__(self, lemmatization=False):\n        CommitModel.__init__(self, lemmatization)\n\n        self.sampler = RandomUnderSampler(random_state=0)\n\n        feature_extractors = [\n            commit_features.file_size(),\n            commit_features.test_added(),\n            commit_features.added(),\n            commit_features.deleted(),\n            commit_features.test_deleted(),\n            commit_features.author_experience(),\n            commit_features.reviewer_experience(),\n            commit_features.reviewers_num(),\n            commit_features.component_touched_prev(),\n            commit_features.directory_touched_prev(),\n            commit_features.file_touched_prev(),\n            commit_features.types(),\n            commit_features.files(),\n            commit_features.components(),\n            commit_features.components_modified_num(),\n            commit_features.directories(),\n            commit_features.directories_modified_num(),\n            commit_features.files_modified_num(),\n        ]\n\n        cleanup_functions = [\n            feature_cleanup.fileref(),\n            feature_cleanup.url(),\n            feature_cleanup.synonyms(),\n        ]\n\n        self.extraction_pipeline = Pipeline(\n            [\n                (\n                    \"commit_extractor\",\n                    commit_features.CommitExtractor(\n                        feature_extractors, cleanup_functions\n                    ),\n                ),\n                (\n                    \"union\",\n                    ColumnTransformer(\n                        [\n                            (\"data\", DictVectorizer(), \"data\"),\n                            (\"desc\", self.text_vectorizer(min_df=0.0001), \"desc\"),\n                        ]\n                    ),\n                ),\n            ]\n        )\n\n        self.clf = xgboost.XGBClassifier(n_jobs=16)\n        self.clf.set_params(predictor=\"cpu_predictor\")\n\n    def get_labels(self):\n        classes = {}\n\n        regressors = set(r[0] for r in labels.get_labels(\"regressor\"))\n\n        for commit_data in repository.get_commits():\n            if commit_data[\"ever_backedout\"]:\n                continue\n\n            node = commit_data[\"node\"]\n            if node in regressors:\n                classes[node] = 1\n            else:\n                push_date = dateutil.parser.parse(commit_data[\"pushdate\"])\n\n                # The labels we have are only from 2016-11-01.\n                # TODO: Automate collection of labels and somehow remove this check.\n                if push_date < datetime(2016, 11, 1):\n                    continue\n\n                # We remove the last 6 months, as there could be regressions which haven't been filed yet.\n                if push_date > datetime.utcnow() - relativedelta(months=6):\n                    continue\n\n                classes[node] = 0\n\n        print(\n            \"{} commits caused regressions\".format(\n                sum(1 for label in classes.values() if label == 1)\n            )\n        )\n\n        print(\n            \"{} commits did not cause regressions\".format(\n                sum(1 for label in classes.values() if label == 0)\n            )\n        )\n\n        return classes, [0, 1]\n\n    def get_feature_names(self):\n        return self.extraction_pipeline.named_steps[\"union\"].get_feature_names()\n", "path": "bugbug/models/regressor.py"}, {"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nfrom collections import defaultdict\n\nimport pandas as pd\nfrom sklearn.base import BaseEstimator, TransformerMixin\n\nEXPERIENCE_TIMESPAN = 90\nEXPERIENCE_TIMESPAN_TEXT = f\"{EXPERIENCE_TIMESPAN}_days\"\nEXPERIENCE_TIMESPAN_TEXT_USER = f\"{EXPERIENCE_TIMESPAN} days\"\n\n\nclass files_modified_num(object):\n    name = \"# of modified files\"\n\n    def __call__(self, commit, **kwargs):\n        return commit[\"files_modified_num\"]\n\n\nclass file_size(object):\n    name = \"File size\"\n\n    def __call__(self, commit, **kwargs):\n        return {\n            \"sum\": commit[\"total_file_size\"],\n            \"avg\": commit[\"average_file_size\"],\n            \"max\": commit[\"maximum_file_size\"],\n            \"min\": commit[\"minimum_file_size\"],\n        }\n\n\nclass added(object):\n    name = \"# of lines added\"\n\n    def __call__(self, commit, **kwargs):\n        return commit[\"added\"]\n\n\nclass test_added(object):\n    name = \"# of lines added in tests\"\n\n    def __call__(self, commit, **kwargs):\n        return commit[\"test_added\"]\n\n\nclass deleted(object):\n    name = \"# of lines deleted\"\n\n    def __call__(self, commit, **kwargs):\n        return commit[\"deleted\"]\n\n\nclass test_deleted(object):\n    name = \"# of lines deleted in tests\"\n\n    def __call__(self, commit, **kwargs):\n        return commit[\"test_deleted\"]\n\n\ndef get_exps(exp_type, commit):\n    items_key = f\"{exp_type}s\" if exp_type != \"directory\" else \"directories\"\n    items_num = len(commit[items_key])\n\n    return {\n        \"sum\": commit[f\"touched_prev_total_{exp_type}_sum\"],\n        \"max\": commit[f\"touched_prev_total_{exp_type}_max\"],\n        \"min\": commit[f\"touched_prev_total_{exp_type}_min\"],\n        \"avg\": commit[f\"touched_prev_total_{exp_type}_sum\"] / items_num\n        if items_num > 0\n        else 0,\n        \"sum backout\": commit[f\"touched_prev_total_{exp_type}_backout_sum\"],\n        \"max backout\": commit[f\"touched_prev_total_{exp_type}_backout_max\"],\n        \"min backout\": commit[f\"touched_prev_total_{exp_type}_backout_min\"],\n        \"avg backout\": commit[f\"touched_prev_total_{exp_type}_backout_sum\"] / items_num\n        if items_num > 0\n        else 0,\n        f\"sum {EXPERIENCE_TIMESPAN_TEXT_USER}\": commit[\n            f\"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_sum\"\n        ],\n        f\"max {EXPERIENCE_TIMESPAN_TEXT_USER}\": commit[\n            f\"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_max\"\n        ],\n        f\"min {EXPERIENCE_TIMESPAN_TEXT_USER}\": commit[\n            f\"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_min\"\n        ],\n        f\"avg {EXPERIENCE_TIMESPAN_TEXT_USER}\": commit[\n            f\"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_sum\"\n        ]\n        / items_num\n        if items_num > 0\n        else 0,\n        f\"sum {EXPERIENCE_TIMESPAN_TEXT_USER} backout\": commit[\n            f\"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_backout_sum\"\n        ],\n        f\"max {EXPERIENCE_TIMESPAN_TEXT_USER} backout\": commit[\n            f\"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_backout_max\"\n        ],\n        f\"min {EXPERIENCE_TIMESPAN_TEXT_USER} backout\": commit[\n            f\"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_backout_min\"\n        ],\n        f\"avg {EXPERIENCE_TIMESPAN_TEXT_USER} backout\": commit[\n            f\"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_{exp_type}_backout_sum\"\n        ]\n        / items_num\n        if items_num > 0\n        else 0,\n    }\n\n\nclass author_experience(object):\n    name = \"Author experience\"\n\n    def __call__(self, commit, **kwargs):\n        return {\n            \"total\": commit[\"touched_prev_total_author_sum\"],\n            EXPERIENCE_TIMESPAN_TEXT_USER: commit[\n                f\"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_author_sum\"\n            ],\n            \"total backouts\": commit[\"touched_prev_total_author_backout_sum\"],\n            f\"{EXPERIENCE_TIMESPAN_TEXT_USER} backouts\": commit[\n                f\"touched_prev_{EXPERIENCE_TIMESPAN_TEXT}_author_backout_sum\"\n            ],\n            \"seniority\": commit[\"seniority_author\"] / 86400,\n        }\n\n\nclass reviewer_experience(object):\n    name = \"Reviewer experience\"\n\n    def __call__(self, commit, **kwargs):\n        return get_exps(\"reviewer\", commit)\n\n\nclass reviewers_num(object):\n    name = \"# of reviewers\"\n\n    def __call__(self, commit, **kwargs):\n        return len(commit[\"reviewers\"])\n\n\nclass components(object):\n    def __call__(self, commit, **kwargs):\n        return commit[\"components\"]\n\n\nclass components_modified_num(object):\n    name = \"# of components modified\"\n\n    def __call__(self, commit, **kwargs):\n        return len(commit[\"components\"])\n\n\nclass component_touched_prev(object):\n    name = \"# of times the components were touched before\"\n\n    def __call__(self, commit, **kwargs):\n        return get_exps(\"component\", commit)\n\n\nclass directories(object):\n    def __call__(self, commit, **kwargs):\n        return commit[\"directories\"]\n\n\nclass directories_modified_num(object):\n    name = \"# of directories modified\"\n\n    def __call__(self, commit, **kwargs):\n        return len(commit[\"directories\"])\n\n\nclass directory_touched_prev(object):\n    name = \"# of times the directories were touched before\"\n\n    def __call__(self, commit, **kwargs):\n        return get_exps(\"directory\", commit)\n\n\nclass files(object):\n    def __init__(self, min_freq=0.00003):\n        self.min_freq = min_freq\n\n    def fit(self, commits):\n        self.count = defaultdict(int)\n\n        for commit in commits:\n            for f in commit[\"files\"]:\n                self.count[f] += 1\n        self.total_files = sum(self.count.values())\n\n    def __call__(self, commit, **kwargs):\n        return [\n            f\n            for f in commit[\"files\"]\n            if (self.count[f] / self.total_files) > self.min_freq\n        ]\n\n\nclass file_touched_prev(object):\n    name = \"# of times the files were touched before\"\n\n    def __call__(self, commit, **kwargs):\n        return get_exps(\"file\", commit)\n\n\nclass types(object):\n    name = \"file types\"\n\n    def __call__(self, commit, **kwargs):\n        return commit[\"types\"]\n\n\nclass CommitExtractor(BaseEstimator, TransformerMixin):\n    def __init__(self, feature_extractors, cleanup_functions):\n        self.feature_extractors = feature_extractors\n        self.cleanup_functions = cleanup_functions\n\n    def fit(self, x, y=None):\n        for feature in self.feature_extractors:\n            if hasattr(feature, \"fit\"):\n                feature.fit(x)\n\n        return self\n\n    def transform(self, commits):\n        results = []\n\n        for commit in commits:\n            data = {}\n\n            for feature_extractor in self.feature_extractors:\n                if \"bug_features\" in feature_extractor.__module__:\n                    if not commit[\"bug\"]:\n                        continue\n\n                    res = feature_extractor(commit[\"bug\"])\n                else:\n                    res = feature_extractor(commit)\n\n                if res is None:\n                    continue\n\n                if hasattr(feature_extractor, \"name\"):\n                    feature_extractor_name = feature_extractor.name\n                else:\n                    feature_extractor_name = feature_extractor.__class__.__name__\n\n                if isinstance(res, dict):\n                    for key, value in res.items():\n                        data[f\"{feature_extractor_name} ({key})\"] = value\n                    continue\n\n                if isinstance(res, list):\n                    for item in res:\n                        data[f\"{item} in {feature_extractor_name}\"] = \"True\"\n                    continue\n\n                if isinstance(res, bool):\n                    res = str(res)\n\n                data[feature_extractor_name] = res\n\n            # TODO: Try simply using all possible fields instead of extracting features manually.\n\n            for cleanup_function in self.cleanup_functions:\n                commit[\"desc\"] = cleanup_function(commit[\"desc\"])\n\n            result = {\"data\": data, \"desc\": commit[\"desc\"]}\n\n            results.append(result)\n\n        return pd.DataFrame(results)\n", "path": "bugbug/commit_features.py"}]}