We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Train the component model on bugs from Fenix
It is now appropriate to include Fenix in the training dataset since it has been moved to Bugzilla.





--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `bugbug/models/component.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 # This Source Code Form is subject to the terms of the Mozilla Public
3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 # You can obtain one at http://mozilla.org/MPL/2.0/.
5 
6 import logging
7 from collections import Counter
8 from datetime import datetime, timezone
9 
10 import dateutil.parser
11 import xgboost
12 from dateutil.relativedelta import relativedelta
13 from sklearn.compose import ColumnTransformer
14 from sklearn.feature_extraction import DictVectorizer
15 from sklearn.pipeline import Pipeline
16 
17 from bugbug import bug_features, bugzilla, feature_cleanup, utils
18 from bugbug.bugzilla import get_product_component_count
19 from bugbug.model import BugModel
20 
21 logging.basicConfig(level=logging.INFO)
22 logger = logging.getLogger(__name__)
23 
24 
25 class ComponentModel(BugModel):
26     PRODUCTS = {
27         "Core",
28         "External Software Affecting Firefox",
29         "DevTools",
30         "Firefox",
31         "Toolkit",
32         "WebExtensions",
33         "Firefox Build System",
34     }
35 
36     PRODUCT_COMPONENTS = {
37         "Cloud Services": {"Server: Firefox Accounts"},
38     }
39 
40     CONFLATED_COMPONENTS = [
41         "Core::Audio/Video",
42         "Core::DOM",
43         "Core::Graphics",
44         "Core::IPC",
45         "Core::JavaScript",
46         "Core::Layout",
47         "Core::Networking",
48         "Core::Print",
49         "Core::WebRTC",
50         "Toolkit::Password Manager",
51         "DevTools",
52         "External Software Affecting Firefox",
53         "WebExtensions",
54         "Firefox Build System",
55     ]
56 
57     CONFLATED_COMPONENTS_MAPPING = {
58         "Core::DOM": "Core::DOM: Core & HTML",
59         "Core::JavaScript": "Core::JavaScript Engine",
60         "Core::Print": "Core::Printing: Output",
61         "DevTools": "DevTools::General",
62         "External Software Affecting Firefox": "External Software Affecting Firefox::Other",
63         "WebExtensions": "WebExtensions::Untriaged",
64         "Firefox Build System": "Firefox Build System::General",
65     }
66 
67     def __init__(self, lemmatization=False):
68         BugModel.__init__(self, lemmatization)
69 
70         self.cross_validation_enabled = False
71         self.calculate_importance = False
72 
73         feature_extractors = [
74             bug_features.HasSTR(),
75             bug_features.Severity(),
76             bug_features.Keywords(),
77             bug_features.HasCrashSignature(),
78             bug_features.HasURL(),
79             bug_features.HasW3CURL(),
80             bug_features.HasGithubURL(),
81             bug_features.Whiteboard(),
82             bug_features.Patches(),
83             bug_features.Landings(),
84         ]
85 
86         cleanup_functions = [
87             feature_cleanup.fileref(),
88             feature_cleanup.url(),
89             feature_cleanup.synonyms(),
90         ]
91 
92         self.extraction_pipeline = Pipeline(
93             [
94                 (
95                     "bug_extractor",
96                     bug_features.BugExtractor(
97                         feature_extractors, cleanup_functions, rollback=True
98                     ),
99                 ),
100             ]
101         )
102 
103         self.clf = Pipeline(
104             [
105                 (
106                     "union",
107                     ColumnTransformer(
108                         [
109                             ("data", DictVectorizer(), "data"),
110                             ("title", self.text_vectorizer(min_df=0.0001), "title"),
111                             (
112                                 "comments",
113                                 self.text_vectorizer(min_df=0.0001),
114                                 "comments",
115                             ),
116                         ]
117                     ),
118                 ),
119                 (
120                     "estimator",
121                     xgboost.XGBClassifier(n_jobs=utils.get_physical_cpu_count()),
122                 ),
123             ]
124         )
125 
126         self.CONFLATED_COMPONENTS_INVERSE_MAPPING = {
127             v: k for k, v in self.CONFLATED_COMPONENTS_MAPPING.items()
128         }
129 
130     def filter_component(self, product, component):
131         full_comp = f"{product}::{component}"
132 
133         if full_comp in self.CONFLATED_COMPONENTS_INVERSE_MAPPING:
134             return self.CONFLATED_COMPONENTS_INVERSE_MAPPING[full_comp]
135 
136         if (product, component) in self.meaningful_product_components:
137             return full_comp
138 
139         for conflated_component in self.CONFLATED_COMPONENTS:
140             if full_comp.startswith(conflated_component):
141                 return conflated_component
142 
143         return None
144 
145     def get_labels(self):
146         product_components = {}
147         for bug_data in bugzilla.get_bugs():
148             if dateutil.parser.parse(bug_data["creation_time"]) < datetime.now(
149                 timezone.utc
150             ) - relativedelta(years=2):
151                 continue
152 
153             product_components[bug_data["id"]] = (
154                 bug_data["product"],
155                 bug_data["component"],
156             )
157 
158         self.meaningful_product_components = self.get_meaningful_product_components(
159             (
160                 (product, component)
161                 for product, component in product_components.values()
162                 if self.is_meaningful(product, component)
163             )
164         )
165 
166         classes = {}
167         for bug_id, (product, component) in product_components.items():
168             component = self.filter_component(product, component)
169 
170             if component:
171                 classes[bug_id] = component
172 
173         component_counts = Counter(classes.values()).most_common()
174         top_components = set(component for component, count in component_counts)
175 
176         logger.info("%d components", len(top_components))
177         for component, count in component_counts:
178             logger.info("%s: %d", component, count)
179 
180         # Assert there is at least one bug for each conflated component.
181         for conflated_component in self.CONFLATED_COMPONENTS:
182             assert any(
183                 conflated_component == component
184                 for component, count in component_counts
185             ), f"There should be at least one bug matching {conflated_component}*"
186 
187         # Assert there is at least one bug for each component the conflated components are mapped to.
188         for conflated_component_mapping in self.CONFLATED_COMPONENTS_MAPPING.values():
189             assert any(
190                 conflated_component_mapping == f"{product}::{component}"
191                 for product, component in product_components.values()
192             ), f"There should be at least one bug in {conflated_component_mapping}"
193 
194         # Assert all conflated components are either in conflated_components_mapping or exist as components.
195         for conflated_component in self.CONFLATED_COMPONENTS:
196             assert conflated_component in self.CONFLATED_COMPONENTS_MAPPING or any(
197                 conflated_component == f"{product}::{component}"
198                 for product, component in product_components.values()
199             ), f"It should be possible to map {conflated_component}"
200 
201         classes = {
202             bug_id: component
203             for bug_id, component in classes.items()
204             if component in top_components
205         }
206 
207         return classes, set(classes.values())
208 
209     def is_meaningful(self, product, component):
210         return (
211             product in self.PRODUCTS
212             and component not in ["General", "Untriaged", "Foxfooding"]
213         ) or (
214             product in self.PRODUCT_COMPONENTS
215             and component in self.PRODUCT_COMPONENTS[product]
216         )
217 
218     def get_meaningful_product_components(self, full_comp_tuples, threshold_ratio=100):
219         """Filter out components which does not have more than 1% of the most common component.
220 
221         Returns:
222             a set of tuples which have at least 1% of the most common tuple
223         """
224         product_component_counts = Counter(full_comp_tuples).most_common()
225 
226         max_count = product_component_counts[0][1]
227         threshold = max_count / threshold_ratio
228 
229         active_product_components = bugzilla.get_active_product_components(
230             list(self.PRODUCTS) + list(self.PRODUCT_COMPONENTS)
231         )
232 
233         return set(
234             product_component
235             for product_component, count in product_component_counts
236             if count > threshold and product_component in active_product_components
237         )
238 
239     def get_feature_names(self):
240         return self.clf.named_steps["union"].get_feature_names_out()
241 
242     def check(self):
243         success = super().check()
244 
245         # Get the number of bugs per full component to fasten up the check
246         bugs_number = get_product_component_count()
247 
248         # Check number 1, check that the most meaningful product components
249         # still have at least a bug in this component. If the check is failing
250         # that could mean that:
251         # - A component has been renamed / removed
252         # - A component is not used anymore by developers
253 
254         for product, component in self.meaningful_product_components:
255             full_comp = f"{product}::{component}"
256 
257             if full_comp not in bugs_number.keys():
258                 logger.warning(
259                     "Component %r of product %r doesn't exists, failure",
260                     component,
261                     product,
262                 )
263                 success = False
264 
265             elif bugs_number[full_comp] <= 0:
266                 logger.warning(
267                     "Component %r of product %r have 0 bugs or less in it, failure",
268                     component,
269                     product,
270                 )
271                 success = False
272 
273         # Check number 2, check that conflated components in
274         # CONFLATED_COMPONENTS match at least one component which has more
275         # than 0 bugs
276 
277         for conflated_component in self.CONFLATED_COMPONENTS:
278             matching_components = [
279                 full_comp
280                 for full_comp in bugs_number
281                 if full_comp.startswith(conflated_component)
282             ]
283 
284             if not matching_components:
285                 logger.warning("%s doesn't match any component", conflated_component)
286                 success = False
287                 continue
288 
289             matching_components_values = [
290                 bugs_number[full_comp]
291                 for full_comp in matching_components
292                 if bugs_number[full_comp] > 0
293             ]
294 
295             if not matching_components_values:
296                 logger.warning(
297                     "%s should match at least one component with more than 0 bugs",
298                     conflated_component,
299                 )
300                 success = False
301 
302         # Check number 3, check that values of CONFLATED_COMPONENTS_MAPPING
303         # still exist as components and have more than 0 bugs
304 
305         for full_comp in self.CONFLATED_COMPONENTS_MAPPING.values():
306             if full_comp not in bugs_number:
307                 logger.warning(
308                     "%s from conflated component mapping doesn't exists, failure",
309                     full_comp,
310                 )
311                 success = False
312             elif bugs_number[full_comp] <= 0:
313                 logger.warning(
314                     "%s from conflated component mapping have less than 1 bug, failure",
315                     full_comp,
316                 )
317                 success = False
318 
319         # Check number 4, conflated components in CONFLATED_COMPONENTS either
320         # exist as components or are in CONFLATED_COMPONENTS_MAPPING
321 
322         for conflated_component in self.CONFLATED_COMPONENTS:
323             in_mapping = conflated_component in self.CONFLATED_COMPONENTS_MAPPING
324 
325             matching_components = [
326                 full_comp
327                 for full_comp in bugs_number
328                 if full_comp.startswith(conflated_component)
329             ]
330 
331             if not (matching_components or in_mapping):
332                 logger.warning("It should be possible to map %s", conflated_component)
333                 success = False
334                 continue
335 
336         # Check number 5, there is no component with many bugs that is not in
337         # meaningful_product_components
338 
339         # Recompute the meaningful components
340 
341         def generate_meaningful_tuples():
342             for full_comp, count in bugs_number.items():
343                 product, component = full_comp.split("::", 1)
344 
345                 if not self.is_meaningful(product, component):
346                     continue
347 
348                 if count > 0:
349                     for i in range(count):
350                         yield (product, component)
351 
352         meaningful_product_components = self.get_meaningful_product_components(
353             generate_meaningful_tuples(), threshold_ratio=10
354         )
355 
356         if not meaningful_product_components.issubset(
357             self.meaningful_product_components
358         ):
359             logger.warning("Meaningful product components mismatch")
360 
361             new_meaningful_product_components = (
362                 meaningful_product_components.difference(
363                     self.meaningful_product_components
364                 )
365             )
366             logger.info(
367                 "New meaningful product components %r",
368                 new_meaningful_product_components,
369             )
370 
371             success = False
372 
373         return success
374 
375     def get_extra_data(self):
376         return {"conflated_components_mapping": self.CONFLATED_COMPONENTS_MAPPING}
377
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/bugbug/models/component.py b/bugbug/models/component.py
--- a/bugbug/models/component.py
+++ b/bugbug/models/component.py
@@ -27,6 +27,7 @@
         "Core",
         "External Software Affecting Firefox",
         "DevTools",
+        "Fenix",
         "Firefox",
         "Toolkit",
         "WebExtensions",
@@ -52,6 +53,7 @@
         "External Software Affecting Firefox",
         "WebExtensions",
         "Firefox Build System",
+        "Fenix",
     ]
 
     CONFLATED_COMPONENTS_MAPPING = {
@@ -62,6 +64,7 @@
         "External Software Affecting Firefox": "External Software Affecting Firefox::Other",
         "WebExtensions": "WebExtensions::Untriaged",
         "Firefox Build System": "Firefox Build System::General",
+        "Fenix": "Fenix::General",
     }
 
     def __init__(self, lemmatization=False):

{"golden_diff": "diff --git a/bugbug/models/component.py b/bugbug/models/component.py\n--- a/bugbug/models/component.py\n+++ b/bugbug/models/component.py\n@@ -27,6 +27,7 @@\n         \"Core\",\n         \"External Software Affecting Firefox\",\n         \"DevTools\",\n+        \"Fenix\",\n         \"Firefox\",\n         \"Toolkit\",\n         \"WebExtensions\",\n@@ -52,6 +53,7 @@\n         \"External Software Affecting Firefox\",\n         \"WebExtensions\",\n         \"Firefox Build System\",\n+        \"Fenix\",\n     ]\n \n     CONFLATED_COMPONENTS_MAPPING = {\n@@ -62,6 +64,7 @@\n         \"External Software Affecting Firefox\": \"External Software Affecting Firefox::Other\",\n         \"WebExtensions\": \"WebExtensions::Untriaged\",\n         \"Firefox Build System\": \"Firefox Build System::General\",\n+        \"Fenix\": \"Fenix::General\",\n     }\n \n     def __init__(self, lemmatization=False):\n", "issue": "Train the component model on bugs from Fenix\nIt is now appropriate to include Fenix in the training dataset since it has been moved to Bugzilla.\r\n\r\n\r\n\r\n\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport logging\nfrom collections import Counter\nfrom datetime import datetime, timezone\n\nimport dateutil.parser\nimport xgboost\nfrom dateutil.relativedelta import relativedelta\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.feature_extraction import DictVectorizer\nfrom sklearn.pipeline import Pipeline\n\nfrom bugbug import bug_features, bugzilla, feature_cleanup, utils\nfrom bugbug.bugzilla import get_product_component_count\nfrom bugbug.model import BugModel\n\nlogging.basicConfig(level=logging.INFO)\nlogger = logging.getLogger(__name__)\n\n\nclass ComponentModel(BugModel):\n    PRODUCTS = {\n        \"Core\",\n        \"External Software Affecting Firefox\",\n        \"DevTools\",\n        \"Firefox\",\n        \"Toolkit\",\n        \"WebExtensions\",\n        \"Firefox Build System\",\n    }\n\n    PRODUCT_COMPONENTS = {\n        \"Cloud Services\": {\"Server: Firefox Accounts\"},\n    }\n\n    CONFLATED_COMPONENTS = [\n        \"Core::Audio/Video\",\n        \"Core::DOM\",\n        \"Core::Graphics\",\n        \"Core::IPC\",\n        \"Core::JavaScript\",\n        \"Core::Layout\",\n        \"Core::Networking\",\n        \"Core::Print\",\n        \"Core::WebRTC\",\n        \"Toolkit::Password Manager\",\n        \"DevTools\",\n        \"External Software Affecting Firefox\",\n        \"WebExtensions\",\n        \"Firefox Build System\",\n    ]\n\n    CONFLATED_COMPONENTS_MAPPING = {\n        \"Core::DOM\": \"Core::DOM: Core & HTML\",\n        \"Core::JavaScript\": \"Core::JavaScript Engine\",\n        \"Core::Print\": \"Core::Printing: Output\",\n        \"DevTools\": \"DevTools::General\",\n        \"External Software Affecting Firefox\": \"External Software Affecting Firefox::Other\",\n        \"WebExtensions\": \"WebExtensions::Untriaged\",\n        \"Firefox Build System\": \"Firefox Build System::General\",\n    }\n\n    def __init__(self, lemmatization=False):\n        BugModel.__init__(self, lemmatization)\n\n        self.cross_validation_enabled = False\n        self.calculate_importance = False\n\n        feature_extractors = [\n            bug_features.HasSTR(),\n            bug_features.Severity(),\n            bug_features.Keywords(),\n            bug_features.HasCrashSignature(),\n            bug_features.HasURL(),\n            bug_features.HasW3CURL(),\n            bug_features.HasGithubURL(),\n            bug_features.Whiteboard(),\n            bug_features.Patches(),\n            bug_features.Landings(),\n        ]\n\n        cleanup_functions = [\n            feature_cleanup.fileref(),\n            feature_cleanup.url(),\n            feature_cleanup.synonyms(),\n        ]\n\n        self.extraction_pipeline = Pipeline(\n            [\n                (\n                    \"bug_extractor\",\n                    bug_features.BugExtractor(\n                        feature_extractors, cleanup_functions, rollback=True\n                    ),\n                ),\n            ]\n        )\n\n        self.clf = Pipeline(\n            [\n                (\n                    \"union\",\n                    ColumnTransformer(\n                        [\n                            (\"data\", DictVectorizer(), \"data\"),\n                            (\"title\", self.text_vectorizer(min_df=0.0001), \"title\"),\n                            (\n                                \"comments\",\n                                self.text_vectorizer(min_df=0.0001),\n                                \"comments\",\n                            ),\n                        ]\n                    ),\n                ),\n                (\n                    \"estimator\",\n                    xgboost.XGBClassifier(n_jobs=utils.get_physical_cpu_count()),\n                ),\n            ]\n        )\n\n        self.CONFLATED_COMPONENTS_INVERSE_MAPPING = {\n            v: k for k, v in self.CONFLATED_COMPONENTS_MAPPING.items()\n        }\n\n    def filter_component(self, product, component):\n        full_comp = f\"{product}::{component}\"\n\n        if full_comp in self.CONFLATED_COMPONENTS_INVERSE_MAPPING:\n            return self.CONFLATED_COMPONENTS_INVERSE_MAPPING[full_comp]\n\n        if (product, component) in self.meaningful_product_components:\n            return full_comp\n\n        for conflated_component in self.CONFLATED_COMPONENTS:\n            if full_comp.startswith(conflated_component):\n                return conflated_component\n\n        return None\n\n    def get_labels(self):\n        product_components = {}\n        for bug_data in bugzilla.get_bugs():\n            if dateutil.parser.parse(bug_data[\"creation_time\"]) < datetime.now(\n                timezone.utc\n            ) - relativedelta(years=2):\n                continue\n\n            product_components[bug_data[\"id\"]] = (\n                bug_data[\"product\"],\n                bug_data[\"component\"],\n            )\n\n        self.meaningful_product_components = self.get_meaningful_product_components(\n            (\n                (product, component)\n                for product, component in product_components.values()\n                if self.is_meaningful(product, component)\n            )\n        )\n\n        classes = {}\n        for bug_id, (product, component) in product_components.items():\n            component = self.filter_component(product, component)\n\n            if component:\n                classes[bug_id] = component\n\n        component_counts = Counter(classes.values()).most_common()\n        top_components = set(component for component, count in component_counts)\n\n        logger.info(\"%d components\", len(top_components))\n        for component, count in component_counts:\n            logger.info(\"%s: %d\", component, count)\n\n        # Assert there is at least one bug for each conflated component.\n        for conflated_component in self.CONFLATED_COMPONENTS:\n            assert any(\n                conflated_component == component\n                for component, count in component_counts\n            ), f\"There should be at least one bug matching {conflated_component}*\"\n\n        # Assert there is at least one bug for each component the conflated components are mapped to.\n        for conflated_component_mapping in self.CONFLATED_COMPONENTS_MAPPING.values():\n            assert any(\n                conflated_component_mapping == f\"{product}::{component}\"\n                for product, component in product_components.values()\n            ), f\"There should be at least one bug in {conflated_component_mapping}\"\n\n        # Assert all conflated components are either in conflated_components_mapping or exist as components.\n        for conflated_component in self.CONFLATED_COMPONENTS:\n            assert conflated_component in self.CONFLATED_COMPONENTS_MAPPING or any(\n                conflated_component == f\"{product}::{component}\"\n                for product, component in product_components.values()\n            ), f\"It should be possible to map {conflated_component}\"\n\n        classes = {\n            bug_id: component\n            for bug_id, component in classes.items()\n            if component in top_components\n        }\n\n        return classes, set(classes.values())\n\n    def is_meaningful(self, product, component):\n        return (\n            product in self.PRODUCTS\n            and component not in [\"General\", \"Untriaged\", \"Foxfooding\"]\n        ) or (\n            product in self.PRODUCT_COMPONENTS\n            and component in self.PRODUCT_COMPONENTS[product]\n        )\n\n    def get_meaningful_product_components(self, full_comp_tuples, threshold_ratio=100):\n        \"\"\"Filter out components which does not have more than 1% of the most common component.\n\n        Returns:\n            a set of tuples which have at least 1% of the most common tuple\n        \"\"\"\n        product_component_counts = Counter(full_comp_tuples).most_common()\n\n        max_count = product_component_counts[0][1]\n        threshold = max_count / threshold_ratio\n\n        active_product_components = bugzilla.get_active_product_components(\n            list(self.PRODUCTS) + list(self.PRODUCT_COMPONENTS)\n        )\n\n        return set(\n            product_component\n            for product_component, count in product_component_counts\n            if count > threshold and product_component in active_product_components\n        )\n\n    def get_feature_names(self):\n        return self.clf.named_steps[\"union\"].get_feature_names_out()\n\n    def check(self):\n        success = super().check()\n\n        # Get the number of bugs per full component to fasten up the check\n        bugs_number = get_product_component_count()\n\n        # Check number 1, check that the most meaningful product components\n        # still have at least a bug in this component. If the check is failing\n        # that could mean that:\n        # - A component has been renamed / removed\n        # - A component is not used anymore by developers\n\n        for product, component in self.meaningful_product_components:\n            full_comp = f\"{product}::{component}\"\n\n            if full_comp not in bugs_number.keys():\n                logger.warning(\n                    \"Component %r of product %r doesn't exists, failure\",\n                    component,\n                    product,\n                )\n                success = False\n\n            elif bugs_number[full_comp] <= 0:\n                logger.warning(\n                    \"Component %r of product %r have 0 bugs or less in it, failure\",\n                    component,\n                    product,\n                )\n                success = False\n\n        # Check number 2, check that conflated components in\n        # CONFLATED_COMPONENTS match at least one component which has more\n        # than 0 bugs\n\n        for conflated_component in self.CONFLATED_COMPONENTS:\n            matching_components = [\n                full_comp\n                for full_comp in bugs_number\n                if full_comp.startswith(conflated_component)\n            ]\n\n            if not matching_components:\n                logger.warning(\"%s doesn't match any component\", conflated_component)\n                success = False\n                continue\n\n            matching_components_values = [\n                bugs_number[full_comp]\n                for full_comp in matching_components\n                if bugs_number[full_comp] > 0\n            ]\n\n            if not matching_components_values:\n                logger.warning(\n                    \"%s should match at least one component with more than 0 bugs\",\n                    conflated_component,\n                )\n                success = False\n\n        # Check number 3, check that values of CONFLATED_COMPONENTS_MAPPING\n        # still exist as components and have more than 0 bugs\n\n        for full_comp in self.CONFLATED_COMPONENTS_MAPPING.values():\n            if full_comp not in bugs_number:\n                logger.warning(\n                    \"%s from conflated component mapping doesn't exists, failure\",\n                    full_comp,\n                )\n                success = False\n            elif bugs_number[full_comp] <= 0:\n                logger.warning(\n                    \"%s from conflated component mapping have less than 1 bug, failure\",\n                    full_comp,\n                )\n                success = False\n\n        # Check number 4, conflated components in CONFLATED_COMPONENTS either\n        # exist as components or are in CONFLATED_COMPONENTS_MAPPING\n\n        for conflated_component in self.CONFLATED_COMPONENTS:\n            in_mapping = conflated_component in self.CONFLATED_COMPONENTS_MAPPING\n\n            matching_components = [\n                full_comp\n                for full_comp in bugs_number\n                if full_comp.startswith(conflated_component)\n            ]\n\n            if not (matching_components or in_mapping):\n                logger.warning(\"It should be possible to map %s\", conflated_component)\n                success = False\n                continue\n\n        # Check number 5, there is no component with many bugs that is not in\n        # meaningful_product_components\n\n        # Recompute the meaningful components\n\n        def generate_meaningful_tuples():\n            for full_comp, count in bugs_number.items():\n                product, component = full_comp.split(\"::\", 1)\n\n                if not self.is_meaningful(product, component):\n                    continue\n\n                if count > 0:\n                    for i in range(count):\n                        yield (product, component)\n\n        meaningful_product_components = self.get_meaningful_product_components(\n            generate_meaningful_tuples(), threshold_ratio=10\n        )\n\n        if not meaningful_product_components.issubset(\n            self.meaningful_product_components\n        ):\n            logger.warning(\"Meaningful product components mismatch\")\n\n            new_meaningful_product_components = (\n                meaningful_product_components.difference(\n                    self.meaningful_product_components\n                )\n            )\n            logger.info(\n                \"New meaningful product components %r\",\n                new_meaningful_product_components,\n            )\n\n            success = False\n\n        return success\n\n    def get_extra_data(self):\n        return {\"conflated_components_mapping\": self.CONFLATED_COMPONENTS_MAPPING}\n", "path": "bugbug/models/component.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport logging\nfrom collections import Counter\nfrom datetime import datetime, timezone\n\nimport dateutil.parser\nimport xgboost\nfrom dateutil.relativedelta import relativedelta\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.feature_extraction import DictVectorizer\nfrom sklearn.pipeline import Pipeline\n\nfrom bugbug import bug_features, bugzilla, feature_cleanup, utils\nfrom bugbug.bugzilla import get_product_component_count\nfrom bugbug.model import BugModel\n\nlogging.basicConfig(level=logging.INFO)\nlogger = logging.getLogger(__name__)\n\n\nclass ComponentModel(BugModel):\n    PRODUCTS = {\n        \"Core\",\n        \"External Software Affecting Firefox\",\n        \"DevTools\",\n        \"Fenix\",\n        \"Firefox\",\n        \"Toolkit\",\n        \"WebExtensions\",\n        \"Firefox Build System\",\n    }\n\n    PRODUCT_COMPONENTS = {\n        \"Cloud Services\": {\"Server: Firefox Accounts\"},\n    }\n\n    CONFLATED_COMPONENTS = [\n        \"Core::Audio/Video\",\n        \"Core::DOM\",\n        \"Core::Graphics\",\n        \"Core::IPC\",\n        \"Core::JavaScript\",\n        \"Core::Layout\",\n        \"Core::Networking\",\n        \"Core::Print\",\n        \"Core::WebRTC\",\n        \"Toolkit::Password Manager\",\n        \"DevTools\",\n        \"External Software Affecting Firefox\",\n        \"WebExtensions\",\n        \"Firefox Build System\",\n        \"Fenix\",\n    ]\n\n    CONFLATED_COMPONENTS_MAPPING = {\n        \"Core::DOM\": \"Core::DOM: Core & HTML\",\n        \"Core::JavaScript\": \"Core::JavaScript Engine\",\n        \"Core::Print\": \"Core::Printing: Output\",\n        \"DevTools\": \"DevTools::General\",\n        \"External Software Affecting Firefox\": \"External Software Affecting Firefox::Other\",\n        \"WebExtensions\": \"WebExtensions::Untriaged\",\n        \"Firefox Build System\": \"Firefox Build System::General\",\n        \"Fenix\": \"Fenix::General\",\n    }\n\n    def __init__(self, lemmatization=False):\n        BugModel.__init__(self, lemmatization)\n\n        self.cross_validation_enabled = False\n        self.calculate_importance = False\n\n        feature_extractors = [\n            bug_features.HasSTR(),\n            bug_features.Severity(),\n            bug_features.Keywords(),\n            bug_features.HasCrashSignature(),\n            bug_features.HasURL(),\n            bug_features.HasW3CURL(),\n            bug_features.HasGithubURL(),\n            bug_features.Whiteboard(),\n            bug_features.Patches(),\n            bug_features.Landings(),\n        ]\n\n        cleanup_functions = [\n            feature_cleanup.fileref(),\n            feature_cleanup.url(),\n            feature_cleanup.synonyms(),\n        ]\n\n        self.extraction_pipeline = Pipeline(\n            [\n                (\n                    \"bug_extractor\",\n                    bug_features.BugExtractor(\n                        feature_extractors, cleanup_functions, rollback=True\n                    ),\n                ),\n            ]\n        )\n\n        self.clf = Pipeline(\n            [\n                (\n                    \"union\",\n                    ColumnTransformer(\n                        [\n                            (\"data\", DictVectorizer(), \"data\"),\n                            (\"title\", self.text_vectorizer(min_df=0.0001), \"title\"),\n                            (\n                                \"comments\",\n                                self.text_vectorizer(min_df=0.0001),\n                                \"comments\",\n                            ),\n                        ]\n                    ),\n                ),\n                (\n                    \"estimator\",\n                    xgboost.XGBClassifier(n_jobs=utils.get_physical_cpu_count()),\n                ),\n            ]\n        )\n\n        self.CONFLATED_COMPONENTS_INVERSE_MAPPING = {\n            v: k for k, v in self.CONFLATED_COMPONENTS_MAPPING.items()\n        }\n\n    def filter_component(self, product, component):\n        full_comp = f\"{product}::{component}\"\n\n        if full_comp in self.CONFLATED_COMPONENTS_INVERSE_MAPPING:\n            return self.CONFLATED_COMPONENTS_INVERSE_MAPPING[full_comp]\n\n        if (product, component) in self.meaningful_product_components:\n            return full_comp\n\n        for conflated_component in self.CONFLATED_COMPONENTS:\n            if full_comp.startswith(conflated_component):\n                return conflated_component\n\n        return None\n\n    def get_labels(self):\n        product_components = {}\n        for bug_data in bugzilla.get_bugs():\n            if dateutil.parser.parse(bug_data[\"creation_time\"]) < datetime.now(\n                timezone.utc\n            ) - relativedelta(years=2):\n                continue\n\n            product_components[bug_data[\"id\"]] = (\n                bug_data[\"product\"],\n                bug_data[\"component\"],\n            )\n\n        self.meaningful_product_components = self.get_meaningful_product_components(\n            (\n                (product, component)\n                for product, component in product_components.values()\n                if self.is_meaningful(product, component)\n            )\n        )\n\n        classes = {}\n        for bug_id, (product, component) in product_components.items():\n            component = self.filter_component(product, component)\n\n            if component:\n                classes[bug_id] = component\n\n        component_counts = Counter(classes.values()).most_common()\n        top_components = set(component for component, count in component_counts)\n\n        logger.info(\"%d components\", len(top_components))\n        for component, count in component_counts:\n            logger.info(\"%s: %d\", component, count)\n\n        # Assert there is at least one bug for each conflated component.\n        for conflated_component in self.CONFLATED_COMPONENTS:\n            assert any(\n                conflated_component == component\n                for component, count in component_counts\n            ), f\"There should be at least one bug matching {conflated_component}*\"\n\n        # Assert there is at least one bug for each component the conflated components are mapped to.\n        for conflated_component_mapping in self.CONFLATED_COMPONENTS_MAPPING.values():\n            assert any(\n                conflated_component_mapping == f\"{product}::{component}\"\n                for product, component in product_components.values()\n            ), f\"There should be at least one bug in {conflated_component_mapping}\"\n\n        # Assert all conflated components are either in conflated_components_mapping or exist as components.\n        for conflated_component in self.CONFLATED_COMPONENTS:\n            assert conflated_component in self.CONFLATED_COMPONENTS_MAPPING or any(\n                conflated_component == f\"{product}::{component}\"\n                for product, component in product_components.values()\n            ), f\"It should be possible to map {conflated_component}\"\n\n        classes = {\n            bug_id: component\n            for bug_id, component in classes.items()\n            if component in top_components\n        }\n\n        return classes, set(classes.values())\n\n    def is_meaningful(self, product, component):\n        return (\n            product in self.PRODUCTS\n            and component not in [\"General\", \"Untriaged\", \"Foxfooding\"]\n        ) or (\n            product in self.PRODUCT_COMPONENTS\n            and component in self.PRODUCT_COMPONENTS[product]\n        )\n\n    def get_meaningful_product_components(self, full_comp_tuples, threshold_ratio=100):\n        \"\"\"Filter out components which does not have more than 1% of the most common component.\n\n        Returns:\n            a set of tuples which have at least 1% of the most common tuple\n        \"\"\"\n        product_component_counts = Counter(full_comp_tuples).most_common()\n\n        max_count = product_component_counts[0][1]\n        threshold = max_count / threshold_ratio\n\n        active_product_components = bugzilla.get_active_product_components(\n            list(self.PRODUCTS) + list(self.PRODUCT_COMPONENTS)\n        )\n\n        return set(\n            product_component\n            for product_component, count in product_component_counts\n            if count > threshold and product_component in active_product_components\n        )\n\n    def get_feature_names(self):\n        return self.clf.named_steps[\"union\"].get_feature_names_out()\n\n    def check(self):\n        success = super().check()\n\n        # Get the number of bugs per full component to fasten up the check\n        bugs_number = get_product_component_count()\n\n        # Check number 1, check that the most meaningful product components\n        # still have at least a bug in this component. If the check is failing\n        # that could mean that:\n        # - A component has been renamed / removed\n        # - A component is not used anymore by developers\n\n        for product, component in self.meaningful_product_components:\n            full_comp = f\"{product}::{component}\"\n\n            if full_comp not in bugs_number.keys():\n                logger.warning(\n                    \"Component %r of product %r doesn't exists, failure\",\n                    component,\n                    product,\n                )\n                success = False\n\n            elif bugs_number[full_comp] <= 0:\n                logger.warning(\n                    \"Component %r of product %r have 0 bugs or less in it, failure\",\n                    component,\n                    product,\n                )\n                success = False\n\n        # Check number 2, check that conflated components in\n        # CONFLATED_COMPONENTS match at least one component which has more\n        # than 0 bugs\n\n        for conflated_component in self.CONFLATED_COMPONENTS:\n            matching_components = [\n                full_comp\n                for full_comp in bugs_number\n                if full_comp.startswith(conflated_component)\n            ]\n\n            if not matching_components:\n                logger.warning(\"%s doesn't match any component\", conflated_component)\n                success = False\n                continue\n\n            matching_components_values = [\n                bugs_number[full_comp]\n                for full_comp in matching_components\n                if bugs_number[full_comp] > 0\n            ]\n\n            if not matching_components_values:\n                logger.warning(\n                    \"%s should match at least one component with more than 0 bugs\",\n                    conflated_component,\n                )\n                success = False\n\n        # Check number 3, check that values of CONFLATED_COMPONENTS_MAPPING\n        # still exist as components and have more than 0 bugs\n\n        for full_comp in self.CONFLATED_COMPONENTS_MAPPING.values():\n            if full_comp not in bugs_number:\n                logger.warning(\n                    \"%s from conflated component mapping doesn't exists, failure\",\n                    full_comp,\n                )\n                success = False\n            elif bugs_number[full_comp] <= 0:\n                logger.warning(\n                    \"%s from conflated component mapping have less than 1 bug, failure\",\n                    full_comp,\n                )\n                success = False\n\n        # Check number 4, conflated components in CONFLATED_COMPONENTS either\n        # exist as components or are in CONFLATED_COMPONENTS_MAPPING\n\n        for conflated_component in self.CONFLATED_COMPONENTS:\n            in_mapping = conflated_component in self.CONFLATED_COMPONENTS_MAPPING\n\n            matching_components = [\n                full_comp\n                for full_comp in bugs_number\n                if full_comp.startswith(conflated_component)\n            ]\n\n            if not (matching_components or in_mapping):\n                logger.warning(\"It should be possible to map %s\", conflated_component)\n                success = False\n                continue\n\n        # Check number 5, there is no component with many bugs that is not in\n        # meaningful_product_components\n\n        # Recompute the meaningful components\n\n        def generate_meaningful_tuples():\n            for full_comp, count in bugs_number.items():\n                product, component = full_comp.split(\"::\", 1)\n\n                if not self.is_meaningful(product, component):\n                    continue\n\n                if count > 0:\n                    for i in range(count):\n                        yield (product, component)\n\n        meaningful_product_components = self.get_meaningful_product_components(\n            generate_meaningful_tuples(), threshold_ratio=10\n        )\n\n        if not meaningful_product_components.issubset(\n            self.meaningful_product_components\n        ):\n            logger.warning(\"Meaningful product components mismatch\")\n\n            new_meaningful_product_components = (\n                meaningful_product_components.difference(\n                    self.meaningful_product_components\n                )\n            )\n            logger.info(\n                \"New meaningful product components %r\",\n                new_meaningful_product_components,\n            )\n\n            success = False\n\n        return success\n\n    def get_extra_data(self):\n        return {\"conflated_components_mapping\": self.CONFLATED_COMPONENTS_MAPPING}\n", "path": "bugbug/models/component.py"}]}